Merge pull request #1694 from UniversityOfHelsinkiCS/trunk

Trying to fix updater api heisenbug

.circleci/config.yml

@@ -30,12 +30,12 @@ jobs:
             if [[ -n "$CIRCLE_TAG" ]]; then
               echo 'export TAG=latest' >> $BASH_ENV;
               echo 'export DEPLOY=true' >> $BASH_ENV;
-              echo 'export CACHE_TAG=staging' >> $BASH_ENV;
+              echo 'export CACHE_TAG=latest' >> $BASH_ENV;
               echo 'USING RELEASE CONFIG';
             elif [[ "$CIRCLE_BRANCH" = "master" && -z "$CIRCLE_PULL_REQUEST" ]]; then
               echo 'export TAG=staging' >> $BASH_ENV;
               echo 'export DEPLOY=true' >> $BASH_ENV;
-              echo 'export CACHE_TAG=trunk' >> $BASH_ENV;
+              echo 'export CACHE_TAG=staging' >> $BASH_ENV;
               echo 'USING STAGING CONFIG';
             else
               echo 'export TAG=trunk' >> $BASH_ENV;
@@ -53,9 +53,9 @@ jobs:
           key: npmcache-v2-{{ checksum "package-lock.json" }}
           paths:
             - ~/.cache
-      - run: TAG=${CACHE_TAG} npm run docker:ci -- pull --ignore-pull-failures
+      - run: npm run docker:ci -- pull --ignore-pull-failures
       - run: npm run docker:ci -- build
-      - run: npm run docker:ci -- up -d db user_db db_kone
+      - run: npm run docker:ci -- up -d db user_db db_kone analytics_db
       - *check_exit_statuses
       - run: git clone --depth 1 git@github.com:UniversityOfHelsinkiCS/anonyymioodi.git
       - run: source ./scripts/scripts.sh && init_dirs && db_anon_setup_full
@@ -67,10 +67,20 @@ jobs:
           command: |
             docker ps -a && npm run docker:ci -- logs -f
       - *check_exit_statuses
-      - run: npm run test_services
+      - run:
+          name: test services
+          command: |
+            if [ "${DEPLOY}" != "true" ]; then
+              npm run test_services
+            fi
       - *check_exit_statuses
-      - run: timeout 5m bash -c 'until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf .; sleep 5; done'
-      - run: npm run cypress:ci
+      - run:
+          name: run cypress tests
+          command: |
+            if [ "${DEPLOY}" != "true" ]; then
+              timeout 5m bash -c 'until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf .; sleep 5; done'
+              npm run cypress:ci
+            fi
       - *check_exit_statuses
       - store_artifacts:
           path: cypress/videos

.dependabot/config.yml

@@ -0,0 +1,6 @@
+version: 1
+update_configs:
+  - package_manager: "javascript"
+    directory: "/"
+    update_schedule: "weekly"
+    target_branch: "trunk"

README.md

@@ -1,6 +1,6 @@
 # oodikone
 
-[![Build Status](https://travis-ci.org/UniversityOfHelsinkiCS/oodikone.svg?branch=master)](https://travis-ci.org/UniversityOfHelsinkiCS/oodikone)
+[![CircleCI](https://circleci.com/gh/UniversityOfHelsinkiCS/oodikone/tree/master.svg?style=svg)](https://circleci.com/gh/UniversityOfHelsinkiCS/oodikone/tree/master)
 
 Simple CLI tool for setting up the development environment for Oodikone. The entire development environment runs inside of a Docker network that's defined in the docker-compose.yml file. The specifics of what this means for setting up the environment and accessing logs is discussed in a later section.
 
@@ -44,10 +44,10 @@ Run E2E tests in with `npm run cypress:run` or `npm run cypress:open` with visua
 ## Docker commands
 
 The development environment is entirely configured in the docker-compose.yml file located in this repository. The file defines the containers/environments for:
+
 - Frontend
 - Backend
 - Userservice
-- Oodilearn
 - The databases used by the services
 - Adminer & other dev tools
 
@@ -63,6 +63,12 @@ npm run docker:up
 
 #### With real data:
 
+```
+npm run start:real
+```
+
+OR
+
 ```
 docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml -f ./docker/docker-compose.dev.real.yml up -d
 ```
@@ -82,22 +88,21 @@ npm run docker:restart:backend  # just backend
 ```
 
 ### View the containers in the running environment
+
 ```
 docker-compose ps
 ```
 
 ### View logs
 
-
 ```
 npm run docker:logs          # all
 
 npm run docker:logs:backend  # just backend
 ```
 
-
-
 ### Attach a terminal shell to a container
+
 ```
 docker exec -it backend bash
 
@@ -105,6 +110,7 @@ docker exec -it <container> <command>
 ```
 
 ### Use `psql` or other database CLI tools
+
 ```
 docker exec -it -u postgres oodi_db psql -d tkt_oodi
 
@@ -118,6 +124,7 @@ docker exec -it -u <username> <container> psql -d <db name>
 This is how the setup script fetches the database dump from production servers. It will require you to have access to both melkki.cs.helsinki.fi as well as oodikone.cs.helsinki.fi. The command uses `scp` to transfer all backup files recursively from a known location on the production server by using the melkki server as a proxy, allowing you to get the dump even when you're not connected to the university network.
 
 #### Command
+
 ```
 scp -r -o ProxyCommand="ssh -W %h:%p melkki.cs.helsinki.fi" oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/* ./backups/
 
@@ -137,7 +144,7 @@ scp -r -o ProxyCommand="ssh -W %h:%p melkki.cs.helsinki.fi" oodikone.cs.helsinki
 >
 > If you use password authentication instead, try with:
 >
->  ```
+> ```
 > scp -o ProxyCommand="ssh -W %h:%p user1@server1" > user2@server2:/<remotePath> <localpath>
 > ```
 >

cypress/integration/Course_population.js

@@ -0,0 +1,67 @@
+/// <reference types="Cypress" />
+
+describe('Course population tests', () => {
+  beforeEach(() => {
+    cy.server({
+      onAnyRequest: function (route, proxy) {
+        if (Cypress.config().baseUrl.includes("http://nginx/")) {
+          proxy.xhr.setRequestHeader('uid', 'tktl')
+          proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
+          proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
+          proxy.xhr.setRequestHeader('edupersonaffiliation', 'asdasd')
+        }
+      }
+    })
+    console.log(Cypress.config().baseUrl)
+    cy.visit(Cypress.config().baseUrl)
+    cy.contains("Course statistics").click()
+    cy.contains("Search for courses")
+  })
+
+  it('Can find course population', () => {
+    cy.url().should('include', '/coursestatistics')
+    cy.contains("Search for courses")
+    cy.get("input[placeholder='Search by entering a course code']").type('TKT20003')
+    cy.contains("tr", "TKT20003").click()
+    cy.contains("Fetch statistics").should('be.enabled').click()
+
+    cy.contains("Käyttöjärjestelmät")
+    cy.contains("TKT20003")
+
+    cy.get(':nth-child(2) > :nth-child(1) > div > .item > .level').click()
+    cy.contains('Population of course Käyttöjärjestelmät 2017-18')
+    cy.contains('TKT20003, 582219, 582640, 582497')
+    cy.contains('Students (all=39)')
+    cy.contains('add').click()
+    cy.contains('Select students that have grade')
+
+    cy.contains('show').click()
+    cy.contains('013614218')
+    cy.contains('014022579')
+  })
+
+  it('Filters are working', ()=> {
+    cy.url().should('include', '/coursestatistics')
+    cy.contains("Search for courses")
+    cy.get("input[placeholder='Search by entering a course code']").type('TKT10004')
+    cy.get(':nth-child(2) > .ten > :nth-child(1)').click()
+    cy.contains("Fetch statistics").should('be.enabled').click()
+
+    cy.get(':nth-child(1) > :nth-child(1) > div > .item > .level').click()
+    cy.contains('Population of course Tietokantojen perusteet 2017-18')
+    cy.contains('add').click()
+    
+    // sex filter
+    cy.get('.inline > :nth-child(2) > :nth-child(1) > label').click()
+    cy.get(':nth-child(8) > .form > .inline > :nth-child(3) > .ui').click()
+
+    cy.contains('Students (all=100)')
+    cy.get(':nth-child(4) > .segment > .selectable > tbody > :nth-child(2) > :nth-child(2)').click()
+
+    cy.contains('Students (all=17)')
+
+    cy.get(':nth-child(5) > .segment > .selectable > tbody > :nth-child(3) > :nth-child(1)').click()
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma, KH50_005').should('not.exist')
+    cy.contains('Students (all=1)')
+  })
+})

cypress/integration/Course_statistics.js

@@ -1,9 +1,10 @@
+/// <reference types="Cypress" />
 
 describe('Course Statistics tests', () => {
   beforeEach(() => {
     cy.server({
       onAnyRequest: function (route, proxy) {
-        if (Cypress.config().baseUrl.includes("http://localhost:1337/")) {
+        if (Cypress.config().baseUrl.includes("http://nginx/")) {
           proxy.xhr.setRequestHeader('uid', 'tktl')
           proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
           proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
@@ -72,4 +73,436 @@ describe('Course Statistics tests', () => {
     cy.contains("Search for courses")
   })
 
+  describe('When searching unified course stats', () => {
+    beforeEach(() => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains("Search for courses")
+      cy.get("input[name='unifyOpenUniCourses']").parent().click()
+      cy.get("input[placeholder='Search by entering a course code']").type('TKT10002')
+      cy.contains("td", "TKT10002, AYTKT10002").click()
+      cy.contains("Fetch statistics").should('be.enabled').click()
+      cy.contains("Search for courses").should('not.exist')
+      cy.contains("TKT10002, 581325, A581325, AYTKT10002 Ohjelmoinnin perusteet")
+    })
+
+    const yearRange = { from: "2000-01", to: "2017-18" }
+    const cumulativeTableContents = [
+      // [time, passed, failed, passrate]
+      ["2017-18", 175, 29, "85.78 %"],
+      ["2016-17", 66, 2, "97.06 %"],
+      ["2015-16", 27, 2, "93.10 %"],
+      ["2014-15", 16, 0, "100.00 %"],
+      ["2013-14", 6, 2, "75.00 %"],
+    ]
+    const studentTableContents = [
+      // time, students, passedfirsttry, passedretry, pass%, failedfirsttry, failedretry, fail%]
+      ["2017-18", 195, 131, 43, "89.23 %", 20, 1, "10.77 %"],
+      ["2016-17", 67, 59, 6, "97.01 %", 2, 0, "2.99 %"],
+      ["2015-16", 27, 24, 2, "96.30 %", 1, 0, "3.70 %"],
+      ["2014-15", 16, 16, 0, "100.00 %", 0, 0, "0.00 %"],
+      ["2013-14", 5, 2, 3, "100.00 %", 0, 0, "0.00 %"],
+    ]
+    const gradesTableContents = [
+      // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
+      ["2017-18", 204, 29, 5, 8, 10, 16, 127, 9],
+      ["2016-17", 68, 2, 0, 1, 6, 10, 49, 0],
+      ["2015-16", 29, 2, 1, 1, 0, 5, 16, 4],
+      ["2014-15", 16, 0, 0, 0, 1, 0, 11, 4],
+      ["2013-14", 8, 2, 0, 0, 2, 0, 4, 0],
+    ]
+
+    it('shows stats', () => {
+      // Time range
+      cy.get("div[name='fromYear']").within(() => {
+        cy.get("div[role='option']").first().should("have.text", yearRange.to)
+        cy.contains("div[role='option']", yearRange.from).should('have.class', 'selected')
+        cy.get("div[role='option']").last().should("have.text", "2000-01")
+        cy.get("div[role='option']").should('have.length', 18)
+      })
+      cy.get("div[name='toYear']").within(() => {
+        cy.get("div[role='option']").first().should("have.text", "2017-18")
+        cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
+        cy.get("div[role='option']").last().should("have.text", yearRange.from)
+        cy.get("div[role='option']").should('have.length', 18)
+      })
+      cy.contains("Show population").should('be.enabled')
+
+      // Filters
+      const primaryContents = [
+        // [studentcount, label, code]
+        [179, "Tietojenkäsittelytieteen kandiohjelma", "KH50_005"],
+        [64, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
+        [24, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
+        [16, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
+        [9, "Matematiikan koulutusohjelma", "520070"],
+      ]
+      const totalStudyProgrammes = 20
+      cy.get("div[name='primary']").within(() => {
+        primaryContents.forEach(([count, label, code], i) => {
+          cy.get("div[role='option']").eq(i).within(() => {
+            cy.contains(label)
+            cy.contains("div>div:last-child>div.label", code)
+            cy.contains("div>div:first-child>div.label", count)
+          })
+        })
+        cy.get("div[role='option']").eq(10).within(() => {
+          cy.contains("Muu")
+          cy.contains("div>div:last-child>div.label", "OTHER")
+          cy.contains("div>div:first-child>div.label", "2")
+        })
+        cy.get("div[role='option']").should('have.length', totalStudyProgrammes)
+      })
+      const comparisonContents = [
+        [289, "All", "ALL"],
+        ...primaryContents
+      ]
+      cy.get("div[name='comparison']").within(() => {
+        comparisonContents.forEach(([count, label, code], i) => {
+          cy.get("div[role='option']").eq(i).within(() => {
+            cy.contains(label)
+            cy.contains("div>div:last-child>div.label", code)
+            cy.contains("div>div:first-child>div.label", count)
+          })
+        })
+        cy.get("div[role='option']").eq(11).within(() => {
+          cy.contains("Muu")
+          cy.contains("div>div:last-child>div.label", "OTHER")
+          cy.contains("div>div:first-child>div.label", "2")
+        })
+        cy.get("div[role='option']").should('have.length', totalStudyProgrammes+1) // +1 comes from already selected option
+      })
+      cy.contains("Select excluded study programmes").should("be.disabled")
+
+      // Statistics
+      cy.contains("#CourseStatPanes a.item", "Table").click()
+      cy.contains("#CourseStatPanes a.item", "Cumulative").click()
+      cy.get("#CourseStatPanes h3+table>tbody").within(() => {
+        cumulativeTableContents.forEach((values, trIndex) => {
+          cy.get("tr").eq(trIndex).within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get("td").eq(tdIndex).contains(value)
+            })
+          })
+        })
+        cy.get("tr").should('have.length', 13)
+      })
+
+      cy.contains("#CourseStatPanes a.item", "Table").click()
+      cy.contains("#CourseStatPanes a.item", "Student").click()
+      cy.get("#CourseStatPanes h3+table>tbody").within(() => {
+        studentTableContents.forEach((values, trIndex) => {
+          cy.get("tr").eq(trIndex).within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get("td").eq(tdIndex).contains(value)
+            })
+          })
+        })
+        cy.get("tr").should('have.length', 13)
+      })
+
+      cy.contains("#CourseStatPanes a.item", "Table").click()
+      cy.contains("#CourseStatPanes a.item", "Grades").click()
+      cy.get("#CourseStatPanes h3+table>tbody").within(() => {
+        gradesTableContents.forEach((values, trIndex) => {
+          cy.get("tr").eq(trIndex).within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get("td").eq(tdIndex).contains(value)
+            })
+          })
+        })
+        cy.get("tr").should('have.length', 13)
+      })
+    })
+
+    it('after changing time range shows same stats', () => {
+      const newYearRange = { from: "2014-15", to: "2016-17" }
+      cy.get("div[name='fromYear']").click().within(() => {
+        cy.contains(newYearRange.from).click()
+      })
+      cy.get("div[name='toYear']").click().within(() => {
+        cy.contains(newYearRange.to).click()
+      })
+
+      // Time range
+      cy.get("div[name='fromYear']").within(() => {
+        cy.get("div[role='option']").first().should("have.text", newYearRange.to)
+        cy.contains("div[role='option']", newYearRange.from).should('have.class', 'selected')
+        cy.get("div[role='option']").last().should("have.text", "2000-01")
+        cy.get("div[role='option']").should('have.length', 17)
+      })
+      cy.get("div[name='toYear']").within(() => {
+        cy.get("div[role='option']").first().should("have.text", "2017-18")
+        cy.contains("div[role='option']", newYearRange.to).should('have.class', 'selected')
+        cy.get("div[role='option']").last().should("have.text", newYearRange.from)
+        cy.get("div[role='option']").should('have.length', 4)
+      })
+      cy.contains("Show population").should('be.enabled')
+
+      // Filters
+      const primaryContentsAfterTimeChange = [
+        // [studentcount, label, code]
+        [79, "Tietojenkäsittelytieteen kandiohjelma", "KH50_005"],
+        [15, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
+        [10, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
+        [7, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
+      ]
+      const totalStudyProgrammesAfterTimeChange = 13
+      cy.get("div[name='primary']").within(() => {
+        primaryContentsAfterTimeChange.forEach(([count, label, code], i) => {
+          cy.get("div[role='option']").eq(i).within(() => {
+            cy.contains(label)
+            cy.contains("div>div:last-child>div.label", code)
+            cy.contains("div>div:first-child>div.label", count)
+          })
+        })
+        cy.get("div[role='option']").eq(5).within(() => {
+          cy.contains("Muu")
+          cy.contains("div>div:last-child>div.label", "OTHER")
+          cy.contains("div>div:first-child>div.label", "2")
+        })
+        cy.get("div[role='option']").should('have.length', totalStudyProgrammesAfterTimeChange)
+      })
+      const comparisonContentsAfterTimeChange = [
+        [107, "All", "ALL"],
+        ...primaryContentsAfterTimeChange
+      ]
+      cy.get("div[name='comparison']").within(() => {
+        comparisonContentsAfterTimeChange.forEach(([count, label, code], i) => {
+          cy.get("div[role='option']").eq(i).within(() => {
+            cy.contains(label)
+            cy.contains("div>div:last-child>div.label", code)
+            cy.contains("div>div:first-child>div.label", count)
+          })
+        })
+        cy.get("div[role='option']").eq(6).within(() => {
+          cy.contains("Muu")
+          cy.contains("div>div:last-child>div.label", "OTHER")
+          cy.contains("div>div:first-child>div.label", "2")
+        })
+        cy.get("div[role='option']").should('have.length', totalStudyProgrammesAfterTimeChange+1) // +1 comes from already selected option
+      })
+      cy.contains("Select excluded study programmes").should("be.disabled")
+
+      // Statistics
+      const timeRangeFilter = (timeRange) => ([time]) => {
+        return timeRange.from <= time && time <= timeRange.to
+      }
+      cy.contains("#CourseStatPanes a.item", "Table").click()
+      cy.contains("#CourseStatPanes a.item", "Cumulative").click()
+      cy.get("#CourseStatPanes h3+table>tbody").within(() => {
+        cumulativeTableContents.filter(timeRangeFilter(newYearRange)).forEach((values, trIndex) => {
+          cy.get("tr").eq(trIndex).within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get("td").eq(tdIndex).contains(value)
+            })
+          })
+        })
+        cy.get("tr").should('have.length', 3)
+      })
+
+      cy.contains("#CourseStatPanes a.item", "Table").click()
+      cy.contains("#CourseStatPanes a.item", "Student").click()
+      cy.get("#CourseStatPanes h3+table>tbody").within(() => {
+        studentTableContents.filter(timeRangeFilter(newYearRange)).forEach((values, trIndex) => {
+          cy.get("tr").eq(trIndex).within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get("td").eq(tdIndex).contains(value)
+            })
+          })
+        })
+        cy.get("tr").should('have.length', 3)
+      })
+
+      cy.contains("#CourseStatPanes a.item", "Table").click()
+      cy.contains("#CourseStatPanes a.item", "Grades").click()
+      cy.get("#CourseStatPanes h3+table>tbody").within(() => {
+        gradesTableContents.filter(timeRangeFilter(newYearRange)).forEach((values, trIndex) => {
+          cy.get("tr").eq(trIndex).within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get("td").eq(tdIndex).contains(value)
+            })
+          })
+        })
+        cy.get("tr").should('have.length', 3)
+      })
+    })
+
+    it('after changing filters shows different stats', () => {
+      cy.get("div[name='primary']").click().within(() => {
+        cy.contains("div[role='option']", "Tietojenkäsittelytieteen kandiohjelma").click()
+      })
+      cy.get("body").type("{esc}") // close dropdown
+
+      // Time range (should not have changed)
+      cy.get("div[name='fromYear']").within(() => {
+        cy.get("div[role='option']").first().should("have.text", yearRange.to)
+        cy.contains("div[role='option']", yearRange.from).should('have.class', 'selected')
+        cy.get("div[role='option']").last().should("have.text", "2000-01")
+        cy.get("div[role='option']").should('have.length', 18)
+      })
+      cy.get("div[name='toYear']").within(() => {
+        cy.get("div[role='option']").first().should("have.text", "2017-18")
+        cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
+        cy.get("div[role='option']").last().should("have.text", yearRange.from)
+        cy.get("div[role='option']").should('have.length', 18)
+      })
+      cy.contains("Show population").should('be.enabled')
+
+      // Filters
+      const primaryContentsAfterFilterChange = [
+        // [studentcount, label, code]
+        [289, "All", "ALL"],
+        [64, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
+        [24, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
+        [16, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
+        [9, "Matematiikan koulutusohjelma", "520070"],
+      ]
+      const totalStudyProgrammesAfterFilterChange = 20
+      cy.get("div[name='primary']").within(() => {
+        primaryContentsAfterFilterChange.forEach(([count, label, code], i) => {
+          cy.get("div[role='option']").eq(i).within(() => {
+            cy.contains(label)
+            cy.contains("div>div:last-child>div.label", code)
+            cy.contains("div>div:first-child>div.label", count)
+          })
+        })
+        cy.get("div[role='option']").should('have.length', totalStudyProgrammesAfterFilterChange)
+      })
+      const comparisonContentsAfterFilterChange = [
+        // [studentcount, label, code]
+        [289, "All", "ALL"],
+        [179, "Tietojenkäsittelytieteen kandiohjelma", "KH50_005"],
+        [122, "Excluded", "EXCLUDED"],
+        [64, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
+        [24, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
+        [16, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
+        [9, "Matematiikan koulutusohjelma", "520070"],
+      ]
+      cy.get("div[name='comparison']").within(() => {
+        comparisonContentsAfterFilterChange.forEach(([count, label, code], i) => {
+          cy.get("div[role='option']").eq(i).within(() => {
+            cy.contains(label)
+            cy.contains("div>div:last-child>div.label", code)
+            cy.contains("div>div:first-child>div.label", count)
+          })
+        })
+        cy.get("div[role='option']").should('have.length', totalStudyProgrammesAfterFilterChange+2) // +2 comes from EXCLUDED and the filtered programme
+      })
+      cy.contains("Select excluded study programmes").should("be.enabled").click()
+
+      // Statistics
+      cy.contains("#CourseStatPanes a.item", "Table").click()
+      cy.contains("#CourseStatPanes a.item", "Cumulative").click()
+      cy.get("#PrimaryDataTable h3+table>tbody").within(() => {
+        const cumulativeTableContents = [
+          // [time, passed, failed, passrate]
+          ["2017-18", 126, 10, "92.65 %"],
+          ["2016-17", 56, 1, "98.25 %"],
+          ["2015-16", 20, 1, "95.24 %"],
+          ["2014-15", 6, 0, "100.00 %"],
+        ]
+        cumulativeTableContents.forEach((values, trIndex) => {
+          cy.get("tr").eq(trIndex).within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get("td").eq(tdIndex).contains(value)
+            })
+          })
+        })
+        cy.get("tr").should('have.length', 13)
+      })
+      cy.get("#ComparisonDataTable h3+table>tbody").within(() => {
+        const cumulativeTableContents = [
+          // [time, passed, failed, passrate]
+          ["2017-18", 52, 19, "73.24 %"],
+          ["2016-17", 15, 1, "93.75 %"],
+          ["2015-16", 9, 1, "90.00 %"],
+          ["2014-15", 11, 0, "100.00 %"],
+        ]
+        cumulativeTableContents.forEach((values, trIndex) => {
+          cy.get("tr").eq(trIndex).within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get("td").eq(tdIndex).contains(value)
+            })
+          })
+        })
+        cy.get("tr").should('have.length', 13)
+      })
+
+      cy.contains("#CourseStatPanes a.item", "Table").click()
+      cy.contains("#CourseStatPanes a.item", "Student").click()
+      cy.get("#PrimaryDataTable h3+table>tbody").within(() => {
+        const studentTableContents = [
+          // time, students, passedfirsttry, passedretry, pass%, failedfirsttry, failedretry, fail%]
+          ["2017-18", 131, 86, 40, "96.18 %", 5, 0, "3.82 %"],
+          ["2016-17", 56, 51, 4, "98.21 %", 1, 0, "1.79 %"],
+          ["2015-16", 19, 17, 2, "100.00 %", 0, 0, "0.00 %"],
+          ["2014-15", 6, 6, 0, "100.00 %", 0, 0, "0.00 %"],
+        ]
+        studentTableContents.forEach((values, trIndex) => {
+          cy.get("tr").eq(trIndex).within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get("td").eq(tdIndex).contains(value)
+            })
+          })
+        })
+        cy.get("tr").should('have.length', 13)
+      })
+      cy.get("#ComparisonDataTable h3+table>tbody").within(() => {
+        const studentTableContents = [
+          // time, students, passedfirsttry, passedretry, pass%, failedfirsttry, failedretry, fail%]
+          ["2017-18", 67, 48, 3, "76.12 %", 15, 1, "23.88 %"],
+          ["2016-17", 15, 11, 3, "93.33 %", 1, 0, "6.67 %"],
+          ["2015-16", 10, 9, 0, "90.00 %", 1, 0, "10.00 %"],
+          ["2014-15", 11, 11, 0, "100.00 %", 0, 0, "0.00 %"],
+        ]
+        studentTableContents.forEach((values, trIndex) => {
+          cy.get("tr").eq(trIndex).within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get("td").eq(tdIndex).contains(value)
+            })
+          })
+        })
+        cy.get("tr").should('have.length', 13)
+      })
+
+      cy.contains("#CourseStatPanes a.item", "Table").click()
+      cy.contains("#CourseStatPanes a.item", "Grades").click()
+      cy.get("#PrimaryDataTable h3+table>tbody").within(() => {
+        const gradesTableContents = [
+          // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
+          ["2017-18", 136, 10, 3, 4, 4, 4, 102, 9],
+          ["2016-17", 57, 1, 0, 1, 6, 8, 41, 0],
+          ["2015-16", 21, 1, 1, 1, 0, 5, 11, 2],
+          ["2014-15", 6, 0, 0, 0, 0, 0, 3, 3],
+          ["2013-14", 1, 0, 0, 0, 0, 0, 1, 0],
+        ]
+        gradesTableContents.forEach((values, trIndex) => {
+          cy.get("tr").eq(trIndex).within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get("td").eq(tdIndex).contains(value)
+            })
+          })
+        })
+        cy.get("tr").should('have.length', 13)
+      })
+      cy.get("#ComparisonDataTable h3+table>tbody").within(() => {
+        const gradesTableContents = [
+          // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
+          ["2017-18", 71, 19, 2, 4, 6, 13, 27, 0],
+          ["2016-17", 16, 1, 0, 0, 0, 3, 12, 0],
+          ["2015-16", 10, 1, 0, 0, 0, 0, 7, 2],
+          ["2014-15", 11, 0, 0, 0, 1, 0, 9, 1],
+          ["2013-14", 8, 2, 0, 0, 2, 0, 4, 0],
+        ]
+        gradesTableContents.forEach((values, trIndex) => {
+          cy.get("tr").eq(trIndex).within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get("td").eq(tdIndex).contains(value)
+            })
+          })
+        })
+        cy.get("tr").should('have.length', 13)
+      })
+    })
+  })
 })

cypress/integration/Population_statistics.js

@@ -1,9 +1,16 @@
+/// <reference types="Cypress" />
+
+const setPopStatsUntil = (until) => {
+  cy.contains("Advanced settings").siblings().get('.toggle').click()
+  cy.contains("Statistics until").siblings().get('.rdt').get('input').eq(1).click().clear().type(until)
+  cy.contains("Fetch population with new settings").click()
+}
 
 describe('Population Statistics tests', () => {
   beforeEach(() => {
     cy.server({
       onAnyRequest: function (route, proxy) {
-        if (Cypress.config().baseUrl.includes("http://localhost:1337/")) {
+        if (Cypress.config().baseUrl.includes("http://nginx/")) {
           proxy.xhr.setRequestHeader('uid', 'tktl')
           proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
           proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
@@ -24,9 +31,7 @@ describe('Population Statistics tests', () => {
       expect(students).to.equal(assertion)
     })
     cy.contains("Credit accumulation").siblings().within(() => {
-      cy.get(".highcharts-series-group").within(() => {
-        cy.get("path").its('length').should('be.equal', (students * 2) + 2) // For each student there should be 2 paths in the graph + 2 for the scrollbar
-      })
+      cy.get(".highcharts-series-group").find("path").should('have.length', students ? (students * 2) + 2 : 0) // For each student there should be 2 paths in the graph + 2 for the scrollbar
     })
   }
 
@@ -53,27 +58,19 @@ describe('Population Statistics tests', () => {
       cy.get("@enrollmentSelect").should('not.have.value', beforeVal)
     })
 
-
-    cy.contains("Statistics until").siblings().within(() => {
-      cy.get("input").click()
-      cy.get("table").contains(`${new Date().getFullYear()}`).click()
-      cy.contains("2018").click()
-      cy.contains("Oct").click()
-      cy.get("input").should("have.value", "October 2018")
-    })
     cy.contains("Select study programme").click().siblings().contains("Tietojenkäsittelytieteen koulutusohjelma").click()
     cy.contains("Select degree").click().siblings().contains("Luonnontieteiden kandidaatti")
-    cy.get(".toggle").click()
-    cy.contains("Exchange students")
-    cy.contains("See population").should('be.enabled')
   })
 
   it('Population statistics is usable on general level', () => {
-    cy.contains("Select study programme").click().siblings().contains("Kasvatustieteiden kandiohjelma").click()
+    cy.contains("Select study programme").click().siblings().contains("Tietojenkäsittelytieteen maisteriohjelma").click()
     cy.contains("See population").click()
+
+    setPopStatsUntil('September 2019')
+
     cy.get(".card").within(() => {
-      cy.contains("Kasvatustieteiden kandiohjelma")
-      cy.contains("Sample size: 26 students")
+      cy.contains("Tietojenkäsittelytieteen maisteriohjelma")
+      cy.contains("Sample size: 29 students")
       cy.contains("Excludes exchange students")
       cy.contains("Excludes students with cancelled study right")
     })
@@ -81,72 +78,67 @@ describe('Population Statistics tests', () => {
     cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
       cy.get("tr").its('length').should('be.gte', 10)
       cy.route('/api/v3/courseyearlystats**').as('coursePage')
-      cy.contains("Opiskelijan digitaidot: orientaatio").siblings().find(".level").click()
+      cy.contains("Laskennan mallit")
+      cy.get(':nth-child(2) > .iconCell > p > .item > .level').click({ force: true })
       cy.wait('@coursePage')
       cy.url().should('include', '/coursestatistics')
     })
-    cy.contains("DIGI-000A")
+    cy.contains("TKT20005")
     cy.go("back")
 
     cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
-      cy.contains("number at least").siblings().within(() => cy.get("input").clear().type("0"))
-      cy.contains("Matematiikan didaktiikka").siblings().eq(2).should("have.text", '9')
+      cy.contains("Ohjelmoinnin perusteet").siblings().eq(2).should("have.text", "15")
     })
 
     cy.contains("add").click()
     cy.contains("Add filters").siblings().within(() => {
-      cy.get(".form").should('have.length', 10)
+      cy.get(".form").should('have.length', 9)
     })
 
-    checkAmountOfStudents(26)
+    checkAmountOfStudents(29)
 
     let filteredStudents = 1328493
     cy.contains("Credits gained during first").parentsUntil(".tab").get("table").within(() => {
-      cy.get("tr").eq(1).find("td").eq(1).invoke("text").then(text => filteredStudents = Number(text))
+      cy.get("tr").eq(2).find("td").eq(1).invoke("text").then(text => filteredStudents = Number(text))
       cy.route('POST', '/api/v2/populationstatistics/courses**').as('courseData')
-      cy.get("tr").eq(1).click()
+      cy.get("tr").eq(2).click()
       cy.wait('@courseData')
     }).then(() => {
       checkAmountOfStudents(filteredStudents)
     })
 
     cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
-      cy.contains("number at least").siblings().within(() => cy.get("input").clear())
-      cy.contains("number at least").siblings().within(() => cy.get("input").type("0"))
-      cy.contains("Matematiikan didaktiikka").siblings().eq(2).should("have.text", '1')
+      cy.contains("Ohjelmoinnin perusteet").siblings().eq(2).should("have.text", "1")
     })
 
     cy.contains("button", "show").click()
     cy.contains("Student names hidden").click()
-    cy.contains("Oinonen").siblings().eq(2).click()
-    cy.contains("Oinonen").invoke('text').then((text) => expect(text).to.equal('Oinonen Heidi Eeva Elisabet, 014473717'))
+    cy.contains("Luoto").siblings().eq(2).click()
+    cy.contains("Luoto").invoke('text').then((text) => expect(text).to.equal('Luoto Veli-Matti, 014824094'))
   })
 
   it('Student list checking works as intended', () => {
-    cy.contains("Select study programme").click().siblings().contains("Kasvatustieteiden kandiohjelma").click()
+    cy.contains("Select study programme").click().siblings().contains("Tietojenkäsittelytieteen maisteriohjelma").click()
     cy.contains("See population").click()
     cy.contains("button", "show").click()
-    cy.contains("010483918")
+    cy.contains("010111264")
     cy.contains("666666666").should('not.exist')
     cy.contains('button', "Check studentnumbers").click()
     cy.contains('Check for studentnumbers')
-    cy.get('textarea').type("010483918").type('{enter}').type("666666666")
+    cy.get('textarea').type("010111264").type('{enter}').type("666666666")
     cy.contains('button', 'check students').click()
-    cy.contains('form', 'Results').within(e => {
-      cy.get('div').eq(0).within(e => {
-        cy.contains('student numbers in list but not in oodi')
-        cy.contains('666666666')
-      })
-      cy.get('div').eq(2).within(e => {
-        cy.contains('student numbers in oodi but not in list')
-        cy.contains('014896381')
-      })
-      cy.contains('010483918').should('not.exist')
+    cy.contains('#checkstudentsresults', 'Results').within(e => {
+      cy.contains('Student numbers in list and in oodi').click()
+      cy.contains('#found', '010111264')
+      cy.contains('Student numbers in list but not in oodi').click()
+      cy.contains('#notfound', '666666666')
+      cy.contains('Student numbers in oodi but not in list').click()
+      cy.contains('#notsearched', '010533091')
     })
   })
 
   it('All filters working', () => {
-    cy.contains("Select study programme", { timeout: 50000 }).click().siblings().contains("Kasvatustieteiden kandiohjelma").click()
+    cy.contains("Select study programme", { timeout: 50000 }).click().siblings().contains("Tietojenkäsittelytieteen maisteriohjelma").click()
     cy.contains("See population").click()
 
     cy.contains("add").click()
@@ -159,40 +151,40 @@ describe('Population Statistics tests', () => {
     cy.contains("Filters").siblings().within(() => {
       cy.contains("Credits at least 15")
     })
-    checkAmountOfStudents(24)
+    checkAmountOfStudents(23)
 
     cy.contains("Show only students with credits less than").parentsUntil("form").contains("set filter").should('be.disabled')
       .parentsUntil("form").find("input").type("100")
     cy.contains("Show only students with credits less than").parentsUntil("form").contains("set filter").click()
 
-    checkAmountOfStudents(23)
+    checkAmountOfStudents(22)
 
     cy.contains("chosen semester").parentsUntil("form").contains("set filter").click()
 
-    checkAmountOfStudents(17)
+    checkAmountOfStudents(11)
 
     cy.contains("select status").click().siblings().contains("present").click()
     cy.contains("select semesters").click().siblings().contains("Fall 2018").click()
     cy.contains("Spring 2018").click()
     cy.contains("Spring 2018").parentsUntil("form").contains("set filter").click()
     cy.contains("Students that were present").should('have.text', "Students that were present during Fall 2018, Spring 2018")
+    cy.get(':nth-child(7) > .form > .inline > :nth-child(2) > .ui > .default').click().siblings().contains('have not').click()
 
-    cy.contains('graduated from Kasvatustieteiden kandiohjelma').contains('have/not').click().siblings().contains('have not').click()
-    cy.contains('graduated from Kasvatustieteiden kandiohjelma').contains('set filter').click()
-
-    cy.contains('Showing students that have not graduated from Kasvatustieteiden kandiohjelma')
-
-    checkAmountOfStudents(17)
+    checkAmountOfStudents(11)
 
     cy.contains("have/haven't").click().siblings().contains('haven\'t').click()
     cy.contains('canceled this studyright').parentsUntil('form').contains('set filter').click()
     cy.contains('Excluded students whose').should('have.text', 'Excluded students whose studyright is cancelled')
 
-    checkAmountOfStudents(17)
+    checkAmountOfStudents(11)
 
     cy.contains('transferred to').parentsUntil("form").contains("set filter").click()
 
-    checkAmountOfStudents(17)
+    checkAmountOfStudents(11)
+
+    cy.contains("Students that").parentsUntil("form").contains("set filter").click()
+    cy.contains('Showing students that have not graduated from Tietojenkäsittelytieteen maisteriohjelma')
+    checkAmountOfStudents(11)
 
     cy.contains("Advanced filters").click()
 
@@ -205,21 +197,22 @@ describe('Population Statistics tests', () => {
     })
 
     cy.contains("course type").click().siblings().contains("Aineopinnot").click()
-    cy.contains("discipline").click().siblings().contains("Kasvatustieteet").click()
+    cy.contains("discipline").click().siblings().contains("Tietojenkäsittelytiede").click()
     cy.contains("and at least").parentsUntil("form").contains("set filter").click()
 
-    cy.contains("Filters").siblings().within(() => {
-      cy.contains('Kehittävä').should('have.text', 'Avoin yo: Kehittävä työntutkimus')
-    })
+    checkAmountOfStudents(0)
 
-    checkAmountOfStudents(1)
-
-    removeFilter('Kehittävä')
+    cy.get('.filter-segment').its('length')
+      .then(originalLength => {
+        cy.contains('.filter-segment', 'Laskennan mallit').should('exist')
+        removeFilter('Laskennan mallit')
+        cy.get('.filter-segment').should('have.length', originalLength - 1)
+      })
 
     cy.contains("select source").click().siblings().contains("Anywhere").click()
-    cy.contains("select target").click().siblings().contains("Kasvatustieteiden kandiohjelma").click()
+    cy.contains("select target").click().siblings().contains("Tietojenkäsittelytieteen maisteriohjelma").click()
     cy.contains("Students that transferred from").parentsUntil("form").contains("set filter").click()
-    cy.contains("Showing students that transferred").should('have.text', 'Showing students that transferred to Kasvatustieteiden kandiohjelma')
+    cy.contains("Showing students that transferred").should('have.text', 'Showing students that transferred to Tietojenkäsittelytieteen maisteriohjelma')
 
     cy.contains('are/not').click().siblings().contains('not').click()
     cy.contains('select graduation').click().siblings().contains('graduated').click()
@@ -230,12 +223,12 @@ describe('Population Statistics tests', () => {
 
     cy.contains("Students that has").parentsUntil("form").within(() => {
       cy.contains("degree").click().siblings().contains("any degree").click()
-      cy.contains("programme").click().siblings().contains("Kasvatustieteiden kandiohjelma")
+      cy.contains("programme").click().siblings().contains("Tietojenkäsittelytieteen maisteriohjelma")
       cy.contains("priority").click().siblings().contains("primary studies").click()
       cy.contains("set filter").click()
     })
 
-    checkAmountOfStudents(3)
+    checkAmountOfStudents(0)
 
     cy.contains("Basic filters").parentsUntil("form").contains("set filter").click()
 
@@ -244,7 +237,7 @@ describe('Population Statistics tests', () => {
     cy.contains("Save current filters as preset").parentsUntil(".dimmer").within(() => { cy.get("button").contains("Save").click() })
 
     cy.visit("/populations")
-    cy.contains("Select study programme", { timeout: 50000 }).click().siblings().contains("Kasvatustieteiden kandiohjelma").click()
+    cy.contains("Select study programme", { timeout: 50000 }).click().siblings().contains("Tietojenkäsittelytieteen maisteriohjelma").click()
     cy.contains("See population").click()
 
     cy.contains("add").click()
@@ -257,23 +250,24 @@ describe('Population Statistics tests', () => {
       cy.wrap($f).parentsUntil("form").contains("set filter").click()
     })
 
-    cy.get(".header").contains("Filters").siblings().within(() => {
-      cy.get("label:contains(Basic filters)").each(($f) => {
-        cy.wrap($f).parentsUntil(".segment").within(() => {
-          cy.get(".trash").click()
-        })
+    cy.get("label:contains(Basic filters)").each(($f) => {
+      cy.wrap($f).parentsUntil(".segment").within(() => {
+        cy.get(".trash").click()
       })
+      cy.root().get("button").contains("Delete for good").click({ force: true })
     })
-    cy.get("button").contains("Delete for good").click({ force: true })
 
-    cy.contains("Filters").siblings().within(() => {
-      cy.get("label:contains(Advanced filters-)").each(($f) => {
-        cy.wrap($f).parentsUntil(".segment").within(() => {
-          cy.get(".trash").click()
-        })
+    cy.get("label:contains(Advanced filters-)").each(($f) => {
+      cy.wrap($f).parentsUntil(".segment").within(() => {
+        cy.get(".trash").click()
       })
+      cy.root().get("button").contains("Delete for good").click({ force: true })
     })
-    cy.get("button").contains("Delete for good").click({ force: true })
-
+  })
+  it('Population statistics wont crash course population', () => {
+    cy.contains("Select study programme").click().siblings().contains("Tietojenkäsittelytieteen maisteriohjelma").click()
+    cy.contains("See population").click()
+    cy.get(':nth-child(3) > .iconCell > p > .item > .level').click({ force: true })
+    cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click({ force: true })
   })
 })

cypress/integration/Student_statistics.js

@@ -1,9 +1,10 @@
+/// <reference types="Cypress" />
 
 describe('Student Statistics tests', () => {
     beforeEach(() => {
         cy.server({
             onAnyRequest: function (route, proxy) {
-                if (Cypress.config().baseUrl.includes("http://localhost:1337/")) {
+                if (Cypress.config().baseUrl.includes("http://nginx/")) {
                     proxy.xhr.setRequestHeader('uid', 'tktl')
                     proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
                     proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
@@ -23,21 +24,21 @@ describe('Student Statistics tests', () => {
         cy.contains('Student number')
         cy.contains('Started')
         cy.contains('Credits')
-        cy.contains('Wallin').should('not.exist')
+        cy.contains('Heidi Eeva').should('not.exist')
 
         cy.get('label').click()
-        cy.contains('Wallin')
+        cy.contains('Heidi Eeva')
 
         cy.get('label').click()
-        cy.contains('Wallin').should('not.exist')
+        cy.contains('Heidi Eeva').should('not.exist')
 
     })
     it('Can get student specific page by clicking student', () => {
         cy.url().should('include', '/students')
         cy.get('.prompt').type('Oinonen')
-        cy.contains('011143561').click()
-        cy.contains('Updated at 25.02.2019')
-        cy.contains('Toinen kotimainen kieli (40061)')
+        cy.contains('014473717').click()
+        cy.contains('Updated at 02.10.2019')
+        cy.contains('Musiikin didaktiikka (EDUK4212)')
         cy.contains('Oinonen').should('not.exist')
 
         cy.get('label').click()
@@ -49,7 +50,7 @@ describe('Student Statistics tests', () => {
 
     it('Can get back to search menu', () => {
         cy.get('.prompt').type('Oinonen')
-        cy.contains('011143561').click()
+        cy.contains('014473717').click()
         cy.get('.remove').click()
         cy.contains('Student number').should('not.exist')
         cy.contains('Credits').should('not.exist')
@@ -57,10 +58,10 @@ describe('Student Statistics tests', () => {
 
     it('Can jump to course', () => {
         cy.get('.prompt').type('Oinonen')
-        cy.contains('011143561').click()
-        cy.contains('Toinen kotimainen kieli (40061)').siblings().within(() => { cy.get('.level').click() })
+        cy.contains('014473717').click()
+        cy.contains('Musiikin didaktiikka (EDUK4212)').siblings().within(() => { cy.get('.level').click() })
         cy.url().should('include', '/coursestatistics')
-        cy.contains('Toinen kotimainen kieli')
+        cy.contains('Musiikin didaktiikka')
     })
 
     it('Searching with bad inputs doesnt yield results', () => {
@@ -72,10 +73,10 @@ describe('Student Statistics tests', () => {
     })
 
     it('Can jump to population page', () => {
-        cy.get('.prompt').type('Oinonen')
-        cy.contains('011143561').click()
+        cy.get('.prompt').type('Veli-Matti')
+        cy.contains('014824094').click()
         cy.get('i.level.up.alternate.icon').eq(0).click()
         cy.contains('Population statistics')
-        cy.contains('Credit accumulation (for 1 students)')
+        cy.contains('Tietojenkäsittelytieteen koulutusohjelma')
     })
 })

cypress/integration/Studyprogramme_overview.js

@@ -1,8 +1,10 @@
+/// <reference types="Cypress" />
+
 describe('Studyprogramme overview', () => {
   beforeEach(() => {
     cy.server({
       onAnyRequest: function (route, proxy) {
-        if (Cypress.config().baseUrl.includes("http://localhost:1337/")) {
+        if (Cypress.config().baseUrl.includes("http://nginx/")) {
           proxy.xhr.setRequestHeader('uid', 'tktl')
           proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
           proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
@@ -30,10 +32,10 @@ describe('Studyprogramme overview', () => {
     cy.contains('tr', 'Test course group').get('i.edit').click()
     cy.contains("Edit group")
     cy.get('.prompt').type("Professori Pekka")
-    cy.contains("Add teacher").parent().contains("9000960")
-    cy.contains("Teachers in group").parent().contains("9000960")
-    cy.get("i.reply.link.icon").click()
+    cy.contains("Add teacher").parent().contains("000960").click()
+    cy.contains("Teachers in group").parent().contains("000960")
 
+    cy.get("i.reply.link.icon").click()
     cy.contains('tr a', 'Test course group').click()
     cy.contains("Total teachers")
     cy.get("i.reply.icon").click()
@@ -41,19 +43,19 @@ describe('Studyprogramme overview', () => {
 
   it('renders progress and productivity tables', () => {
     cy.contains("Tietojenkäsittelytieteen kandiohjelma").click()
-    cy.get('button:contains("Recalculate")').eq(0).click()
-    cy.get('button:contains("Recalculate")').eq(1).click()
-    cy.contains('Loading...').should('not.be.visible').then(() => {
-      cy.wait(1000)
-      cy.reload()
-      cy.get('table').should('have.length', 2)
-      cy.contains('Population progress')
-      cy.contains('Yearly productivity')
-      cy.contains("2018-2019").siblings().contains('9')
-      cy.contains("2017-2018").siblings().contains('12').siblings().contains('7').siblings().contains('2')
-      cy.get('table').eq(1).contains('2018').siblings().contains('443')
-      cy.get('table').eq(1).contains('2017').siblings().contains('381')
-    })
+    cy.contains("Admin").click()
+    cy.contains("productivity").click()
+    cy.contains("throughput").click()
+
+    cy.wait(1000)
+    cy.get('.attached > :nth-child(1)').click()
+    cy.get('table').should('have.length', 2)
+    cy.contains('Population progress')
+    cy.contains('Yearly productivity')
+    cy.contains("2017-2018").siblings().contains('219').siblings().contains('171')
+
+    cy.get('table').eq(1).contains('2018').siblings().contains('3023.00').siblings().contains('2616.00')
+    cy.get('table').eq(1).contains('2017').siblings().contains('3159.00').siblings().contains("2889.00")
   })
 
   it('can add and delete mandatory courses and labels', () => {
@@ -111,8 +113,71 @@ describe('Studyprogramme overview', () => {
   })
 
   it('can move to Population statistics page by clickin', () => {
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+    cy.contains('Tietojenkäsittelytieteen maisteriohjelma').click()
     cy.get('i.level.up.alternate.icon').eq(0).click()
-    cy.contains('Credit accumulation (for 9 students)')
+    cy.contains('Credit accumulation (for 29 students)')
+  })
+  it('can create and delete tags for population', () => {
+    cy.contains('Tietojenkäsittelytieteen maisteriohjelma').click()
+    cy.get('.attached > :nth-child(6)').click()
+    cy.get(':nth-child(1) > .field > .ui > input').type('tykittäjät')
+    cy.get('.form-control').type('2018')
+    cy.contains('Create new tag').click()
+    cy.contains('tykittäjät')
+    cy.contains('2018')
+    cy.get('tr > :nth-child(3) > .field > .ui').click()
+    cy.contains('Are you sure you want to delete tag')
+    cy.contains('Confirm').click()
+    cy.contains('tykittäjät').should('not.exist')
+  })
+  it('can create personal tags', () => {
+    cy.contains('Tietojenkäsittelytieteen maisteriohjelma').click()
+    cy.get('.attached > :nth-child(6)').click()
+    cy.get(':nth-child(1) > .field > .ui > input').type('epätykittäjät')
+    cy.get('.form-control').type('2018')
+    cy.get('.ui > label').click()
+    cy.contains('Create new tag').click()
+    cy.get('.purple')
+    cy.contains('epätykittäjät')
+    cy.get('tr > :nth-child(3) > .field > .ui').click()
+    cy.contains('Are you sure you want to delete tag')
+    cy.contains('Confirm').click()
+    cy.contains('epätykittäjät').should('not.exist')
+
+  })
+  it('can add tags to students', () => {
+    cy.contains('Tietojenkäsittelytieteen maisteriohjelma').click()
+    cy.get('.attached > :nth-child(6)').click()
+    cy.get(':nth-child(1) > .field > .ui > input').type('tykittäjät')
+    cy.get('.form-control').type('2018')
+    cy.contains('Create new tag').click()
+    cy.contains('tykittäjät')
+
+    cy.contains('Add tags to students').click()
+    cy.get('.form > .field > .dropdown').click().get('.ui > .search').type('tykittäjät').click()
+    
+    cy.get('.form > .field > .dropdown').contains('tykittäjät').click()
+
+    cy.get('textarea').type('014495339')
+    cy.get('.positive').click()
+
+    cy.contains('Student statistics').click()
+    cy.get('.prompt').type('014495339')
+    cy.contains('10').click()
+    cy.contains('tykittäjät')
+
+    cy.go('back')
+    cy.go('back')
+
+    cy.get('tr > :nth-child(3) > .field > .ui').click()
+    cy.contains('Are you sure you want to delete tag')
+    cy.contains('Confirm').click()
+    cy.contains('tykittäjät').should('not.exist')
+
+    cy.contains('Student statistics').click()
+    cy.get('.prompt').type('014495339')
+    cy.contains('10').click()
+    cy.contains('tykittäjät').should('not.exist')
+
   })
 })

cypress/integration/Tags.js

@@ -0,0 +1,53 @@
+/// <reference types="Cypress" />
+
+describe('Tag tests', () => {
+  beforeEach(() => {
+    cy.server({
+      onAnyRequest: function (route, proxy) {
+        if (Cypress.config().baseUrl.includes("http://nginx/")) {
+          proxy.xhr.setRequestHeader('uid', 'admink')
+          proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
+          proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
+          proxy.xhr.setRequestHeader('edupersonaffiliation', 'asdasd')
+        }
+      }
+    })
+    console.log(Cypress.config().baseUrl)
+    cy.visit(Cypress.config().baseUrl)
+    cy.reload()
+    cy.contains("Study programme").click()
+    cy.contains("Overview").click()
+    cy.contains("Datatieteen maisteriohjelma").click()
+    cy.get('.attached > :nth-child(6)').click()
+
+    cy.get(':nth-child(1) > .field > .ui > input').type('test tag')
+    cy.get('.form-control').type('2016')
+    cy.contains('Create new tag').click()
+    cy.contains('test tag')
+    cy.contains('2016')
+
+
+  })
+
+  it("Tagged population works", () => {
+    cy.contains('Add tags to students').click()
+    cy.get('.form > .field > .dropdown').click().get('.ui > .search').type('test tag').click()
+
+    cy.get('.form > .field > .dropdown').contains('test tag').click()
+
+    cy.get('textarea').type('014022579\n011122249')
+    cy.get('.positive').click()
+    cy.get('.level').click()
+
+    cy.contains('Credit accumulation (for 2 students)')
+    cy.get(':nth-child(5) > .dividing > :nth-child(1)').click()
+    cy.contains('014022579')
+    cy.contains('011122249')
+
+    cy.go('back')
+    cy.get('tr > :nth-child(3) > .field > .ui').click()
+    cy.contains('Are you sure you want to delete tag')
+    cy.contains('Confirm').click()
+    cy.contains('test tag').should('not.exist')
+  })
+})

cypress/integration/Teachers.js

@@ -1,9 +1,10 @@
+/// <reference types="Cypress" />
 
 describe('Teachers page tests', () => {
   beforeEach(() => {
     cy.server({
       onAnyRequest: function (route, proxy) {
-        if (Cypress.config().baseUrl.includes("http://localhost:1337/")) {
+        if (Cypress.config().baseUrl.includes("http://nginx/")) {
           proxy.xhr.setRequestHeader('uid', 'tktl')
           proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
           proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
@@ -43,8 +44,8 @@ describe('Teachers page tests', () => {
     cy.url().should('include', '/teachers')
     cy.get('.borderless > :nth-child(3)').click()
     cy.get('.prompt').type('Pekka')
-    cy.contains('Saren').click()
-    cy.contains('English Academic & Professional Skills: Reading, Writing & Spoken Communication (CEFR B2)*')
+    cy.contains('Leinonen').click()
+    cy.contains('Academic and Professional Communication in English 1 & 2 (CEFR B2)')
   })
 
   it("Can check teacher page if teacher doesn't have courses", () => {
@@ -65,7 +66,7 @@ describe('Teachers page tests', () => {
     cy.get(':nth-child(1) > .ui > .search').click()
     cy.contains('2017-18').click()
     cy.contains("Passed")
-    cy.contains("Valtonen")
-    cy.contains("Wahlström")
+    cy.contains("Antero")
+    cy.contains("Sassmannshausen")
   })
 })

cypress/integration/Users.js

@@ -1,9 +1,10 @@
+/// <reference types="Cypress" />
 
 describe('Users tests', () => {
   beforeEach(() => {
     cy.server({
       onAnyRequest: function (route, proxy) {
-        if (Cypress.config().baseUrl.includes("http://localhost:1337/")) {
+        if (Cypress.config().baseUrl.includes("http://nginx/")) {
           proxy.xhr.setRequestHeader('uid', 'admink')
           proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
           proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
@@ -21,7 +22,7 @@ describe('Users tests', () => {
   it("Admin mocking normal user shows only the mocked user's programmes", () => {
     cy.contains("mocking").should('not.exist')
     cy.contains("tr", "Normaalikäyttäjä").within(($row) => {
-      cy.contains('button', 'Edit').click()
+      cy.contains('.button', 'Edit').click()
     })
     cy.contains("Access rights").siblings().within(($row) => {
       cy.get("div[role='listitem']").should('have.length', 1).contains("Tietojenkäsittelytieteen kandiohjelma")

docker-compose.yml

@@ -11,25 +11,25 @@ services:
       - "4222:4222"
     container_name: nats
     volumes:
-      - ./datastore:/datastore
+      - datastore:/datastore
 
   analytics_db:
     image: postgres:9.6.3
     ports:
       - "5423:5432"
     volumes:
-      - ./analytics_pgdata:/var/lib/postgresql/data
+      - analytics_pgdata:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
     container_name: oodi_analytics_db
     environment:
       - POSTGRES_USER=postgres
-      - POSTGRES_DB=analytics_db
 
   user_db:
     image: postgres:9.6.3
     ports:
       - "5422:5432"
     volumes:
-      - ./user_pgdata:/var/lib/postgresql/data
+      - user_pgdata:/var/lib/postgresql/data
     container_name: oodi_user_db
     environment:
       - POSTGRES_USER=postgres
@@ -39,7 +39,7 @@ services:
     ports:
       - "5424:5432"
     volumes:
-      - ./usage_pgdata:/var/lib/postgresql/data
+      - usage_pgdata:/var/lib/postgresql/data
     container_name: oodi_usage_db
     environment:
       - POSTGRES_USER=postgres
@@ -50,7 +50,7 @@ services:
     ports:
       - "5421:5432"
     volumes:
-      - ./pgdata:/var/lib/postgresql/data
+      - pgdata:/var/lib/postgresql/data
       - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
     container_name: oodi_db
     environment:
@@ -61,7 +61,7 @@ services:
     ports:
       - "5425:5432"
     volumes:
-      - ./pgdata_kone:/var/lib/postgresql/data
+      - pgdata_kone:/var/lib/postgresql/data
       - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
     container_name: db_kone
     environment:
@@ -74,7 +74,7 @@ services:
     ports:
       - "6379:6379"
     volumes:
-      - ./redis-data:/data
+      - redis-data:/data
     container_name: redis
 
   backend:
@@ -90,7 +90,6 @@ services:
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
       TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test
       TOKEN_SECRET: IWannaBeTheVeryBest
-      OODILEARN_URL: http://oodilearn:5000
       USERSERVICE_URL: http://userservice:4567
       USAGESERVICE_URL: http://usageservice:545
       USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
@@ -98,6 +97,11 @@ services:
       ANALYTICS_URL: http://analytics:4568
       NATS_URI: nats://nats:4222
       UPDATER_URL: http://updater_scheduler:3678
+      # Set these in production:
+      # ANALYTICS_INFLUXDB_URL
+      # ANALYTICS_INFLUXDB_DB
+      # ANALYTICS_INFLUXDB_PASSWORD
+      # ANALYTICS_INFLUXDB_USER
     container_name: backend
     depends_on:
       - db
@@ -146,9 +150,8 @@ services:
       - nats
     environment:
       - NATS_URI=nats://nats:4222
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
+      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi/test
       - TOKEN=jotaindiibadaabaa
-    #container_name: updater_api
 
   updater_writer:
     build:
@@ -162,12 +165,11 @@ services:
       DB_URL: postgres://postgres@db:5432/tkt_oodi
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
       TEST_DB_SCHEMA: updater_test
-    #container_name: updater_writer
 
   scheduler_mongo:
     image: mongo:4.0.10
     volumes:
-      - ./mongodata:/data/db
+      - mongodata:/data/db
     expose:
       - "27017"
     container_name: scheduler_mongo
@@ -195,3 +197,13 @@ services:
     depends_on:
       - analytics_db
     container_name: analytics
+
+volumes:
+  datastore:
+  analytics_pgdata:
+  user_pgdata:
+  usage_pgdata:
+  pgdata:
+  pgdata_kone:
+  redis-data:
+  mongodata:

docker/docker-compose.cypress.yml

@@ -0,0 +1,12 @@
+version: '3.2'
+services:
+  cypress:
+    image: "cypress/included:3.4.1"
+    environment:
+      - CYPRESS_baseUrl=${CYPRESS_baseUrl}
+      - CYPRESS_videoCompression=false
+    working_dir: /e2e
+    volumes:
+      - ./cypress:/e2e/cypress
+      - ./cypress.json:/e2e/cypress.json
+    container_name: cypress

docker/docker-compose.dev.real.yml

@@ -1,14 +1,36 @@
 version: '3.2'
 
 services:
+  nats:
+    volumes:
+      - datastore-real:/datastore
+
   backend:
     environment:
       DB_URL: postgres://postgres@db:5432/tkt_oodi_real
       DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone_real
+
   userservice:
     environment:
       DB_URL: postgres://postgres@user_db:5432/user_db_real
 
+  analytics:
+    environment:
+      DB_URL: postgres://postgres@analytics_db:5432/analytics_db_real
+
   updater_writer:
     environment:
       DB_URL: postgres://postgres@db:5432/tkt_oodi_real
+
+  redis:
+    volumes:
+      - redis-data-real:/data
+
+  scheduler_mongo:
+    volumes:
+      - mongodata-real:/data/db
+
+volumes:
+  datastore-real:
+  redis-data-real:
+  mongodata-real:

docker/docker-compose.dev.yml

@@ -6,6 +6,11 @@ services:
       - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real
     restart: unless-stopped
 
+  analytics_db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=analytics_db,analytics_db_real
+    restart: unless-stopped
+
   db:
     environment:
       - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
@@ -15,6 +20,7 @@ services:
     build:
       args:
         NODE_ENV: 'development'
+        TAG: 'development'
     command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     environment:
       FRONT_URL: http://localhost:8000
@@ -29,6 +35,7 @@ services:
     build:
       args:
         NODE_ENV: 'development'
+        TAG: 'development'
     command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run docker']
     ports:
       - "8081:8081"
@@ -112,6 +119,9 @@ services:
     container_name: adminer
     restart: unless-stopped
 
+networks:
+  default:
+
   # zookeeper:
   #   image: wurstmeister/zookeeper
   #   ports:
@@ -141,23 +151,7 @@ services:
   # mongo_db:
   #   image: mongo
   #   volumes:
-  #     - ./backups/mongo/oodilearn:/dump
   #     - ./mongodata:/data/db
   #   ports:
   #     - "27017:27017"
   #   container_name: mongo_db
-
-  # oodilearn:
-  #   build:
-  #     context: services/oodilearn/server
-  #     dockerfile: Dockerfile
-  #   volumes:
-  #     - ./services/oodilearn/data:/data
-  #     - ./services/oodilearn/server:/src
-  #     - ./services/oodilearn/models:/models
-  #   ports:
-  #     - "5000:5000"
-  #   environment:
-  #     - ENV=development
-  #     - MONGO_URI=mongodb://mongo_db:27017/oodilearn
-  #   container_name: oodilearn

docker/docker-compose.lateste2e.yml

@@ -9,8 +9,15 @@ services:
     environment:
       - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
 
+  analytics_db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=analytics_db
+
   backend:
     image: toska/oodikone2-backend:${TAG}
+    build:
+      args:
+        TAG: ${TAG}
     build:
       cache_from:
         - toska/oodikone2-backend:trunk
@@ -27,6 +34,7 @@ services:
     build:
       args:
         BASE_PATH: /
+        TAG: ${TAG}
       cache_from:
         - toska/oodikone2-frontend:trunk
         - toska/oodikone2-frontend:staging

package-lock.json

@@ -1316,6 +1316,14 @@
         "cli-cursor": "^1.0.2"
       }
     },
+    "lru-cache": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
+      "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
+      "requires": {
+        "yallist": "^3.0.2"
+      }
+    },
     "map-age-cleaner": {
       "version": "0.1.3",
       "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz",
@@ -2197,6 +2205,11 @@
       "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
       "dev": true
     },
+    "yallist": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.0.tgz",
+      "integrity": "sha512-6gpP93MR+VOOehKbCPchro3wFZNSNmek8A2kbkOAZLIZAYx1KP/zAqwO0sOHi3xJEb+UBz8NaYt/17UNit1Q9w=="
+    },
     "yargs": {
       "version": "12.0.5",
       "resolved": "https://registry.npmjs.org/yargs/-/yargs-12.0.5.tgz",

package.json

@@ -7,15 +7,19 @@
     "cli": "./scripts/run.sh",
     "cypress": "cypress",
     "cypress:open": "cypress open --config baseUrl=http://localhost:8081/",
-    "cypress:run": "cypress run --config video=true,videoCompression=false,baseUrl=http://localhost:8081/",
-    "cypress:ci": "cypress run --config video=true,videoCompression=false,baseUrl=http://localhost:1337/",
+    "cypress:run": "CYPRESS_baseUrl=http://frontend:8081/ npm run docker:cypress",
+    "cypress:ci": "CYPRESS_baseUrl=http://nginx/ npm run docker:ci:cypress",
     "concurrently": "concurrently",
-    "test": "./scripts/run_all_tests.sh",
-    "test_services": "./scripts/run_service_tests.sh",
-    "lint": "./scripts/run_service_lints.sh",
+    "services": "./scripts/exec_command_for_services.sh",
+    "lint": "npm run services -- -c lint",
+    "lintfix": "npm run services -- -c lintfix",
+    "test_services": "npm run services -- -e -c test",
+    "test": "./scripts/test.sh",
     "docker": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml",
+    "docker:cypress": "npm run docker -- -f ./docker/docker-compose.cypress.yml run --rm cypress",
     "docker:real": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml -f ./docker/docker-compose.dev.real.yml",
     "docker:ci": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml",
+    "docker:ci:cypress": "npm run docker:ci -- -f ./docker/docker-compose.cypress.yml run --rm cypress",
     "docker:build": "npm run docker -- build",
     "docker:up": "npm run docker -- up -d",
     "docker:up:real": "npm run docker:real -- up -d",
@@ -25,7 +29,8 @@
     "docker:restart": "npm run docker -- restart",
     "docker:restart:backend": "npm run docker -- restart backend",
     "docker:obliterate": "docker stop $(docker ps -aq) ; docker rmi -f $(docker images -a -q) ; docker system prune -a",
-    "start": "npm run docker:up && npm run docker:logs"
+    "start": "npm run docker:up && npm run docker:logs",
+    "start:real": "npm run docker:real -- up -d && npm run docker:real -- logs -f"
   },
   "husky": {
     "hooks": {
@@ -47,5 +52,7 @@
     "cypress": "^3.4.1",
     "husky": "^1.3.1"
   },
-  "dependencies": {}
+  "dependencies": {
+    "lru-cache": "^5.1.1"
+  }
 }

scripts/exec_command_for_services.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# This script is used to execute npm commands inside multiple running docker services.
+#
+# Params:
+#   -c (required). The npm command to execute in the services.
+#     Example:
+#       ./exec_command_for_services.sh -c lint
+#
+#   -e (optional). Sets the script to stop on first error.
+#     Example:
+#       ./exec_command_for_services.sh -e -c test
+#
+#   -p (optional). Additional params for the npm command.
+#
+# You can also run f.ex. "./exec_command_for_services.sh -c lint backend frontend"
+# to run the specified command in some specific services (in this case, backend & frontend).
+
+while getopts 'ec:p::' flag; do
+  case "${flag}" in
+    # Stop on first error
+    e) set -e ;;
+    c) COMMAND=${OPTARG};;
+    p) PARAMS=${OPTARG};;
+  esac
+done
+
+# Check that command is given
+if [ -z $COMMAND ]; then
+  echo "-c (command) missing"
+  exit
+fi
+
+# Remove opts from argument list
+shift $(( OPTIND - 1 ))
+
+# Service names given as arguments
+WHITELISTED_SERVICES=${@}
+
+if [ ! -z "$WHITELISTED_SERVICES" ]; then
+  services=($WHITELISTED_SERVICES)
+else
+  services=(
+    backend
+    updater_writer
+    updater_api
+    updater_scheduler
+    userservice
+    frontend
+    analytics
+    userservice
+    usageservice
+  )
+fi
+
+RED='\033[0;31m'
+Green='\033[0;32m'
+NC='\033[0m'
+
+echo -e "Executing command ${RED}$COMMAND${NC} $([ -n "$PARAMS" ] && echo "with param(s) ${RED}${PARAMS[@]}${NC} ")to services ${Green}${services[@]}${NC}"
+
+# Execute the given npm command in given services
+for service in "${services[@]}";
+  do docker-compose exec -T $service npm run "$COMMAND" --if-present -- "${PARAMS[@]}"
+done

scripts/run_all_tests.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-# stop on first error
-set -e
-
-# Async version -- the hare. 
-# npm run concurrently "npm run test_services" "npm run cypress:run"
-
-# Sync version -- the tortoise.
-npm run test_services && npm run cypress:run
-

scripts/run_service_lints.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# stop on first error
-set -e
-docker-compose exec -T backend npm run lint
-docker-compose exec -T updater_writer npm  run lint
-docker-compose exec -T updater_api npm run lint
-docker-compose exec -T updater_scheduler npm run lint
-docker-compose exec -T userservice npm run lint
-docker-compose exec -T frontend npm run lint
-docker-compose exec -T analytics npm run lint
-docker-compose exec -T userservice npm run lint

scripts/run_service_tests.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-# stop on first error
-set -e
-docker-compose exec -T backend npm test
-docker-compose exec -T updater_writer npm test
-docker-compose exec -T updater_api npm test
-docker-compose exec -T updater_scheduler npm test
-docker-compose exec -T userservice npm test
-docker-compose exec -T frontend npm test
-docker-compose exec -T analytics npm test
-docker-compose exec -T userservice npm test
-docker-compose exec -T usageservice npm test

scripts/scripts.sh

@@ -3,12 +3,16 @@
 DIR_PATH=$(dirname "$0")
 ANONDB_DIR=anonyymioodi
 BACKUP_DIR=backups
+
 PSQL_DB_BACKUP="$ANONDB_DIR/anon.sqz"
 USER_DB_BACKUP="$ANONDB_DIR/user-dump.sqz"
 KONE_DB_BACKUP="$ANONDB_DIR/anon_kone.sqz"
+ANALYTICS_DB_BACKUP="$ANONDB_DIR/analytics-dump.sqz"
+
 PSQL_REAL_DB_BACKUP="$BACKUP_DIR/latest-pg.sqz"
 KONE_REAL_DB_BACKUP="$BACKUP_DIR/latest-kone-pg.sqz"
 USER_REAL_DB_BACKUP="$BACKUP_DIR/latest-user-pg.sqz"
+ANALYTICS_REAL_DB_BACKUP="$BACKUP_DIR/latest-analytics-pg.sqz"
 
 docker-compose-dev () {
     docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml "$@"
@@ -46,7 +50,7 @@ get_anon_oodikone() {
       chmod 400 private.key
     fi
     rm -rf anonyymioodi
-    GIT_SSH_COMMAND='ssh -i private.key' git clone git@github.com:UniversityOfHelsinkiCS/anonyymioodi.git
+    GIT_SSH_COMMAND='ssh -i private.key' git clone --depth=1 git@github.com:UniversityOfHelsinkiCS/anonyymioodi.git
 }
 
 restore_psql_from_backup () {
@@ -54,11 +58,6 @@ restore_psql_from_backup () {
     docker exec $2 pg_restore -U postgres --no-owner -F c --dbname=$3 -j4 /asd.sqz
 }
 
-# oodilearn
-# restore_mongodb_from_backup () {
-#     docker exec -t mongo_db mongorestore -d oodilearn "/dump"
-# }
-
 ping_psql () {
     drop_psql $1 $2
     echo "Creating psql in container $1 with db name $2"
@@ -80,6 +79,8 @@ db_setup_full () {
     restore_psql_from_backup $KONE_REAL_DB_BACKUP db_kone db_kone_real
     ping_psql "oodi_user_db" "user_db_real"
     restore_psql_from_backup $USER_REAL_DB_BACKUP oodi_user_db user_db_real
+    ping_psql "oodi_analytics_db" "analytics_db_real"
+    restore_psql_from_backup $ANALYTICS_REAL_DB_BACKUP oodi_analytics_db analytics_db_real
     # echo "Restoring MongoDB from backup"
     # retry restore_mongodb_from_backup
     echo "Database setup finished"
@@ -90,11 +91,16 @@ db_anon_setup_full () {
     ping_psql "oodi_db" "tkt_oodi"
     ping_psql "oodi_db" "tkt_oodi_test"
     restore_psql_from_backup $PSQL_DB_BACKUP oodi_db tkt_oodi
+
     ping_psql "db_kone" "db_kone"
     ping_psql "db_kone" "db_kone_test"
     restore_psql_from_backup $KONE_DB_BACKUP db_kone db_kone
+
     ping_psql "oodi_user_db" "user_db"
     restore_psql_from_backup $USER_DB_BACKUP oodi_user_db user_db
+
+    ping_psql "oodi_analytics_db" "analytics_db"
+    restore_psql_from_backup $ANALYTICS_DB_BACKUP oodi_analytics_db analytics_db
     # echo "Restoring MongoDB from backup"
     # retry restore_mongodb_from_backup
     echo "Database setup finished"
@@ -102,14 +108,14 @@ db_anon_setup_full () {
 
 reset_real_db () {
     docker-compose-dev down
-    docker-compose-dev up -d db user_db db_kone
+    docker-compose-dev up -d db user_db db_kone analytics_db
     db_setup_full
     docker-compose-dev down
 }
 
 reset_db () {
     docker-compose-dev down
-    docker-compose-dev up -d db user_db db_kone
+    docker-compose-dev up -d db user_db db_kone analytics_db
     db_anon_setup_full
     docker-compose-dev down
 }
@@ -144,7 +150,7 @@ run_full_setup () {
     echo "Building images"
     docker-compose-dev build
     echo "Setup oodikone db from dump."
-    docker-compose-dev up -d db user_db db_kone
+    docker-compose-dev up -d db user_db db_kone analytics_db
     db_setup_full
     db_anon_setup_full
     docker-compose-dev down
@@ -161,7 +167,7 @@ run_anon_full_setup () {
     echo "Building images"
     docker-compose-dev build
     echo "Setup oodikone db from dump."
-    docker-compose-dev up -d db user_db db_kone
+    docker-compose-dev up -d db user_db db_kone analytics_db
     db_anon_setup_full
     docker-compose-dev down
     show_instructions
@@ -177,7 +183,7 @@ run_e2e_setup () {
     echo "Building images"
     TAG=$2 SENTRY_RELEASE_VERSION=$3 docker-compose -f docker-compose.yml -f $1 build
     echo "Setup oodikone db from dump."
-    TAG=$2 SENTRY_RELEASE_VERSION=$3 docker-compose -f docker-compose.yml -f $1 up -d db user_db db_kone
+    TAG=$2 SENTRY_RELEASE_VERSION=$3 docker-compose -f docker-compose.yml -f $1 up -d db user_db db_kone analytics_db
     db_anon_setup_full
     echo "Starting services."
     TAG=$2 SENTRY_RELEASE_VERSION=$3 docker-compose -f docker-compose.yml -f $1 up -d

scripts/test.sh

@@ -0,0 +1,7 @@
+
+#!/bin/bash
+
+# stop on first error
+set -e
+
+npm run test_services && npm run cypress:run

services/backend/oodikone2-backend/.eslintrc.json

@@ -1,28 +1,13 @@
 {
-    "env": {
-        "es6": true,
-        "node": true,
-        "jest": true
-    },
-    "parser": "babel-eslint",
-    "extends": "eslint:recommended",
-    "rules": {
-        "indent": [
-            "error",
-            2
-        ],
-        "linebreak-style": [
-            "error",
-            "unix"
-        ],
-        "quotes": [
-            "error",
-            "single"
-        ],
-        "semi": [
-            "error",
-            "never"
-        ],
-        "no-console": 0
-    }
-}
+  "env": {
+    "es6": true,
+    "node": true,
+    "jest": true
+  },
+  "parser": "babel-eslint",
+  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
+  "rules": {
+    "no-console": 0,
+    "prettier/prettier": "error"
+  }
+}

services/backend/oodikone2-backend/.prettierrc

@@ -0,0 +1,9 @@
+{
+  "semi": false,
+  "arrowParens": "avoid",
+  "singleQuote": true,
+  "printWidth": 120,
+  "tabWidth": 2,
+  "useTabs": false,
+  "jsxSingleQuote": false
+}

services/backend/oodikone2-backend/Dockerfile

@@ -12,6 +12,8 @@ RUN npm ci
 
 ARG NODE_ENV=production
 ENV NODE_ENV=${NODE_ENV}
+ARG TAG=production
+ENV TAG=${TAG}
 RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
 
 COPY ./oodikone2-backend /usr/src/app

services/backend/oodikone2-backend/README.md

@@ -146,8 +146,6 @@ Environment variable _INCREMENT_ (currently not set) controls how many new stude
 
 At the moment script uses the local postgres (see .env) to save student numbers.
 
-Use _docker-compose run -d CONTAINER_NAME node src/services/doo_api_database_updater/student_list_updater.js /updater_files/all_student_numbers.txt_ to save student numbers in textfile. 
-
 ### Student info updater (database updater)
 
 To update your database run `docker-compose run -d CONTAINER_NAME npm run update_database` with optional args `file="<filename>"` and `index=<number>` with former telling the studentnumber filename to update and latter telling which row to start from.

services/backend/oodikone2-backend/index.js

@@ -1,6 +1,6 @@
 //require('./src/tester')
 require('./src/app')
 
-process.on('unhandledRejection', (reason) => {
+process.on('unhandledRejection', reason => {
   console.log(reason)
-})
+})

services/backend/oodikone2-backend/package-lock.json

@@ -2312,6 +2312,12 @@
         }
       }
     },
+    "@types/eslint-visitor-keys": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz",
+      "integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==",
+      "dev": true
+    },
     "@types/events": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.0.tgz",
@@ -2354,6 +2360,12 @@
         "@types/istanbul-lib-report": "*"
       }
     },
+    "@types/json-schema": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.3.tgz",
+      "integrity": "sha512-Il2DtDVRGDcqjDtE+rF8iqg1CArehSK84HZJCT7AMITlyXRBpuPhqGLDQMowraqqu1coEaimg4ZOqggt6L6L+A==",
+      "dev": true
+    },
     "@types/minimatch": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
@@ -2377,6 +2389,59 @@
       "integrity": "sha512-SOhuU4wNBxhhTHxYaiG5NY4HBhDIDnJF60GU+2LqHAdKKer86//e4yg69aENCtQ04n0ovz+tq2YPME5t5yp4pw==",
       "dev": true
     },
+    "@typescript-eslint/experimental-utils": {
+      "version": "1.13.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-1.13.0.tgz",
+      "integrity": "sha512-zmpS6SyqG4ZF64ffaJ6uah6tWWWgZ8m+c54XXgwFtUv0jNz8aJAVx8chMCvnk7yl6xwn8d+d96+tWp7fXzTuDg==",
+      "dev": true,
+      "requires": {
+        "@types/json-schema": "^7.0.3",
+        "@typescript-eslint/typescript-estree": "1.13.0",
+        "eslint-scope": "^4.0.0"
+      },
+      "dependencies": {
+        "eslint-scope": {
+          "version": "4.0.3",
+          "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz",
+          "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==",
+          "dev": true,
+          "requires": {
+            "esrecurse": "^4.1.0",
+            "estraverse": "^4.1.1"
+          }
+        }
+      }
+    },
+    "@typescript-eslint/parser": {
+      "version": "1.13.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-1.13.0.tgz",
+      "integrity": "sha512-ITMBs52PCPgLb2nGPoeT4iU3HdQZHcPaZVw+7CsFagRJHUhyeTgorEwHXhFf3e7Evzi8oujKNpHc8TONth8AdQ==",
+      "dev": true,
+      "requires": {
+        "@types/eslint-visitor-keys": "^1.0.0",
+        "@typescript-eslint/experimental-utils": "1.13.0",
+        "@typescript-eslint/typescript-estree": "1.13.0",
+        "eslint-visitor-keys": "^1.0.0"
+      }
+    },
+    "@typescript-eslint/typescript-estree": {
+      "version": "1.13.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-1.13.0.tgz",
+      "integrity": "sha512-b5rCmd2e6DCC6tCTN9GSUAuxdYwCM/k/2wdjHGrIRGPSJotWMCe/dGpi66u42bhuh8q3QBzqM4TMA1GUUCJvdw==",
+      "dev": true,
+      "requires": {
+        "lodash.unescape": "4.0.1",
+        "semver": "5.5.0"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "5.5.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz",
+          "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==",
+          "dev": true
+        }
+      }
+    },
     "abab": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.0.tgz",
@@ -2415,9 +2480,9 @@
       }
     },
     "acorn-jsx": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.0.1.tgz",
-      "integrity": "sha512-HJ7CfNHrfJLlNTzIEUTj43LNWGkqpRLxm3YjAlcD0ACydk9XynzYsCBHxut+iqt+1aBXkx9UP/w/ZqMr13XIzg==",
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.0.2.tgz",
+      "integrity": "sha512-tiNTrP1MP0QrChmD2DdupCr6HWSFeKVw5d/dHTu4Y7rkAkRhU/Dt7dphAfIUyxtHpl/eBVip5uTNSpQJHylpAw==",
       "dev": true
     },
     "acorn-walk": {
@@ -4053,9 +4118,9 @@
       }
     },
     "commander": {
-      "version": "2.20.0",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz",
-      "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==",
+      "version": "2.20.3",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+      "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
       "dev": true,
       "optional": true
     },
@@ -4065,6 +4130,12 @@
       "integrity": "sha1-zVL28HEuC6q5fW+XModPIvR3UsA=",
       "dev": true
     },
+    "common-tags": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.0.tgz",
+      "integrity": "sha512-6P6g0uetGpW/sdyUy/iQQCbFF0kWVMSIVSyYz7Zgjcgh8mgw8PQzDNZeyZ5DQ2gM7LBoZPHmnjz8rUthkBG5tw==",
+      "dev": true
+    },
     "component-emitter": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz",
@@ -4129,8 +4200,7 @@
     "console-control-strings": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
-      "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=",
-      "optional": true
+      "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
     },
     "content-disposition": {
       "version": "0.5.3",
@@ -4582,6 +4652,12 @@
         "path-type": "^4.0.0"
       }
     },
+    "dlv": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz",
+      "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==",
+      "dev": true
+    },
     "doctrine": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
@@ -4856,26 +4932,6 @@
           "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
           "dev": true
         },
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "dev": true,
-          "requires": {
-            "color-convert": "^1.9.0"
-          }
-        },
-        "chalk": {
-          "version": "2.4.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
-          "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
-          "dev": true,
-          "requires": {
-            "ansi-styles": "^3.2.1",
-            "escape-string-regexp": "^1.0.5",
-            "supports-color": "^5.3.0"
-          }
-        },
         "cross-spawn": {
           "version": "6.0.5",
           "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
@@ -4908,34 +4964,12 @@
             "estraverse": "^4.1.1"
           }
         },
-        "globals": {
-          "version": "11.12.0",
-          "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
-          "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==",
-          "dev": true
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
-          "dev": true
-        },
         "js-tokens": {
           "version": "4.0.0",
           "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
           "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
           "dev": true
         },
-        "js-yaml": {
-          "version": "3.13.1",
-          "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz",
-          "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==",
-          "dev": true,
-          "requires": {
-            "argparse": "^1.0.7",
-            "esprima": "^4.0.0"
-          }
-        },
         "ms": {
           "version": "2.1.2",
           "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -4943,9 +4977,9 @@
           "dev": true
         },
         "semver": {
-          "version": "5.7.0",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
-          "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==",
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
           "dev": true
         },
         "strip-ansi": {
@@ -4956,18 +4990,27 @@
           "requires": {
             "ansi-regex": "^3.0.0"
           }
-        },
-        "supports-color": {
-          "version": "5.5.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
-          "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
-          "dev": true,
-          "requires": {
-            "has-flag": "^3.0.0"
-          }
         }
       }
     },
+    "eslint-config-prettier": {
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-6.3.0.tgz",
+      "integrity": "sha512-EWaGjlDAZRzVFveh2Jsglcere2KK5CJBhkNSa1xs3KfMUGdRiT7lG089eqPdvlzWHpAqaekubOsOMu8W8Yk71A==",
+      "dev": true,
+      "requires": {
+        "get-stdin": "^6.0.0"
+      }
+    },
+    "eslint-plugin-prettier": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.1.0.tgz",
+      "integrity": "sha512-XWX2yVuwVNLOUhQijAkXz+rMPPoCr7WFiAl8ig6I7Xn+pPVhDhzg4DxHpmbeb0iqjO9UronEA3Tb09ChnFVHHA==",
+      "dev": true,
+      "requires": {
+        "prettier-linter-helpers": "^1.0.0"
+      }
+    },
     "eslint-scope": {
       "version": "3.7.1",
       "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-3.7.1.tgz",
@@ -5492,8 +5535,7 @@
         "ansi-regex": {
           "version": "2.1.1",
           "bundled": true,
-          "dev": true,
-          "optional": true
+          "dev": true
         },
         "aproba": {
           "version": "1.2.0",
@@ -5514,14 +5556,12 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true,
-          "optional": true
+          "dev": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -5536,20 +5576,17 @@
         "code-point-at": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true,
-          "optional": true
+          "dev": true
         },
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true,
-          "optional": true
+          "dev": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true,
-          "optional": true
+          "dev": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -5666,8 +5703,7 @@
         "inherits": {
           "version": "2.0.3",
           "bundled": true,
-          "dev": true,
-          "optional": true
+          "dev": true
         },
         "ini": {
           "version": "1.3.5",
@@ -5679,7 +5715,6 @@
           "version": "1.0.0",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -5694,7 +5729,6 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -5702,14 +5736,12 @@
         "minimist": {
           "version": "0.0.8",
           "bundled": true,
-          "dev": true,
-          "optional": true
+          "dev": true
         },
         "minipass": {
           "version": "2.3.5",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.2",
             "yallist": "^3.0.0"
@@ -5728,7 +5760,6 @@
           "version": "0.5.1",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -5809,8 +5840,7 @@
         "number-is-nan": {
           "version": "1.0.1",
           "bundled": true,
-          "dev": true,
-          "optional": true
+          "dev": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -5822,7 +5852,6 @@
           "version": "1.4.0",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -5908,8 +5937,7 @@
         "safe-buffer": {
           "version": "5.1.2",
           "bundled": true,
-          "dev": true,
-          "optional": true
+          "dev": true
         },
         "safer-buffer": {
           "version": "2.1.2",
@@ -5945,7 +5973,6 @@
           "version": "1.0.2",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -5965,7 +5992,6 @@
           "version": "3.0.1",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "ansi-regex": "^2.0.0"
           }
@@ -6009,14 +6035,12 @@
         "wrappy": {
           "version": "1.0.2",
           "bundled": true,
-          "dev": true,
-          "optional": true
+          "dev": true
         },
         "yallist": {
           "version": "3.0.3",
           "bundled": true,
-          "dev": true,
-          "optional": true
+          "dev": true
         }
       }
     },
@@ -6099,6 +6123,12 @@
         }
       }
     },
+    "get-stdin": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-6.0.0.tgz",
+      "integrity": "sha512-jp4tHawyV7+fkkSKyvjuLZswblUtz+SQKzSWnBbii16BuZksJlU1wuBYXY75r+duh/llF1ur6oNwi+2ZzjKZ7g==",
+      "dev": true
+    },
     "get-stream": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
@@ -6234,9 +6264,9 @@
       "dev": true
     },
     "handlebars": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz",
-      "integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==",
+      "version": "4.4.5",
+      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.4.5.tgz",
+      "integrity": "sha512-0Ce31oWVB7YidkaTq33ZxEbN+UDxMMgThvCe8ptgQViymL5DPis9uLdTA13MiRPhgvqyxIegugrP97iK3JeBHg==",
       "dev": true,
       "requires": {
         "neo-async": "^2.6.0",
@@ -6278,6 +6308,15 @@
         "function-bind": "^1.1.1"
       }
     },
+    "has-ansi": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
+      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
+      "dev": true,
+      "requires": {
+        "ansi-regex": "^2.0.0"
+      }
+    },
     "has-flag": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz",
@@ -6533,9 +6572,9 @@
       "integrity": "sha1-BTfLedr1m1mhpRff9wbIbsA5Fi4="
     },
     "inquirer": {
-      "version": "6.5.0",
-      "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.0.tgz",
-      "integrity": "sha512-scfHejeG/lVZSpvCXpsB4j/wQNPM5JC8kiElOI0OUTwmc1RTpXr4H32/HOlQHcZiYl2z2VElwuCVDRG8vFmbnA==",
+      "version": "6.5.2",
+      "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz",
+      "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==",
       "dev": true,
       "requires": {
         "ansi-escapes": "^3.2.0",
@@ -6565,32 +6604,6 @@
           "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==",
           "dev": true
         },
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "dev": true,
-          "requires": {
-            "color-convert": "^1.9.0"
-          }
-        },
-        "chalk": {
-          "version": "2.4.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
-          "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
-          "dev": true,
-          "requires": {
-            "ansi-styles": "^3.2.1",
-            "escape-string-regexp": "^1.0.5",
-            "supports-color": "^5.3.0"
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
-          "dev": true
-        },
         "strip-ansi": {
           "version": "5.2.0",
           "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz",
@@ -6599,15 +6612,6 @@
           "requires": {
             "ansi-regex": "^4.1.0"
           }
-        },
-        "supports-color": {
-          "version": "5.5.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
-          "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
-          "dev": true,
-          "requires": {
-            "has-flag": "^3.0.0"
-          }
         }
       }
     },
@@ -9829,6 +9833,12 @@
       "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=",
       "dev": true
     },
+    "lodash.unescape": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/lodash.unescape/-/lodash.unescape-4.0.1.tgz",
+      "integrity": "sha1-vyJJiGzlFM2hEvrpIYzcBlIR/Jw=",
+      "dev": true
+    },
     "log-symbols": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz",
@@ -9857,6 +9867,49 @@
         }
       }
     },
+    "loglevel": {
+      "version": "1.6.4",
+      "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.6.4.tgz",
+      "integrity": "sha512-p0b6mOGKcGa+7nnmKbpzR6qloPbrgLcnio++E+14Vo/XffOGwZtRpUhr8dTH/x2oCMmEoIU0Zwm3ZauhvYD17g==",
+      "dev": true
+    },
+    "loglevel-colored-level-prefix": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/loglevel-colored-level-prefix/-/loglevel-colored-level-prefix-1.0.0.tgz",
+      "integrity": "sha1-akAhj9x64V/HbD0PPmdsRlOIYD4=",
+      "dev": true,
+      "requires": {
+        "chalk": "^1.1.3",
+        "loglevel": "^1.4.1"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "2.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
+          "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
+          "dev": true
+        },
+        "chalk": {
+          "version": "1.1.3",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^2.2.1",
+            "escape-string-regexp": "^1.0.2",
+            "has-ansi": "^2.0.0",
+            "strip-ansi": "^3.0.0",
+            "supports-color": "^2.0.0"
+          }
+        },
+        "supports-color": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
+          "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
+          "dev": true
+        }
+      }
+    },
     "long": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/long/-/long-1.1.2.tgz",
@@ -12791,6 +12844,75 @@
       "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=",
       "dev": true
     },
+    "prettier": {
+      "version": "1.18.2",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.18.2.tgz",
+      "integrity": "sha512-OeHeMc0JhFE9idD4ZdtNibzY0+TPHSpSSb9h8FqtP+YnoZZ1sl8Vc9b1sasjfymH3SonAF4QcA2+mzHPhMvIiw==",
+      "dev": true
+    },
+    "prettier-eslint": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/prettier-eslint/-/prettier-eslint-9.0.0.tgz",
+      "integrity": "sha512-0dael2aMpMAxAwClnLi2Coc30v3BubsTX6clqseZ8NFCJZnbZlwxZGHHESYBlqTyN9lvZDHHv+XdeHW0fKhxJQ==",
+      "dev": true,
+      "requires": {
+        "@typescript-eslint/parser": "^1.10.2",
+        "common-tags": "^1.4.0",
+        "core-js": "^3.1.4",
+        "dlv": "^1.1.0",
+        "eslint": "^5.0.0",
+        "indent-string": "^4.0.0",
+        "lodash.merge": "^4.6.0",
+        "loglevel-colored-level-prefix": "^1.0.0",
+        "prettier": "^1.7.0",
+        "pretty-format": "^23.0.1",
+        "require-relative": "^0.8.7",
+        "typescript": "^3.2.1",
+        "vue-eslint-parser": "^2.0.2"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
+          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
+          "dev": true
+        },
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "core-js": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.2.1.tgz",
+          "integrity": "sha512-Qa5XSVefSVPRxy2XfUC13WbvqkxhkwB3ve+pgCQveNgYzbM/UxZeu1dcOX/xr4UmfUd+muuvsaxilQzCyUurMw==",
+          "dev": true
+        },
+        "pretty-format": {
+          "version": "23.6.0",
+          "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-23.6.0.tgz",
+          "integrity": "sha512-zf9NV1NSlDLDjycnwm6hpFATCGl/K1lt0R/GdkAK2O5LN/rwJoB+Mh93gGJjut4YbmecbfgLWVGSTCr0Ewvvbw==",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^3.0.0",
+            "ansi-styles": "^3.2.0"
+          }
+        }
+      }
+    },
+    "prettier-linter-helpers": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz",
+      "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==",
+      "dev": true,
+      "requires": {
+        "fast-diff": "^1.1.2"
+      }
+    },
     "pretty-format": {
       "version": "24.8.0",
       "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.8.0.tgz",
@@ -13332,6 +13454,12 @@
       "integrity": "sha1-WhtS63Dr7UPrmC6XTIWrWVceVvo=",
       "dev": true
     },
+    "require-relative": {
+      "version": "0.8.7",
+      "resolved": "https://registry.npmjs.org/require-relative/-/require-relative-0.8.7.tgz",
+      "integrity": "sha1-eZlTn8ngR6N5KPoZb44VY9q9Nt4=",
+      "dev": true
+    },
     "resolve": {
       "version": "1.11.1",
       "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.1.tgz",
@@ -13453,9 +13581,9 @@
       "dev": true
     },
     "rxjs": {
-      "version": "6.5.2",
-      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.2.tgz",
-      "integrity": "sha512-HUb7j3kvb7p7eCUHE3FqjoDsC1xfZQ4AHFWfTKSpZ+sAhhz5X1WX0ZuUqWbzB2QhSLp3DoLUG+hMdEDKqWo2Zg==",
+      "version": "6.5.3",
+      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.3.tgz",
+      "integrity": "sha512-wuYsAYYFdWTAnAaPoKGNhfpWwKZbJW+HgAJ+mImp+Epl7BG8oNWBCTyRM8gba9k4lk8BgWdoYm21Mo/RYhhbgA==",
       "dev": true,
       "requires": {
         "tslib": "^1.9.0"
@@ -13882,25 +14010,25 @@
       }
     },
     "sequelize": {
-      "version": "5.10.2",
-      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-5.10.2.tgz",
-      "integrity": "sha512-A7QTOuT/tqPChkT6TW7ujHO0hbngctBIHU9BoGuFkzMN7cMkzkxpGJdztOl3rxe9EQnQFRbUOHbvUvPBZI4CPA==",
+      "version": "5.21.1",
+      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-5.21.1.tgz",
+      "integrity": "sha512-JI+53MwcClfCFUPJT/l2dDzSpEzWAueyCZus33L/yhJxKTisfdd9OHrUPQ6/dI5nR5eIYT/EafrjkqTAlEQS2w==",
       "requires": {
         "bluebird": "^3.5.0",
         "cls-bluebird": "^2.1.0",
         "debug": "^4.1.1",
         "dottie": "^2.0.0",
         "inflection": "1.12.0",
-        "lodash": "^4.17.11",
+        "lodash": "^4.17.15",
         "moment": "^2.24.0",
         "moment-timezone": "^0.5.21",
-        "retry-as-promised": "^3.1.0",
-        "semver": "^6.1.1",
+        "retry-as-promised": "^3.2.0",
+        "semver": "^6.3.0",
         "sequelize-pool": "^2.3.0",
         "toposort-class": "^1.0.1",
-        "uuid": "^3.2.1",
+        "uuid": "^3.3.3",
         "validator": "^10.11.0",
-        "wkx": "^0.4.6"
+        "wkx": "^0.4.8"
       },
       "dependencies": {
         "debug": {
@@ -13920,6 +14048,11 @@
           "version": "6.3.0",
           "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
           "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
+        },
+        "uuid": {
+          "version": "3.3.3",
+          "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz",
+          "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ=="
         }
       }
     },
@@ -14565,9 +14698,9 @@
       "dev": true
     },
     "table": {
-      "version": "5.4.4",
-      "resolved": "https://registry.npmjs.org/table/-/table-5.4.4.tgz",
-      "integrity": "sha512-IIfEAUx5QlODLblLrGTTLJA7Tk0iLSGBvgY8essPRVNGHAzThujww1YqHLs6h3HfTg55h++RzLHH5Xw/rfv+mg==",
+      "version": "5.4.6",
+      "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz",
+      "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==",
       "dev": true,
       "requires": {
         "ajv": "^6.10.2",
@@ -14582,26 +14715,6 @@
           "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==",
           "dev": true
         },
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "dev": true,
-          "requires": {
-            "color-convert": "^1.9.0"
-          }
-        },
-        "slice-ansi": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz",
-          "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==",
-          "dev": true,
-          "requires": {
-            "ansi-styles": "^3.2.0",
-            "astral-regex": "^1.0.0",
-            "is-fullwidth-code-point": "^2.0.0"
-          }
-        },
         "string-width": {
           "version": "3.1.0",
           "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz",
@@ -15040,14 +15153,20 @@
         "is-typedarray": "^1.0.0"
       }
     },
+    "typescript": {
+      "version": "3.6.3",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.6.3.tgz",
+      "integrity": "sha512-N7bceJL1CtRQ2RiG0AQME13ksR7DiuQh/QehubYcghzv20tnh+MQnQIuJddTmsbqYj+dztchykemz0zFzlvdQw==",
+      "dev": true
+    },
     "uglify-js": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.0.tgz",
-      "integrity": "sha512-W+jrUHJr3DXKhrsS7NUVxn3zqMOFn0hL/Ei6v0anCIMoKC93TjcflTagwIHLW7SfMFfiQuktQyFVCFHGUE0+yg==",
+      "version": "3.6.4",
+      "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.4.tgz",
+      "integrity": "sha512-9Yc2i881pF4BPGhjteCXQNaXx1DCwm3dtOyBaG2hitHjLWOczw/ki8vD1bqyT3u6K0Ms/FpCShkmfg+FtlOfYA==",
       "dev": true,
       "optional": true,
       "requires": {
-        "commander": "~2.20.0",
+        "commander": "~2.20.3",
         "source-map": "~0.6.1"
       },
       "dependencies": {
@@ -15084,6 +15203,11 @@
         "debug": "^2.2.0"
       }
     },
+    "unfuck-utf8-headers-middleware": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/unfuck-utf8-headers-middleware/-/unfuck-utf8-headers-middleware-1.0.1.tgz",
+      "integrity": "sha512-C+0mNfriMuyP9LUPvNghKLI/7xLa42vJ7d4O2vd1ubKPwZeZ5oEfLXh1tF1OWzCKfI7ObFSfUm7bepWNNUJ+LQ=="
+    },
     "unicode-canonical-property-names-ecmascript": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz",
@@ -15550,6 +15674,70 @@
         "extsprintf": "^1.2.0"
       }
     },
+    "vue-eslint-parser": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-2.0.3.tgz",
+      "integrity": "sha512-ZezcU71Owm84xVF6gfurBQUGg8WQ+WZGxgDEQu1IHFBZNx7BFZg3L1yHxrCBNNwbwFtE1GuvfJKMtb6Xuwc/Bw==",
+      "dev": true,
+      "requires": {
+        "debug": "^3.1.0",
+        "eslint-scope": "^3.7.1",
+        "eslint-visitor-keys": "^1.0.0",
+        "espree": "^3.5.2",
+        "esquery": "^1.0.0",
+        "lodash": "^4.17.4"
+      },
+      "dependencies": {
+        "acorn": {
+          "version": "5.7.3",
+          "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz",
+          "integrity": "sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw==",
+          "dev": true
+        },
+        "acorn-jsx": {
+          "version": "3.0.1",
+          "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-3.0.1.tgz",
+          "integrity": "sha1-r9+UiPsezvyDSPb7IvRk4ypYs2s=",
+          "dev": true,
+          "requires": {
+            "acorn": "^3.0.4"
+          },
+          "dependencies": {
+            "acorn": {
+              "version": "3.3.0",
+              "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz",
+              "integrity": "sha1-ReN/s56No/JbruP/U2niu18iAXo=",
+              "dev": true
+            }
+          }
+        },
+        "debug": {
+          "version": "3.2.6",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
+          "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
+          "dev": true,
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        },
+        "espree": {
+          "version": "3.5.4",
+          "resolved": "https://registry.npmjs.org/espree/-/espree-3.5.4.tgz",
+          "integrity": "sha512-yAcIQxtmMiB/jL32dzEp2enBeidsB7xWPLNiw3IIkpVds1P+h7qF9YwJq1yUNzp2OKXgAprs4F61ih66UsoD1A==",
+          "dev": true,
+          "requires": {
+            "acorn": "^5.5.0",
+            "acorn-jsx": "^3.0.0"
+          }
+        },
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+          "dev": true
+        }
+      }
+    },
     "w3c-hr-time": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.1.tgz",
@@ -15752,9 +15940,9 @@
       }
     },
     "wkx": {
-      "version": "0.4.7",
-      "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.7.tgz",
-      "integrity": "sha512-pHf546L96TK8RradLt1cWaIffstgv/zXZ14CGz5KnBs1AxBX0wm+IDphjJw0qrEqRv8P9W9CdTt8Z1unMRZ19A==",
+      "version": "0.4.8",
+      "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.8.tgz",
+      "integrity": "sha512-ikPXMM9IR/gy/LwiOSqWlSL3X/J5uk9EO2hHNRXS41eTLXaUFEVw9fn/593jW/tE5tedNg8YjT5HkCa4FqQZyQ==",
       "requires": {
         "@types/node": "*"
       }

services/backend/oodikone2-backend/package.json

@@ -18,15 +18,11 @@
     "test": "npm run lint && npm run jest",
     "jest": "NODE_ENV=test jest --runInBand --verbose",
     "test_docker": "docker exec -t backend /bin/bash -c 'npm test'",
-    "update_database_dev": "NODE_ENV=dev node src/services/doo_api_database_updater/run_updater.js",
-    "update_database": "node src/services/doo_api_database_updater/run_updater.js",
-    "update_database_anon": "NODE_ENV=anon node src/services/doo_api_database_updater/run_updater.js",
     "init_redis": "node src/util/init_redis.js",
     "seed_redis": "node src/util/add_json_duplicates_to_redis.js && node src/util/map_course_codes.js",
-    "studentnumbers_to_file": "NODE_ENV=dev node src/services/doo_api_database_updater/db_studentnumbers_to_file.js",
-    "lint": "eslint --parser babel-eslint .",
+    "lint": "eslint .",
+    "lintfix": "prettier --write '**/*.{js,json}' && eslint --fix .",
     "force_sync_database": "node scripts/force_sync_database.js",
-    "update_studentnumbers_dev": "NODE_ENV=dev node src/services/doo_api_database_updater/student_list_updater.js",
     "anonymize": "node src/anonymizer/anonymizer.js",
     "seed_migrations": "node scripts/seed_migrations.js",
     "update_top_teachers": "node scripts/update_top_teachers.js",
@@ -34,6 +30,7 @@
     "refresh_studyright_associations": "node scripts/refresh_studyright_associations.js",
     "update_attainment_dates": "node scripts/update_max_attainment_dates.js",
     "create_defa_report": "node scripts/create_defa_report.js",
+    "create_fmsei_report": "node scripts/create_fmsei_report.js",
     "seed_course_groups": "node scripts/create_course_groups_kayttis.js"
   },
   "author": "Matti Luukkainen",
@@ -57,8 +54,9 @@
     "pg": "^7.11.0",
     "raven": "^2.6.3",
     "redis": "^2.8.0",
-    "sequelize": "^5.10.2",
+    "sequelize": "^5.21.1",
     "umzug": "^2.2.0",
+    "unfuck-utf8-headers-middleware": "^1.0.1",
     "uuid": "^3.3.2",
     "winston": "^3.0.0",
     "winston-log2gelf": "^2.0.4",
@@ -68,12 +66,16 @@
     "ava": "^2.2.0",
     "babel-eslint": "^8.2.6",
     "eslint": "^5.16.0",
+    "eslint-config-prettier": "^6.3.0",
+    "eslint-plugin-prettier": "^3.1.0",
     "faker": "^4.1.0",
     "jest": "^24.8.0",
     "nock": "^10.0.6",
     "node-status": "^1.0.0",
     "nodemon": "^1.19.1",
     "nyc": "^13.3.0",
+    "prettier": "^1.18.2",
+    "prettier-eslint": "^9.0.0",
     "supertest": "^3.4.2"
   },
   "ava": {

services/backend/oodikone2-backend/scripts/create_course_groups_kayttis.js

@@ -31,4 +31,4 @@ const groups = [
   'Svenska studier'
 ]
 
-run()
+run()

services/backend/oodikone2-backend/scripts/create_defa_report.js

@@ -1,179 +0,0 @@
-const fs = require('fs')
-const { getCourseUnitRealisation, getStudyAttainments } = require('../src/services/doo_api_database_updater/oodi_interface')
-
-const parseInfo = response => {
-  const { students } = response
-  return {
-    enrolled: students.length,
-    student_numbers: students.map(student => student.student_number)
-  }
-}
-
-const getStudents = (includedCourses, requiredCourses, timeframe) => async studentNumbers => {
-  const responses = await Promise.all(
-    studentNumbers.map(sn => getStudyAttainments(sn))
-  )
-  return responses.map((response, index) => parseStudentInfo(includedCourses, requiredCourses, timeframe)(response, studentNumbers[index]))
-}
-
-const optionalAYprefixRegex = course => new RegExp(`^(AY)?${course}$`)
-
-const parseStudentInfo = (includedCourses, requiredCourses, timeframe) => (response, studentNumber) => {
-  const studyAttainments = response.map(studyAttainment => ({
-    date: new Date(studyAttainment.attainment_date),
-    grade: Number(studyAttainment.grade[0].text),
-    credits: studyAttainment.credits,
-    course_code: studyAttainment.learningopportunity_id
-  }))
-  const defaCourses = studyAttainments
-    .filter(
-      studyAttainment => Boolean(includedCourses.find(
-        course => Boolean(studyAttainment.course_code.match(optionalAYprefixRegex(course)))
-      ))
-    )
-    .filter(studyAttainment => studyAttainment.grade > 0)
-    .filter(studyAttainment => withinTimeframe(timeframe)(studyAttainment.date))
-  const creditsTotal = defaCourses.reduce((acc, studyAttainment) => acc + studyAttainment.credits, 0)
-  const hasRequired = checkRequiredCourses(requiredCourses)(defaCourses)
-  const courseCompletions = includedCourses.map(course => Boolean(
-    defaCourses.find(courseCompletion => Boolean(courseCompletion.course_code.match(optionalAYprefixRegex(course))))
-  ))
-  return {
-    student_number: studentNumber,
-    credits: creditsTotal,
-    hasRequired,
-    courseCompletions
-  }
-}
-
-const checkRequiredCourses = requiredCourses => studyAttainments => requiredCourses.reduce(
-  (acc, course) => acc && Boolean(studyAttainments.find(
-    studyAttainment => Boolean(studyAttainment.course_code.match(optionalAYprefixRegex(course)))
-  )),
-  true
-)
-
-const withinTimeframe = timeframe => time => time > timeframe.start && time < timeframe.end
-
-const HEADER_ROW_STUB = '"student number","DEFA credits","required courses done"'
-
-const toCsv = includedCourses => students => {
-  const creditTotals = [0,0,0,0,0,0,0,0,0,0,0,0,0]
-  const courseTotals = includedCourses.map(() => 0)
-  students.forEach(student => {
-    let creditTotalIndex
-    if (student.credits >= 60) {
-      creditTotalIndex = 12
-    } else {
-      creditTotalIndex = (student.credits - student.credits % 5) / 5
-    }
-    creditTotals[creditTotalIndex] += 1
-    student.courseCompletions.forEach((complete, index) => {
-      if (complete) {
-        courseTotals[index] += 1
-      }
-    })
-  })
-
-  const allCredits = students.reduce((acc, student) => acc + student.credits,0)
-  const totalRequired = students.filter(student => student.hasRequired).length
-  const reportCsv = [
-    `${HEADER_ROW_STUB},${includedCourses.join(',')}`,
-    ...students.map(
-      student => `"${student.student_number}",${student.credits},${Number(student.hasRequired)},${student.courseCompletions.map(bool => Number(bool)).join(',')}`
-    ),
-    ['total', allCredits, totalRequired, ...courseTotals].join(',')
-  ].join('\n')
-
-  const creditTotalHeaders = ['0-4','5-9','10-14','15-19','20-24','25-29','30-34','35-39','40-44','45-49','50-54','55-59','60+']
-  const creditTotalCsv = [
-    creditTotalHeaders.join(','),
-    creditTotals.join(',')
-  ].join('\n')
-
-  return [reportCsv, creditTotalCsv]
-}
-
-const getStudentNumbers = async courseIds => {
-  const responses = await Promise.all(
-    courseIds.map(id => getCourseUnitRealisation(id))
-  )
-  const courseInfos = responses.map(response => parseInfo(response))
-  const studentNumbers = courseInfos
-    .reduce((acc, info) => [...acc, ...info.student_numbers], [])
-    .filter((studentNumber, index, array) => array.indexOf(studentNumber) === index)
-  return studentNumbers
-}
-
-const createReport = async (paramFileName) => {
-  // Get parameters
-  if (!paramFileName) {
-    console.error('No parameter file name was provided.')
-    process.exit(1)
-  }
-  const params = JSON.parse(String(fs.readFileSync(paramFileName)))
-  if (!params.out) {
-    console.error('No output files were defined in parameter file.')
-    process.exit(1)
-  }
-  if (!params.in) {
-    console.error('No input files were defined in parameter file.')
-    process.exit(1)
-  }
-  if (!params.in.course_ids) {
-    console.error('No file was defined as input for DEFA course unit realisation codes. Set it as in.course_ids in the parameter file.')
-    process.exit(1)
-  }
-  const courseIds = String(fs.readFileSync(params.in.course_ids)).trim().split('\n')
-  if (courseIds.length === 1 && courseIds[0].length === 0) {
-    console.error('DEFA course unit realisation codes input file is empty.')
-    process.exit(1)
-  }
-  if (!params.in.included_courses) {
-    console.error('No file was defined as input for included course codes. Set it as in.included_courses in the parameter file.')
-    process.exit(1)
-  }
-  const includedCourses = String(fs.readFileSync(params.in.included_courses)).trim().split('\n')
-  if (includedCourses.length === 1 && includedCourses[0].length === 0) {
-    console.error('Included course codes input file is empty.')
-    process.exit(1)
-  }
-  if (!params.in.required_courses) {
-    console.error('No file was defined as input for required course codes. Set it as in.required_courses in the parameter file.')
-    process.exit(1)
-  }
-  const requiredCourses = String(fs.readFileSync(params.in.required_courses)).trim().split('\n')
-  if (requiredCourses.length === 1 && requiredCourses[0].length === 0) {
-    console.error('Required course codes input file is empty.')
-    process.exit(1)
-  }
-  if (!params.in.timeframe || !params.in.timeframe.start || !params.in.timeframe.end) {
-    console.error('Input in.timeframe must be an object with fields "start" and "end".')
-    process.exit(1)
-  }
-  const timeframe = {
-    start: new Date(params.in.timeframe.start),
-    end: new Date(params.in.timeframe.end)
-  }
-
-  // Get information from api
-  const studentNumbers = await getStudentNumbers(courseIds)
-  const defaStudents = await getStudents(includedCourses, requiredCourses, timeframe)(studentNumbers)
-
-  // Print out into output files
-  const [reportCsv, statsCsv] = toCsv(includedCourses)(defaStudents)
-  if (params.out.report) {
-    fs.writeFileSync(params.out.report, reportCsv)
-  } else {
-    console.log('No file defined for report output. You can set it as out.report in the parameter file.')
-  }
-  if (params.out.stats) {
-    fs.writeFileSync(params.out.stats, statsCsv)
-  } else {
-    console.log('No file defined for stats output. You can set it as out.stats in the parameter file.')
-  }
-  console.log('Successfully created a DEFA report.')
-  process.exit(0)
-}
-
-createReport(...process.argv.slice(2))

services/backend/oodikone2-backend/scripts/create_fmsei_report.js

@@ -0,0 +1,125 @@
+/* eslint-disable no-console */
+
+const { Student, Credit, Studyright } = require('../src/models')
+const Sequelize = require('sequelize')
+const Op = Sequelize.Op
+
+const startDate = new Date(2019, 1, 1, 0, 0, 0, 0)
+
+const wasFMSEIStudent = (student, date) => {
+  const degreeright = student.studyrights.find(sr => sr.extentcode !== 9 && sr.startdate <= date && sr.enddate >= date)
+  return !degreeright
+}
+
+const courseCodes = [
+  'AYTKT200041',
+  'AYTKT21007',
+  'TKT21007',
+  'AYTKT21025en',
+  'AYTKT21009',
+  'TKT21009',
+  'AYTKT21010',
+  'TKT21010',
+  'AYTKT21018',
+  'AYTKT21018fi',
+  'AYTKT21018sv',
+  'AY5823951',
+  'AY5823952',
+  'AY5823953',
+  'AY5823954',
+  'AY5823955',
+  'AY5823956'
+]
+
+const createReport = async () => {
+  console.log('SPLIT')
+  const { fmseiStudentnumbers, otherStudentnumbers } = await courseCodes.reduce(
+    async (accPromise, courseCode) => {
+      const credits = await Credit.findAll({
+        where: {
+          course_code: courseCode,
+          attainment_date: {
+            [Op.gte]: startDate
+          }
+        }
+      })
+
+      const { fmseiStudentnumbers, otherStudentnumbers } = await credits.reduce(
+        async (accPromise, credit) => {
+          const student = await Student.findOne({
+            where: { studentnumber: credit.student_studentnumber },
+            include: [{ model: Studyright }]
+          })
+          const acc = await accPromise
+
+          if (wasFMSEIStudent(student, credit.attainment_date)) {
+            // FMSEI student fmsei credits
+            console.log(
+              `${credit.student_studentnumber};${credit.course_code};${credit.grade};${credit.credits};${new Date(
+                credit.attainment_date
+              ).toISOString()}`
+            )
+            return {
+              fmseiStudentnumbers: acc.fmseiStudentnumbers.concat(student.studentnumber),
+              otherStudentnumbers: acc.otherStudentnumbers
+            }
+          }
+          return {
+            fmseiStudentnumbers: acc.fmseiStudentnumbers,
+            otherStudentnumbers: acc.otherStudentnumbers.concat(student.studentnumber)
+          }
+        },
+        { fmseiStudentnumbers: [], otherStudentnumbers: [] }
+      )
+
+      const acc = await accPromise
+
+      return {
+        courseStats: acc.courseStats.concat({
+          courseCode: courseCode,
+          fmsei: fmseiStudentnumbers.length,
+          other: otherStudentnumbers.length
+        }),
+        fmseiStudentnumbers: acc.fmseiStudentnumbers.concat(fmseiStudentnumbers),
+        otherStudentnumbers: acc.otherStudentnumbers.concat(otherStudentnumbers)
+      }
+    },
+    { courseStats: [], fmseiStudentnumbers: [], otherStudentnumbers: [] }
+  )
+  const fmseiStudents = await Student.findAll({
+    where: { studentnumber: fmseiStudentnumbers },
+    include: [
+      {
+        model: Credit,
+        separate: true
+      }
+    ]
+  })
+
+  const otherStudents = await Student.findAll({
+    where: { studentnumber: otherStudentnumbers }
+  })
+  console.log('SPLIT')
+  otherStudents.forEach(s => console.log(`${new Date(s.birthdate).getFullYear()};${s.gender_fi}`))
+  console.log('SPLIT')
+  fmseiStudents.forEach(s =>
+    console.log(
+      `${s.studentnumber};${s.email || ''};${s.zipcode || ''};${s.city_fi || ''};${new Date(
+        s.birthdate
+      ).getFullYear()};${s.country_fi || ''};${s.home_country_fi || ''};${s.gender_fi}`
+    )
+  )
+  console.log('SPLIT')
+  // FMSEI all credits
+  fmseiStudents.forEach(s => {
+    s.credits.forEach(c =>
+      console.log(
+        `${c.student_studentnumber};${c.course_code};${c.grade};${c.credits};${new Date(
+          c.attainment_date
+        ).toISOString()}`
+      )
+    )
+  })
+}
+
+createReport()

services/backend/oodikone2-backend/scripts/force_sync_database.js

@@ -13,4 +13,4 @@ const sync = async () => {
   }
 }
 
-sync()
+sync()

services/backend/oodikone2-backend/scripts/refresh_studyright_associations.js

@@ -5,4 +5,4 @@ const run = async () => {
   process.exit(0)
 }
 
-run()
+run()

services/backend/oodikone2-backend/scripts/seed_migrations.js

@@ -10,4 +10,4 @@ const run = async () => {
   }
 }
 
-run()
+run()

services/backend/oodikone2-backend/scripts/update_max_attainment_dates.js

@@ -1,18 +0,0 @@
-const logger = require('../src/util/logger')
-const { updateAttainmentDates } = require('../src/services/doo_api_database_updater/update_attainment_dates')
-
-const run = async () => {
-  try {
-    logger.info('starting')
-
-    await updateAttainmentDates()
-
-    logger.info('complete')
-    process.exit(0)
-  } catch (err) {
-    logger.error('Update failed', err)
-    process.exit(1)
-  }
-}
-
-run()

services/backend/oodikone2-backend/scripts/update_students_not_modified_since.js

@@ -1,85 +0,0 @@
-const logger = require('../src/util/logger')
-const { Student } = require('../src/models/index')
-const { updateStudentsTaskPooled } = require('../src/services/doo_api_database_updater/database_updater')
-const { Op } = require('sequelize')
-
-const studentsNotModifiedSince = async since => {
-  const students = Student.findAll({
-    attributes: ['studentnumber'],
-    where: {
-      updatedAt: {
-        [Op.lt]: since
-      }
-    },
-    raw: true
-  })
-  return students.map(({ studentnumber }) => studentnumber)
-}
-
-const basiclogger = (studentnumbers, nstamps = 100) => {
-
-  const total = studentnumbers.length
-  const divisor = Math.floor(total/nstamps)
-
-  const dolog = iter => {
-    if (iter % divisor === 0) {
-      logger.info(`${new Date()} Students updated: ${iter}/${total}`)
-    }
-  }
-
-  let iter = 0
-
-  const start = () => {
-    logger.info(`Updater started, students to update: ${total}. `)
-  }
-
-  const onUpdate = () => {
-    iter += 1
-    dolog(iter)
-  }
-
-  const stop = () => {
-    logger.info('Updater finished. ')
-  }
-
-  return {
-    start,
-    stop,
-    onUpdate
-  }
-}
-
-const parseDateString = datestring => {
-  if (!datestring) {
-    throw Error('No datestring argument passed to script. Please pass a single datestring, e.g. 2018-08-20.')
-  } else if (!Date.parse(datestring)) {
-    throw Error(`${datestring} is not a valid Date. `)
-  } else {
-    return new Date(datestring)
-  }
-}
-
-const updateStudents = async datestring => {
-  const date = parseDateString(datestring)
-  const studentnumbers = await studentsNotModifiedSince(date)
-  if (!studentnumbers) {
-    logger.verbose(`No studentnumbers found who have not been updated since ${datestring}. `)
-  } else {
-    const logger = basiclogger(studentnumbers)
-    logger.start()
-    await updateStudentsTaskPooled(studentnumbers, 50, logger.onUpdate)
-    logger.stop()
-  }
-}
-
-const run = async () => {
-  try {
-    await updateStudents(process.argv[2])
-    process.exit(0)
-  } catch (e) {
-    logger.error(`Updater failed, error: ${e.message}`)
-    process.exit(1)
-  }
-}
-
-run()

services/backend/oodikone2-backend/scripts/update_top_teachers.js

@@ -8,15 +8,16 @@ const findAndSaveTeachers = async (startcode, endcode) => {
   }
 }
 
-const parseargs = (args, SEPARATOR='=')=> args
-  .filter(arg => arg.includes(SEPARATOR))
-  .reduce((acc, arg) => {
-    const [ key, val ] = arg.split(SEPARATOR)
-    return { 
-      ...acc, 
-      [key.trim()]: val.trim()
-    }
-  }, {})
+const parseargs = (args, SEPARATOR = '=') =>
+  args
+    .filter(arg => arg.includes(SEPARATOR))
+    .reduce((acc, arg) => {
+      const [key, val] = arg.split(SEPARATOR)
+      return {
+        ...acc,
+        [key.trim()]: val.trim()
+      }
+    }, {})
 
 const run = async () => {
   try {
@@ -34,4 +35,4 @@ const run = async () => {
   }
 }
 
-run()
+run()

services/backend/oodikone2-backend/src/anonymizer/anonDisciplines.js

@@ -112,4 +112,4 @@ const disciplines = {
     ]
   }
 }
-module.exports = {disciplines}
+module.exports = { disciplines }

services/backend/oodikone2-backend/src/anonymizer/anonStudyRights.js

@@ -60,4 +60,4 @@ const studyRightCodes = {
     code: '666mast'
   }
 }
-module.exports = {studyRightCodes}
+module.exports = { studyRightCodes }

services/backend/oodikone2-backend/src/anonymizer/anon_oodi_interface.js

@@ -2,7 +2,6 @@ const axios = require('axios')
 const https = require('https')
 require('dotenv').config()
 
-
 const instance = axios.create({
   httpsAgent: new https.Agent({
     rejectUnauthorized: false
@@ -10,16 +9,17 @@ const instance = axios.create({
 })
 
 axios.defaults.auth = {
-  username: 'tktl',
+  username: 'tktl'
 }
 
-instance.interceptors.request.use((config) => {
-  config.params = {token: process.env.TOKEN}
+instance.interceptors.request.use(config => {
+  config.params = { token: process.env.TOKEN }
   return config
 })
 
-const getStudent = (studentNumber) => {
-  return instance.get(`${process.env.OODI_ADDR}/students/${studentNumber}/info`)
+const getStudent = studentNumber => {
+  return instance
+    .get(`${process.env.OODI_ADDR}/students/${studentNumber}/info`)
     .then(response => {
       return response.data.data
     })
@@ -54,7 +54,6 @@ const getStudyattainmentStatusCodes = async () => {
   return response.data.data
 }
 
-
 const getSemesters = async () => {
   const url = `${process.env.OODI_ADDR}/codes/semesters`
   const response = await instance.get(url)
@@ -67,14 +66,13 @@ const getSemesterEnrollments = async studentnumber => {
   return response.data.data
 }
 
-
 const getCourseRealisationTypes = async () => {
   const url = `${process.env.OODI_ADDR}/codes/courseunitrealisations/types`
   const response = await instance.get(url)
   return response.data.data
 }
 
-const getLearningOpportunity = async (id) => {
+const getLearningOpportunity = async id => {
   const url = `${process.env.OODI_ADDR}/learningopportunities/${id}`
   const response = await instance.get(url)
   return response.data.data
@@ -103,4 +101,4 @@ module.exports = {
   getCourseRealisationTypes,
   courseUnitRealisations,
   getCourseUnitRealisation
-}
+}

services/backend/oodikone2-backend/src/anonymizer/anonymizer.js

@@ -1,5 +1,3 @@
-
-
 const faker = require('faker')
 const moment = require('moment')
 const mkdirp = require('mkdirp')
@@ -15,7 +13,6 @@ const status = require('node-status')
 require('dotenv').config()
 const startTime = moment()
 
-
 let student_numbers = []
 
 // few objects to save connections between anonymized and real data to keep relations.
@@ -27,17 +24,21 @@ const startCodesStatusBar = () => {
   status.start({ pattern: 'Running: {uptime.time} {spinner.monkey.blue} | {codes.bar} | codes anonymized: {codes}' })
 }
 const startStudentsStatusBar = () => {
-  status.start({ pattern: 'Running: {uptime.time} {spinner.hearts.magenta} | {students.bar} | students anonymized: {students}' })
+  status.start({
+    pattern: 'Running: {uptime.time} {spinner.hearts.magenta} | {students.bar} | students anonymized: {students}'
+  })
 }
 const startRealisationsStatusBar = () => {
-  status.start({ pattern: 'Running: {uptime.time} {spinner.clock.red} | {realisations.bar} | realisations anonymized: {realisations}' })
+  status.start({
+    pattern: 'Running: {uptime.time} {spinner.clock.red} | {realisations.bar} | realisations anonymized: {realisations}'
+  })
 }
 const stopStatusBar = () => {
   status.stamp()
   status.stop()
 }
 
-const getAnonymizedTeacher = async (teacherId) => {
+const getAnonymizedTeacher = async teacherId => {
   if (teacherRelations[teacherId]) {
     return teacherRelations[teacherId]
   }
@@ -61,7 +62,7 @@ const getAnonymizedTeacher = async (teacherId) => {
 
 const generateRandomId = () => String(Math.floor(Math.random() * (999999999 - 100000000 + 1) + 100000000))
 
-const anonymizeStudentInfo = async (id) => {
+const anonymizeStudentInfo = async id => {
   let student = await oodi.getStudent(id)
   let anonStudent = undefined
   if (student) {
@@ -72,7 +73,16 @@ const anonymizeStudentInfo = async (id) => {
       ...student,
       first_names: faker.name.firstName(),
       last_name: faker.name.lastName(),
-      birthdate: new Date(moment(moment('1940', 'YYYY').valueOf() + Math.random() * (moment().subtract(17, 'years').valueOf() - moment('1940', 'YYYY').valueOf()))),
+      birthdate: new Date(
+        moment(
+          moment('1940', 'YYYY').valueOf() +
+            Math.random() *
+              (moment()
+                .subtract(17, 'years')
+                .valueOf() -
+                moment('1940', 'YYYY').valueOf())
+        )
+      ),
       email: faker.internet.email(),
       mobile_phone: faker.phone.phoneNumberFormat(),
       national_student_number: null,
@@ -111,19 +121,19 @@ const anonymizeStudentInfo = async (id) => {
 
   return anonStudent
 }
-const getRandomAmount = (attainments) => {
-  const rate = (Math.random() * (0.375 - 0.05) + 0.05)
+const getRandomAmount = attainments => {
+  const rate = Math.random() * (0.375 - 0.05) + 0.05
   const amount = Math.floor(rate * attainments.length)
   return amount
 }
-const dropRandomAttainments = (attainments) => {
+const dropRandomAttainments = attainments => {
   if (student_numbers.length < 1) {
     return attainments
   }
   const dropAmount = getRandomAmount(attainments)
   let i = 0
   while (i < dropAmount) {
-    let randomIndex = Math.floor(Math.random() * (attainments.length))
+    let randomIndex = Math.floor(Math.random() * attainments.length)
     attainments.splice(randomIndex, 1)
     i++
   }
@@ -138,7 +148,11 @@ const addRandomAttainments = (attainments, studentnumber_attainments) => {
   let i = 0
   while (i < addAmount) {
     let randomStudentNumber = _.sample(studentnumbers)
-    let randomAttainment = _.sample(_.flattenDeep(studentnumber_attainments.filter(s => s.student_number === randomStudentNumber).map(s => s.attainments)))
+    let randomAttainment = _.sample(
+      _.flattenDeep(
+        studentnumber_attainments.filter(s => s.student_number === randomStudentNumber).map(s => s.attainments)
+      )
+    )
     if (!randomAttainment) {
       continue
     }
@@ -148,7 +162,7 @@ const addRandomAttainments = (attainments, studentnumber_attainments) => {
   }
   return attainments
 }
-const populateAnonymizedTeachers = async (teachers) => {
+const populateAnonymizedTeachers = async teachers => {
   let anonTeachers = []
   for (let teacher of teachers) {
     let anonTeacher = await getAnonymizedTeacher(teacher.teacher_id)
@@ -172,7 +186,7 @@ const anonymizeStudentAttainments = async (id, studentInfo, studentnumber_attain
       }
       anonLearningOpportunity = {
         ...learningOpportunity,
-        disciplines: [disciplines.data.data[Math.floor(Math.random() * 12)]] ,
+        disciplines: [disciplines.data.data[Math.floor(Math.random() * 12)]],
         learningopportunity_id: generateRandomId(),
         names: [
           {
@@ -188,17 +202,26 @@ const anonymizeStudentAttainments = async (id, studentInfo, studentnumber_attain
         materials: [],
         organisations: [],
         learningopportunity_internal: generateRandomId(),
-        start_date: moment(learningOpportunity.start_date).add(_.sample([-3, 0, 3]), 'months').add(_.sample(Math.floor(Math.random() * 100) - 50, 'days')),
-        end_date: moment(learningOpportunity.start_date).add(_.sample([-3, 0, 3]), 'months').add(_.sample(Math.floor(Math.random() * 100) - 50, 'days')).add(100, 'years'),
+        start_date: moment(learningOpportunity.start_date)
+          .add(_.sample([-3, 0, 3]), 'months')
+          .add(_.sample(Math.floor(Math.random() * 100) - 50, 'days')),
+        end_date: moment(learningOpportunity.start_date)
+          .add(_.sample([-3, 0, 3]), 'months')
+          .add(_.sample(Math.floor(Math.random() * 100) - 50, 'days'))
+          .add(100, 'years')
       }
       learningOpportunityRelations[attainment.learningopportunity_id] = anonLearningOpportunity
-      APIWriter(`./src/anonymized_API/learningopportunities/${anonLearningOpportunity.learningopportunity_id}`, { data: { data: anonLearningOpportunity } })
+      APIWriter(`./src/anonymized_API/learningopportunities/${anonLearningOpportunity.learningopportunity_id}`, {
+        data: { data: anonLearningOpportunity }
+      })
     }
     let teachers = []
     teachers = await populateAnonymizedTeachers(attainment.teachers)
 
     // semester code is calculated as "code * 6 months since 1950-1-1", Im bucketing all study attainments in 3 month periods by semester code abstracting real study attainment dates
-    const attainment_date = new Date(moment(moment('1950', 'YYYY').add((attainment.semester_code * 6) + _.sample([-3, 0, 3]), 'months')))
+    const attainment_date = new Date(
+      moment(moment('1950', 'YYYY').add(attainment.semester_code * 6 + _.sample([-3, 0, 3]), 'months'))
+    )
     attainment = {
       ...attainment,
       studyattainment_id: generateRandomId(),
@@ -223,40 +246,42 @@ const anonymizeStudentStudyRights = async (id, studentInfo) => {
       program = _.sample(['i', 'j', 'k', 'l', 'm', 'h'])
       degree = 'master'
     }
-    const elements = studyright.elements.filter(element => element.element_id === 10 || element.element_id === 20).map(element => {
-      if (element.element_id === 10) {
-        return {
-          ...element,
-          code: studyRightCodes[degree].code,
-          name: [
-            {
-              langcode: 'en',
-              text: studyRightCodes[degree].name
-            },
-            {
-              langcode: 'fi',
-              text: studyRightCodes[degree].name
-            }
-          ]
+    const elements = studyright.elements
+      .filter(element => element.element_id === 10 || element.element_id === 20)
+      .map(element => {
+        if (element.element_id === 10) {
+          return {
+            ...element,
+            code: studyRightCodes[degree].code,
+            name: [
+              {
+                langcode: 'en',
+                text: studyRightCodes[degree].name
+              },
+              {
+                langcode: 'fi',
+                text: studyRightCodes[degree].name
+              }
+            ]
+          }
+        } else if (element.element_id === 20) {
+          return {
+            ...element,
+            code: studyRightCodes[program].code,
+            name: [
+              {
+                langcode: 'en',
+                text: studyRightCodes[program].name
+              },
+              {
+                langcode: 'fi',
+                text: studyRightCodes[program].name
+              }
+            ]
+          }
         }
-      } else if (element.element_id === 20) {
-        return {
-          ...element,
-          code: studyRightCodes[program].code,
-          name: [
-            {
-              langcode: 'en',
-              text: studyRightCodes[program].name
-            },
-            {
-              langcode: 'fi',
-              text: studyRightCodes[program].name
-            }
-          ]
-        }
-      }
-      return
-    })
+        return
+      })
 
     let anonStudyright = {
       ...studyright,
@@ -278,8 +303,10 @@ const anonymizeStudentStudyRights = async (id, studentInfo) => {
   }
   return anonStudyRights
 }
-const getElapsedTime = () => `${moment().diff(startTime, 'minutes')} m, ${moment().diff(startTime, 'seconds') / (moment().diff(startTime, 'minutes') * 60)} s.`
-const anonymizeSemesterEnrollments = async (id) => {
+const getElapsedTime = () =>
+  `${moment().diff(startTime, 'minutes')} m, ${moment().diff(startTime, 'seconds') /
+    (moment().diff(startTime, 'minutes') * 60)} s.`
+const anonymizeSemesterEnrollments = async id => {
   const data = await oodi.getSemesterEnrollments(id)
   let anonymizedData = []
   for (const enrollment of data) {
@@ -299,8 +326,16 @@ const anonymizeStudent = async (id, student_numbers) => {
   const info = await anonymizeStudentInfo(id)
 
   const attainments = await anonymizeStudentAttainments(id, info, studentnumber_attainments)
-  studentnumber_attainments = studentnumber_attainments.concat({ student_number: info.student_number, attainments: attainments })
-  info.studyattainments = attainments.filter(attainment => ['Hyv.', '1', '2', '3', '4', '5'].includes(attainment.grade[0].text) && attainment.credits < 25).map(attainment => attainment.credits).reduce((acc, curr) => acc + curr, 0)
+  studentnumber_attainments = studentnumber_attainments.concat({
+    student_number: info.student_number,
+    attainments: attainments
+  })
+  info.studyattainments = attainments
+    .filter(
+      attainment => ['Hyv.', '1', '2', '3', '4', '5'].includes(attainment.grade[0].text) && attainment.credits < 25
+    )
+    .map(attainment => attainment.credits)
+    .reduce((acc, curr) => acc + curr, 0)
 
   const studyrights = await anonymizeStudentStudyRights(id, info)
 
@@ -319,14 +354,14 @@ const anonymizeStudent = async (id, student_numbers) => {
   APIWriter(semesterEnrollmentPath, { data: { data: semesterEnrollments } })
   return student_numbers
 }
-const anonymizeCourseUnitRealisations = async () => { 
+const anonymizeCourseUnitRealisations = async () => {
   const courseUnitRealisations = await oodi.courseUnitRealisations()
   const realisationsCounter = status.addItem('realisations', { max: courseUnitRealisations.length })
   startRealisationsStatusBar()
   let anonRealisations = []
   for (const realisation of courseUnitRealisations) {
     continue
-    realisationsCounter.inc()// eslint-disable-line
+    realisationsCounter.inc() // eslint-disable-line
     if (!learningOpportunityRelations[realisation.learningopportunity_id]) {
       continue
     }
@@ -373,30 +408,42 @@ const anonymizeCourseUnitRealisations = async () => {
       descriptions: [],
       languages: []
     }
-    await APIWriter(`src/anonymized_API/courseunitrealisations/${anonCourseUnitRealisation.course_id}`, { data: { data: anonCourseUnitRealisation } })
+    await APIWriter(`src/anonymized_API/courseunitrealisations/${anonCourseUnitRealisation.course_id}`, {
+      data: { data: anonCourseUnitRealisation }
+    })
   }
-  await APIWriter('src/anonymized_API/courseunitrealisations/changes/ids/0000-12-24', { data: { data: anonRealisations } })
+  await APIWriter('src/anonymized_API/courseunitrealisations/changes/ids/0000-12-24', {
+    data: { data: anonRealisations }
+  })
   stopStatusBar()
   return
 }
 
 const APIWriter = async (path, data) => {
   mkdirp(getDirName(path), async () => {
-    fs.writeFile(path, JSON.stringify(data, null, 4), (error) => { if (error) { console.log(error) } })
+    fs.writeFile(path, JSON.stringify(data, null, 4), error => {
+      if (error) {
+        console.log(error)
+      }
+    })
     return 'succeeee'
   })
 }
 
-
 let studentnumber_attainments = []
 
-
 const anonymize = async () => {
   logger.verbose(`starting time: ${startTime.format('HH:mm:ss')}`)
   const filename = './studentnumbers.txt'
   const readStudentNumbersFromFile = async filename => {
-    let studentnumbers = fs.readFileSync(filename, 'utf-8').split('\n').map(s => s.replace(' ', ''))
-    const randomCopies = _.sampleSize(studentnumbers, Math.floor((Math.random() * (1 - 0.33) + 0.33) * studentnumbers.length))
+    let studentnumbers = fs
+      .readFileSync(filename, 'utf-8')
+      .split('\n')
+      .map(s => s.replace(' ', ''))
+    const randomCopies = _.sampleSize(
+      studentnumbers,
+      Math.floor((Math.random() * (1 - 0.33) + 0.33) * studentnumbers.length)
+    )
     studentnumbers = studentnumbers.concat(randomCopies)
     studentnumbers = _.shuffle(studentnumbers)
     return studentnumbers.filter(studentnumber => !!studentnumber)
@@ -411,7 +458,7 @@ const anonymize = async () => {
   codeCounter.inc()
   const courseTypeCodes = { data: { data: await oodi.getCourseTypeCodes() } }
   codeCounter.inc()
-  const courseDisciplines = disciplines 
+  const courseDisciplines = disciplines
   codeCounter.inc()
   await APIWriter('./src/anonymized_API/codes/semesters', semesters)
   await APIWriter('./src/anonymized_API/codes/courseunitrealisations/types', courseUnitRealisationsTypes)
@@ -439,4 +486,4 @@ const anonymize = async () => {
   logger.verbose(`end time: ${moment()}, total run-time:  ${getElapsedTime()}`)
 }
 
-anonymize()
+anonymize()

services/backend/oodikone2-backend/src/app.js

@@ -8,39 +8,41 @@ const { startCron } = require('./events')
 const { PORT } = conf
 const { initializeDatabaseConnection } = require('./database/connection')
 
-initializeDatabaseConnection().then(() => {
-  const app = express()
+initializeDatabaseConnection()
+  .then(() => {
+    const app = express()
 
-  startCron()
+    startCron()
 
-  Raven.config(process.env.SENTRY_ADDR, { captureUnhandledRejections: true }).install()
+    Raven.config(process.env.SENTRY_ADDR, { captureUnhandledRejections: true, environment: process.env.TAG }).install()
 
-  app.use(cors({ credentials: true, origin: conf.frontend_addr }))
-  app.use(bodyParser.json())
+    app.use(cors({ credentials: true, origin: conf.frontend_addr }))
+    app.use(bodyParser.json())
 
-  app.get('/ping', async (req, res) => {
-    res.json({ data: 'pong' })
-  })
+    app.get('/ping', async (req, res) => {
+      res.json({ data: 'pong' })
+    })
 
-  let BASE_URL = ''
-  if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'test') {
-    BASE_URL = '/api'
-  }
+    let BASE_URL = ''
+    if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'test') {
+      BASE_URL = '/api'
+    }
 
-  routes(app, BASE_URL)
+    routes(app, BASE_URL)
 
-  app.get('*', async (req, res) => {
-    const results = { error: 'unknown endpoint' }
-    res.status(404).json(results)
-  })
+    app.get('*', async (req, res) => {
+      const results = { error: 'unknown endpoint' }
+      res.status(404).json(results)
+    })
 
-  const server = app.listen(PORT, () => console.log(`Backend listening on port ${PORT}!`))
-  process.on('SIGTERM', () => {
-    server.close(() => {
-      console.log('Process terminated')
+    const server = app.listen(PORT, () => console.log(`Backend listening on port ${PORT}!`))
+    process.on('SIGTERM', () => {
+      server.close(() => {
+        console.log('Process terminated')
+      })
     })
   })
-}).catch(e => {
-  process.exitCode = 1
-  console.log(e)
-})
+  .catch(e => {
+    process.exitCode = 1
+    console.log(e)
+  })

services/backend/oodikone2-backend/src/conf-backend.js

@@ -7,8 +7,7 @@ let DB_URL_KONE = process.env.DB_URL_KONE
 if (isTest) {
   DB_URL = process.env.TEST_DB
   DB_URL_KONE = process.env.TEST_DB_KONE
-}
-else if (NODE_ENV === 'anon') {
+} else if (NODE_ENV === 'anon') {
   DB_URL = process.env.ANON_DB
   DB_URL_KONE = process.env.ANON_DB_KONE
 }
@@ -19,29 +18,34 @@ const DB_SCHEMA = process.env.DB_SCHEMA || 'public'
 const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
 const CERT_PATH = process.env.CERT_PATH // production/staging only
 const KEY_PATH = process.env.KEY_PATH // production/staging only
-const OODILEARN_URL = process.env.OODILEARN_URL
 const USERSERVICE_URL = process.env.USERSERVICE_URL
 const UPDATER_URL = process.env.UPDATER_URL
 const USAGESERVICE_URL = process.env.USAGESERVICE_URL
 const ANALYTICS_URL = process.env.ANALYTICS_URL
 const PORT = isTest ? 8079 : 8080
 const OODI_SECRET = process.env.OODI_SECRET
+const ANALYTICS_INFLUXDB_URL = process.env.ANALYTICS_INFLUXDB_URL
+const ANALYTICS_INFLUXDB_USER = process.env.ANALYTICS_INFLUXDB_USER
+const ANALYTICS_INFLUXDB_PASSWORD = process.env.ANALYTICS_INFLUXDB_PASSWORD
+const ANALYTICS_INFLUXDB_DB = process.env.ANALYTICS_INFLUXDB_DB
 
 const FEATURES = {
   ERROR_HANDLER: false
 }
 
 const formatURL = url => {
-  return (!!url && !url.startsWith('http')) ? `http://${url}` : url
+  return !!url && !url.startsWith('http') ? `http://${url}` : url
 }
 
 if (process.env.NODE_ENV === 'dev' && process.env.FEATURES) {
   const toggled = process.env.FEATURES.split(',')
-  toggled.map(toggle => toggle.trim()).forEach(feature => {
-    if (FEATURES[feature] !== undefined) {
-      FEATURES[feature] = true
-    }
-  })
+  toggled
+    .map(toggle => toggle.trim())
+    .forEach(feature => {
+      if (FEATURES[feature] !== undefined) {
+        FEATURES[feature] = true
+      }
+    })
 }
 
 const OODI = {
@@ -73,7 +77,6 @@ module.exports = {
   CERT_PATH,
   KEY_PATH,
   FEATURES,
-  OODILEARN_URL,
   USERSERVICE_URL: formatURL(USERSERVICE_URL),
   ACCESS_TOKEN_HEADER_KEY,
   PORT,
@@ -83,5 +86,9 @@ module.exports = {
   OODI_SECRET,
   OODI_SECRET_HEADER_KEY,
   isTest,
-  UPDATER_URL
-}
+  UPDATER_URL,
+  ANALYTICS_INFLUXDB_URL: formatURL(ANALYTICS_INFLUXDB_URL),
+  ANALYTICS_INFLUXDB_USER,
+  ANALYTICS_INFLUXDB_PASSWORD,
+  ANALYTICS_INFLUXDB_DB
+}

services/backend/oodikone2-backend/src/database/connection.js

@@ -16,7 +16,6 @@ const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
 })
 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)
 
-
 const initializeDatabaseConnection = async () => {
   const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
   const waitSeconds = 60
@@ -45,14 +44,11 @@ const initializeDatabaseConnection = async () => {
         },
         logging: console.log,
         migrations: {
-          params: [
-            sequelizeKone.getQueryInterface(),
-            Sequelize
-          ],
+          params: [sequelizeKone.getQueryInterface(), Sequelize],
           path: `${process.cwd()}/src/database/migrations_kone`,
           pattern: /\.js$/,
           schema: conf.DB_SCHEMA_KONE
-        },
+        }
       })
       const migrations = await migrator.up()
 
@@ -84,4 +80,4 @@ module.exports = {
   sequelizeKone,
   initializeDatabaseConnection,
   forceSyncDatabase
-}
+}

services/backend/oodikone2-backend/src/database/seed_migrations.js

@@ -8,12 +8,12 @@ const DEFAULT_PATH_KONE = `${process.cwd()}/src/database/migrations_kone`
 const saveMigrationsToDatabase = Migration => async filenames =>
   Promise.all(filenames.map(name => Migration.upsert({ name })))
 
-const seedMigrations = async (migrationfilepath=DEFAULT_PATH) => {
+const seedMigrations = async (migrationfilepath = DEFAULT_PATH) => {
   const filenames = fs.readdirSync(migrationfilepath)
   await saveMigrationsToDatabase(Migration)(filenames)
 }
 
-const seedMigrationsKone = async (migrationfilepath=DEFAULT_PATH_KONE) => {
+const seedMigrationsKone = async (migrationfilepath = DEFAULT_PATH_KONE) => {
   const filenames = fs.readdirSync(migrationfilepath)
   await saveMigrationsToDatabase(MigrationKone)(filenames)
 }
@@ -25,4 +25,4 @@ const seedAllMigrations = async () => {
 
 module.exports = {
   seedAllMigrations
-}
+}

services/backend/oodikone2-backend/src/events.js

@@ -3,7 +3,15 @@ const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
 const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studytrack')
 const { calculateFacultyYearlyStats } = require('./services/faculties')
-const { setProductivity, setThroughput, patchProductivity, patchThroughput, patchFacultyYearlyStats } = require('./services/analyticsService')
+const topteachers = require('./services/topteachers')
+
+const {
+  setProductivity,
+  setThroughput,
+  patchProductivity,
+  patchThroughput,
+  patchFacultyYearlyStats
+} = require('./services/analyticsService')
 
 const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
 
@@ -32,37 +40,29 @@ const refreshOverview = async () => {
     const codes = (await getAllProgrammes()).map(p => p.code)
     let ready = 0
     for (const code of codes) {
+      let programmeStatsSince = new Date('2017-07-31')
+      if (code.includes('MH') || code.includes('KH')) {
+        programmeStatsSince = new Date('2017-07-31')
+      } else {
+        programmeStatsSince = new Date('2000-07-31')
+      }
       try {
         await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-        const since = new Date().getFullYear() - 5
-        const data = await throughputStatsForStudytrack(
-          code,
-          since
-        )
-        await setThroughput(
-          data
-        )
+        const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
+        await setThroughput(data)
       } catch (e) {
         try {
           await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
         } catch (e) {
-          console.error(
-            e
-          )
+          console.error(e)
         }
         console.error(e)
-        console.log(
-          `Failed to update throughput stats for code: ${code}, reason: ${
-            e.message
-          }`
-        )
+        console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
       }
       try {
         await patchProductivity({ [code]: { status: 'RECALCULATING' } })
-        const since = '2017-08-01'
-        const data = await productivityStatsForStudytrack(code, since)
+        const data = await productivityStatsForStudytrack(code, programmeStatsSince)
         await setProductivity(data)
-
       } catch (e) {
         try {
           await patchProductivity({
@@ -75,23 +75,31 @@ const refreshOverview = async () => {
         console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
       }
       ready += 1
-      console.log(
-        `RefreshOverview ${ready}/${
-          codes.length
-        } done`
-      )
+      console.log(`RefreshOverview ${ready}/${codes.length} done`)
     }
   } catch (e) {
     console.error(e)
   }
 }
 
+const refrestTeacherLeaderboard = async () => {
+  try {
+    const startyearcode = new Date().getFullYear() - 1950
+    const endyearcode = startyearcode + 1
+    console.log('Refreshing teacher leaderboard...')
+    await topteachers.findAndSaveTeachers(startyearcode, endyearcode)
+  } catch (e) {
+    console.log(e)
+  }
+}
+
 const startCron = () => {
   if (process.env.NODE_ENV === 'production') {
     schedule('0 6 * * *', async () => {
       await refreshFacultyYearlyStats()
       await refreshStudyrightAssociations()
       await refreshOverview()
+      await refrestTeacherLeaderboard()
     })
   }
 }

services/backend/oodikone2-backend/src/middleware/accesslogger.js

@@ -1,9 +1,34 @@
 const morgan = require('morgan')
+const usageService = require('../services/usageService')
+const logger = require('../util/logger')
 
+// So this appears to be a hack to get neatly formatted stats like response-time etc. from morgan
+// without actually using morgan what it's used for (LOGGING REQUESTS!).
+// Return undefined from here so that stuff isn't printed to console
+// so don't e.g. make it into an async function or it'll print [object Promise] on every request
 const accessLogger = morgan((tokens, req, res) => {
   const fields = ['method', 'url', 'status', 'response-time', 'remote-addr', 'remote-user', 'user-agent', 'referrer']
   const meta = req.decodedToken
-  fields.forEach(field => meta[field] = tokens[field](req, res))
+  fields.forEach(field => (meta[field] = tokens[field](req, res)))
+  fields['time'] = tokens['date'](req, res, 'iso')
+  if (req.route) {
+    meta['req-route'] = req.route.path
+  }
+
+  const message = [
+    req.decodedToken.name,
+    ':',
+    tokens['method'](req, res),
+    tokens['url'](req, res),
+    tokens['status'](req, res),
+    '-',
+    tokens['response-time'](req, res),
+    'ms'
+  ].join(' ')
+
+  usageService.log(message, meta).catch(e => {
+    logger.error('Failed to save usage stats!', e)
+  })
 })
 
-module.exports = accessLogger
+module.exports = accessLogger

services/backend/oodikone2-backend/src/middleware/auth.js

@@ -1,7 +1,7 @@
 const jwt = require('jsonwebtoken')
 const conf = require('../conf-backend')
 const blacklist = require('../services/blacklist')
-const { getAccessGroupCodesFor, getCodesFromElementDetails } = require('../services/userService')
+const { getUserDataFor } = require('../services/userService')
 const { ACCESS_TOKEN_HEADER_KEY } = require('../conf-backend')
 const { hasRequiredGroup, parseHyGroups } = require('../util/utils')
 
@@ -13,11 +13,13 @@ const checkAuth = async (req, res, next) => {
   if (!token) return res.status(403).json({ error: 'No token in headers' })
   jwt.verify(token, conf.TOKEN_SECRET, async (err, decoded) => {
     if (err) return res.status(403).json(err)
-    if (decoded.version !== TOKEN_VERSION) return res.status(401).json({ error: 'Token needs to be refreshed - invalid version' })
-    if (decoded.mockedBy ? decoded.mockedBy !== uid : decoded.userId !== uid) return res.status(403).json({ error: 'User shibboleth id and token id did not match' })
+    if (decoded.version !== TOKEN_VERSION)
+      return res.status(401).json({ error: 'Token needs to be refreshed - invalid version' })
+    if (decoded.mockedBy ? decoded.mockedBy !== uid : decoded.userId !== uid)
+      return res.status(403).json({ error: 'User shibboleth id and token id did not match' })
+    const userData = await getUserDataFor(decoded.userId)
     req.decodedToken = decoded
-    req.roles = await getAccessGroupCodesFor(decoded.userId)
-    req.rights = await getCodesFromElementDetails(decoded.userId)
+    Object.assign(req, userData)
     next()
   })
 }
@@ -30,7 +32,7 @@ const roles = requiredRoles => async (req, res, next) => {
       return
     }
   }
-  console.log(`Missing required roles ${ requiredRoles }`)
+  console.log(`Missing required roles ${requiredRoles}`)
   res.status(403).json({ error: 'missing required roles' })
 }
 
@@ -60,5 +62,8 @@ const checkUserBlacklisting = async (req, res, next) => {
 }
 
 module.exports = {
-  checkAuth, roles, checkRequiredGroup, checkUserBlacklisting
+  checkAuth,
+  roles,
+  checkRequiredGroup,
+  checkUserBlacklisting
 }

services/backend/oodikone2-backend/src/routes.js

@@ -1,3 +1,4 @@
+const shibbolethHeadersFix = require('unfuck-utf8-headers-middleware')
 const accessLogger = require('./middleware/accesslogger')
 const log = require('./routes/log')
 const courses = require('./routes/courses')
@@ -13,8 +14,8 @@ const teachers = require('./routes/teachers')
 const usage = require('./routes/usage')
 const providers = require('./routes/providers')
 const faculties = require('./routes/faculties')
+const facultyStats = require('./routes/facultyStats')
 const semesters = require('./routes/semesters')
-const oodilearn = require('./routes/oodilearn')
 const courseGroups = require('./routes/courseGroups')
 const mandatoryCourses = require('./routes/mandatorycourses')
 const mandatoryCourseLabels = require('./routes/mandatorycourselabels')
@@ -24,8 +25,11 @@ const task = require('./routes/tasks')
 const feedback = require('./routes/feedback')
 const tags = require('./routes/tags')
 const updater = require('./routes/updater')
+const tsaAnalytics = require('./routes/tsaAnalytics')
+const customPopulationSearch = require('./routes/customPopulationSearch')
 
 module.exports = (app, url) => {
+  app.use(shibbolethHeadersFix(['hyGroupCn', 'SHIB_LOGOUT_URL', 'eduPersonAffiliation', 'uid', 'displayName', 'mail']))
   app.use(url, log)
   app.use(url, login)
   app.use(`${url}/superlogin`, superlogin)
@@ -37,18 +41,20 @@ module.exports = (app, url) => {
   app.use(url, population)
   app.use(url, language)
   app.use(url, providers)
-  app.use(url, faculties)
   app.use(url, semesters)
   app.use(url, tags)
+  app.use(`${url}/faculties`, faculties)
+  app.use(`${url}/facultystats`, auth.roles(['faculties']), facultyStats)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
   app.use(`${url}/users`, auth.roles(['users']), users)
   app.use(`${url}/feedback`, feedback)
   app.use(`${url}/usage`, auth.roles(['usage']), usage)
-  app.use(`${url}/oodilearn`, auth.roles(['oodilearn']), oodilearn)
   app.use(`${url}/course-groups`, auth.roles(['coursegroups']), courseGroups)
   app.use(`${url}/mandatory_courses`, mandatoryCourses)
   app.use(`${url}/mandatory-course-labels`, mandatoryCourseLabels)
   app.use(`${url}/oodi`, auth.roles(['dev']), oodi)
+  app.use(`${url}/tsa`, tsaAnalytics)
+  app.use(`${url}/custom-population-search`, auth.roles(['dev', 'admin']), customPopulationSearch)
   app.use(url, auth.roles(['dev', 'admin']), task)
-}
+}

services/backend/oodikone2-backend/src/routes/courseGroups.js

@@ -7,7 +7,9 @@ router.get('/programme/:programmeId/:force?', async (req, res) => {
   const semesterCode = req.query.semester
   if (rights.includes(programmeId) || (roles && roles.includes('admin'))) {
     const courseGroups = await CourseGroupService.getCourseGroupsWithTotals(
-      programmeId, semesterCode, force === 'force'
+      programmeId,
+      semesterCode,
+      force === 'force'
     )
     return res.json(courseGroups)
   }

services/backend/oodikone2-backend/src/routes/courses.js

@@ -1,5 +1,6 @@
 const router = require('express').Router()
 const Course = require('../services/courses')
+const { validateParamLength } = require('../util')
 const logger = require('../util/logger')
 
 router.get('/courses', async (req, res) => {
@@ -14,16 +15,25 @@ router.get('/courses', async (req, res) => {
 router.get('/coursesmulti', async (req, res) => {
   let results = []
   if (req.query.name || req.query.discipline || req.query.type) {
-    results = await Course.bySearchTermTypeAndDiscipline(req.query.name, req.query.type, req.query.discipline, req.query.language) // eslint-disable-line
+    results = await Course.bySearchTermTypeAndDiscipline(
+      req.query.name,
+      req.query.type,
+      req.query.discipline,
+      req.query.language
+    ) // eslint-disable-line
   }
   res.json(results)
 })
 
 router.get('/v2/coursesmulti', async (req, res) => {
-  let results = []
-  if (req.query.name || req.query.code) {
-    results = await Course.byNameAndOrCodeLike(req.query.name, req.query.code)
+  let results = { courses: [], groups: {}, groupMeta: {} }
+  const { name, code } = req.query
+
+  if (!(validateParamLength(name, 5) || validateParamLength(code, 2))) {
+    return res.status(400).json({ error: 'name or code invalid' })
   }
+
+  results = await Course.byNameAndOrCodeLike(name, code)
   res.json(results)
 })
 
@@ -49,8 +59,9 @@ router.get('/v2/courseyearlystats', async (req, res) => {
   if (req.query.start && req.query.codes && req.query.end) {
     const { codes } = req.query
     const years = { start: req.query.start, end: req.query.end }
-    results = await Promise.all(JSON.parse(codes)
-      .map(code => Course.yearlyStatsOf(code, years, req.query.separate, req.query.language)))
+    results = await Promise.all(
+      JSON.parse(codes).map(code => Course.yearlyStatsOf(code, years, req.query.separate, req.query.language))
+    )
   }
   res.json(results)
 })
@@ -58,7 +69,7 @@ router.get('/v2/courseyearlystats', async (req, res) => {
 router.get('/v3/courseyearlystats', async (req, res) => {
   try {
     const { rights, roles } = req
-    const admin = roles.includes('admin')
+    const admin = roles && roles.includes('admin')
     if (!admin) {
       // If user has rights to see at least one programme, then they are allowed
       // to see all of them
@@ -66,12 +77,14 @@ router.get('/v3/courseyearlystats', async (req, res) => {
         return res.status(403).json({ error: 'No programmes so no access to course stats' })
       }
     }
-    const { codes, separate: sep } = req.query
+    const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
+
     const separate = !sep ? false : JSON.parse(sep)
+    const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
     if (!codes) {
       res.status(422).send('Missing required query parameters')
     } else {
-      const results = await Course.courseYearlyStats(codes, separate)
+      const results = await Course.courseYearlyStats(codes, separate, unifyOpenUniCourses)
       res.json(results)
     }
   } catch (e) {

services/backend/oodikone2-backend/src/routes/customPopulationSearch.js

@@ -0,0 +1,60 @@
+const router = require('express').Router()
+const {
+  getCustomPopulationSearchesByUser,
+  createCustomPopulationSearch,
+  updateCustomPopulationSearch
+} = require('../services/customPopulationSearch')
+
+router.get('/', async (req, res) => {
+  try {
+    const {
+      decodedToken: { id }
+    } = req
+    const customPopulationSearches = await getCustomPopulationSearchesByUser(id)
+    res.json(customPopulationSearches)
+  } catch (e) {
+    console.error('e', e)
+    res.status(500).json({ error: 'error' })
+  }
+})
+
+router.post('/', async (req, res) => {
+  try {
+    const {
+      decodedToken: { id },
+      body: { name, students }
+    } = req
+
+    if (!name) return res.status(400).json({ error: 'name missing' })
+    if (students && !Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
+
+    const customPopulationSearch = await createCustomPopulationSearch(name, id, students || [])
+    res.json(customPopulationSearch)
+  } catch (e) {
+    console.error('e', e)
+    res.status(500).json({ error: 'error' })
+  }
+})
+
+router.put('/', async (req, res) => {
+  try {
+    const {
+      decodedToken: { id: userId },
+      body: { id, students }
+    } = req
+
+    if (!id) return res.status(400).json({ error: 'id missing' })
+    if (!students) return res.status(400).json({ error: 'students missing' })
+    if (!Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
+
+    const updatedPopulationSearch = await updateCustomPopulationSearch(userId, id, students)
+    if (!updatedPopulationSearch) return res.status(404).json({ error: 'custom population search not found' })
+
+    res.json(updatedPopulationSearch)
+  } catch (e) {
+    console.error('e', e)
+    res.status(500).json({ error: 'error' })
+  }
+})
+
+module.exports = router

services/backend/oodikone2-backend/src/routes/elementdetails.js

@@ -3,8 +3,17 @@ const { getAllDegreesAndProgrammes, getAllProgrammes, getAllElementDetails } = r
 const MandatoryCourses = require('../services/mandatoryCourses')
 const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('../services/studytrack')
 const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
-const { getProductivity, setProductivity, getThroughput, setThroughput,
-  patchProductivity, patchThroughput, ping } = require('../services/analyticsService')
+const {
+  getProductivity,
+  setProductivity,
+  getThroughput,
+  setThroughput,
+  patchProductivity,
+  patchThroughput,
+  ping
+} = require('../services/analyticsService')
+
+const programmeStatsSince = new Date('2017-07-31')
 
 router.get('/elementdetails/all', async (req, res) => {
   try {
@@ -54,8 +63,7 @@ router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
     }
     if (!data) {
       try {
-        const since = '2017-08-01'
-        const stats = await productivityStatsForStudytrack(code, since)
+        const stats = await productivityStatsForStudytrack(code, programmeStatsSince)
         data = await setProductivity(stats)
       } catch (e) {
         console.error(e)
@@ -73,10 +81,12 @@ router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
   console.log('Productivity stats recalculation starting')
   const codes = code ? [code] : (await getAllProgrammes()).map(p => p.code)
   try {
-    await patchProductivity(codes.reduce((acc, c) => {
-      acc[c] = { status: 'RECALCULATING' }
-      return acc
-    }, {}))
+    await patchProductivity(
+      codes.reduce((acc, c) => {
+        acc[c] = { status: 'RECALCULATING' }
+        return acc
+      }, {})
+    )
     res.status(200).end()
   } catch (e) {
     console.error(e)
@@ -84,11 +94,15 @@ router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
   }
 
   let ready = 0
-  for(const code of codes) {
+  for (const code of codes) {
     try {
-      const since = '2017-08-01'
-      const data = await productivityStatsForStudytrack(code, since)
-      await setProductivity(data)
+      if (code.includes('MH') || code.includes('KH')) {
+        const data = await productivityStatsForStudytrack(code, new Date('2017-07-31'))
+        await setProductivity(data)
+      } else {
+        const data = await productivityStatsForStudytrack(code, new Date('2000-07-31'))
+        await setProductivity(data)
+      }
     } catch (e) {
       try {
         await patchProductivity({
@@ -119,8 +133,7 @@ router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
     }
     if (!data) {
       try {
-        const since = req.params.since ? req.params.since : new Date().getFullYear() - 5
-        const result = await throughputStatsForStudytrack(req.params.id, since)
+        const result = await throughputStatsForStudytrack(req.params.id, programmeStatsSince.getFullYear())
         data = await setThroughput(result)
       } catch (e) {
         console.error(e)
@@ -138,7 +151,7 @@ router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
   console.log('Throughput stats recalculation starting')
   const codes = code ? [code] : (await getAllProgrammes()).map(p => p.code)
   try {
-    const data = codes.reduce((acc, id) => ({ ...acc, [id]: { status: 'RECALCULATING' }}), {})
+    const data = codes.reduce((acc, id) => ({ ...acc, [id]: { status: 'RECALCULATING' } }), {})
     await patchThroughput(data)
     res.status(200).end()
   } catch (e) {
@@ -147,14 +160,18 @@ router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
   }
 
   let ready = 0
-  for(const code of codes) {
+  for (const code of codes) {
     try {
-      const since = req.params.since ? req.params.since : new Date().getFullYear() - 5
-      const data = await throughputStatsForStudytrack(code, since)
-      await setThroughput(data)
+      if (code.includes('MH') || code.includes('KH')) {
+        const data = await throughputStatsForStudytrack(code, 2017)
+        await setThroughput(data)
+      } else {
+        const data = await throughputStatsForStudytrack(code, 2000)
+        await setThroughput(data)
+      }
     } catch (e) {
       try {
-        await patchThroughput({ [code]:  { status: 'RECALCULATION ERRORED' }})
+        await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
       } catch (e) {
         console.error(e)
         return

services/backend/oodikone2-backend/src/routes/faculties.js

@@ -1,29 +1,9 @@
 const router = require('express').Router()
 const { all } = require('../services/organisations')
-const { byType: getElementDetailsByType } = require('../services/elementdetails')
-const { getFacultyYearlyStats } = require('../services/analyticsService')
 
-router.get('/faculties', async (req, res) => {
+router.get('/', async (req, res) => {
   const faculties = await all()
   res.json(faculties)
 })
 
-router.get('/faculties/yearlystats', async (req, res) => {
-  try {
-    const facultyYearlyStats = await getFacultyYearlyStats()
-    res.status(200).json(facultyYearlyStats)
-  } catch (e) {
-    res.status(400).json({ error: e })
-  }
-})
-
-router.get('/faculties/programmes', async (req, res) => {
-  try {
-    const facultyProgrammes = await getElementDetailsByType(20)
-    res.json(facultyProgrammes)
-  } catch (e) {
-    res.status(400).json({ error: e })
-  }
-})
-
-module.exports = router
+module.exports = router

services/backend/oodikone2-backend/src/routes/facultyStats.js

@@ -0,0 +1,34 @@
+const router = require('express').Router()
+const { all } = require('../services/organisations')
+const { byType: getElementDetailsByType } = require('../services/elementdetails')
+const { getFacultyYearlyStats } = require('../services/analyticsService')
+
+router.get('/faculties', async (req, res) => {
+  const { roles, faculties } = req
+  const allFaculties = await all()
+  res.json(roles.includes('admin') ? allFaculties : allFaculties.filter(({ code }) => faculties.has(code)))
+})
+
+router.get('/yearlystats', async (req, res) => {
+  try {
+    const { roles, faculties } = req
+    const facultyYearlyStats = await getFacultyYearlyStats()
+    res.json(roles.includes('admin') ? facultyYearlyStats : facultyYearlyStats.filter(({ id }) => faculties.has(id)))
+  } catch (e) {
+    res.status(400).json({ error: e })
+  }
+})
+
+router.get('/programmes', async (req, res) => {
+  try {
+    const { roles } = req
+    const isAdmin = roles.includes('admin')
+    const facultyProgrammes = await getElementDetailsByType(20)
+    const userRightsSet = new Set(req.rights)
+    res.json(facultyProgrammes.filter(({ code }) => userRightsSet.has(code) || isAdmin))
+  } catch (e) {
+    res.status(400).json({ error: e })
+  }
+})
+
+module.exports = router

services/backend/oodikone2-backend/src/routes/feedback.js

@@ -6,21 +6,30 @@ router.post('/email', async (req, res) => {
   const { content } = req.body
   const { uid } = req.headers
   const { email, full_name } = await userService.byUsername(uid)
-  const formattedEmail = content.split('\n\n').map(line => `<p> ${line} </p>`).join('')
+  const formattedEmail = content
+    .split('\n\n')
+    .map(line => `<p> ${line} </p>`)
+    .join('')
   const feedback = mailservice.feedback(formattedEmail, uid, email, full_name)
 
   if (process.env.SMTP !== undefined && content) {
-    await mailservice.transporter.sendMail(feedback, (error) => {
+    await mailservice.transporter.sendMail(feedback, error => {
       if (error) {
         console.log('Error occured')
-        res.status(400).json(error).end()
+        res
+          .status(400)
+          .json(error)
+          .end()
       } else {
         console.log('Message sent succesfully!')
-        res.status(200).json('success').end()
+        res
+          .status(200)
+          .json('success')
+          .end()
       }
       mailservice.transporter.close()
     })
   }
 })
 
-module.exports = router
+module.exports = router

services/backend/oodikone2-backend/src/routes/language.js

@@ -4,7 +4,7 @@ const userService = require('../services/userService')
 router.post('/language', async (req, res) => {
   const { language } = req.body
   const { userId } = req.decodedToken
-  if (!['fi','sv','en'].includes(language)) {
+  if (!['fi', 'sv', 'en'].includes(language)) {
     return res.status(400).json('invalid language')
   }
   try {

services/backend/oodikone2-backend/src/routes/login.js

@@ -3,10 +3,10 @@ const mailservice = require('../services/mailservice')
 const userService = require('../services/userService')
 const { parseHyGroups } = require('../util/utils')
 
-const sendEmail = async (uid) => {
+const sendEmail = async uid => {
   if (process.env.SMTP !== undefined) {
     const message = mailservice.message1(uid)
-    await mailservice.transporter.sendMail(message, (error) => {
+    await mailservice.transporter.sendMail(message, error => {
       if (error) {
         console.log('Error occurred')
         console.log(error.message)
@@ -33,10 +33,13 @@ router.post('/login', async (req, res) => {
       isNew && sendEmail(uid)
       res.status(200).json({ token })
     } else {
-      res.status(401).json({
-        message: `Not enough headers login, uid: ${req.headers.uid}
+      res
+        .status(401)
+        .json({
+          message: `Not enough headers login, uid: ${req.headers.uid}
         session-id ${req.headers['shib-session-id']}`
-      }).end()
+        })
+        .end()
     }
   } catch (err) {
     console.log(err)
@@ -49,13 +52,18 @@ router.delete('/logout', async (req, res) => {
     const logoutUrl = req.headers.shib_logout_url
     const { returnUrl } = req.body
     if (logoutUrl) {
-      return res.status(200).send({ logoutUrl: `${logoutUrl}?return=${returnUrl}` }).end()
+      return res
+        .status(200)
+        .send({ logoutUrl: `${logoutUrl}?return=${returnUrl}` })
+        .end()
     }
-    res.status(200).send({ logoutUrl: returnUrl }).end()
+    res
+      .status(200)
+      .send({ logoutUrl: returnUrl })
+      .end()
   } catch (err) {
     res.status(500).json({ message: 'Error with logout', err })
   }
 })
 
-
 module.exports = router

services/backend/oodikone2-backend/src/routes/mandatorycourselabels.js

@@ -67,4 +67,4 @@ router.post('/:programme/labels', async (req, res) => {
   }
 })
 
-module.exports = router
+module.exports = router

services/backend/oodikone2-backend/src/routes/mandatorycourses.js

@@ -37,17 +37,15 @@ router.post('/:programme/label/:course', async (req, res) => {
     await MandatoryCourses.updateLabel(programme, course, label)
     const updatedCourse = await MandatoryCourses.find(programme, course)
     if (!updatedCourse) return res.status(400).end()
-    res
-      .status(200)
-      .json({
-        name: updatedCourse.course.name,
-        code: updatedCourse.course_code,
-        label: updatedCourse.mandatory_course_label
-      })
+    res.status(200).json({
+      name: updatedCourse.course.name,
+      code: updatedCourse.course_code,
+      label: updatedCourse.mandatory_course_label
+    })
   } catch (err) {
     console.error(err)
     res.status(500).json(err.message)
   }
 })
 
-module.exports = router
+module.exports = router

services/backend/oodikone2-backend/src/routes/oodi.js

@@ -1,8 +1,32 @@
+const axios = require('axios')
+const https = require('https')
+const fs = require('fs')
 const { wrapper, router } = require('./router').routerWithWrapper()
-const { getOodiApi } = require('../services/doo_api_database_updater/oodi_interface')
-const { OODI_SECRET, OODI_SECRET_HEADER_KEY } = require('../conf-backend')
+const { KEY_PATH, CERT_PATH, OODI_ADDR, OODI_SECRET, OODI_SECRET_HEADER_KEY } = require('../conf-backend')
 const logger = require('../util/logger')
 
+const agent =
+  KEY_PATH && CERT_PATH
+    ? new https.Agent({
+        cert: fs.readFileSync(CERT_PATH, 'utf8'),
+        key: fs.readFileSync(KEY_PATH, 'utf8')
+      })
+    : new https.Agent({
+        rejectUnauthorized: false
+      })
+
+const instance = axios.create({
+  httpsAgent: agent
+})
+
+const getUrl = instance.get
+
+const getOodiApi = async relative => {
+  const route = OODI_ADDR + relative
+  const stuff = await getUrl(route)
+  return stuff
+}
+
 const validOodApiRequest = request => {
   const secret = request.headers[OODI_SECRET_HEADER_KEY]
   return !!OODI_SECRET && OODI_SECRET === secret
@@ -23,4 +47,4 @@ wrapper.post('', async (req, res) => {
   }
 })
 
-module.exports = router
+module.exports = router

services/backend/oodikone2-backend/src/routes/oodilearn.js

@@ -1,81 +0,0 @@
-const { router, wrapper } = require('./router').routerWithWrapper()
-const oodilearn = require('../services/oodilearn')
-const courses = require('../services/courses')
-
-const AVAILABLE_COURSES = [
-  'DIGI-100A',
-  'MAT11001',
-  'MAT11002',
-  'MAT11003',
-  'MAT11004',
-  'MAT12003',
-  'TKT10001',
-  'TKT10002',
-  'TKT10003',
-  'TKT10004',
-  'TKT10005',
-  'TKT20001',
-  'TKT50003'
-]
-
-wrapper.get('/ping', async (req, res) => {
-  const result = await oodilearn.ping()
-  res.json(result.data)
-})
-
-wrapper.get('/query', async (req, res) => {
-  const result = await oodilearn.query(req.query.query)
-  res.json(result.data)
-})
-
-wrapper.get('/suggest_course', async (req, res) => {
-  const result = await oodilearn.suggestCourse(req.query.doneCourses, req.query.period)
-  res.json(result.data)
-})
-
-wrapper.get('/suggest_route', async (req, res) => {
-  const result = await oodilearn.suggestRoute()
-  res.json(result.data)
-})
-
-wrapper.get('/student/:id', async (req, res) => {
-  const result = await oodilearn.getStudentData(req.params.id)
-  res.status(200).json(result.data)
-})
-
-wrapper.get('/student', async (req, res) => {
-  const { searchTerm } = req.query
-  if (!searchTerm) {
-    return res.status(400).send('Missing searchTerm query parameter')
-  } else {
-    const matches = await oodilearn.matchingStudents(searchTerm)
-    return res.json(matches)
-  }
-})
-
-wrapper.get('/courses', async (req, res) => {
-  const result = await courses.byCodes(AVAILABLE_COURSES)
-  res.json(result)
-})
-
-wrapper.get('/populations', async (req, res) => {
-  const result = await oodilearn.getPopulations()
-  res.json(result)
-})
-
-wrapper.get('/populations/:population', async (req, res) => {
-  const result = await oodilearn.getPopulation(req.params.population)
-  res.json(result)
-})
-
-wrapper.get('/:code', async (req, res) => {
-  const result = await oodilearn.getCluster(req.params.code)
-  res.status(200).json(result.data)
-})
-
-wrapper.get('/courses/:id', async (req, res) => {
-  const result = await oodilearn.courseGradeData(req.params.id)
-  res.json(result)
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/population.js

@@ -1,3 +1,4 @@
+const crypto = require('crypto')
 const router = require('express').Router()
 const Population = require('../services/populations')
 const Filters = require('../services/filters')
@@ -7,29 +8,67 @@ const { isValidStudentId } = require('../util/index')
 const Student = require('../services/students')
 const StudyrightService = require('../services/studyrights')
 const UserService = require('../services/userService')
+const TagService = require('../services/tags')
+const CourseService = require('../services/courses')
+const StatMergeService = require('../services/statMerger')
+
+const filterPersonalTags = (population, userId) => {
+  return {
+    ...population,
+    students: population.students.map(student => {
+      student.tags = student.tags.filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
+      return student
+    })
+  }
+}
 
 // POST instead of GET because of too long params and "sensitive" data
 router.post('/v2/populationstatistics/courses', async (req, res) => {
   try {
-    if (!req.body.startYear || !req.body.semesters || !req.body.studyRights) {
+    if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
       res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
       return
     }
-    if (!Array.isArray(req.body.studyRights)) { // studyRights should always be an array
+    if (!Array.isArray(req.body.studyRights)) {
+      // studyRights should always be an array
       req.body.studyRights = [req.body.studyRights]
     }
 
     if (req.body.months == null) {
       req.body.months = 12
     }
-    const result = await Population.bottlenecksOf(req.body)
 
-    if (result.error) {
-      res.status(400).json(result)
-      return
+    if (req.body.years) {
+      const upperYearBound = new Date().getFullYear() + 1
+      const multicoursestatPromises = Promise.all(
+        req.body.years.map(year => {
+          if (req.body.selectedStudentsByYear) {
+            req.body.selectedStudents = req.body.selectedStudentsByYear[year]
+          }
+          const newMonths = (upperYearBound - Number(year)) * 12
+          const query = { ...req.body, year, months: newMonths }
+          const coursestatistics = Population.bottlenecksOf(query)
+          return coursestatistics
+        })
+      )
+      const multicoursestats = await multicoursestatPromises
+      const result = StatMergeService.populationCourseStatsMerger(multicoursestats)
+      if (result.error) {
+        res.status(400).json(result)
+        return
+      }
+
+      res.json(result)
+    } else {
+      const result = await Population.bottlenecksOf(req.body)
+
+      if (result.error) {
+        res.status(400).json(result)
+        return
+      }
+
+      res.json(result)
     }
-
-    res.json(result)
   } catch (e) {
     console.log(e)
     res.status(500).json({ error: e })
@@ -38,14 +77,20 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
 
 router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => {
   try {
-    if (!req.body.yearcode || !req.body.coursecodes) {
-      res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
-      return
+    const { from, to, coursecodes } = req.body
+    if (!from || !to || !coursecodes) {
+      return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
+    }
+    const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(coursecodes)
+    if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
+      return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
     }
-    const { coursecodes, yearcode } = req.body
     let studentnumberlist
-    const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, yearcode)
-    const { decodedToken: { userId }, roles } = req
+    const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
+    const {
+      decodedToken: { userId },
+      roles
+    } = req
 
     if (roles && roles.includes('admin')) {
       studentnumberlist = studentnumbers
@@ -55,13 +100,15 @@ router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) =>
       studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
     }
 
-    const result = await Population.bottlenecksOf({
-      startYear: 1900,
-      endYear: 2200,
-      studyRights: [],
-      semesters: ['FALL', 'SPRING'],
-      months: 10000
-    }, studentnumberlist)
+    const result = await Population.bottlenecksOf(
+      {
+        year: 1900,
+        studyRights: [],
+        semesters: ['FALL', 'SPRING'],
+        months: 10000
+      },
+      studentnumberlist
+    )
 
     if (result.error) {
       res.status(400).json(result)
@@ -84,7 +131,10 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
     }
     let studentnumberlist
     const studentnumbers = await Student.findByTag(tag)
-    const { decodedToken: { userId }, roles } = req
+    const {
+      decodedToken: { userId },
+      roles
+    } = req
     if (roles && roles.includes('admin')) {
       studentnumberlist = studentnumbers
     } else {
@@ -92,14 +142,16 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
       const codes = unitsUserCanAccess.map(unit => unit.id)
       studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
     }
-    const result = await Population.bottlenecksOf({
-      startYear: 1900,
-      endYear: 2200,
-      studyRights: [],
-      semesters: ['FALL', 'SPRING'],
-      months: 10000,
-      tag
-    }, studentnumberlist)
+    const result = await Population.bottlenecksOf(
+      {
+        year: 1900,
+        studyRights: [],
+        semesters: ['FALL', 'SPRING'],
+        months: 10000,
+        tag
+      },
+      studentnumberlist
+    )
 
     if (result.error) {
       console.log(result.error)
@@ -109,7 +161,6 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
 
     console.log(`request completed ${new Date()}`)
     res.json(result)
-
   } catch (e) {
     console.log(e)
     res.status(500).json({ error: e })
@@ -123,7 +174,10 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
       return
     }
     let studentnumberlist
-    const { decodedToken: { userId }, roles } = req
+    const {
+      decodedToken: { userId },
+      roles
+    } = req
 
     if (roles && roles.includes('admin')) {
       studentnumberlist = req.body.studentnumberlist
@@ -133,20 +187,21 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
       studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
     }
 
-    const result = await Population.bottlenecksOf({
-      startYear: 1900,
-      endYear: 2200,
-      studyRights: [],
-      semesters: ['FALL', 'SPRING'],
-      months: 10000
-    }, studentnumberlist)
+    const result = await Population.bottlenecksOf(
+      {
+        year: 1900,
+        studyRights: [],
+        semesters: ['FALL', 'SPRING'],
+        months: 10000
+      },
+      studentnumberlist
+    )
     if (result.error) {
       res.status(400).json(result)
       return
     }
 
     res.json(result)
-
   } catch (err) {
     console.log(err)
     res.status(500).json({ error: err })
@@ -154,9 +209,10 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
 })
 
 router.get('/v3/populationstatistics', async (req, res) => {
-  const { startYear, semesters, studyRights: studyRightsJSON } = req.query
+  const { year, semesters, studyRights: studyRightsJSON } = req.query
+  const { decodedToken } = req
   try {
-    if (!startYear || !semesters || !studyRightsJSON) {
+    if (!year || !semesters || !studyRightsJSON) {
       res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
       return
     }
@@ -180,16 +236,44 @@ router.get('/v3/populationstatistics', async (req, res) => {
     if (req.query.months == null) {
       req.query.months = 12
     }
-    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
 
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
+    if (req.query.years) {
+      const upperYearBound = new Date().getFullYear() + 1
+      const multipopulationstudentPromises = Promise.all(
+        req.query.years.map(year => {
+          const newMonths = (upperYearBound - Number(year)) * 12
+          const populationStudents = Population.optimizedStatisticsOf({
+            ...req.query,
+            studyRights,
+            year,
+            months: newMonths
+          })
+          return populationStudents
+        })
+      )
+      const multipopulationstudents = await multipopulationstudentPromises
+
+      const result = StatMergeService.populationStudentsMerger(multipopulationstudents)
+
+      if (result.error) {
+        console.log(result.error)
+        res.status(400).end()
+        return
+      }
+
+      console.log(`request completed ${new Date()}`)
+      res.json(filterPersonalTags(result, decodedToken.id))
+    } else {
+      const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
+      if (result.error) {
+        console.log(result.error)
+        res.status(400).end()
+        return
+      }
+
+      console.log(`request completed ${new Date()}`)
+      res.json(filterPersonalTags(result, decodedToken.id))
     }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(result)
   } catch (e) {
     console.log(e)
     res.status(500).json({ error: e })
@@ -197,15 +281,20 @@ router.get('/v3/populationstatistics', async (req, res) => {
 })
 
 router.get('/v3/populationstatisticsbytag', async (req, res) => {
-  const { tag } = req.query
-  if (!tag) {
-    res.status(400).json({ error: 'The query should have a tag defined' })
-    return
-  }
+  const { tag, studyRights: studyRightsJSON, months, year } = req.query
+  const { decodedToken } = req
+
+  if (!tag) return res.status(400).json({ error: 'The query should have a tag defined' })
+  const foundTag = await TagService.findTagById(tag)
+  if (!foundTag) return res.status(404).json({ error: 'Tag not found' })
+
   const semesters = ['FALL', 'SPRING']
   let studentnumberlist
   const studentnumbers = await Student.findByTag(tag)
-  const { decodedToken: { userId }, roles } = req
+  const {
+    decodedToken: { userId },
+    roles
+  } = req
   if (roles && roles.includes('admin')) {
     studentnumberlist = studentnumbers
   } else {
@@ -214,13 +303,20 @@ router.get('/v3/populationstatisticsbytag', async (req, res) => {
     studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
   }
   try {
-    const result = await Population.optimizedStatisticsOf({
-      startYear: 1900,
-      endYear: 2200,
-      studyRights: [],
-      semesters,
-      months: 10000
-    }, studentnumberlist)
+    const studyRights = JSON.parse(studyRightsJSON)
+    const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
+    const yearDifference = Number(year) - Number(newStartYear)
+    const newMonths = Number(months) + 12 * yearDifference
+    const result = await Population.optimizedStatisticsOf(
+      {
+        year: newStartYear,
+        studyRights,
+        semesters,
+        months: newMonths,
+        tag: foundTag
+      },
+      studentnumberlist
+    )
 
     if (result.error) {
       console.log(result.error)
@@ -229,7 +325,7 @@ router.get('/v3/populationstatisticsbytag', async (req, res) => {
     }
 
     console.log(`request completed ${new Date()}`)
-    res.json(result)
+    res.json(filterPersonalTags(result, decodedToken.id))
   } catch (e) {
     console.log(e)
     res.status(500).json({ error: e })
@@ -237,31 +333,67 @@ router.get('/v3/populationstatisticsbytag', async (req, res) => {
 })
 
 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
-  const { coursecodes, yearcode } = req.query
-  if (!coursecodes || !yearcode) {
-    res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
-    return
-  }
-  const semesters = ['FALL', 'SPRING']
-  let studentnumberlist
-  const studentnumbers = await Student.findByCourseAndSemesters(JSON.parse(coursecodes), yearcode)
-  const { decodedToken: { userId }, roles } = req
+  const { coursecodes, from, to, separate: sep } = req.query
+  const { decodedToken, roles } = req
+  const separate = sep ? JSON.parse(sep) : false
 
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = studentnumbers
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
+  if (!coursecodes || !from || !to) {
+    return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
   }
+
+  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(coursecodes))
+  const toFromDiff = Math.abs(to - from + 1)
+  // 2 semesters = 1 year
+  const requestedYearsToCreatePopulationFrom = Math.ceil(separate ? toFromDiff / 2 : toFromDiff)
+  if (requestedYearsToCreatePopulationFrom > maxYearsToCreatePopulationFrom) {
+    return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
+  }
+
+  const semesters = ['FALL', 'SPRING']
+  const studentnumbers = await Student.findByCourseAndSemesters(JSON.parse(coursecodes), from, to, separate)
+
   try {
-    const result = await Population.optimizedStatisticsOf({
-      startYear: 1900,
-      endYear: 2200,
-      studyRights: [],
-      semesters,
-      months: 10000
-    }, studentnumberlist)
+    const result = await Population.optimizedStatisticsOf(
+      {
+        // Useless, because studentnumbers are already filtered above by from & to.
+        // We should probably refactor this to avoid more confusement.
+        year: 1900,
+        studyRights: [],
+        semesters,
+        months: 10000
+      },
+      studentnumbers
+    )
+
+    let studentsUserCanAccess
+    if (roles && roles.includes('admin')) {
+      studentsUserCanAccess = new Set(studentnumbers)
+    } else {
+      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(decodedToken.userId)
+      const codes = unitsUserCanAccess.map(unit => unit.id)
+      studentsUserCanAccess = new Set(await Student.filterStudentnumbersByAccessrights(studentnumbers, codes))
+    }
+
+    const randomHash = crypto.randomBytes(12).toString('hex')
+    const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
+      courses,
+      studyrights,
+      gender_code,
+      studentNumber: crypto
+        .createHash('md5')
+        .update(`${studentNumber}${randomHash}`)
+        .digest('hex'),
+      firstnames: '',
+      lastname: '',
+      name: '',
+      email: '',
+      started: null,
+      credits: '',
+      tags: [],
+      obfuscated: true
+    })
+
+    result.students = result.students.map(s => (studentsUserCanAccess.has(s.studentNumber) ? s : obfuscateStudent(s)))
 
     if (result.error) {
       console.log(result.error)
@@ -270,7 +402,7 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
     }
 
     console.log(`request completed ${new Date()}`)
-    res.json(result)
+    res.json(filterPersonalTags(result, decodedToken.id))
   } catch (e) {
     console.log(e)
     res.status(500).json({ error: e })
@@ -279,7 +411,7 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
 
 router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
   const { studentnumberlist } = req.body
-  const { roles } = req
+  const { roles, decodedToken } = req
 
   if (!(roles && roles.includes('admin'))) {
     console.log('no rights')
@@ -287,19 +419,21 @@ router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
     return
   } else {
     try {
-      const result = await Population.optimizedStatisticsOf({
-        startYear: 1900,
-        endYear: 2200,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000
-      }, studentnumberlist)
+      const result = await Population.optimizedStatisticsOf(
+        {
+          year: 1900,
+          studyRights: [],
+          semesters: ['FALL', 'SPRING'],
+          months: 10000
+        },
+        studentnumberlist
+      )
       if (result.error) {
         console.log(result.error)
         res.status(400).end()
         return
       }
-      res.status(200).json(result)
+      res.status(200).json(filterPersonalTags(result, decodedToken.id))
     } catch (err) {
       console.log(err)
       res.status(400).end()
@@ -308,21 +442,19 @@ router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
 })
 
 router.get('/v2/populationstatistics/filters', async (req, res) => {
-
   let results = []
   let rights = req.query.studyRights
-  if (!Array.isArray(rights)) { // studyRights should always be an array
+  if (!Array.isArray(rights)) {
+    // studyRights should always be an array
     rights = [rights]
   }
   try {
     results = await Filters.findForPopulation(rights)
     res.status(200).json(results)
-
   } catch (err) {
     console.log(err)
     res.status(400).end()
   }
-
 })
 router.post('/v2/populationstatistics/filters', async (req, res) => {
   let results = []
@@ -331,12 +463,10 @@ router.post('/v2/populationstatistics/filters', async (req, res) => {
   try {
     results = await Filters.createNewFilter(filter)
     res.status(200).json(results)
-
   } catch (err) {
     console.log(err)
     res.status(400).end()
   }
-
 })
 router.delete('/v2/populationstatistics/filters', async (req, res) => {
   let results = []
@@ -344,11 +474,9 @@ router.delete('/v2/populationstatistics/filters', async (req, res) => {
   try {
     results = await Filters.deleteFilter(filter)
     res.status(200).json(results)
-
   } catch (err) {
     res.status(400).end()
   }
-
 })
 
 router.post('/updatedatabase', async (req, res) => {
@@ -391,4 +519,14 @@ router.get('/v3/populationstatistics/studyprogrammes/unfiltered', async (req, re
   }
 })
 
+router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
+  try {
+    const { courseCodes } = req.query
+    const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(courseCodes))
+    return res.json(maxYearsToCreatePopulationFrom)
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
 module.exports = router

services/backend/oodikone2-backend/src/routes/providers.js

@@ -6,4 +6,4 @@ router.get('/providers', async (req, res) => {
   res.json(providers)
 })
 
-module.exports = router
+module.exports = router

services/backend/oodikone2-backend/src/routes/router.js

@@ -1,8 +1,8 @@
 const express = require('express')
 const logger = require('../util/logger')
 
-const wrapWithErrorHandler = fn => (req, res) => Promise.resolve(fn(req, res))
-  .catch(error => {
+const wrapWithErrorHandler = fn => (req, res) =>
+  Promise.resolve(fn(req, res)).catch(error => {
     logger.error(error)
     res.status(500).send('Internal server error')
   })
@@ -18,4 +18,4 @@ const routerWithWrapper = () => {
   }
 }
 
-module.exports = { routerWithWrapper }
+module.exports = { routerWithWrapper }

services/backend/oodikone2-backend/src/routes/semesters.js

@@ -1,9 +1,9 @@
 const router = require('express').Router()
 const { getSemestersAndYears } = require('../services/semesters')
 
-router.get('/semesters/codes', async(req, res) => {
+router.get('/semesters/codes', async (req, res) => {
   const providers = await getSemestersAndYears()
   res.json(providers)
 })
 
-module.exports = router
+module.exports = router

services/backend/oodikone2-backend/src/routes/students.js

@@ -3,50 +3,92 @@ const Student = require('../services/students')
 const userService = require('../services/userService')
 const Unit = require('../services/units')
 
+const filterStudentTags = (student, userId) => {
+  return {
+    ...student,
+    tags: student.tags.filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
+  }
+}
+
 router.get('/students', async (req, res) => {
-  const { roles, decodedToken: { userId } } = req
+  const {
+    roles,
+    decodedToken: { userId },
+    query: { searchTerm }
+  } = req
+
+  const trimmedSearchTerm = searchTerm ? searchTerm.trim() : undefined
+
+  if (
+    trimmedSearchTerm &&
+    !Student.splitByEmptySpace(trimmedSearchTerm)
+      .slice(0, 2)
+      .find(t => t.length > 3)
+  ) {
+    return res.status(400).json({ error: 'at least one search term must be longer than 3 characters' })
+  }
 
   if (roles && roles.includes('admin')) {
     let results = []
-    if (req.query.searchTerm) {
-      results = await Student.bySearchTerm(req.query.searchTerm)
+    if (trimmedSearchTerm) {
+      results = await Student.bySearchTerm(trimmedSearchTerm)
     }
-    return res.json(results)
+    return res
+      .status(200)
+      .json(results)
+      .end()
   } else {
     const unitsUserCanAccess = await userService.getUnitsFromElementDetails(userId)
     const codes = unitsUserCanAccess.map(unit => unit.id)
-    const matchingStudents = await Student.bySearchTermAndElements(req.query.searchTerm, codes)
+    const matchingStudents = await Student.bySearchTermAndElements(trimmedSearchTerm, codes)
     res.json(matchingStudents)
   }
 })
 
 router.get('/students/:id', async (req, res) => {
   const { id: studentId } = req.params
-  const { roles } = req
+  const { roles, decodedToken } = req
 
   if (roles && roles.includes('admin')) {
     const results = await Student.withId(studentId)
-    return results.error ?
-      res.status(400).json({ error: 'error finding student' }) :
-      res.json(results)
+    return results.error
+      ? res
+          .status(400)
+          .json({ error: 'error finding student' })
+          .end()
+      : res
+          .status(200)
+          .json(filterStudentTags(results, decodedToken.id))
+          .end()
   }
 
   const uid = req.decodedToken.userId
   const student = await Student.withId(studentId)
   if (student.error) {
-    return res.status(400).json({ error: 'error finding student' })
+    return res
+      .status(400)
+      .json({ error: 'error finding student' })
+      .end()
   }
   const units = await userService.getUnitsFromElementDetails(uid)
 
-  const rights = await Promise.all(units.map(async unit => {
-    const jtn = await Unit.hasStudent(unit.id, student.studentNumber)
-    return jtn
-  }))
+  const rights = await Promise.all(
+    units.map(async unit => {
+      const jtn = await Unit.hasStudent(unit.id, student.studentNumber)
+      return jtn
+    })
+  )
 
   if (rights.some(right => right !== null)) {
-    res.json(student).end()
+    res
+      .status(200)
+      .json(filterStudentTags(student, decodedToken.id))
+      .end()
   } else {
-    res.status(400).json({ error: 'error finding student' })
+    res
+      .status(400)
+      .json({ error: 'error finding student' })
+      .end()
   }
 })
 

services/backend/oodikone2-backend/src/routes/superlogin.js

@@ -9,10 +9,13 @@ router.post('/:uid', async (req, res) => {
       const token = await userService.superlogin(uid, asUser)
       res.status(200).json(token)
     } else {
-      res.status(401).json({
-        message: `Not enough headers login, uid:
+      res
+        .status(401)
+        .json({
+          message: `Not enough headers login, uid:
         ${req.headers.uid} session-id ${req.headers['shib-session-id']}`
-      }).end()
+        })
+        .end()
     }
   } catch (err) {
     console.log(err.message)

services/backend/oodikone2-backend/src/routes/tags.js

@@ -4,6 +4,14 @@ const TagStudent = require('../services/tagstudent')
 const Students = require('../services/students')
 const { difference } = require('lodash')
 
+const filterRelevantTags = (tags, userId) => {
+  return tags.filter(tag => !tag.personal_user_id || tag.personal_user_id === userId)
+}
+
+const filterRelevantStudentTags = (studentTags, userId) => {
+  return studentTags.filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
+}
+
 router.get('/tags', async (req, res) => {
   try {
     const tags = await Tags.findTags()
@@ -17,12 +25,12 @@ router.get('/tags', async (req, res) => {
 router.get('/tags/:studytrack', async (req, res) => {
   try {
     const { studytrack } = req.params
-    const { rights, roles } = req
+    const { rights, roles, decodedToken } = req
 
     if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
     const tags = await Tags.findTagsByStudytrack(studytrack)
-    res.status(200).json(tags)
+    res.status(200).json(filterRelevantTags(tags, decodedToken.id))
   } catch (err) {
     console.log(err)
     res.status(500).json(err)
@@ -31,14 +39,16 @@ router.get('/tags/:studytrack', async (req, res) => {
 
 router.post('/tags', async (req, res) => {
   try {
-    const { tag } = req.body
-    const { rights, roles } = req
+    const {
+      tag: { studytrack, tagname, year, personal_user_id }
+    } = req.body
+    const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(tag.studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    await Tags.createNewTag(tag)
-    const tags = await Tags.findTagsByStudytrack(tag.studytrack)
-    res.status(200).json(tags)
+    await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
+    const tags = await Tags.findTagsByStudytrack(studytrack)
+    res.status(200).json(filterRelevantTags(tags, decodedToken.id))
   } catch (err) {
     console.log(err)
     res.status(500).json(err.message)
@@ -48,18 +58,22 @@ router.post('/tags', async (req, res) => {
 router.delete('/tags', async (req, res) => {
   try {
     const { tag } = req.body
-    const { rights, roles } = req
+    const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(tag.studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+    if (
+      !rights.includes(tag.studytrack) &&
+      !(roles && roles.includes('admin')) &&
+      !(tag.personal_user_id === decodedToken.id)
+    )
+      return res.status(403).end()
 
     await Tags.deleteTag(tag)
     const t = await Tags.findTagsByStudytrack(tag.studytrack)
-    res.status(200).json(t)
+    res.status(200).json(filterRelevantTags(t, decodedToken.id))
   } catch (err) {
     console.log(err)
     res.status(500).json(err)
   }
-
 })
 
 router.get('/studenttags', async (req, res) => {
@@ -75,12 +89,12 @@ router.get('/studenttags', async (req, res) => {
 router.get('/studenttags/:studytrack', async (req, res) => {
   try {
     const { studytrack } = req.params
-    const { rights, roles } = req
+    const { rights, roles, decodedToken } = req
 
     if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
     const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(result)
+    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
   } catch (err) {
     console.log(err)
     res.status(500).json(err)
@@ -90,8 +104,9 @@ router.get('/studenttags/:studytrack', async (req, res) => {
 router.get('/studenttags/:studentnumber', async (req, res) => {
   try {
     const { studentnumber } = req.params
+    const { decodedToken } = req
     const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
-    res.status(200).json(result)
+    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
   } catch (err) {
     console.log(err)
     res.status(500).json(err)
@@ -101,13 +116,13 @@ router.get('/studenttags/:studentnumber', async (req, res) => {
 router.post('/studenttags/:studentnumber', async (req, res) => {
   try {
     const { tag, studytrack } = req.body
-    const { rights, roles } = req
+    const { rights, roles, decodedToken } = req
 
     if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
     await TagStudent.createStudentTag(tag)
     const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(result)
+    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
   } catch (err) {
     console.log(err)
     res.status(500).json(err)
@@ -117,7 +132,7 @@ router.post('/studenttags/:studentnumber', async (req, res) => {
 router.post('/studenttags', async (req, res) => {
   try {
     const { tags, studytrack } = req.body
-    const { rights, roles } = req
+    const { rights, roles, decodedToken } = req
 
     if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
@@ -129,21 +144,24 @@ router.post('/studenttags', async (req, res) => {
     const studentnumbers = tags.map(t => t.studentnumber)
     const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
     const missingStudents = difference(studentnumbers, students)
-    if (missingStudents.length !== 0) return res.status(400).json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
+    if (missingStudents.length !== 0)
+      return res
+        .status(400)
+        .json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
 
     await TagStudent.createMultipleStudentTags(tags)
     const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(result)
+    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
   } catch (err) {
     console.log(err)
     res.status(500).json(err)
   }
 })
 
-router.delete('/studenttags', async (req, res) => {
+router.delete('/studenttags/delete_one', async (req, res) => {
   try {
     const { tag_id, studentnumber, studytrack } = req.body
-    const { rights, roles } = req
+    const { rights, roles, decodedToken } = req
 
     if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
@@ -152,11 +170,30 @@ router.delete('/studenttags', async (req, res) => {
 
     await TagStudent.deleteStudentTag(studentnumber, tag_id)
     const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(result)
+    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
   } catch (err) {
     console.log(err)
     res.status(500).json(err)
   }
 })
 
+router.delete('/studenttags/delete_many', async (req, res) => {
+  try {
+    const { tagId, studentnumbers, studytrack } = req.body
+    const { rights, roles, decodedToken } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
+    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
+
+    await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
+    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  } catch (err) {
+    console.log(err)
+    res.status(400).json(err)
+  }
+})
+
 module.exports = router

services/backend/oodikone2-backend/src/routes/tasks.js

@@ -17,4 +17,4 @@ router.post('/tasks/topteachers', async (req, res) => {
   }
 })
 
-module.exports = router
+module.exports = router

services/backend/oodikone2-backend/src/routes/teachers.js

@@ -8,7 +8,17 @@ const router = FEATURES.ERROR_HANDLER ? r.wrapper : r.router
 
 router.get('/', async (req, res) => {
   const { searchTerm } = req.query
-  const result = await teachers.bySearchTerm(searchTerm)
+  if (!searchTerm) return res.status(400).json({ error: 'searchTerm missing' })
+
+  const trimmedSearchTerm = searchTerm.trim()
+  if (
+    !teachers.splitByEmptySpace(trimmedSearchTerm).find(t => t.length >= 4) ||
+    (Number(trimmedSearchTerm) && trimmedSearchTerm.length < 6)
+  ) {
+    return res.status(400).json({ error: 'invalid searchTerm' })
+  }
+
+  const result = await teachers.bySearchTerm(trimmedSearchTerm)
   res.json(result)
 })
 
@@ -44,7 +54,7 @@ router.get('/stats', async (req, res) => {
   if (!(providers.every(p => providerRights.includes(p)) || roles.includes('admin'))) {
     return res.status(403).send('You do not have rights to see this data')
   }
-  const result = await teachers.yearlyStatistics(providers, semesterStart, semesterEnd||semesterStart + 1)
+  const result = await teachers.yearlyStatistics(providers, semesterStart, semesterEnd || semesterStart + 1)
   res.json(result)
 })
 
@@ -57,4 +67,4 @@ router.get('/:id', async (req, res) => {
   res.json(result)
 })
 
-module.exports = r.router
+module.exports = r.router

services/backend/oodikone2-backend/src/routes/tsaAnalytics.js

@@ -0,0 +1,27 @@
+const Raven = require('raven')
+const router = require('express').Router()
+const tsaService = require('../services/tsaService')
+
+router.post('/event', (req, res) => {
+  const { group, name, label, value } = req.body
+
+  if (!group || !name || !value) {
+    return res.status(400).json({ error: 'group, name and value are required' })
+  }
+
+  // don't await here because frontend doesn't care if it succeeds
+  tsaService.sendTsaEvent(req.decodedToken.userId, { group, name, label, value }).catch(e => {
+    let ravenOpts = { req }
+    // if error was caused by a 400, dig out the JSON response for Sentry
+    if (e && e.isAxiosError && e.response && e.response.data) {
+      ravenOpts.extra = {
+        errorData: e.response.data
+      }
+    }
+    Raven.captureException(e, ravenOpts)
+  })
+
+  res.status(200).end()
+})
+
+module.exports = router

services/backend/oodikone2-backend/src/routes/updater.js

@@ -9,7 +9,8 @@ const {
   updateStudentlist,
   rescheduleScheduled,
   rescheduleFetched,
-  updateNoStudents
+  updateNoStudents,
+  updateDaily
 } = require('../services/updaterService')
 
 router.post('/update/oldest', async (req, res) => {
@@ -126,4 +127,15 @@ router.post('/reschedule/fetched', async (req, res) => {
   }
 })
 
+router.post('/update/daily', async (req, res) => {
+  try {
+    const response = await updateDaily()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
 module.exports = router

services/backend/oodikone2-backend/src/routes/usage.js

@@ -1,11 +1,14 @@
 const router = require('express').Router()
 const UsageService = require('../services/usageService')
 
+const S_TO_MS = 1000
+
 router.get('/', async (req, res) => {
-  const from = req.query.from || 1
-  const to = req.query.to || Number((new Date().getTime() / 1000 + 60).toFixed(0))
-  const results = await UsageService.get(from, to)
-  res.json(results)
+  const from = req.query.from ? new Date(req.query.from) : new Date(1 * S_TO_MS)
+  const to = req.query.to ? new Date(req.query.to) : new Date(new Date().getTime() + 60 * S_TO_MS)
+  const resultStream = await UsageService.getStream(from, to)
+  res.contentType('json')
+  resultStream.pipe(res)
 })
 
-module.exports = router
+module.exports = router

services/backend/oodikone2-backend/src/routes/users.js

@@ -2,6 +2,15 @@ const router = require('express').Router()
 const userService = require('../services/userService')
 const mailservice = require('../services/mailservice')
 const blacklist = require('../services/blacklist')
+const { userDataCache } = require('../services/cache')
+
+const addUserToBlacklist = async (user, decodedToken) => {
+  if (user) {
+    const { username } = user
+    userDataCache.del(username)
+    if (username != decodedToken.userId) await blacklist.addUserToBlacklist(username)
+  }
+}
 
 router.get('/', async (req, res) => {
   const results = await userService.findAll()
@@ -18,27 +27,36 @@ router.post('/modifyaccess', async (req, res) => {
     const { uid } = req.body
     const result = await userService.modifyAccess(req.body)
     const user = await userService.byId(uid)
-    if (user && user.username != req.decodedToken.userId) await blacklist.addUserToBlacklist(user.username)
+    await addUserToBlacklist(user, req.decodedToken)
     res.status(200).json(result)
   } catch (e) {
     res.status(400).json(e)
   }
 })
 
+router.get('/email/preview', (req, res) => {
+  const { subject, html } = mailservice.message2(null)
+  res.json({ subject, html })
+})
+
 router.post('/email', async (req, res) => {
+  if (!process.env.SMTP) {
+    return res.status(500).json({ error: 'Email system has not been configured' })
+  }
+
   const email = req.body.email
-  if (process.env.SMTP !== undefined && email) {
-    const message = mailservice.message2(email)
-    await mailservice.transporter.sendMail(message, (error) => {
-      if (error) {
-        console.log('Error occurred')
-        res.status(400).end()
-      } else {
-        console.log('Message sent successfully!')
-        res.status(200).end()
-      }
-      mailservice.transporter.close()
-    })
+  if (!email) {
+    return res.status(400).json({ error: 'email is missing' })
+  }
+
+  const message = mailservice.message2(email)
+  try {
+    const info = await mailservice.transporter.sendMail(message)
+    console.log('Message sent successfully', info)
+    res.status(200).end()
+  } catch (e) {
+    console.error('Error occurred while sending user email', e)
+    res.status(500).json({ error: e.message })
   }
 })
 
@@ -46,7 +64,7 @@ router.post('/:uid/elements', async (req, res) => {
   const { uid } = req.params
   const { codes } = req.body
   const user = await userService.enableElementDetails(uid, codes)
-  if (user && user.username != req.decodedToken.userId) await blacklist.addUserToBlacklist(user.username)
+  await addUserToBlacklist(user, req.decodedToken)
   res.json(user)
 })
 
@@ -54,7 +72,7 @@ router.delete('/:uid/elements', async (req, res) => {
   const { uid } = req.params
   const { codes } = req.body
   const user = await userService.removeElementDetails(uid, codes)
-  if (user && user.username != req.decodedToken.userId) await blacklist.addUserToBlacklist(user.username)
+  await addUserToBlacklist(user, req.decodedToken)
   res.json(user)
 })
 
@@ -62,8 +80,8 @@ router.post('/:uid/faculties', async (req, res) => {
   const { uid } = req.params
   const { faculties } = req.body
   const user = await userService.setFaculties(uid, faculties)
-  if (user && user.username != req.decodedToken.userId) await blacklist.addUserToBlacklist(user.username)
+  await addUserToBlacklist(user, req.decodedToken)
   res.json(user)
 })
 
-module.exports = router
+module.exports = router

services/backend/oodikone2-backend/src/services/analyticsService.js

@@ -8,42 +8,42 @@ const ping = async () => {
   return response.data
 }
 
-const getProductivity = async (id) => {
+const getProductivity = async id => {
   const response = await client.get(`/productivity/${id}`)
   return response.data
 }
 
-const setProductivity = async (data) => {
+const setProductivity = async data => {
   const response = await client.post('/productivity', { data })
   return response.data
 }
 
-const patchProductivity = async (data) => {
+const patchProductivity = async data => {
   const response = await client.patch('/productivity', { data })
   return response.data
 }
 
-const getThroughput = async (id) => {
+const getThroughput = async id => {
   const response = await client.get(`/throughput/${id}`)
   return response.data
 }
 
-const setThroughput = async (data) => {
+const setThroughput = async data => {
   const response = await client.post('/throughput', { data })
   return response.data
 }
 
-const patchThroughput = async (data) => {
+const patchThroughput = async data => {
   const response = await client.patch('/throughput', { data })
   return response.data
 }
 
-const patchFacultyYearlyStats = async (data) => {
+const patchFacultyYearlyStats = async data => {
   const response = await client.patch('/facultystats', { data })
   return response.data
 }
 
-const getFacultyYearlyStats = async (data) => {
+const getFacultyYearlyStats = async data => {
   const response = await client.get('/facultystats', { data })
   return response.data
 }
@@ -58,4 +58,4 @@ module.exports = {
   patchThroughput,
   patchFacultyYearlyStats,
   getFacultyYearlyStats
-}
+}

services/backend/oodikone2-backend/src/services/cache.js

@@ -0,0 +1,12 @@
+const LRU = require('lru-cache')
+
+// Max 25 users can be stored in the cache
+const userDataCache = new LRU({
+  max: 25,
+  length: () => 1,
+  maxAge: 1000 * 60 * 60
+})
+
+module.exports = {
+  userDataCache
+}

services/backend/oodikone2-backend/src/services/courseGroups.js

@@ -17,28 +17,26 @@ const ACADEMIC_YEAR_START_SEMESTER = 111 // academic year 2005-06
 
 const COURSE_GROUP_STATISTICS_KEY = (programmeId, semesterCode) =>
   `course_group_statistics_${programmeId}_${semesterCode}`
-const TEACHER_COURSES_KEY = (teacherId, semesterCode) =>
-  `course_group_courses_${teacherId}_${semesterCode}`
+const TEACHER_COURSES_KEY = (teacherId, semesterCode) => `course_group_courses_${teacherId}_${semesterCode}`
 const REDIS_CACHE_TTL = 12 * 60 * 60
 
-const getAcademicYears = async () =>
-  getAcademicYearsFrom(ACADEMIC_YEAR_START_SEMESTER)
+const getAcademicYears = async () => getAcademicYearsFrom(ACADEMIC_YEAR_START_SEMESTER)
 
 const getCurrentAcademicYearSemesterCode = async () => {
   const academicYear = await getCurrentAcademicYear()
   return academicYear[0].semestercode
 }
 
-const getTeachersForCourseGroup = async (courseGroupId) => {
+const getTeachersForCourseGroup = async courseGroupId => {
   const courseGroups = await CourseGroup.findAll({
     model: CourseGroup,
     include: {
       model: TeacherCourseGroup,
-      required: true,
+      required: true
     },
     where: {
       id: courseGroupId
-    },
+    }
   })
   const teacherIds = flatMap(courseGroups, e => e.teacher_course_groups.map(tcg => tcg.teacher_id))
   const teachers = await Teacher.findAll({
@@ -47,12 +45,12 @@ const getTeachersForCourseGroup = async (courseGroupId) => {
       id: {
         [Op.in]: teacherIds
       }
-    },
+    }
   })
   return teachers
 }
 
-const createCourseGroup = (name) => CourseGroup.create({ name })
+const createCourseGroup = name => CourseGroup.create({ name })
 
 const getCourseGroupsWithTotals = async (programmeId, semesterCode, forceRefresh = false) => {
   if (!forceRefresh) {
@@ -61,20 +59,20 @@ const getCourseGroupsWithTotals = async (programmeId, semesterCode, forceRefresh
       return JSON.parse(cachedStats)
     }
   }
-  const startSemester = semesterCode || await getCurrentAcademicYearSemesterCode()
+  const startSemester = semesterCode || (await getCurrentAcademicYearSemesterCode())
   const result = await getAcademicYearStatisticsForStudyProgramme(programmeId, startSemester)
 
   await redisClient.setAsync(
     COURSE_GROUP_STATISTICS_KEY(programmeId, semesterCode),
     JSON.stringify(result),
     'EX',
-    6*4*7*24*60*60 // 6 months in seconds
+    6 * 4 * 7 * 24 * 60 * 60 // 6 months in seconds
   )
   return result
 }
 
 const getCourseGroup = async (courseGroupId, semesterCode) => {
-  const startSemester = semesterCode || await getCurrentAcademicYearSemesterCode()
+  const startSemester = semesterCode || (await getCurrentAcademicYearSemesterCode())
   const teacherBasicInfo = await getTeachersForCourseGroup(courseGroupId)
   const courseGroup = await CourseGroup.findByPk(courseGroupId)
 
@@ -99,23 +97,21 @@ const getCourseGroup = async (courseGroupId, semesterCode) => {
 
   const { credits, students, courses } = statistics[0]
 
-  const mergedTeachers = [...teacherBasicInfo.map(e => e.get()), ...teacherStats]
-    .reduce((acc, cur) => {
-      if (acc[cur.id]) {
-        acc[cur.id] = { ...cur, ...acc[cur.id] }
-      } else {
-        acc[cur.id] = cur
-      }
-      return acc
-    }, {})
+  const mergedTeachers = [...teacherBasicInfo.map(e => e.get()), ...teacherStats].reduce((acc, cur) => {
+    if (acc[cur.id]) {
+      acc[cur.id] = { ...cur, ...acc[cur.id] }
+    } else {
+      acc[cur.id] = cur
+    }
+    return acc
+  }, {})
 
-  const teachers = Object.values(mergedTeachers)
-    .map(t => ({
-      ...t,
-      courses: t.courses ? Number(t.courses) : 0,
-      credits: t.credits ? Number(t.credits) : 0,
-      students: t.students ? Number(t.students) : 0
-    }))
+  const teachers = Object.values(mergedTeachers).map(t => ({
+    ...t,
+    courses: t.courses ? Number(t.courses) : 0,
+    credits: t.credits ? Number(t.credits) : 0,
+    students: t.students ? Number(t.students) : 0
+  }))
 
   const courseGroupInfo = {
     id: courseGroupId,
@@ -131,7 +127,7 @@ const getCourseGroup = async (courseGroupId, semesterCode) => {
 }
 
 const getCoursesByTeachers = async (teacherIds, semesterCode) => {
-  const startSemester = semesterCode || await getCurrentAcademicYearSemesterCode()
+  const startSemester = semesterCode || (await getCurrentAcademicYearSemesterCode())
   const cacheKey = TEACHER_COURSES_KEY(teacherIds, startSemester)
   const cachedData = await redisClient.getAsync(cacheKey)
 
@@ -142,7 +138,7 @@ const getCoursesByTeachers = async (teacherIds, semesterCode) => {
   const courses = await getAcademicYearCoursesByTeacherIds(teacherIds, startSemester)
 
   // Sequelize returns credis sums as string
-  courses.forEach((course) => {
+  courses.forEach(course => {
     course.credits = Number(course.credits)
     course.students = Number(course.students)
   })

services/backend/oodikone2-backend/src/services/course_stats_counter.js

@@ -1,10 +1,16 @@
 const lengthOf = obj => Object.keys(obj).length
-const percentageOf = (num, denom) => Number((100 * num / denom).toFixed(2))
+const percentageOf = (num, denom) => Math.round(((100 * num) / denom) * 100) / 100
+
+const fall = []
+const spring = []
+for (let i = 0; i < 7; i++) {
+  fall[i] = `${i}-FALL`
+  spring[i] = `${i}-SPRING`
+}
 
 class CourseStatsCounter {
-
-  constructor(code, name, allstudents) {
-    this.studentsInTotal = Object.keys(allstudents).length
+  constructor(code, name, studentsInTotal) {
+    this.studentsInTotal = studentsInTotal
     this.course = {
       code,
       name,
@@ -18,8 +24,6 @@ class CourseStatsCounter {
       retryPassed: {},
       failedMany: {},
       improvedPassedGrade: {},
-      notParticipated: { ...allstudents },
-      notParticipatedOrFailed: { ...allstudents },
       markedToSemester: {}
     }
     this.stats = {
@@ -45,8 +49,8 @@ class CourseStatsCounter {
     }
 
     for (let i = 0; i < 7; i++) {
-      passingSemesters[`${i}-FALL`] = 0
-      passingSemesters[`${i}-SPRING`] = 0
+      passingSemesters[fall[i]] = 0
+      passingSemesters[spring[i]] = 0
     }
 
     return passingSemesters
@@ -56,10 +60,6 @@ class CourseStatsCounter {
     this.stats.attempts = this.stats.attempts + 1
   }
 
-  markParticipation(studentnumber) {
-    delete this.students.notParticipated[studentnumber]
-  }
-
   markToAll(studentnumber) {
     this.students.all[studentnumber] = true
   }
@@ -75,7 +75,6 @@ class CourseStatsCounter {
   markCredit(studentnumber, grade, passed, failed, improved, semester) {
     // studentnumber = `${shajs('sha256').update(process.env.key + studentnumber).digest('hex')}`
     this.markAttempt()
-    this.markParticipation(studentnumber)
     this.markGrade(grade, passed, failed, improved)
     this.markToAll(studentnumber)
     if (passed) {
@@ -92,11 +91,11 @@ class CourseStatsCounter {
   }
 
   failedBefore(studentnumber) {
-    return (this.students.failed[studentnumber] !== undefined)
+    return this.students.failed[studentnumber] !== undefined
   }
 
   passedBefore(studentnumber) {
-    return (this.students.passed[studentnumber] !== undefined)
+    return this.students.passed[studentnumber] !== undefined
   }
 
   removeFromFailed(studentnumber) {
@@ -109,7 +108,6 @@ class CourseStatsCounter {
   }
 
   markPassingGrade(studentnumber) {
-    delete this.students.notParticipatedOrFailed[studentnumber]
     this.students.passed[studentnumber] = true
     if (this.failedBefore(studentnumber)) {
       this.students.retryPassed[studentnumber] = true
@@ -151,15 +149,15 @@ class CourseStatsCounter {
   getPassingSemestersCumulative() {
     const passingSemesters = this.stats.passingSemesters
     const cumulativeStats = {
-      'BEFORE': passingSemesters['BEFORE']
+      BEFORE: passingSemesters['BEFORE']
     }
 
     cumulativeStats['0-FALL'] = passingSemesters['BEFORE'] + passingSemesters['0-FALL']
     cumulativeStats['0-SPRING'] = cumulativeStats['0-FALL'] + passingSemesters['0-SPRING']
 
     for (let i = 1; i < 7; i++) {
-      cumulativeStats[`${i}-FALL`] = cumulativeStats[`${i - 1}-SPRING`] + passingSemesters[`${i}-FALL`]
-      cumulativeStats[`${i}-SPRING`] = cumulativeStats[`${i}-FALL`] + passingSemesters[`${i}-SPRING`]
+      cumulativeStats[fall[i]] = cumulativeStats[spring[i - 1]] + passingSemesters[fall[i]]
+      cumulativeStats[spring[i]] = cumulativeStats[fall[i]] + passingSemesters[spring[i]]
     }
 
     cumulativeStats['LATER'] = cumulativeStats['6-SPRING'] + passingSemesters['LATER']
@@ -190,9 +188,8 @@ class CourseStatsCounter {
       grades: this.grades
     }
   }
-
 }
 
 module.exports = {
   CourseStatsCounter
-}
+}

services/backend/oodikone2-backend/src/services/course_stats_counter.test.js

@@ -1,37 +1,31 @@
 const { CourseStatsCounter } = require('./course_stats_counter')
 
 describe('CourseStatsCounter tests', () => {
-  
   const studentnumbers = ['1', '2', '3', '4', '5']
-  const allstudents = studentnumbers.reduce((numbers, num) => ({ ...numbers, [num]: true }), {})
   const coursecode = 'COURSE_01'
   const coursename = 'COURSE NAME'
 
-  const createCounter = () => new CourseStatsCounter(coursecode, coursename, allstudents)
+  const createCounter = () => new CourseStatsCounter(coursecode, coursename, studentnumbers.length)
 
   describe('counter constructor should set course values correctly', () => {
-
     const counter = createCounter()
 
     test('course name should be set after creation', () => {
       expect(counter.course.name).toBe(coursename)
     })
-    
+
     test('course code should be set after creation', () => {
       expect(counter.course.code).toBe(coursecode)
     })
-    
   })
 
   describe('mark passed credit for student', () => {
-
     const student = studentnumbers[0]
     const grade = '5'
     const counter = createCounter()
 
     counter.markCredit(student, grade, true, false, false)
-    const { all, failed, passed, notParticipated, notParticipatedOrFailed } = counter.students
-
+    const { all, failed, passed } = counter.students
 
     test('all should have studentnumber as key', () => {
       expect(all).toHaveProperty(student)
@@ -44,26 +38,16 @@ describe('CourseStatsCounter tests', () => {
     test('failed students should not have studentnumber as key', () => {
       expect(failed).not.toHaveProperty(student)
     })
-
-    test('notParticipated should not have studentnumber as key', () => {
-      expect(notParticipated).not.toHaveProperty(student)
-    })
-
-    test('notParticipatedOrFailed should not have studentnumber as key', () => {
-      expect(notParticipatedOrFailed).not.toHaveProperty(student)
-    })
-
   })
 
   describe('mark failed credit for student', () => {
-
     const student = studentnumbers[0]
     const grade = '0'
     const counter = createCounter()
 
     counter.markCredit(student, grade, false, true, false)
 
-    const { all, failed, passed, notParticipated, notParticipatedOrFailed } = counter.students
+    const { all, failed, passed } = counter.students
 
     test('all should have studentnumber as key', () => {
       expect(all).toHaveProperty(student)
@@ -76,26 +60,16 @@ describe('CourseStatsCounter tests', () => {
     test('passed students should not have studentnumber as key', () => {
       expect(passed).not.toHaveProperty(student)
     })
-
-    test('notParticipated should not have studentnumber as key', () => {
-      expect(notParticipated).not.toHaveProperty(student)
-    })
-
-    test('notParticipatedOrFailed should have studentnumber as key', () => {
-      expect(notParticipatedOrFailed).toHaveProperty(student)
-    })
-
   })
 
   describe('mark failed credit then passed credit for student', () => {
-    
     const student = studentnumbers[0]
     const counter = createCounter()
 
     counter.markCredit(student, '0', false, true, false)
     counter.markCredit(student, '5', true, false, false)
 
-    const { passed, failed, retryPassed, notParticipatedOrFailed } = counter.students
+    const { passed, failed, retryPassed } = counter.students
 
     test('failed students should not have studentnumber as key', () => {
       expect(failed).not.toHaveProperty(student)
@@ -108,22 +82,16 @@ describe('CourseStatsCounter tests', () => {
     test('retryPassed should have studentnumber as key', () => {
       expect(retryPassed).toHaveProperty(student)
     })
-
-    test('notParticipatedOrFailed should not have studentnumber as key', () => {
-      expect(notParticipatedOrFailed).not.toHaveProperty(student)
-    })
-
   })
 
   describe('mark passed credit then failed credit for student', () => {
-    
     const student = studentnumbers[0]
     const counter = createCounter()
 
     counter.markCredit(student, '5', true, false, false)
     counter.markCredit(student, '0', false, true, false)
 
-    const { passed, failed, retryPassed, notParticipatedOrFailed } = counter.students
+    const { passed, failed, retryPassed } = counter.students
 
     test('failed students should not have studentnumber as key', () => {
       expect(failed).not.toHaveProperty(student)
@@ -136,15 +104,9 @@ describe('CourseStatsCounter tests', () => {
     test('retryPassed should have student', () => {
       expect(retryPassed).toHaveProperty(student)
     })
-
-    test('notParticipatedOrFailed should not have student', () => {
-      expect(notParticipatedOrFailed).not.toHaveProperty(student)
-    })
-
   })
 
   describe('mark failed credit then failed credit for student', () => {
-
     const student = studentnumbers[0]
     const counter = createCounter()
 
@@ -164,19 +126,17 @@ describe('CourseStatsCounter tests', () => {
     test('failed many students should have students', () => {
       expect(failedMany).toHaveProperty(student)
     })
-
   })
 
   describe('mark passed then failed then failed for student', () => {
-
     const student = studentnumbers[0]
     const counter = createCounter()
 
     counter.markCredit(student, '5', true, false, false)
     counter.markCredit(student, '0', false, true, false)
     counter.markCredit(student, '0', false, true, false)
-    
-    const { passed, failed, failedMany, notParticipatedOrFailed, retryPassed } = counter.students
+
+    const { passed, failed, failedMany, retryPassed } = counter.students
 
     test('passed students should have studentnumber', () => {
       expect(passed).toHaveProperty(student)
@@ -190,18 +150,12 @@ describe('CourseStatsCounter tests', () => {
       expect(failedMany).not.toHaveProperty(student)
     })
 
-    test('notParticipatedOrFailed should not have studentnumber', () => {
-      expect(notParticipatedOrFailed).not.toHaveProperty(student)
-    })
-
     test('retryPassed should have studentnumber', () => {
       expect(retryPassed).toHaveProperty(student)
     })
-
   })
 
   describe('mark passed then improved grade for student', () => {
-
     const student = studentnumbers[0]
     const counter = createCounter()
 
@@ -217,11 +171,9 @@ describe('CourseStatsCounter tests', () => {
     test('improvedPassedGrade should have student', () => {
       expect(improvedPassedGrade).toHaveProperty(student)
     })
-
   })
 
   describe('mark improved grade then failed for student', () => {
-
     const student = studentnumbers[0]
     const counter = createCounter()
 
@@ -237,11 +189,9 @@ describe('CourseStatsCounter tests', () => {
     test('improvedPassedGrade should contain student', () => {
       expect(improvedPassedGrade).toHaveProperty(student)
     })
-    
+
     test('failed should not contain student', () => {
       expect(failed).not.toHaveProperty(student)
     })
-
   })
-
 })

services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js

@@ -6,7 +6,6 @@ const CATEGORY = {
 }
 
 class CourseYearlyStatsCounter {
-
   constructor() {
     this.groups = {}
     this.programmes = {}
@@ -18,7 +17,7 @@ class CourseYearlyStatsCounter {
   }
 
   initProgramme(code, name) {
-    this.programmes[code] = { name, students: [] }
+    this.programmes[code] = { name, students: {} }
   }
 
   initGroup(groupcode, name, coursecode, yearcode) {
@@ -45,18 +44,17 @@ class CourseYearlyStatsCounter {
     return { attempted, passed, failed }
   }
 
-  markStudyProgramme(code, name, studentnumber) {
+  markStudyProgramme(code, name, studentnumber, yearcode) {
     if (!this.programmes[code]) {
       this.initProgramme(code, name)
     }
-    if (!this.programmes[code].students.includes(studentnumber)) {
-      this.programmes[code].students.push(studentnumber)
-    }
+    this.programmes[code].students[yearcode] = this.programmes[code].students[yearcode] || []
+    this.programmes[code].students[yearcode].push(studentnumber)
   }
 
-  markStudyProgrammes(studentnumber, programmes) {
+  markStudyProgrammes(studentnumber, programmes, yearcode) {
     programmes.forEach(({ code, name }) => {
-      this.markStudyProgramme(code, name, studentnumber)
+      this.markStudyProgramme(code, name, studentnumber, yearcode)
     })
   }
 
@@ -117,14 +115,14 @@ class CourseYearlyStatsCounter {
   formatStudentStatistics(students) {
     const grades = {}
     const classes = {}
-    const studentnumbers = []
+    const studentnumbers = new Set()
     Object.entries(students).forEach(([studentnumber, stat]) => {
       const { grade, category } = stat
       grades[grade] = grades[grade] ? grades[grade].concat(studentnumber) : [studentnumber]
       classes[category] = classes[category] ? classes[category].concat(studentnumber) : [studentnumber]
-      studentnumbers.includes(studentnumber) ? null : studentnumbers.push(studentnumber)
+      studentnumbers.add(studentnumber)
     })
-    return { grades, classes, studentnumbers }
+    return { grades, classes, studentnumbers: [...studentnumbers] }
   }
 
   formatGroupStatistics() {
@@ -140,7 +138,6 @@ class CourseYearlyStatsCounter {
       statistics: this.formatGroupStatistics()
     }
   }
-
 }
 
-module.exports = { CourseYearlyStatsCounter, CATEGORY }
+module.exports = { CourseYearlyStatsCounter, CATEGORY }

services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.test.js

@@ -1,11 +1,9 @@
 const { CourseYearlyStatsCounter, CATEGORY } = require('./course_yearly_stats_counter')
 
 describe('Marking credit to history', () => {
-
   const studentnumber = '12345678'
 
   describe('Student history to empty counter', () => {
-
     const counter = new CourseYearlyStatsCounter()
     const history = counter.studentHistory(studentnumber)
 
@@ -14,11 +12,9 @@ describe('Marking credit to history', () => {
       expect(history.failed).toBe(false)
       expect(history.failed).toBe(false)
     })
-
   })
 
   describe('Mark passed credit to empty counter', () => {
-    
     const counter = new CourseYearlyStatsCounter()
     counter.markCreditToHistory(studentnumber, true)
     const history = counter.studentHistory(studentnumber)
@@ -28,11 +24,9 @@ describe('Marking credit to history', () => {
       expect(history.passed).toBe(true)
       expect(history.failed).toBe(false)
     })
-
   })
 
   describe('Mark failed credit to empty counter', () => {
-
     const counter = new CourseYearlyStatsCounter()
     counter.markCreditToHistory(studentnumber, false)
     const history = counter.studentHistory(studentnumber)
@@ -42,11 +36,9 @@ describe('Marking credit to history', () => {
       expect(history.passed).toBe(false)
       expect(history.failed).toBe(true)
     })
-
   })
 
   describe('Mark failed credit after passed credit', () => {
-
     const counter = new CourseYearlyStatsCounter()
     counter.markCreditToHistory(studentnumber, true)
     counter.markCreditToHistory(studentnumber, false)
@@ -59,11 +51,9 @@ describe('Marking credit to history', () => {
     test('student history failed should be false', () => {
       expect(history.failed).toBe(false)
     })
-
   })
 
   describe('Mark passed credit after failed credit', () => {
-
     const counter = new CourseYearlyStatsCounter()
     counter.markCreditToHistory(studentnumber, false)
     counter.markCreditToHistory(studentnumber, true)
@@ -77,11 +67,9 @@ describe('Marking credit to history', () => {
       expect(history.failed).toBe(false)
     })
   })
-
 })
 
 describe('Marking credit to group', () => {
-
   const studentnumber = '123456789'
   const groupcode = 'GROUP'
   const groupname = 'Group name'
@@ -92,19 +80,17 @@ describe('Marking credit to group', () => {
   const markFail = counter => counter.markCreditToGroup(studentnumber, false, FAILGRADE, groupcode, groupname)
 
   describe('Marking passed credit to group for empty counter', () => {
-
     const counter = new CourseYearlyStatsCounter()
     markPass(counter)
 
     describe('group students tests', () => {
-
       const { students } = counter.groups[groupcode]
       const { category, grade, passed } = students[studentnumber]
 
       test('student category in group should be PASS_FIRST', () => {
         expect(category).toBe(CATEGORY.PASS_FIRST)
       })
-    
+
       test('student grade in group should be the given grade', () => {
         expect(grade).toBe(PASSGRADE)
       })
@@ -131,18 +117,14 @@ describe('Marking credit to group', () => {
       test('failed should not contain studentnumber', () => {
         expect(failed).not.toContain(studentnumber)
       })
-
     })
-
   })
 
   describe('Marking failed credit to group for empty counter', () => {
-
     const counter = new CourseYearlyStatsCounter()
     markFail(counter)
 
     describe('group students tests', () => {
-    
       const { students } = counter.groups[groupcode]
       const { category, grade, passed } = students[studentnumber]
 
@@ -157,11 +139,9 @@ describe('Marking credit to group', () => {
       test('student passed should be false', () => {
         expect(passed).toBe(false)
       })
-
     })
 
     describe('group attempts tests', () => {
-
       const { attempts } = counter.groups[groupcode]
       const { passed, failed } = attempts.classes
 
@@ -172,7 +152,6 @@ describe('Marking credit to group', () => {
       test('failed should contain studentnumber', () => {
         expect(failed).toContain(studentnumber)
       })
-
     })
 
     test('student history should contain correct values for student', () => {
@@ -181,11 +160,9 @@ describe('Marking credit to group', () => {
       expect(history.passed).toBe(false)
       expect(history.failed).toBe(true)
     })
-
   })
 
   describe('Mark failed credit then passed credit to same group', () => {
-    
     const counter = new CourseYearlyStatsCounter()
     markFail(counter)
     markPass(counter)
@@ -220,7 +197,5 @@ describe('Marking credit to group', () => {
       expect(history.failed).toBe(false)
       expect(history.attempted).toBe(true)
     })
-
   })
-
 })

services/backend/oodikone2-backend/src/services/courses.js

@@ -1,90 +1,99 @@
 const Sequelize = require('sequelize')
 const moment = require('moment')
 const {
-  sequelize, Student, Credit,
-  Course, CourseType, Discipline,
-  ElementDetails, StudyrightElement,
-  Studyright, Semester,
+  sequelize,
+  Student,
+  Credit,
+  Course,
+  CourseType,
+  Discipline,
+  ElementDetails,
+  StudyrightElement,
+  Studyright,
+  Semester
 } = require('../models')
-const {
-  sequelizeKone, CourseDuplicates,
-} = require('../models/models_kone')
+const { sequelizeKone, CourseDuplicates } = require('../models/models_kone')
 const Op = Sequelize.Op
 const { CourseYearlyStatsCounter } = require('../services/course_yearly_stats_counter')
 const _ = require('lodash')
 
-const byNameOrCode = (searchTerm, language) => Course.findAll({
-  where: {
-    [Op.or]: [
-      {
-        name: {
-          [language]: {
-            [Op.iLike]: searchTerm
+const byNameOrCode = (searchTerm, language) =>
+  Course.findAll({
+    where: {
+      [Op.or]: [
+        {
+          name: {
+            [language]: {
+              [Op.iLike]: searchTerm
+            }
+          }
+        },
+        {
+          code: {
+            [Op.like]: searchTerm
           }
         }
-      },
-      {
-        code: {
-          [Op.like]: searchTerm
+      ]
+    }
+  })
+
+const byName = (name, language) =>
+  Course.findAll({
+    where: {
+      name: {
+        [language]: {
+          [Op.eq]: name
         }
       }
-    ]
-  }
-})
-
-const byName = (name, language) => Course.findAll({
-  where: {
-    name: {
-      [language]: {
-        [Op.eq]: name
-      }
-    }
-  },
-  order: [
-    ['latest_instance_date', 'DESC']
-  ],
-  limit: 1
-})
+    },
+    order: [['latest_instance_date', 'DESC']],
+    limit: 1
+  })
 
 const byNameOrCodeTypeAndDiscipline = (searchTerm, type, discipline, language) => {
-  const includeDiscipline = discipline ? {
-    include: {
-      model: Discipline,
-      where: {
-        discipline_id: {
-          [Op.eq]: discipline
-        }
-      }
-    }
-  } : null
-
-  const whereNameOrCode = searchTerm ? {
-    [Op.or]: [
-      {
-        name: {
-          [language]: {
-            [Op.iLike]: `%${searchTerm}%`
+  const includeDiscipline = discipline
+    ? {
+        include: {
+          model: Discipline,
+          where: {
+            discipline_id: {
+              [Op.eq]: discipline
+            }
           }
         }
-      },
-      {
-        code: {
-          [Op.iLike]: `%${searchTerm}%`
-        }
       }
-    ],
+    : null
 
-  } : null
-
-  const whereType = type ? {
-    [Op.and]: [
-      {
-        coursetypecode: {
-          [Op.eq]: type
-        }
+  const whereNameOrCode = searchTerm
+    ? {
+        [Op.or]: [
+          {
+            name: {
+              [language]: {
+                [Op.iLike]: `%${searchTerm}%`
+              }
+            }
+          },
+          {
+            code: {
+              [Op.iLike]: `%${searchTerm}%`
+            }
+          }
+        ]
       }
-    ]
-  } : null
+    : null
+
+  const whereType = type
+    ? {
+        [Op.and]: [
+          {
+            coursetypecode: {
+              [Op.eq]: type
+            }
+          }
+        ]
+      }
+    : null
 
   return Course.findAll({
     ...includeDiscipline,
@@ -97,51 +106,52 @@ const byNameOrCodeTypeAndDiscipline = (searchTerm, type, discipline, language) =
 
 const byCode = code => Course.findByPk(code)
 
-const creditsForCourses = (codes) => Credit.findAll({
-  include: [
-    {
-      model: Student,
-      attributes: ['studentnumber'],
-      include: {
-        model: StudyrightElement,
-        attributes: ['code', 'startdate'],
-        include: [
-          {
-            model: ElementDetails,
-            attributes: ['name', 'type'],
-            where: {
-              type: {
-                [Op.eq]: 20
+const creditsForCourses = codes =>
+  Credit.findAll({
+    include: [
+      {
+        model: Student,
+        attributes: ['studentnumber'],
+        include: {
+          model: StudyrightElement,
+          attributes: ['code', 'startdate'],
+          include: [
+            {
+              model: ElementDetails,
+              attributes: ['name', 'type'],
+              where: {
+                type: {
+                  [Op.eq]: 20
+                }
+              }
+            },
+            {
+              model: Studyright,
+              attributes: ['prioritycode'],
+              where: {
+                prioritycode: {
+                  [Op.eq]: 1
+                }
               }
             }
-          },
-          {
-            model: Studyright,
-            attributes: ['prioritycode'],
-            where: {
-              prioritycode: {
-                [Op.eq]: 1
-              }
-            }
-          }
-        ],
+          ]
+        }
+      },
+      {
+        model: Semester,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname']
       }
-    }, {
-      model: Semester,
-      attributes: ['semestercode', 'name', 'yearcode', 'yearname']
-    }],
-  where: {
-    course_code: {
-      [Op.in]: codes
-    }
-  },
-  order: [
-    ['attainment_date', 'ASC']
-  ]
-})
+    ],
+    where: {
+      course_code: {
+        [Op.in]: codes
+      }
+    },
+    order: [['attainment_date', 'ASC']]
+  })
 
 const bySearchTerm = async (term, language) => {
-  const formatCourse = (course) => ({
+  const formatCourse = course => ({
     name: course.name[language],
     code: course.code,
     date: course.latest_instance_date
@@ -158,12 +168,12 @@ const bySearchTerm = async (term, language) => {
 }
 
 const bySearchTermTypeAndDiscipline = async (term, type, discipline, language) => {
-  const formatCourse = (course) => ({
+  const formatCourse = course => ({
     name: course.name[language],
     code: course.code,
     date: course.latest_instance_date
   })
-  const removeDuplicates = (courses) => {
+  const removeDuplicates = courses => {
     let newList = []
     courses.map(course => {
       const nameDuplicates = courses.filter(c => course.name === c.name)
@@ -189,9 +199,8 @@ const bySearchTermTypeAndDiscipline = async (term, type, discipline, language) =
   }
 }
 
-const creditsOf = async (codes) => {
-
-  const formatCredit = (credit) => {
+const creditsOf = async codes => {
+  const formatCredit = credit => {
     const credits = [credit]
     return {
       id: credit.id,
@@ -216,56 +225,75 @@ const creditsOf = async (codes) => {
 }
 
 const oneYearStats = (instances, year, separate, allInstancesUntilYear) => {
-
   const calculateStats = (thisSemester, allInstancesUntilSemester) => {
-    const studentsThatPassedThisYear = _.uniq(_.flattenDeep(thisSemester
-      .map(inst => inst.credits.filter(Credit.passed).map(c => c.student))))
+    const studentsThatPassedThisYear = _.uniq(
+      _.flattenDeep(thisSemester.map(inst => inst.credits.filter(Credit.passed).map(c => c.student)))
+    )
 
-    const gradeDistribution = _.groupBy(_.uniq(_.flattenDeep(thisSemester
-      .map(inst => inst.credits))), 'grade')
+    const gradeDistribution = _.groupBy(_.uniq(_.flattenDeep(thisSemester.map(inst => inst.credits))), 'grade')
 
-    const studentsThatFailedThisYear = _.uniq(_.flattenDeep(thisSemester
-      .map(inst => inst.credits.filter(Credit.failed).map(c => c.student))))
+    const studentsThatFailedThisYear = _.uniq(
+      _.flattenDeep(thisSemester.map(inst => inst.credits.filter(Credit.failed).map(c => c.student)))
+    )
 
-    const allStudentsThatFailedEver = _.flattenDeep(allInstancesUntilSemester
-      .map(inst => inst.credits.filter(Credit.failed).map(c => c.student)))
+    const allStudentsThatFailedEver = _.flattenDeep(
+      allInstancesUntilSemester.map(inst => inst.credits.filter(Credit.failed).map(c => c.student))
+    )
 
-    const passedStudentsThatFailedBefore = _.uniq(studentsThatPassedThisYear
-      .filter(student => allStudentsThatFailedEver.map(s => s.studentnumber).includes(student.studentnumber)))
+    const passedStudentsThatFailedBefore = _.uniq(
+      studentsThatPassedThisYear.filter(student =>
+        allStudentsThatFailedEver.map(s => s.studentnumber).includes(student.studentnumber)
+      )
+    )
 
     const passedStudentsOnFirstTry = _.difference(studentsThatPassedThisYear, passedStudentsThatFailedBefore)
-    const failedStudentsThatFailedBefore = _.uniq(_.flattenDeep(studentsThatFailedThisYear.filter(student =>
-      Object.entries(_.countBy(allStudentsThatFailedEver, 'studentnumber'))
-        .some(([number, count]) => number === student.studentnumber && count > 1))))
+    const failedStudentsThatFailedBefore = _.uniq(
+      _.flattenDeep(
+        studentsThatFailedThisYear.filter(student =>
+          Object.entries(_.countBy(allStudentsThatFailedEver, 'studentnumber')).some(
+            ([number, count]) => number === student.studentnumber && count > 1
+          )
+        )
+      )
+    )
 
     const failedStudentsOnFirstTry = _.difference(studentsThatFailedThisYear, failedStudentsThatFailedBefore)
 
     return {
-      studentsThatPassedThisYear, studentsThatFailedThisYear, allStudentsThatFailedEver,
-      passedStudentsThatFailedBefore, passedStudentsOnFirstTry, failedStudentsThatFailedBefore,
-      failedStudentsOnFirstTry, gradeDistribution, studentnumbers: thisSemester.studentnumbers
+      studentsThatPassedThisYear,
+      studentsThatFailedThisYear,
+      allStudentsThatFailedEver,
+      passedStudentsThatFailedBefore,
+      passedStudentsOnFirstTry,
+      failedStudentsThatFailedBefore,
+      failedStudentsOnFirstTry,
+      gradeDistribution,
+      studentnumbers: thisSemester.studentnumbers
     }
   }
   const stats = []
   if (separate === 'true') {
-    const fallInstances = instances.
-      filter(inst => moment(inst.date).isBetween(String(year) + '-08-01', String(year + 1) + '-01-15'))
+    const fallInstances = instances.filter(inst =>
+      moment(inst.date).isBetween(String(year) + '-08-01', String(year + 1) + '-01-15')
+    )
 
-    const allInstancesUntilFall = allInstancesUntilYear
-      .filter(inst => moment(inst.date).isBefore(String(year + 1) + '-01-15'))
+    const allInstancesUntilFall = allInstancesUntilYear.filter(inst =>
+      moment(inst.date).isBefore(String(year + 1) + '-01-15')
+    )
 
-    const springInstances = instances
-      .filter(inst => moment(inst.date).isBetween(String(year + 1) + '-01-15', String(year + 1) + '-08-01'))
+    const springInstances = instances.filter(inst =>
+      moment(inst.date).isBetween(String(year + 1) + '-01-15', String(year + 1) + '-08-01')
+    )
 
     let fallStatistics = calculateStats(fallInstances, allInstancesUntilFall)
 
     let springStatistics = calculateStats(springInstances, allInstancesUntilYear)
 
-    const passedF = fallInstances.reduce((a, b) => b.pass ? a = a.concat(b.credits[0].student) : a, [])
-    const failedF = fallInstances.reduce((a, b) => b.fail ? a = a.concat(b.credits[0].student) : a, [])
+    const passedF = fallInstances.reduce((a, b) => (b.pass ? (a = a.concat(b.credits[0].student)) : a), [])
+    const failedF = fallInstances.reduce((a, b) => (b.fail ? (a = a.concat(b.credits[0].student)) : a), [])
 
-    const passedS = springInstances.reduce((a, b) => b.pass ? a = a.concat(b.credits[0].student) : a, [])
-    const failedS = springInstances.reduce((a, b) => b.fail ? a = a.concat(b.credits[0].student) : a, [])
+    const passedS = springInstances.reduce((a, b) => (b.pass ? (a = a.concat(b.credits[0].student)) : a), [])
+    const failedS = springInstances.reduce((a, b) => (b.fail ? (a = a.concat(b.credits[0].student)) : a), [])
 
     if (fallStatistics.studentsThatPassedThisYear.length + fallStatistics.studentsThatFailedThisYear.length > 0) {
       stats.push({
@@ -295,14 +323,14 @@ const oneYearStats = (instances, year, separate, allInstancesUntilYear) => {
         time: String(year + 1) + ' Spring'
       })
     }
-
   } else {
-    const yearInst = instances
-      .filter(inst => moment(inst.date).isBetween(String(year) + '-08-01', String(year + 1) + '-08-01'))
+    const yearInst = instances.filter(inst =>
+      moment(inst.date).isBetween(String(year) + '-08-01', String(year + 1) + '-08-01')
+    )
 
     let statistics = calculateStats(yearInst, allInstancesUntilYear)
-    const passed = yearInst.reduce((a, b) => b.pass ? a = a.concat(b.credits[0].student) : a, [])
-    const failed = yearInst.reduce((a, b) => b.fail ? a = a.concat(b.credits[0].student) : a, [])
+    const passed = yearInst.reduce((a, b) => (b.pass ? (a = a.concat(b.credits[0].student)) : a), [])
+    const failed = yearInst.reduce((a, b) => (b.fail ? (a = a.concat(b.credits[0].student)) : a), [])
 
     stats.push({
       studentsThatPassedThisYear: statistics.studentsThatPassedThisYear || 0,
@@ -322,22 +350,25 @@ const oneYearStats = (instances, year, separate, allInstancesUntilYear) => {
 }
 
 const yearlyStatsOf = async (code, year, separate, language) => {
-  const getProgrammesFromStats = (stats) => _.flattenDeep(stats
-    .map(year =>
-      _.union(year.courseLevelPassed, year.courseLevelFailed)
-        .map(s => s.studyright_elements
-          .map(e => e))))
-    .reduce((b, a) => {
-      b[a.code] = b[a.code] ?
-        { ...b[a.code], amount: b[a.code].amount + 1 } :
-        { name: a.element_detail.name, amount: 1 }
+  const getProgrammesFromStats = stats =>
+    _.flattenDeep(
+      stats.map(year =>
+        _.union(year.courseLevelPassed, year.courseLevelFailed).map(s => s.studyright_elements.map(e => e))
+      )
+    ).reduce((b, a) => {
+      if (b[a.code]) {
+        b[a.code].amount++
+      } else {
+        b[a.code] = { name: a.element_detail.name, amount: 1 }
+      }
       return b
     }, resultProgrammes)
 
   const codes = await alternativeCodes(code)
   const allInstances = await creditsOf(codes)
-  const yearInst = allInstances
-    .filter(inst => moment(new Date(inst.date)).isBetween(year.start + '-09-01', year.end + '-08-01'))
+  const yearInst = allInstances.filter(inst =>
+    moment(new Date(inst.date)).isBetween(year.start + '-09-01', year.end + '-08-01')
+  )
 
   const allInstancesUntilYear = allInstances.filter(inst => moment(new Date(inst.date)).isBefore(year.end + '-08-01'))
   const name = (await Course.findOne({ where: { code: { [Op.eq]: code } } })).dataValues.name[language]
@@ -357,19 +388,23 @@ const yearlyStatsOf = async (code, year, separate, language) => {
     return {
       code,
       alternativeCodes: codes.filter(cd => cd !== code),
-      start, end, separate,
-      stats: resultStats, programmes: resultProgrammes,
+      start,
+      end,
+      separate,
+      stats: resultStats,
+      programmes: resultProgrammes,
       name
     }
   }
   return
 }
 
-const createCourse = async (code, name, latest_instance_date) => Course.create({
-  code,
-  name,
-  latest_instance_date
-})
+const createCourse = async (code, name, latest_instance_date) =>
+  Course.create({
+    code,
+    name,
+    latest_instance_date
+  })
 
 const findDuplicates = async (oldPrefixes, newPrefixes) => {
   let oldPrefixQuery = ''
@@ -409,7 +444,7 @@ const getMainCodes = () => {
   return CourseDuplicates.findAll()
 }
 
-const deleteDuplicateCode = async (code) => {
+const deleteDuplicateCode = async code => {
   try {
     await CourseDuplicates.destroy({
       where: {
@@ -437,23 +472,31 @@ const getDuplicateCodesWithCourses = async () => {
       }
     }
   })
-  const codeToCourse = courses.reduce((acc, c) => { acc[c.code] = c; return acc }, {})
+  const codeToCourse = courses.reduce((acc, c) => {
+    acc[c.code] = c
+    return acc
+  }, {})
   return courseDuplicates.map(cd => ({ ...cd.get(), course: codeToCourse[cd.coursecode] }))
 }
 
 const getDuplicatesToIdMap = () => {
-  return getMainCodes().then(res => res.reduce((acc, e) => {
-    acc[e.coursecode] = e.groupid
-    return acc
-  }, {}))
+  return getMainCodes().then(res =>
+    res.reduce((acc, e) => {
+      acc[e.coursecode] = e.groupid
+      return acc
+    }, {})
+  )
 }
 
 const getIdToDuplicatesMapWithCourse = () => {
-  return getDuplicateCodesWithCourses().then(res => res.reduce((acc, e) => {
-    acc[e.groupid] = acc[e.groupid] || []
-    acc[e.groupid].push(e.course)
-    return acc
-  }, {}))
+  return getDuplicateCodesWithCourses().then(res =>
+    res.reduce((acc, e) => {
+      if (!e.course) return acc
+      acc[e.groupid] = acc[e.groupid] || []
+      acc[e.groupid].push(e.course)
+      return acc
+    }, {})
+  )
 }
 
 const getMainCodeToDuplicates = async () => {
@@ -462,13 +505,13 @@ const getMainCodeToDuplicates = async () => {
     const main = _.orderBy(
       courses,
       [
-        (c) => {
+        c => {
           if (c.code.match(/^A/)) return 4 // open university codes come last
           if (c.code.match(/^\d/)) return 2 // old numeric codes come second
           if (c.code.match(/^[A-Za-z]/)) return 1 // new letter based codes come first
           return 3 // unknown, comes before open uni?
         },
-        c => c.latest_instance_date || new Date,
+        c => c.latest_instance_date || new Date(),
         'code'
       ],
       ['asc', 'desc', 'desc']
@@ -486,7 +529,7 @@ const getCodeToMainCourseMap = async () => {
   try {
     const maincodeToDuplicates = await getMainCodeToDuplicates()
     const codeToMainCode = Object.values(maincodeToDuplicates).reduce((acc, d) => {
-      d.duplicates.forEach((c) => {
+      d.duplicates.forEach(c => {
         acc[c.code] = d.maincourse
       })
       return acc
@@ -520,7 +563,7 @@ const getMainCourseToCourseMap = async (/*programme*/) => {
   return {}
 }
 
-const getDuplicateCodes = async (code) => {
+const getDuplicateCodes = async code => {
   const [mainCodeToDuplicates, codeToMainCourseMap] = await getMainCodeToDuplicatesAndCodeToMainCode()
   const maincourse = codeToMainCourseMap[code]
   if (!maincourse) return null
@@ -542,11 +585,9 @@ const setDuplicateCode = async (code1, code2) => {
             groupid = Math.max(0, ...Object.values(all).filter(e => e))
             groupid = groupid && !isNaN(groupid) ? groupid + 1 : 1
           }
-          await CourseDuplicates.bulkCreate([
-            { groupid, coursecode: code1 },
-            { groupid, coursecode: code2 }
-          ],
-          { ignoreDuplicates: true })
+          await CourseDuplicates.bulkCreate([{ groupid, coursecode: code1 }, { groupid, coursecode: code2 }], {
+            ignoreDuplicates: true
+          })
         } else {
           // both have a group, must merge groups
           await CourseDuplicates.update({ groupid: all[code1] }, { where: { groupid: all[code2] } })
@@ -566,6 +607,15 @@ const alternativeCodes = async code => {
   return alternatives ? alternatives : [code]
 }
 
+const getGroupId = async code => {
+  const duplicates = await CourseDuplicates.findOne({
+    where: {
+      coursecode: code
+    }
+  })
+  return duplicates ? duplicates.groupid : code
+}
+
 const formatStudyrightElement = ({ code, element_detail, startdate }) => ({
   code,
   name: element_detail.name,
@@ -590,28 +640,55 @@ const parseCredit = credit => {
   }
 }
 
-const yearlyStatsOfNew = async (coursecode, separate) => {
+const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses) => {
   const codes = await alternativeCodes(coursecode)
-  const [credits, course] = await Promise.all([creditsForCourses(codes), Course.findByPk(coursecode)])
+
+  if (unifyOpenUniCourses) {
+    const nonOpenUniCodes = _.uniq(codes.map(unifyOpenUniversity))
+
+    const matchingOpenUniCourseCodes = nonOpenUniCodes.length
+      ? await Course.findAll({
+          where: {
+            code: {
+              [Op.regexp]: {
+                [Op.any]: nonOpenUniCodes.map(c => `^AY?${c}(en|fi|sv)?$`)
+              }
+            }
+          }
+        }).map(course => course.code)
+      : []
+
+    codes.push(...matchingOpenUniCourseCodes)
+  }
+
+  const uniqueCodes = _.uniq(codes)
+  const [credits, course] = await Promise.all([creditsForCourses(uniqueCodes), Course.findByPk(coursecode)])
   const counter = new CourseYearlyStatsCounter()
   for (let credit of credits) {
     const {
-      studentnumber, grade, passed,
-      semestercode, semestername,
-      yearcode, yearname, programmes,
+      studentnumber,
+      grade,
+      passed,
+      semestercode,
+      semestername,
+      yearcode,
+      yearname,
+      programmes,
       coursecode
     } = parseCredit(credit)
     const groupcode = separate ? semestercode : yearcode
     const groupname = separate ? semestername : yearname
-    const unknownProgramme = [{
-      code: 'OTHER',
-      name: {
-        en: 'Other',
-        fi: 'Muu',
-        sv: 'Andra'
+    const unknownProgramme = [
+      {
+        code: 'OTHER',
+        name: {
+          en: 'Other',
+          fi: 'Muu',
+          sv: 'Andra'
+        }
       }
-    }]
-    counter.markStudyProgrammes(studentnumber, programmes.length === 0 ? unknownProgramme : programmes)
+    ]
+    counter.markStudyProgrammes(studentnumber, programmes.length === 0 ? unknownProgramme : programmes, yearcode)
     counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode)
     counter.markCreditToHistory(studentnumber, passed)
   }
@@ -619,18 +696,64 @@ const yearlyStatsOfNew = async (coursecode, separate) => {
   return {
     ...statistics,
     coursecode,
-    alternatives: codes,
+    alternatives: uniqueCodes,
     name: course.name.fi
   }
 }
 
-const courseYearlyStats = async (coursecodes, separate) => {
-  const stats = await Promise.all(coursecodes
-    .map(code => yearlyStatsOfNew(code, separate)))
+const maxYearsToCreatePopulationFrom = async coursecodes => {
+  const maxAttainmentDate = new Date(
+    Math.max(
+      ...(await Course.findAll({
+        where: {
+          code: {
+            [Op.in]: coursecodes
+          }
+        },
+        attributes: ['max_attainment_date']
+      }).map(c => new Date(c.max_attainment_date).getTime()))
+    )
+  )
+  const attainmentThreshold = new Date(maxAttainmentDate.getFullYear(), 0, 1)
+  attainmentThreshold.setFullYear(attainmentThreshold.getFullYear() - 6)
+
+  const credits = await Credit.findAll({
+    where: {
+      course_code: {
+        [Op.in]: coursecodes
+      },
+      attainment_date: {
+        [Op.gt]: attainmentThreshold
+      }
+    },
+    order: [['attainment_date', 'ASC']]
+  })
+
+  const yearlyStudents = Object.values(
+    credits.reduce((res, credit) => {
+      const attainmentYear = new Date(credit.attainment_date).getFullYear()
+      if (!res[attainmentYear]) res[attainmentYear] = 0
+      res[attainmentYear]++
+      return res
+    }, {})
+  )
+  const maxYearsToCreatePopulationFrom = Math.max(
+    Math.floor(
+      1200 / // Lower this value to get a smaller result if necessary
+        (yearlyStudents.reduce((acc, curr) => acc + curr, 0) / yearlyStudents.length)
+    ),
+    1
+  )
+
+  return maxYearsToCreatePopulationFrom
+}
+
+const courseYearlyStats = async (coursecodes, separate, unifyOpenUniCourses) => {
+  const stats = await Promise.all(coursecodes.map(code => yearlyStatsOfNew(code, separate, unifyOpenUniCourses)))
   return stats
 }
 
-const nameLikeTerm = (name) => {
+const nameLikeTerm = name => {
   if (!name) {
     return undefined
   }
@@ -652,18 +775,32 @@ const nameLikeTerm = (name) => {
   }
 }
 
-const codeLikeTerm = (code) => !code ? undefined : {
-  code: {
-    [Op.iLike]: `%${code.trim()}%`
-  }
+const codeLikeTerm = code =>
+  !code
+    ? undefined
+    : {
+        code: {
+          [Op.iLike]: `%${code.trim()}%`
+        }
+      }
+
+const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+
+const unifyOpenUniversity = code => {
+  const regexresult = isOpenUniCourseCode(code)
+  if (!regexresult) return code
+  return regexresult[1]
 }
 
-const byNameAndOrCodeLike = (name, code) => {
-  return Course.findAll({
+const byNameAndOrCodeLike = async (name, code) => {
+  const courses = await Course.findAll({
     attributes: [
-      'name', 'code',
+      'name',
+      'code',
       ['latest_instance_date', 'date'],
-      'startdate', 'enddate', 'max_attainment_date',
+      'startdate',
+      'enddate',
+      'max_attainment_date',
       'min_attainment_date'
     ],
     where: {
@@ -671,9 +808,35 @@ const byNameAndOrCodeLike = (name, code) => {
       ...codeLikeTerm(code)
     }
   })
+
+  const groups = {}
+  const groupMeta = {}
+  const codeToMainCourseMap = await getCodeToMainCourseMap()
+  await Promise.all(
+    courses.map(
+      course =>
+        new Promise(async res => {
+          const formattedCode = unifyOpenUniversity(course.code)
+          const groupid = await getGroupId(formattedCode)
+          groups[course.code] = groupid
+          if (!groupMeta[groupid]) {
+            const mainCourse = codeToMainCourseMap[formattedCode]
+            if ((mainCourse && !isOpenUniCourseCode(course.code)) || !isOpenUniCourseCode(course.code)) {
+              groupMeta[groupid] = {
+                name: mainCourse ? mainCourse.name : course.name,
+                code: mainCourse ? mainCourse.code : course.code
+              }
+            }
+          }
+          res()
+        })
+    )
+  )
+
+  return { courses, groups, groupMeta }
 }
 
-const byCodes = (codes) => {
+const byCodes = codes => {
   return Course.findAll({
     where: {
       code: {
@@ -701,5 +864,7 @@ module.exports = {
   courseYearlyStats,
   byNameAndOrCodeLike,
   byCodes,
-  getMainCodeToDuplicatesAndCodeToMainCode
-}
+  getMainCodeToDuplicatesAndCodeToMainCode,
+  maxYearsToCreatePopulationFrom,
+  unifyOpenUniversity
+}

services/backend/oodikone2-backend/src/services/customPopulationSearch.js

@@ -0,0 +1,36 @@
+const { CustomPopulationSearch } = require('../models/models_kone')
+
+const getCustomPopulationSearchesByUser = async userId =>
+  CustomPopulationSearch.findAll({
+    where: {
+      userId
+    }
+  })
+
+const createCustomPopulationSearch = async (name, userId, students) =>
+  CustomPopulationSearch.create({
+    name,
+    userId,
+    students
+  })
+
+const updateCustomPopulationSearch = async (userId, id, students) => {
+  const targetCustomPopulationSearch = await CustomPopulationSearch.findOne({
+    where: {
+      id,
+      userId
+    }
+  })
+
+  if (!targetCustomPopulationSearch) return null
+
+  return targetCustomPopulationSearch.update({
+    students
+  })
+}
+
+module.exports = {
+  getCustomPopulationSearchesByUser,
+  createCustomPopulationSearch,
+  updateCustomPopulationSearch
+}

services/backend/oodikone2-backend/src/services/doo_api_database_updater/database_updater.js

@@ -1,405 +0,0 @@
-const Oodi = require('./oodi_interface')
-const OrganisationService = require('../organisations')
-const mapper = require('./oodi_data_mapper')
-const {
-  Student, Studyright, ElementDetails, StudyrightElement, Credit, Course,
-  Teacher, Organisation, StudyrightExtent, CourseType, CourseDisciplines,
-  Discipline, CreditType, Semester, SemesterEnrollment, Provider, CourseProvider,
-  Transfers, CourseRealisationType, CourseRealisation, CourseEnrollment, sequelize,
-  CreditTeacher, ErrorData
-} = require('../../../src/models/index')
-
-const _ = require('lodash')
-const { taskpool } = require('../../util/taskpool')
-const { updateAttainmentDates } = require('./update_attainment_dates')
-
-let attainmentIds = new Set()
-let courseIds = new Set()
-let elementDetailsIds = new Set()
-
-const getAllStudentInformationFromApi = async studentnumber => {
-  const [student, studyrights, studyattainments, semesterEnrollments, courseEnrollments] = await Promise.all([
-    Oodi.getStudent(studentnumber),
-    Oodi.getStudentStudyRights(studentnumber),
-    Oodi.getStudyAttainments(studentnumber),
-    Oodi.getSemesterEnrollments(studentnumber),
-    Oodi.getCourseEnrollments(studentnumber)
-  ])
-  return {
-    student,
-    studyrights,
-    studyattainments,
-    studentnumber,
-    semesterEnrollments,
-    courseEnrollments
-  }
-}
-
-const createOrUpdateStudyrightTransfers = async (apiStudyright, studentnumber) => {
-  const transfers = mapper.getTransfersFromData(apiStudyright, studentnumber)
-  await Promise.all(transfers.map(transfer => Transfers.upsert(transfer)))
-}
-
-const updateStudyrights = async (api, studentnumber) => {
-  console.log(`update studyrights called for ${studentnumber}`)
-  if (api.studyrights.length === 0) {
-    console.log(`No studyrights for ${studentnumber}`)
-    return
-  }
-  for (let data of api.studyrights) {
-    await StudyrightExtent.upsert(mapper.studyrightDataToExtent(data))
-    console.log('got studyrightextent')
-    const [studyright] = await Studyright.upsert(mapper.getStudyRightFromData(data, studentnumber), { returning: true })
-    console.log('got studyright')
-    let i = 0
-    for (let element of data.elements) {
-      i = i + 1
-      const elementDetail = mapper.elementDetailFromData(element)
-      const studyrightElement = mapper.studyrightElementFromData(element, studyright.studyrightid, studentnumber)
-      if (!elementDetailsIds.has(elementDetail.code)) {
-        await ElementDetails.upsert(elementDetail)
-        elementDetailsIds.add(elementDetail.code)
-      }
-      await StudyrightElement.upsert(studyrightElement)
-      console.log(`${i}/${data.elements.length} studyright elements updated`)
-    }
-    await createOrUpdateStudyrightTransfers(data, studentnumber)
-    console.log(`studyrights updated for ${studentnumber}`)
-  }
-}
-
-const getTeachersFromApi = teacherids => Promise.all(teacherids.map(id => Oodi.getTeacherInfo(id)))
-
-const createTeachers = async (teachers) => {
-  await Promise.all(teachers.map(teacher => Teacher.upsert(teacher)))
-}
-
-const attainmentAlreadyInDb = attainment => attainmentIds.has(String(attainment.studyattainment_id))
-
-const createCourse = async course => {
-  if (!courseIds.has(course.code)) {
-    courseIds.add(course.code)
-    await Course.upsert(course)
-  }
-}
-
-const createCourseEnrollment = async (data, studentnumber) => {
-  const { courserealisation, courseenrollment, course } = mapper.studentEnrollmentToModels(data, studentnumber)
-  await Course.upsert(course)
-  await CourseRealisation.upsert(courserealisation)
-  await CourseEnrollment.upsert(courseenrollment)
-}
-
-const updateCourseEnrollments = async (apidata, studentnumber) => {
-  await Promise.all(apidata.courseEnrollments.map(enrollment => createCourseEnrollment(enrollment, studentnumber)))
-  console.log(`course enrollments updated for ${studentnumber}`)
-}
-
-const parseAttainmentData = (data, studentnumber) => {
-  return {
-    credit: mapper.attainmentDataToCredit(data, studentnumber),
-    teachers: mapper.attainmentDataToTeachers(data),
-    course: mapper.attainmentDataToCourse(data)
-  }
-}
-
-const createCreditTeachers = async (credit, teachers) => {
-  const creditTeachers = teachers.map(teacher => ({
-    credit_id: credit.id,
-    teacher_id: teacher.id
-  }))
-  await Promise.all(creditTeachers.map(ct => CreditTeacher.upsert(ct)))
-}
-
-const updateStudyattainments = async (api, studentnumber) => {
-  console.log(`update studyattainments called for ${studentnumber}`)
-  if (api.studyattainments.length === 0) {
-    console.log(`No study attainments for ${studentnumber}`)
-    return
-  }
-  for (let data of api.studyattainments) {
-    const { credit, teachers, course } = parseAttainmentData(data, studentnumber)
-    console.log(credit)
-    if (!attainmentAlreadyInDb(credit)) {
-      await createCourse(course)
-      if (!credit.semestercode) {
-        await ErrorData.upsert({
-          id: credit.id || Math.round(String(Math.random() * 34893723)) + String(666),
-          data: credit
-        })
-        const tamperedCredit = { ...credit, semestercode: mapper.getSemesterCode(credit.attainment_date) }
-        await Credit.upsert(tamperedCredit)
-      } else {
-        await Credit.upsert(credit)
-      }
-      await createTeachers(teachers)
-      await createCreditTeachers(credit, teachers)
-    }
-    console.log(`Studyattainments updated for ${studentnumber}`)
-  }
-}
-
-const updateSemesterEnrollments = async (apidata, studentnumber) => {
-  await Promise.all(apidata.semesterEnrollments.map(apiEnrollment => {
-    const semesterEnrollment = mapper.semesterEnrollmentFromData(apiEnrollment, studentnumber)
-    return SemesterEnrollment.upsert(semesterEnrollment)
-  }))
-  console.log(`semester enrollments updated for ${studentnumber}`)
-}
-
-const deleteStudentStudyrights = async studentnumber => {
-  await Studyright.destroy({
-    where: {
-      student_studentnumber: studentnumber
-    }
-  })
-  await StudyrightElement.destroy({
-    where: {
-      studentnumber
-    }
-  })
-}
-
-const updateStudent = async (studentnumber) => {
-  const api = await getAllStudentInformationFromApi(studentnumber)
-  if (api.student === null || api.student === undefined) {
-    console.log(`API returned ${api.student} for studentnumber ${studentnumber}.    `)
-  } else {
-    console.log(`TRYING TO UPDATE ${studentnumber}`)
-    await Student.upsert(mapper.getStudentFromData(api.student, api.studyrights))
-    await deleteStudentStudyrights(studentnumber)
-    await Promise.all([
-      updateStudyrights(api, studentnumber),
-      updateStudyattainments(api, studentnumber),
-      updateSemesterEnrollments(api, studentnumber),
-      updateCourseEnrollments(api, studentnumber)
-    ])
-    console.log(`SUCCESSFULLY UPDATED ${studentnumber}`)
-  }
-}
-
-const updateStudentFromData = async (api) => {
-  if (api.student === null || api.student === undefined) {
-    console.log(`API returned ${api.student} for studentnumber ${api.studentnumber}.    `)
-  } else {
-    const { studentnumber } = api
-    console.log(`TRYING TO UPDATE ${studentnumber}`)
-    await Student.upsert(mapper.getStudentFromData(api.student, api.studyrights))
-    await deleteStudentStudyrights(studentnumber)
-    await Promise.all([
-      updateStudyrights(api, studentnumber),
-      updateStudyattainments(api, studentnumber),
-      updateSemesterEnrollments(api, studentnumber),
-      updateCourseEnrollments(api, studentnumber)
-    ])
-  }
-}
-
-const getStudentsDataFromApi = numbers => Promise.all(numbers.map(studentnumber =>
-  getAllStudentInformationFromApi(studentnumber)))
-
-const updateStudents = async (studentnumbers, chunksize = 1, onUpdateStudent = undefined) => {
-  const runOnUpdate = _.isFunction(onUpdateStudent)
-  const remaining = studentnumbers.slice(0)
-  while (remaining.length > 0) {
-    console.log(`remaining: ${remaining.length}`)
-    const nextchunk = remaining.splice(0, chunksize)
-    await Promise.all(nextchunk.map(async studentnumber => {
-      await updateStudent(studentnumber)
-      if (runOnUpdate) {
-        onUpdateStudent()
-      }
-    }))
-  }
-}
-
-const updateStudentsTaskPooled = async (studentnumbers, chunksize = 1, onUpdateStudent = undefined) => {
-  const remaining = studentnumbers.slice(0)
-  const runOnUpdate = _.isFunction(onUpdateStudent)
-  const pool = taskpool(5)
-  while (remaining.length > 0) {
-    const nextchunk = remaining.splice(0, chunksize)
-    const apidata = await getStudentsDataFromApi(nextchunk)
-    await pool.enqueue(() => Promise.all(apidata.map(async data => {
-      await updateStudentFromData(data, onUpdateStudent)
-      if (runOnUpdate) {
-        onUpdateStudent()
-      }
-    })))
-  }
-  await pool.complete()
-}
-
-const getFaculties = () => {
-  return Promise.all([OrganisationService.all(), Oodi.getFaculties()])
-}
-
-const updateFaculties = async () => {
-  const [dbFacultiesArray, apiFacultiesArray] = await getFaculties()
-  const dbFacultyCodes = new Set(dbFacultiesArray.map(faculty => faculty.code))
-  await Promise.all(apiFacultiesArray.map(async faculty => {
-    if (dbFacultyCodes.has(faculty.code)) {
-      return
-    }
-    await Organisation.upsert(mapper.getOrganisationFromData(faculty))
-  }))
-}
-
-const createCourseType = data => {
-  const coursetype = mapper.courseTypeFromData(data)
-  return CourseType.upsert(coursetype)
-}
-
-const updateCourseTypeCodes = async () => {
-  const apiCourseTypes = await Oodi.getCourseTypeCodes()
-  await Promise.all(apiCourseTypes.map(createCourseType))
-}
-
-const createOrUpdateTeacher = async teacher => {
-  if (teacher !== null) {
-    return Teacher.upsert(mapper.getTeacherFromData(teacher))
-  }
-}
-
-const updateTeacherInfo = async (teacherids, chunksize = 1) => {
-  const teacherchunks = _.chunk(teacherids, chunksize)
-  for (let chunk of teacherchunks) {
-    const apidata = await getTeachersFromApi(chunk)
-    await Promise.all(apidata.map(createOrUpdateTeacher))
-  }
-}
-
-const updateTeacherInfoTaskPooled = async (teacherids, chunksize = 1) => {
-  const teacherchunks = _.chunk(teacherids, chunksize)
-  const pool = taskpool()
-  for (let chunk of teacherchunks) {
-    const apidata = await getTeachersFromApi(chunk)
-    await pool.enqueue(() => sequelize.transaction(() => Promise.all(apidata.map(createOrUpdateTeacher))))
-  }
-  await pool.complete()
-}
-
-const updateTeachersInDb = async (chunksize = 50, usetaskpool = true) => {
-  const dbteachers = await Teacher.findAll({ attributes: ['id'] })
-  if (usetaskpool === true) {
-    await updateTeacherInfoTaskPooled(dbteachers.map(teacher => teacher.id), chunksize)
-  } else {
-    await updateTeacherInfo(dbteachers.map(teacher => teacher.id), chunksize)
-  }
-}
-
-const getLearningOpportunityFromApi = (courseids) => {
-  return Promise.all(courseids.map(courseid => Oodi.getLearningOpportunity(courseid)))
-}
-
-const createOrUpdateCourseFromLearningOpportunityData = async data => {
-  await Course.upsert(mapper.learningOpportunityDataToCourse(data))
-  await Promise.all((mapper.learningOpportunityDataToCourseDisciplines(data).map(coursediscipline =>
-    CourseDisciplines.upsert(coursediscipline))))
-}
-
-const createOrUpdateCourseProviders = async data => {
-  const { providers, courseproviders } = mapper.learningOpportunityDataToCourseProviders(data)
-  await Promise.all(providers.map(provider => Provider.upsert(provider)))
-  await Promise.all(courseproviders.map(courseprovider => CourseProvider.upsert(courseprovider)))
-}
-
-const updateCourseInformationAndProviders = async (courseids, chunksize = 1) => {
-  const coursechunks = _.chunk(courseids, chunksize)
-  for (let chunk of coursechunks) {
-    const apidata = await getLearningOpportunityFromApi(chunk)
-    await Promise.all(apidata.map(async data => {
-      if (data) {
-        await createOrUpdateCourseFromLearningOpportunityData(data)
-        await createOrUpdateCourseProviders(data)
-      }
-    }))
-  }
-}
-
-const updateCoursesAndProvidersInDb = async (chunksize = 1) => {
-  const dbcourses = await Course.findAll({ attributes: ['code'] })
-  await updateCourseInformationAndProviders(dbcourses.map(course => course.code), chunksize)
-}
-
-const updateCreditTypeCodes = async () => {
-  const apiStudyAttainmentStatusCodes = await Oodi.getStudyattainmentStatusCodes()
-  const creditTypes = apiStudyAttainmentStatusCodes.map(mapper.studyattainmentStatusCodeToCreditType)
-  await Promise.all(creditTypes.map(type => CreditType.upsert(type)))
-}
-
-const updateCourseDisciplines = async () => {
-  const apiCourseDisciplines = await Oodi.getCourseDisciplines()
-  const courseDisciplines = apiCourseDisciplines.map(mapper.disciplineFromData)
-  await Promise.all(courseDisciplines.map(discipline => Discipline.upsert(discipline)))
-}
-
-const updateSemesters = async () => {
-  const apiSemesters = await Oodi.getSemesters()
-  return await Promise.all(apiSemesters.map(data => Semester.upsert(mapper.semesterFromData(data))))
-}
-
-const updateCourseRealisationTypes = async () => {
-  const apiTypes = await Oodi.getCourseRealisationTypes()
-  await Promise.all(apiTypes.map(data => CourseRealisationType.upsert(mapper.courseRealisationTypeFromData(data))))
-}
-
-const getExistingCourseRealisationCodes = async (since, courseids) => {
-  const validIds = new Set(courseids)
-  const isValidCourse = data => (
-    data !== undefined
-    && data.deleted !== 'true'
-    && validIds.has(data.learningopportunity_id)
-  )
-  const all = await Oodi.courseUnitRealisationsSince(since)
-  return all.filter(isValidCourse).map(data => `${data.course_id}`)
-}
-
-const updateCourseRealisationsAndEnrollments = async (courseids, since = '0000-01-01', chunksize = 50) => {
-  const apidata = await getExistingCourseRealisationCodes(since, courseids)
-  const chunks = _.chunk(apidata, chunksize)
-  const pool = taskpool(5)
-  for (let chunk of chunks) {
-    const datas = await Promise.all(chunk.map(courserealisation_id =>
-      Oodi.getCourseUnitRealisation(courserealisation_id)))
-    await pool.enqueue(() => Promise.all(datas.map(async data => {
-      if (!data) {
-        return
-      }
-      const { course, courserealisation, courseenrollments, students } = mapper.courseUnitRealisationDataToModels(data)
-      await Course.upsert(course)
-      await CourseRealisation.upsert(courserealisation)
-      await Promise.all(students.map(student => Student.upsert(student)))
-      await Promise.all(courseenrollments.map(enrollment => CourseEnrollment.upsert(enrollment)))
-    })))
-  }
-  await pool.complete()
-}
-
-const updateCourseRealisationsForCoursesInDb = async () => {
-  const courses = await Course.findAll()
-  await updateCourseRealisationsAndEnrollments(courses.map(course => course.code))
-}
-
-const updateDatabase = async (studentnumbers, onUpdateStudent) => {
-  if (process.env.NODE_ENV !== 'anon') {
-    await updateFaculties()
-  }
-  await updateCourseRealisationTypes()
-  await updateSemesters()
-  await updateCreditTypeCodes()
-  await updateCourseTypeCodes()
-  await updateCourseDisciplines()
-  await updateStudents(studentnumbers, 25, onUpdateStudent)
-  await updateTeachersInDb(100, true)
-  await updateCoursesAndProvidersInDb(100)
-  await updateAttainmentDates()
-}
-
-module.exports = {
-  updateDatabase, updateFaculties, updateStudents, updateCourseInformationAndProviders,
-  updateCreditTypeCodes, updateCourseDisciplines, updateSemesters, updateCourseRealisationTypes,
-  updateTeachersInDb, updateStudentsTaskPooled, updateCourseRealisationsAndEnrollments,
-  getExistingCourseRealisationCodes, updateCourseRealisationsForCoursesInDb,
-  deleteStudentStudyrights
-}

services/backend/oodikone2-backend/src/services/doo_api_database_updater/database_updater.test.js

@@ -1,31 +0,0 @@
-const { sequelize, Student, Studyright, StudyrightElement } = require('../../models')
-const { readFileSync } = require('fs')
-const path = require('path')
-const { deleteStudentStudyrights } = require('./database_updater')
-
-const SNUMBER = '012345678'
-
-beforeAll(async () => {
-  const filepath = path.resolve(__dirname, './database_updater.test.sql')
-  const query = readFileSync(filepath, 'utf8')
-  await sequelize.query(query)
-})
-
-const findStudent = snumber => Student.findByPk(snumber, {
-  include: {
-    model: Studyright,
-    include: StudyrightElement
-  }
-})
-
-test('deleteStudentStudyrights deletes associated studyrights and elements', async () => {
-  const studentBefore = await findStudent(SNUMBER)
-  expect(studentBefore).toBeTruthy()
-  expect(studentBefore.studyrights.length).toBe(1)
-  await deleteStudentStudyrights(SNUMBER)
-  const studentAfter = await findStudent(SNUMBER)
-  expect(studentAfter).toBeTruthy()
-  expect(studentAfter.studyrights.length).toBe(0)
-  const elements = await StudyrightElement.findAll({ where: { studentnumber: SNUMBER }})
-  expect(elements.length).toBe(0)
-})

services/backend/oodikone2-backend/src/services/doo_api_database_updater/database_updater.test.sql

@@ -1,35 +0,0 @@
-DO $$
-DECLARE
-  ts TIMESTAMP = CURRENT_TIMESTAMP;
-  snumber VARCHAR(50) = '012345678';
-  sr_id BIGINT = 1234;
-  sre_one VARCHAR(50) = 'SRE1';
-  sre_two VARCHAR(50) = 'SRE2';
-BEGIN
-
-  INSERT INTO element_details
-  ("code", "createdAt", "updatedAt")
-  VALUES
-  (sre_one, ts, ts),
-  (sre_two, ts, ts)
-  ;
-  INSERT INTO student
-  ("studentnumber", "createdAt", "updatedAt")
-  VALUES
-  (snumber, ts, ts)
-  ;
-
-  INSERT INTO studyright
-  ("studyrightid", "student_studentnumber")
-  VALUES
-  (sr_id, snumber)
-  ;
-
-  INSERT INTO studyright_elements
-  ("createdAt", "updatedAt", "studyrightid", "studentnumber", "code")
-  VALUES
-  (ts, ts, sr_id, snumber, sre_one),
-  (ts, ts, sr_id, snumber, sre_two)
-  ;
-
-END $$;

services/backend/oodikone2-backend/src/services/doo_api_database_updater/db_studentnumbers_to_file.js

@@ -1,15 +0,0 @@
-const fs = require('fs')
-const { Student } = require('../../models/index')
-
-const getStudentNumbers = async (writefilepath='./studentnumbers.txt') => {
-  const file = fs.openSync(writefilepath, 'w')
-  const students = await Student.findAll()
-  for (let student of students) {
-    const data = `${student.studentnumber} \n`
-    fs.appendFileSync(file, data)
-  }
-  fs.closeSync(file)
-  process.exit(0)
-}
-
-getStudentNumbers()

services/backend/oodikone2-backend/src/services/doo_api_database_updater/oodi_data_mapper.js

@@ -1,352 +0,0 @@
-const moment = require('moment')
-const _ = require('lodash')
-
-const getStudyRightIdStrings = (data) =>
-  data['data'].map(elements => elements[0])
-
-const getTextsByLanguage = texts => {
-  const names = {}
-  texts.forEach(text => names[text.langcode] = text.text)
-  return { fi: null, sv: null, en: null, ...names }
-}
-
-const defaultNameFromTexts = texts => {
-  const names = getTextsByLanguage(texts)
-  return names.fi || names.en || names.sv
-}
-
-const jsonNamesFromTexts = texts => {
-  const names = getTextsByLanguage(texts)
-  return names
-}
-
-const universityEnrollmentDateFromStudyRights = studyRightArray => {
-  return _.sortBy(studyRightArray.map(s => s.start_date), n =>
-    moment(n).valueOf())[0]
-}
-
-const getStudentFromData = (student, studyrights) => {
-  const city = getTextsByLanguage(student.city)
-  const language = getTextsByLanguage(student.language)
-  const country = getTextsByLanguage(student.country)
-  const gender = getTextsByLanguage(student.gender)
-  return {
-    studentnumber: student.student_number,
-    email: student.email,
-    phone: student.phone,
-    city_fi: city.fi,
-    city_sv: city.sv,
-    national_student_number: student.national_student_number,
-    zipcode: student.zipcode,
-    address: student.address1,
-    creditcount: student.studyattainments,
-    address2: student.address2,
-    birthdate: getDate(student.birth_date),
-    language_fi: language.fi,
-    language_sv: language.sv,
-    language_en: language.en,
-    age: student.age_years,
-    lastname: student.last_name,
-    mobile: student.mobile_phone,
-    home_county_id: student.home_county_id,
-    country_fi: country.fi,
-    country_sv: country.sv,
-    country_en: country.en,
-    firstnames: student.first_names,
-    communicationlanguage: language.en || language.fi || language.sv,
-    dateofuniversityenrollment: universityEnrollmentDateFromStudyRights(studyrights),
-    matriculationexamination: null,
-    abbreviatedname: [student.last_name, student.first_names].join(' '),
-    gender_code: student.gender_code,
-    gender_fi: gender.fi,
-    gender_sv: gender.sv,
-    gender_en: gender.en
-  }
-}
-
-const getSemesterCode = attainment_date => ((moment(attainment_date).diff(moment('1950', 'YYYY'), 'months')) / 6)
-
-const getDate = (date, format = 'DD.MM.YYYY') => {
-  if (!date) return null
-  return moment(date, format).format('YYYY-MM-DD')
-}
-
-const getOrganisationFromData = ({ name, code }) => {
-  return {
-    code,
-    name: jsonNamesFromTexts(name)
-  }
-}
-
-const attainmentDataToCredit = (attainment, studentnumber) => {
-  return {
-    id: String(attainment.studyattainment_id),
-    grade: defaultNameFromTexts(attainment.grade),
-    credits: attainment.credits,
-    ordering: getDate(attainment.attainment_date, null),
-    credittypecode: attainment.attainment_status_code,
-    student_studentnumber: studentnumber,
-    attainment_date: parseDate(attainment.attainment_date),
-    course_code: `${attainment.learningopportunity_id}`,
-    semestercode: attainment.semester_code,
-    isStudyModule: attainment.is_leaf === 'false'
-  }
-}
-
-const attainmentDataToCourse = (attainment) => {
-  const { learningopportunity_name, attainment_date } = attainment
-  return {
-    code: attainment.learningopportunity_id,
-    name: jsonNamesFromTexts(learningopportunity_name),
-    latest_instance_date: parseDate(attainment_date),
-    is_study_module: attainment.is_leaf === 'false'
-  }
-}
-
-const getTeacherFromData = teacher => ({
-  id: teacher.teacher_id,
-  code: teacher.userid,
-  name: teacher.full_name
-})
-
-const ELEMENT_ID = {
-  DEGREE_TITLE: 10,
-  DEGREE_STUDY_PROGRAM: 20,
-  DEGREE_MAJOR: 40,
-}
-
-const highlevelnameFromElements = elements => {
-  let subject
-  elements.forEach(element => {
-    const name = defaultNameFromTexts(element.name)
-    switch (element.element_id) {
-    case ELEMENT_ID.DEGREE_STUDY_PROGRAM:
-      subject = name
-      break
-    case ELEMENT_ID.DEGREE_MAJOR:
-      if (subject === undefined) {
-        subject = name
-      }
-      break
-    default:
-      break
-    }
-  })
-  return `${subject}`
-}
-
-const parseDate = (date, format='YYYY-MM-DD') => date && moment.utc(date, format).toDate()
-
-const getStudyRightFromData = (data, studentNumber) => {
-  return {
-    studyrightid: `${data.studyright_id}`,
-    canceldate: data.cancel_date,
-    cancelorganisation: data.organisation_code,
-    enddate: parseDate(data.end_date),
-    extentcode: data.extent_code,
-    givendate: parseDate(data.admission_date),
-    graduated: Number(data.degree_date !== null),
-    graduation_date: data.degree_date,
-    highlevelname: highlevelnameFromElements(data.elements),
-    prioritycode: data.priority,
-    startdate: parseDate(data.start_date),
-    studystartdate: parseDate(data.study_start_date),
-    organization_code: data.organisation_code,
-    student_studentnumber: studentNumber
-  }
-}
-
-const elementDetailFromData = element => {
-  const { code, name, element_id } = element
-  const names = getTextsByLanguage(name)
-  return {
-    code,
-    name: names,
-    type: element_id
-  }
-}
-
-const studyrightElementFromData = (element, studyrightid, studentnumber) => {
-  const { start_date, end_date, code } = element
-  return {
-    code,
-    studyrightid,
-    studentnumber,
-    startdate: start_date,
-    enddate: end_date
-  }
-}
-
-const attainmentDataToTeachers = data => data.teachers.map(teacherdata => ({
-  id: teacherdata.teacher_id,
-}))
-
-const studyrightDataToExtent = data => ({
-  extentcode: data.extent_code,
-  name: getTextsByLanguage(data.extent)
-})
-
-const courseTypeFromData = data => ({
-  coursetypecode: data.code,
-  name: getTextsByLanguage(data.name)
-})
-
-const disciplineFromData = data => ({
-  discipline_id: data.discipline_id,
-  name: getTextsByLanguage(data.name)
-})
-
-const learningOpportunityDataToCourseDisciplines = data => data.disciplines.map(discipline => ({
-  discipline_id: discipline.discipline_id,
-  course_id: data.learningopportunity_id
-}))
-
-const learningOpportunityDataToCourse = data => ({
-  code: data.learningopportunity_id,
-  coursetypecode: data.learningopportunity_type_code,
-  disciplines: data.disciplines,
-  name: getTextsByLanguage(data.names),
-  startdate: parseDate(data.start_date),
-  enddate: parseDate(data.end_date)
-})
-
-const learningOpportunityDataToCourseProviders = data => {
-  const providers = data.organisations.map(organisation => ({
-    providercode: organisation.code,
-    name: getTextsByLanguage(organisation.name)
-  }))
-  const courseproviders = providers.map(provider => ({
-    providercode: provider.providercode,
-    coursecode: data.learningopportunity_id
-  }))
-  return {
-    providers,
-    courseproviders
-  }
-}
-
-const studyattainmentStatusCodeToCreditType = data => ({
-  credittypecode: data.code,
-  name: getTextsByLanguage(data.name)
-})
-
-const semesterFromData = data => ({
-  semestercode: data.semester_code,
-  name: getTextsByLanguage(data.name),
-  startdate: parseDate(data.start_date),
-  enddate: parseDate(data.end_date),
-  yearcode: data.year_code,
-  yearname: data.year_name
-})
-
-const semesterEnrollmentFromData = (data, studentnumber) => ({
-  enrollmenttype: data.semester_enrollment_type_code,
-  semestercode: data.semester_code,
-  enrollment_date: parseDate(data.semester_enrollment_date),
-  studentnumber
-})
-
-const getTransfersFromData = (data, studentnumber) => {
-  const studytracks = data.elements.filter(element => element.element_id === 20)
-  const sorted = _.sortBy(studytracks, 'end_date')
-  let transfers = []
-  for (let i = 1; i < sorted.length; ++i) {
-    let target = sorted[i].code
-    let source = sorted[i - 1].code
-    let transferdate = sorted[i - 1].end_date
-
-    transfers = transfers.concat({
-      sourcecode: source,
-      targetcode: target,
-      transferdate,
-      studentnumber,
-      studyrightid: `${data.studyright_id}`})
-  }
-  return(transfers)
-}
-
-const courseRealisationTypeFromData = data => ({
-  realisationtypecode: `${data.code}`,
-  name: getTextsByLanguage(data.name)
-})
-
-const studentEnrollmentToModels = (data, studentnumber) => {
-  const courserealisation = {
-    courserealisation_id: `${data.course_id}`,
-    startdate: parseDate(data.start_date),
-    enddate: parseDate(data.end_date),
-    coursecode: `${data.learningopportunity_id}`,
-    name: getTextsByLanguage(data.name),
-    realisationtypecode: `${data.type_code}`,
-    parent: `${data.parent_id}`,
-    root: `${data.root_id}`
-  }
-  const courseenrollment = {
-    studentnumber,
-    courserealisation_id: courserealisation.courserealisation_id
-  }
-  const course = {
-    code: data.learningopportunity_id
-  }
-  return {
-    courserealisation,
-    courseenrollment,
-    course
-  }
-}
-
-const courseUnitRealisationDataToModels = data => {
-  const courserealisation = {
-    courserealisation_id: `${data.course_id}`,
-    startdate: parseDate(data.start_date),
-    enddate: parseDate(data.end_date),
-    coursecode: `${data.learningopportunity_id}`,
-    name: getTextsByLanguage(data.realisation_name),
-    realisationtypecode: `${data.realisation_type_code}`,
-    parent: `${data.parent_id}`,
-  }
-  const course = {
-    code: courserealisation.coursecode,
-  }
-  const students = data.students.map(({ student_number }) => ({
-    studentnumber: student_number
-  }))
-  const courseenrollments = students.map(student => ({
-    studentnumber: student.studentnumber,
-    courserealisation_id: courserealisation.courserealisation_id
-  }))
-  return {
-    courserealisation,
-    course,
-    students,
-    courseenrollments
-  }
-}
-
-module.exports = {
-  getStudentFromData,
-  getStudyRightIdStrings,
-  getStudyRightFromData,
-  getDate,
-  getTeacherFromData,
-  elementDetailFromData,
-  studyrightElementFromData,
-  attainmentDataToCourse,
-  attainmentDataToCredit,
-  getOrganisationFromData,
-  attainmentDataToTeachers,
-  studyrightDataToExtent,
-  courseTypeFromData,
-  learningOpportunityDataToCourse,
-  studyattainmentStatusCodeToCreditType,
-  disciplineFromData,
-  learningOpportunityDataToCourseDisciplines,
-  semesterFromData,
-  semesterEnrollmentFromData,
-  learningOpportunityDataToCourseProviders,
-  getTransfersFromData,
-  courseRealisationTypeFromData,
-  studentEnrollmentToModels,
-  courseUnitRealisationDataToModels,
-  getSemesterCode
-}

services/backend/oodikone2-backend/src/services/doo_api_database_updater/oodi_data_mapper.test.js

@@ -1,209 +0,0 @@
-const mapper = require('./oodi_data_mapper')
-
-describe('studyattainment data mapping tests', () => {
-
-  const data = {
-    'attainment_status_code': 4,
-    'attainment_date': '2018-05-30T21:00:00.000Z',
-    'studyattainment_id': 123456789,
-    'attainment_lang': null,
-    'person_id': 1000002,
-    'textual_assessment': null,
-    'teachers': [
-      {
-        'teacher_id': 'TID_01',
-        'person_id': 1000001,
-        'short_name': 'Matti Meikäläinen',
-        'last_names': 'Meikäläinen',
-        'first_names': 'Matti Teppo'
-      }
-    ],
-    'studyright_id': null,
-    'semester_code': 136,
-    'grade': [
-      {
-        'langcode': 'fi',
-        'text': 'Hyv.'
-      },
-      {
-        'langcode': 'sv',
-        'text': 'Godk'
-      },
-      {
-        'langcode': 'en',
-        'text': 'Pass'
-      }],
-    'credits': 10,
-    'expiry_date': null,
-    'learningopportunity_name': [
-      {
-        'langcode': 'fi',
-        'text': 'learningopportunity_name FI'
-      },
-      {
-        'langcode': 'sv',
-        'text': 'learningopportunity_name SV'
-      },
-      {
-        'langcode': 'en',
-        'text': 'learningopportunity_name EN'
-      }
-    ],
-    'learningopportunity_id': 'LID_001'
-  }
-
-  describe('credit mapping tests', () => {
-
-    const studentnumber = '0876654321'
-    const credit = mapper.attainmentDataToCredit(data, studentnumber)
-
-    describe('studyattainment date tests', () => {
-      const { attainment_date } = credit
-      const MAY = 4
-
-      test('attainment date is truthy', () => {
-        expect(attainment_date).toBeTruthy()
-      })
-
-      test('attainment date is instance of Date', () => {
-        expect(attainment_date instanceof Date).toBe(true)
-      })
-
-      test('attainment date has correct month, year and date', () => {
-        expect(attainment_date.getMonth()).toBe(MAY)
-        expect(attainment_date.getFullYear()).toBe(2018)
-        expect(attainment_date.getDate()).toBe(30)
-      })
-    })
-
-    test('course code matches learningopportunity_id', () => {
-      expect(credit.course_code).toBe('LID_001')
-    })
-
-    test('credit id should attainment id as string', () => {
-      expect(credit.id).toBe('123456789')
-    })
-
-    test('studentnumber is correct', () => {
-      expect(credit.student_studentnumber).toBe(studentnumber)
-    })
-
-    test('semestercode is correct', () => {
-      expect(credit.semestercode).toBe(136)
-    })
-
-  })
-
-  describe('teacher mapping tests', () => {
-
-    const teachers = mapper.attainmentDataToTeachers(data)
-
-    test('amount of mapped teachers is correct (1)', () => {
-      expect(teachers.length).toBe(1)
-    })
-
-    test('teacher should have correct id', () => {
-      expect(teachers[0].id).toBe('TID_01')
-    })
-
-  })
-
-})
-
-describe('semester enrollment data mapping tests', () => {
-
-  const studentnumber = '12345678'
-
-  const data = {
-    'full_time_student': 'true',
-    'yths_code': null,
-    'semester_enrollment_type_code': 1,
-    'absence_reason_code': null,
-    'payment_amount': null,
-    'yths_description': null,
-    'student_union_member': 'true',
-    'semester_enrollment_date': '2011-08-30T21:00:00.000Z',
-    'payment_date': null,
-    'semester_code': 123,
-    'doctoral_current_year_estimate': null,
-    'doctoral_previous_year': null
-  }
-
-  describe('mapping of enrolled data', () => {
-    const mapping = mapper.semesterEnrollmentFromData(data, studentnumber)
-
-    test('mapped semestercode should be integer 123', () => {
-      expect(mapping.semestercode).toBe(123)
-    })
-
-    test('mapped enrollmenttype code should be integer 1', () => {
-      expect(mapping.enrollmenttype).toBe(1)
-    })
-
-    test('mapped studentnumber should be correct', () => {
-      expect(mapping.studentnumber).toBe(studentnumber)
-    })
-
-    test('mapped enrollment_date should be instance of Date', () => {
-      expect(mapping.enrollment_date instanceof Date).toBe(true)
-    })
-
-    test('mapped enrollment_date should be equal to Date object with same day, month and year', () => {
-      expect(mapping.enrollment_date).toEqual(new Date('2011-08-30'))
-    })
-
-  })
-
-})
-
-describe('semester code data mapping tests', () => {
-
-  const data = {
-    'teaching_start_date': '1950-09-14T22:00:00.000Z',
-    'name': [
-      {
-        'langcode': 'fi',
-        'text': 'Syksy 1950'
-      },
-      {
-        'langcode': 'sv',
-        'text': 'Hösten 1950'
-      },
-      {
-        'langcode': 'en',
-        'text': 'Autumn 1950'
-      }
-    ],
-    'year_name': '1950-51',
-    'enroll_end_date': '1950-09-29T22:00:00.000Z',
-    'enroll_start_date': '1950-05-31T22:00:00.000Z',
-    'end_date': '1950-12-30T22:00:00.000Z',
-    'semester_code': 1,
-    'start_date': '1900-07-31T22:20:11.000Z',
-    'teaching_end_date': '1950-12-20T22:00:00.000Z',
-    'year_code': 1
-  }
-
-  const semester = mapper.semesterFromData(data)
-
-  test('semester code should be correct', () => {
-    expect(semester.semestercode).toBe(1)
-  })
-  
-  test('semester name should have all language keys set as properties', () => {
-    expect(semester.name).toHaveProperty('fi')
-    expect(semester.name).toHaveProperty('en')
-    expect(semester.name).toHaveProperty('sv')
-  })
-
-  test('semester start and end date should be instances of Date', () => {
-    expect(semester.startdate instanceof Date).toBe(true)
-    expect(semester.enddate instanceof Date).toBe(true)
-  })
-
-  test('semester yearcode and yearname should be correct', () => {
-    expect(semester.yearcode).toBe(1)
-    expect(semester.yearname).toBe('1950-51')
-  })
-
-})

services/backend/oodikone2-backend/src/services/doo_api_database_updater/oodi_interface.js

@@ -1,173 +0,0 @@
-require('dotenv').config()
-const axios = require('axios')
-const { OODI_ADDR, KEY_PATH, CERT_PATH } = require('../../conf-backend')
-const https = require('https')
-const fs = require('fs')
-const base_url = OODI_ADDR
-
-const agent = KEY_PATH && CERT_PATH ?
-  new https.Agent({
-    cert: fs.readFileSync(process.env.CERT_PATH, 'utf8'),
-    key: fs.readFileSync(process.env.KEY_PATH, 'utf8'),
-  }) :
-  new https.Agent({
-    rejectUnauthorized: false
-  })
-
-const instance = axios.create({
-  httpsAgent: agent
-})
-
-const getUrl = instance.get
-
-const getOodiApi = async relative => {
-  const route = base_url + relative
-  const stuff = await getUrl(route)
-  return stuff
-}
-
-const attemptGetFor = async (url, attempts = 5) => {
-  let attempt = 0
-  let response = 0
-  while (attempt <= attempts) {
-    attempt += 1
-    try {
-      response = await getUrl(url)
-      return response
-    } catch (error) {
-      if (attempt === attempts) {
-        throw error
-      }
-    }
-  }
-}
-
-if (process.env.NODE_ENV === 'dev') {
-
-  axios.defaults.auth = {
-    username: 'tktl',
-    password: process.env.OODI_PW
-  }
-  instance.interceptors.request.use((config) => {
-    config.params = {token: process.env.TOKEN}
-    return config
-  })
-
-}
-
-const getStudent = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/info`
-  const response = await attemptGetFor(url)
-  const data = response.data.data
-  return data
-}
-
-const getStudentStudyRights = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/studyrights`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const getFaculties = async () => {
-  const url = `${base_url}/codes/faculties`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const getStudyAttainments = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/studyattainments`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const getTeacherInfo = async id => {
-  const url = `${base_url}/teachers/${id}/info`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const getCourseTypeCodes = async () => {
-  const url = `${base_url}/codes/learningopportunities/types`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const getLearningOpportunity = async (id) => {
-  const url = `${base_url}/learningopportunities/${id}`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const getStudyattainmentStatusCodes = async () => {
-  const url = `${base_url}/codes/studyattainments/statuses`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const getCourseDisciplines = async () => {
-  const url = `${base_url}/codes/learningopportunities/disciplines`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const getSemesters = async () => {
-  const url = `${base_url}/codes/semesters`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const getSemesterEnrollments = async studentnumber => {
-  const url = `${base_url}/students/${studentnumber}/semesterenrollments`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const getCourseEnrollments = async studentnumber => {
-  const url = `${base_url}/students/${studentnumber}/enrollments`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const getCourseRealisationTypes = async () => {
-  const url = `${base_url}/codes/courseunitrealisations/types`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const courseUnitRealisationsSince = async sinceDate => {
-  const url = `${base_url}/courseunitrealisations/changes/ids/${sinceDate}`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const learningOpportunitiesSince = async (sinceDate='0000-01-01') => {
-  const url = `${base_url}/learningopportunities/changes/ids/${sinceDate}`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-const getCourseUnitRealisation = async id => {
-  const url = `${base_url}/courseunitrealisations/${id}`
-  const response = await attemptGetFor(url)
-  return response.data.data
-}
-
-module.exports = {
-  getStudentStudyRights,
-  getStudent,
-  getFaculties,
-  getStudyAttainments,
-  getTeacherInfo,
-  getCourseTypeCodes,
-  getLearningOpportunity,
-  getStudyattainmentStatusCodes,
-  getCourseDisciplines,
-  getSemesters,
-  getSemesterEnrollments,
-  getCourseEnrollments,
-  getCourseRealisationTypes,
-  courseUnitRealisationsSince,
-  getCourseUnitRealisation,
-  learningOpportunitiesSince,
-  getOodiApi
-}

services/backend/oodikone2-backend/src/services/doo_api_database_updater/run_updater.js

@@ -1,129 +0,0 @@
-const { updateDatabase } = require('./database_updater')
-const fs = require('fs')
-const logger = require('../../util/logger')
-const status = require('node-status')
-const { getStudentNumbers } = require('../../models/queries')
-
-const readStudentNumbersFromFile = async filename => {
-  const studentnumbers = fs.readFileSync(filename, 'utf-8').split('\n').map(s => s.replace(/\D/g, ''))
-  return studentnumbers.filter(studentnumber => !!studentnumber).map(s => s.startsWith('0') ? s : '0' + s)
-}
-
-const readStudentNumbersFromDb = async () => {
-  const studentnumbers = await getStudentNumbers().map(s => s.studentnumber)
-  return studentnumbers.filter(studentnumber => !!studentnumber).map(s => s.startsWith('0') ? s : '0' + s)
-}
-
-
-const parseArguments = (args) => {
-  return args.slice(2).reduce((args, arg) => {
-    const split = arg.split('=')
-    if (split[0] === 'file') {
-      args.file = split[1]
-    } else if (split[0] === 'index') {
-      args.index = split[1]
-    }
-    return args
-  }, {})
-
-}
-
-const fancylogger = (studentnumbers) => {
-
-  const counter = status.addItem('students', { max: studentnumbers.length })
-
-  const start = () => {
-    status.start({
-      pattern: 'Running: {uptime.time} {spinner.earth.green} | {students.bar} | students updated: {students}'
-    })
-  }
-
-  const stop = () => {
-    status.stamp()
-    status.stop()
-  }
-
-  const onUpdate = () => {
-    counter.inc()
-  }
-
-  return {
-    start,
-    stop,
-    onUpdate
-  }
-}
-
-const basiclogger = (studentnumbers, nstamps = 100) => {
-
-  const total = studentnumbers.length
-  const divisor = Math.floor(total / nstamps)
-
-  const dolog = iter => {
-    if (iter % divisor === 0) {
-      logger.info(`${new Date()} Students updated: ${iter}/${total}`)
-    }
-  }
-
-  let iter = 0
-
-  const start = () => {
-    logger.info('Updater started. ')
-  }
-
-  const onUpdate = () => {
-    iter += 1
-    dolog(iter)
-  }
-
-  const stop = () => {
-    logger.info('Updater finished. ')
-  }
-
-  return {
-    start,
-    stop,
-    onUpdate
-  }
-}
-
-const run = async (index = 0, basiclogging = true) => {
-
-  const args = parseArguments(process.argv)
-  index = args.index || index
-  basiclogging = args.basiclogging || basiclogging
-
-  let studentnumberssource
-  let readStudentnumbers
-
-  if (args.file) {
-    studentnumberssource = args.file
-    readStudentnumbers = await readStudentNumbersFromFile(studentnumberssource)
-  } else {
-    studentnumberssource = 'database'
-    readStudentnumbers = await readStudentNumbersFromDb()
-  }
-
-  const studentnumbers = readStudentnumbers.slice(index)
-
-  logger.info('Student number source is: ', studentnumberssource)
-
-  const statuslogger = basiclogging ? basiclogger(studentnumbers) : fancylogger(studentnumbers)
-
-  const started = new Date()
-  let exitcode = 0
-
-  try {
-    statuslogger.start()
-    await updateDatabase(studentnumbers, statuslogger.onUpdate)
-    statuslogger.stop()
-  } catch (e) {
-    logger.verbose(e.message)
-    exitcode = 1
-  }
-  const ended = new Date()
-  logger.info(`Updater started/ended: \n${started} \n${ended}`)
-  process.exit(exitcode)
-}
-
-run()

services/backend/oodikone2-backend/src/services/doo_api_database_updater/student_list_updater.js

@@ -1,102 +0,0 @@
-const _ = require('lodash')
-const axios = require('axios')
-const https = require('https')
-const fs = require('fs')
-const util = require('../../util')
-const { Student, StudentList } = require('../../models')
-const logger = require('../../util/logger')
-const STUDENT_SET_KEY = process.env.STUDENT_SET || 'cached_students'
-const timestamp = () => {
-  const d = new Date()
-  const z = (t) => t < 10 ? '0' + t : t
-  return `${z(d.getHours())}:${z(d.getMinutes())}:${z(d.getSeconds())}`
-}
-const arraysEqual = (a1, a2) => {
-  return JSON.stringify(a1.sort()) === JSON.stringify(a2.sort())
-}
-async function save(file) {
-  const cached = await StudentList.findOne({
-    where: { key: STUDENT_SET_KEY }
-  })
-  const numbers = cached.student_numbers.sort().reverse()
-  fs.writeFile(file, numbers.join('\n',), (err) => {
-    if (err) {
-      logger.info(err)
-    } else {
-      logger.info(`saved ${numbers.length} students to ${file} `)
-    }
-    process.exit(0)
-  })
-    
-}
-async function run() {
-  const agent = new https.Agent({
-    cert: fs.readFileSync(process.env.CERT_PATH, 'utf8'),
-    key: fs.readFileSync(process.env.KEY_PATH, 'utf8'),
-  })
-  const instance = axios.create()
-  instance.defaults.httpsAgent = agent
-  const requestStudent = async (studentNumber) => {
-    const url = `${process.env.OODI_ADDR}/students/${studentNumber}/info`
-    const response = await instance.get(url)
-    return response
-  }
-  const validStudents = []
-    
-  let cached = await StudentList.findOne({
-    where: { key: STUDENT_SET_KEY }
-  })
-  
-  const oldMax = () => 
-    Number((cached.student_numbers.sort().reverse()[0]/10).toFixed(0))
-                                        
-  const minStudentNumber = (cached) ? oldMax() : 1010000
-  const numberOfStudents = Number(process.env.INCREMENT) || 500000
-  const maxStudentNumber = process.env.STUDENTS_TO || minStudentNumber + numberOfStudents || 1510000
-  const step = process.env.STEP || 500
-  const range = maxStudentNumber - minStudentNumber
-  logger.info('student numbers from ' + minStudentNumber +
-  ' to ' + maxStudentNumber + ' total of ' + range + ' student numbers')
-  logger.info('log message every ' + step + ' student')
-  for (let i = minStudentNumber; i < maxStudentNumber; i++) {
-    let studentNumber = '0' + i + util.getStudentNumberChecksum(String(i))
-    if (i % step === 1) {
-      const iteration = (1 + (i - minStudentNumber) / step).toFixed()
-      logger.info('student list updater: ' + iteration + '/' + (range / step + 1).toFixed(0)  + ' ' + timestamp())
-    }
-    
-    const response = await requestStudent(studentNumber)
-    if (response.data.data != null) {
-      Student.upsert({studentnumber: studentNumber})
-      validStudents.push(studentNumber)
-    }
-  
-  }
-  logger.info('found and saved ' + validStudents.length + ' students')
-  if (cached === null) {
-    cached = StudentList.build({
-      key: STUDENT_SET_KEY,
-      student_numbers: validStudents,
-      max: maxStudentNumber
-    })
-  } else if (!arraysEqual(cached.student_numbers, validStudents)) {
-    logger.info('DIFFERENCE in student lists. There were ' + cached.student_numbers.length + ' students')
-    logger.info('found: valid ' + validStudents.length + ' out of ' + (maxStudentNumber - minStudentNumber))
-    cached.student_numbers = _.uniq(_.concat(validStudents, cached.student_numbers))
-    cached.max = maxStudentNumber
-    
-    logger.info('total number of valid students in db ' + cached.student_numbers.length )
-    logger.info('student list cached')
-    await cached.save()
-    process.exit(0)
-  }
-  await cached.save()
-  logger.info('found: valid ' + validStudents.length + ' out of ' + (maxStudentNumber - minStudentNumber))
-  logger.info('student list cached')
-  process.exit(0)
-}
-if ( process.argv.length===2) {
-  run()
-} else {
-  save(process.argv[2])
-}

services/backend/oodikone2-backend/src/services/doo_api_database_updater/update_attainment_dates.js

@@ -1,22 +0,0 @@
-const { sequelize } = require('../../database/connection')
-
-const updateAttainmentDates = async () => {
-  await sequelize.query(
-    `UPDATE course
-     SET max_attainment_date = cr.max
-     FROM (select course_code, max(attainment_date) from credit group by course_code) cr
-     WHERE course.code=cr.course_code`,
-    { type: sequelize.QueryTypes.UPDATE }
-  )
-  await sequelize.query(
-    `UPDATE course
-     SET min_attainment_date = cr.min
-     FROM (select course_code, min(attainment_date) from credit group by course_code) cr
-     WHERE course.code=cr.course_code`,
-    { type: sequelize.QueryTypes.UPDATE }
-  )
-}
-
-module.exports = {
-  updateAttainmentDates
-}

services/backend/oodikone2-backend/src/services/elementdetails.js

@@ -3,18 +3,20 @@ const { Op } = require('sequelize')
 
 const byId = id => ElementDetails.findByPk(id)
 
-const byType = type => ElementDetails.findAll({
-  where: {
-    type
-  }
-})
-
-const byCodes = codes => ElementDetails.findAll({
-  where: {
-    code: {
-      [Op.in]: codes
+const byType = type =>
+  ElementDetails.findAll({
+    where: {
+      type
     }
-  }
-})
+  })
 
-module.exports = { byId, byType, byCodes }
+const byCodes = codes =>
+  ElementDetails.findAll({
+    where: {
+      code: {
+        [Op.in]: codes
+      }
+    }
+  })
+
+module.exports = { byId, byType, byCodes }

services/backend/oodikone2-backend/src/services/faculties.js

@@ -4,69 +4,89 @@ const AsyncLock = require('async-lock')
 const { USERSERVICE_URL } = require('../conf-backend')
 const { Credit, StudyrightElement } = require('../models')
 
-const client = axios.create({ baseURL: USERSERVICE_URL, headers: { 'secret': process.env.USERSERVICE_SECRET } })
+const client = axios.create({ baseURL: USERSERVICE_URL, headers: { secret: process.env.USERSERVICE_SECRET } })
 
 const calculateFacultyYearlyStats = async () => {
   const { data: facultyProgrammes } = await client.get('/faculty_programmes')
   const res = {}
+  const studentSets = {}
   const lock = new AsyncLock()
 
   let amountDone = 0
-  await Promise.all(facultyProgrammes.map(({ faculty_code, programme_code }) => (
-    new Promise(async (facultyRes) => {
-      if (!res[faculty_code]) res[faculty_code] = {}
-      if (!res[faculty_code][programme_code]) res[faculty_code][programme_code] = {}
+  await Promise.all(
+    facultyProgrammes.map(
+      ({ faculty_code, programme_code }) =>
+        new Promise(async facultyRes => {
+          if (!res[faculty_code]) {
+            res[faculty_code] = {}
+            studentSets[faculty_code] = {}
+          }
+          if (!res[faculty_code][programme_code]) {
+            res[faculty_code][programme_code] = {}
+            studentSets[faculty_code][programme_code] = {}
+          }
 
-      const facultyStudents = await StudyrightElement.findAll({
-        where: {
-          code: programme_code
-        },
-        attributes: ['studentnumber']
-      })
-
-      await Promise.all(facultyStudents.map((student) => (
-        new Promise(async (studentRes) => {
-          const filteredStudentCredits = await Credit.findAll({
+          const facultyStudents = await StudyrightElement.findAll({
             where: {
-              student_studentnumber: student.studentnumber,
-              credittypecode: [4, 10],
-              isStudyModule: false
-            }
+              code: programme_code
+            },
+            attributes: ['studentnumber']
           })
 
-          filteredStudentCredits.forEach((c) => {
-            const attainmentYear = moment(c.attainment_date).year()
-            if (!res[faculty_code][attainmentYear]) {
-              lock.acquire(faculty_code, (done) => {
-                if (!res[faculty_code][programme_code][attainmentYear]) {
-                  const facultyYearStats = {}
-                  facultyYearStats.studentCredits = 0
-                  facultyYearStats.coursesPassed = 0
-                  facultyYearStats.coursesFailed = 0
-                  res[faculty_code][programme_code][attainmentYear] = facultyYearStats
-                }
-                done()
-              })
-            }
-            const facultyYearStats = res[faculty_code][programme_code][attainmentYear]
-            if (c.credittypecode === 4) {
-              facultyYearStats.studentCredits += c.credits
-              facultyYearStats.coursesPassed += 1
-            } else {
-              facultyYearStats.coursesFailed += 1
-            }
-          })
-          studentRes()
+          await Promise.all(
+            facultyStudents.map(
+              student =>
+                new Promise(async studentRes => {
+                  const filteredStudentCredits = await Credit.findAll({
+                    where: {
+                      student_studentnumber: student.studentnumber,
+                      credittypecode: [4, 10],
+                      isStudyModule: false
+                    }
+                  })
+
+                  filteredStudentCredits.forEach(c => {
+                    const attainmentYear = moment(c.attainment_date).year()
+                    if (!res[faculty_code][attainmentYear]) {
+                      lock.acquire(faculty_code, done => {
+                        if (!res[faculty_code][programme_code][attainmentYear]) {
+                          const facultyYearStats = {}
+                          facultyYearStats.studentCredits = 0
+                          facultyYearStats.coursesPassed = 0
+                          facultyYearStats.coursesFailed = 0
+                          facultyYearStats.students = 0
+                          res[faculty_code][programme_code][attainmentYear] = facultyYearStats
+                          studentSets[faculty_code][programme_code][attainmentYear] = new Set()
+                        }
+                        done()
+                      })
+                    }
+                    const facultyYearStats = res[faculty_code][programme_code][attainmentYear]
+                    const studentSet = studentSets[faculty_code][programme_code][attainmentYear]
+                    if (!studentSet.has(c.student_studentnumber)) {
+                      facultyYearStats.students++
+                      studentSet.add(c.student_studentnumber)
+                    }
+                    if (c.credittypecode === 4) {
+                      facultyYearStats.studentCredits += c.credits
+                      facultyYearStats.coursesPassed += 1
+                    } else {
+                      facultyYearStats.coursesFailed += 1
+                    }
+                  })
+                  studentRes()
+                })
+            )
+          )
+          amountDone += 1
+          console.log(`Faculty programmes done ${amountDone}/${facultyProgrammes.length}`)
+          facultyRes()
         })
-      )))
-      amountDone += 1
-      console.log(`Faculty programmes done ${amountDone}/${facultyProgrammes.length}`)
-      facultyRes()
-    })
-  )))
+    )
+  )
   return res
 }
 
 module.exports = {
   calculateFacultyYearlyStats
-}
+}

services/backend/oodikone2-backend/src/services/filters.js

@@ -3,19 +3,19 @@ const { Filters } = require('../models/models_kone')
 
 const Op = Sequelize.Op
 
-const findForPopulation = (pops) => Filters.findAll({
-  where: {
-    population: {
-      [Op.eq]: pops
+const findForPopulation = pops =>
+  Filters.findAll({
+    where: {
+      population: {
+        [Op.eq]: pops
+      }
     }
-  }
-})
+  })
 
-
-const createNewFilter = async (filter) => {
+const createNewFilter = async filter => {
   return Filters.create(filter)
 }
-const deleteFilter = async (filter) => {
+const deleteFilter = async filter => {
   Filters.destroy({
     where: {
       id: {
@@ -23,10 +23,10 @@ const deleteFilter = async (filter) => {
       }
     }
   })
-  return(filter)
+  return filter
 }
 module.exports = {
   createNewFilter,
   deleteFilter,
   findForPopulation
-}
+}