Add student page sisu link test

- Return sis_person_id for single student from backend
- Add link to Sisu from student infocard
- Make Sisu links open in new tab

.github/dependabot.yml

@@ -1,15 +1,47 @@
 version: 2
 updates:
-- package-ecosystem: npm
-  directory: "/"
-  schedule:
-    interval: weekly
-    time: "03:00"
-  open-pull-requests-limit: 10
-  target-branch: "trunk"
-- package-ecosystem: "github-actions"
-  directory: "/"
-  schedule:
-    interval: weekly
-    time: "03:00"
-  target-branch: "trunk"
+  - package-ecosystem: npm
+    directory: '/'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'infra'
+  - package-ecosystem: npm
+    directory: '/updater/sis-updater-scheduler'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'updater'
+  - package-ecosystem: npm
+    directory: '/updater/sis-updater-worker'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'updater'
+  - package-ecosystem: npm
+    directory: '/services/backend'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'back'
+  - package-ecosystem: npm
+    directory: '/services/frontend'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'front'
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: weekly
+      time: '03:00'

.github/workflows/release-sis-updater-scheduler.yaml

@@ -1,54 +0,0 @@
-name: "Release sis-updater-scheduler"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-sis-updater-scheduler.yaml'
-      - 'updater/sis-updater-scheduler/**'
-
-jobs:
-  build_and_release:
-    name: "Build and release sis-updater-scheduler"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: docker/setup-buildx-action@v1
-    - name: "Login to DockerHub"
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: "Build and push"
-      uses: docker/build-push-action@v2
-      with:
-        cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
-        cache-to: type=inline
-        context: updater/sis-updater-scheduler/
-        push: true
-        tags: toska/sis-updater-scheduler:latest
-        target: production
-    - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-      with:
-        webhook-url: ${{ secrets.WEBHOOK_URL }}
-        message-type: deployment
-        softa-url: https://oodikone.cs.helsinki.fi
-        deployment-target: production
-        softa-to-deploy: sis-updater-scheduler
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: production
-          softa-to-deploy: sis-updater-scheduler

.github/workflows/release-sis-updater-worker.yaml

@@ -1,54 +0,0 @@
-name: "Release sis-updater-worker"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-sis-updater-worker.yaml'
-      - 'updater/sis-updater-worker/**'
-
-jobs:
-  build_and_release:
-    name: "Build and release sis-updater-worker"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: docker/setup-buildx-action@v1
-    - name: "Login to DockerHub"
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: "Build and push"
-      uses: docker/build-push-action@v2
-      with:
-        cache-from: type=registry,ref=toska/sis-updater-worker:latest
-        cache-to: type=inline
-        context: updater/sis-updater-worker/
-        push: true
-        tags: toska/sis-updater-worker:latest
-        target: production
-    - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-      with:
-        webhook-url: ${{ secrets.WEBHOOK_URL }}
-        message-type: deployment
-        softa-url: https://oodikone.cs.helsinki.fi
-        deployment-target: production
-        softa-to-deploy: sis-updater-worker
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: production
-          softa-to-deploy: sis-updater-worker

.github/workflows/test-and-release-updater.yaml

@@ -0,0 +1,157 @@
+name: 'Test and release updater'
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - '.github/workflows/test-and-release-updater.yaml'
+      - 'updater/sis-updater-worker/**'
+      - 'updater/sis-updater-scheduler/**'
+
+jobs:
+  build_worker:
+    name: 'Build sis-updater-worker'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Build the image'
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=production
+          context: updater/sis-updater-worker/
+          target: production
+          push: false
+          outputs: type=docker,dest=/tmp/worker.tar
+          tags: toska/sis-updater-worker:latest
+          cache-from: type=registry,ref=toska/sis-updater-worker:latest
+          cache-to: type=inline
+      - name: 'Store built image'
+        uses: actions/upload-artifact@v2
+        with:
+          path: /tmp/worker.tar
+
+  build_scheduler:
+    name: 'Build sis-updater-scheduler'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Build the image'
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=production
+          context: updater/sis-updater-scheduler/
+          target: production
+          push: false
+          outputs: type=docker,dest=/tmp/scheduler.tar
+          tags: toska/sis-updater-scheduler:latest
+          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
+          cache-to: type=inline
+      - name: 'Store built image'
+        uses: actions/upload-artifact@v2
+        with:
+          path: /tmp/scheduler.tar
+
+  test_updater:
+    name: 'Test updater'
+    runs-on: ubuntu-latest
+    needs:
+      - build_worker
+      - build_scheduler
+    steps:
+      - uses: actions/checkout@v2
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Download built images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load worker image'
+        run: docker load --input /tmp/artifact/worker.tar
+      - name: 'Load scheduler image'
+        run: docker load --input /tmp/artifact/scheduler.tar
+      - name: 'Start worker'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
+      - name: 'Run hourly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: hourly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run weekly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: weekly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run prepurge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: prepurge
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run purge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: purge
+          EXIT_AFTER_IMMEDIATES: yes
+
+  release_updater:
+    name: 'Release updater'
+    runs-on: ubuntu-latest
+    needs:
+      - test_updater
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Login to DockerHub'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: 'Build and push scheduler'
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
+          cache-to: type=inline
+          context: updater/sis-updater-scheduler/
+          push: true
+          tags: toska/sis-updater-scheduler:latest
+          target: production
+      - name: 'Build and push worker'
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=registry,ref=toska/sis-updater-worker:latest
+          cache-to: type=inline
+          context: updater/sis-updater-worker/
+          push: true
+          tags: toska/sis-updater-worker:latest
+          target: production
+      - name: 'Create Sentry release'
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: updater
+        with:
+          environment: production
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/static/js'
+          version: ${{ steps.extract_version.outputs.VERSION }}
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone.cs.helsinki.fi
+          deployment-target: production
+          softa-to-deploy: sis-updater

.github/workflows/test-oodikone.yaml

@@ -9,6 +9,7 @@ on:
       - 'cypress/**'
       - 'cypress.json'
       - 'services/**'
+      - '!services/backend/src/rapodiff.js'
       - 'docker-compose.ci.yml'
       - 'docker-compose.test.yml'
       - 'nginx.conf'

README.md

@@ -8,7 +8,7 @@ To run Oodikone locally, you will need the following:
 
 1. Applications:
    - [Docker](https://docs.docker.com/install/) (version 20.10+),
-   - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
+   - [Docker Compose](https://docs.docker.com/compose/install/)(version 1.29+) and
    - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
 2. Access to Toska Docker Hub,
 3. For real data setup, access to Toskas production servers.
@@ -48,6 +48,10 @@ What different CLI options do:
 
 ## ⌨️ Development
 
+### Architecture
+
+![Oodikone architecture](Oodikone.png)
+
 ### Basics
 
 The development environment is entirely configured in the `docker-compose.yml` file located in this repository. The file defines the services for oodikone's two main components: **oodikone** and **updater**.

cypress/integration/Course_statistics.js

@@ -20,15 +20,13 @@ describe('Course Statistics tests', () => {
 
       cy.contains('.tabular.menu a', 'Table').click()
       cy.contains('All')
-      cy.contains('.modeSelectorRow a', 'Attempts').click()
-      cy.contains('.modeSelectorRow a', 'Students').click()
+      cy.cs('viewMode-Attempts').click()
+      cy.cs('viewMode-Students').click()
 
       cy.contains('.tabular.menu a', 'Pass rate chart').click()
-      cy.get('div.modeSelectorContainer').click()
       cy.contains('svg', 'Pass rate chart')
 
       cy.contains('.tabular.menu a', 'Grade distribution chart').click()
-      cy.get('div.modeSelectorContainer').click()
       cy.contains('svg', 'Grades')
 
       cy.contains('a', 'New query').click()
@@ -98,7 +96,7 @@ describe('Course Statistics tests', () => {
       cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
 
       cy.contains('Tietokantojen perusteet')
-      cy.contains('Avoin yo: Tietokantojen perusteet')
+      cy.contains('Tietokantojen perusteet')
       cy.contains('A581328')
       cy.contains('AYTKT10004')
       cy.contains('TKT10004, 581328')
@@ -107,46 +105,39 @@ describe('Course Statistics tests', () => {
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
 
-      cy.contains('TKT10004, 581328 Tietokantojen perusteet')
+      cy.contains('TKT10004, 581328, AYTKT10004, A581328 Tietokantojen perusteet')
       cy.get('.right').click()
       cy.contains('No results')
+    })
 
+    it('Provider organization toggle works', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
       cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
-      cy.contains('td', /^AYTKT10004/).click()
 
+      cy.contains('td', /^TKT10004/).click()
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
-      cy.contains('AYTKT10004 Avoin yo: Tietokantojen perusteet')
+
+      cy.contains('TKT10004, 581328, AYTKT10004, A581328 Tietokantojen perusteet')
+      cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.contains('TKT10004, 581328 Tietokantojen perusteet')
     })
 
     it('Searching course by name displays right courses, 10 credit courses', () => {
       cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
-
       cy.contains('Tietorakenteet ja algoritmit')
-      cy.contains('Avoin yo: Tietorakenteet ja algoritmit')
       cy.contains('AYTKT20001')
       cy.contains('TKT20001, 58131')
       cy.contains('td', /^TKT20001/).click()
-
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
 
-      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
       cy.get('.right').click()
       cy.contains('No results')
-
-      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
-      cy.contains('td', /^AYTKT20001/).click()
-
-      cy.contains('Fetch statistics').should('be.enabled').click()
-      cy.contains('Search for courses').should('not.exist')
-      cy.contains('AYTKT20001 Avoin yo: Tietorakenteet ja algoritmit')
-
-      cy.get('.right').click()
-      cy.contains('No results')
-      cy.get("input[name='unifyOpenUniCourses']").parent().click()
       cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
       cy.contains('td', 'TKT20001, 58131, AYTKT20001').click()
 
@@ -161,12 +152,11 @@ describe('Course Statistics tests', () => {
       cy.get("input[placeholder='Search by a course code']").type('TKT20003')
       cy.contains('tr', 'TKT20003').click()
       cy.contains('Fetch statistics').should('be.enabled').click()
-
       cy.contains('Käyttöjärjestelmät')
       cy.contains('TKT20003')
 
-      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
-      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020 (open and normal')
       cy.contains('TKT20003')
 
       cy.contains('Students (127)').click()
@@ -176,22 +166,23 @@ describe('Course Statistics tests', () => {
 
     it('Population of course shows grades for each student', () => {
       cy.get("input[placeholder='Search by a course code']").type('TKT20001')
-      cy.contains(/^TKT20001$/).click()
+      cy.contains(/^TKT20001, AYTKT20001$/).click()
       cy.contains('Fetch statistics').should('be.enabled').click()
-      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
-      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
-      cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019')
-      cy.contains('Students (197)').click()
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
+      cy.get('tbody >:nth-child(3) > :nth-child(2) .level').click()
+      cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019 (open and normal)')
+      cy.contains('Students (204)').click()
       cy.contains('td', '010262566').siblings().contains('4')
       cy.contains('td', '010674989').siblings().contains('1')
     })
 
-    it.skip('Language distribution is correct', () => {
+    it('Language distribution is correct', () => {
       cy.get("input[placeholder='Search by a course code']").type('TKT20003')
       cy.contains(/^TKT20003$/).click()
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
-      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
       cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
       cy.contains('Language distribution').click()
       cy.contains('td', 'finnish').siblings().contains('122')
@@ -202,7 +193,6 @@ describe('Course Statistics tests', () => {
       beforeEach(() => {
         cy.url().should('include', '/coursestatistics')
         cy.contains('Search for courses')
-        cy.get("input[name='unifyOpenUniCourses']").parent().click()
         cy.get("input[placeholder='Search by a course code']").type('TKT10002')
         cy.contains('td', 'TKT10002, AYTKT10002').click()
         cy.contains('Fetch statistics').should('be.enabled').click()
@@ -214,18 +204,28 @@ describe('Course Statistics tests', () => {
       const yearRange = { from: '2000-2001', to: '2020-2021' }
       const attemptsTableContents = [
         // [time, passed, failed, passrate]
-        ['Total', 511, 486, 25],
+        ['Total', null, 511, 486, 25],
+      ]
+
+      const attemptsTableContentsOpen = [
+        // [time, passed, failed, passrate]
+        ['Total', null, 175, 175, 0],
+      ]
+
+      const attemptsTableContentsReqular = [
+        // [time, passed, failed, passrate]
+        ['Total', null, 312, 298, 14],
       ]
 
       const gradesTableContents = [
         // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-        ['Total', 511, 25, 11, 17, 21, 59, 365, 13],
-        ['2020-2021', 1, 0, 0, 0, 0, 0, 1, 0],
-        ['2019-2020', 54, 4, 3, 1, 3, 4, 39, 0],
-        ['2018-2019', 154, 7, 2, 6, 5, 15, 119, 0],
-        ['2017-2018', 186, 9, 3, 5, 6, 16, 139, 8],
-        ['2016-2017', 71, 2, 1, 2, 5, 13, 48, 0],
-        ['2015-2016', 21, 1, 1, 1, 0, 7, 8, 3],
+        ['Total', null, 511, 25, 11, 17, 21, 59, 365, 13],
+        ['2020-2021', null, 1, 0, 0, 0, 0, 0, 1, 0],
+        ['2019-2020', null, 54, 4, 3, 1, 3, 4, 39, 0],
+        ['2018-2019', null, 154, 7, 2, 6, 5, 15, 119, 0],
+        ['2017-2018', null, 186, 9, 3, 5, 6, 16, 139, 8],
+        ['2016-2017', null, 71, 2, 1, 2, 5, 13, 48, 0],
+        ['2015-2016', null, 21, 1, 1, 1, 0, 7, 8, 3],
       ]
 
       it('shows stats', () => {
@@ -252,6 +252,7 @@ describe('Course Statistics tests', () => {
               .eq(trIndex)
               .within(() => {
                 values.forEach((value, tdIndex) => {
+                  if (value === null) return
                   cy.get('td').eq(tdIndex).contains(value)
                 })
               })
@@ -265,6 +266,7 @@ describe('Course Statistics tests', () => {
               .eq(trIndex)
               .within(() => {
                 values.forEach((value, tdIndex) => {
+                  if (value === null) return
                   cy.get('td').eq(tdIndex).contains(value)
                 })
               })
@@ -273,7 +275,45 @@ describe('Course Statistics tests', () => {
         })
       })
 
-      it('after changing time range shows same stats', () => {
+      it('Toggling course provider changes stats correctly', () => {
+        cy.get('[data-cy=unify_radio_reqular]').click()
+        cy.get("div[name='toYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', '2020-2021')
+          cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', yearRange.from)
+          cy.get("div[role='option']").should('have.length', 21)
+        })
+        cy.contains('#CourseStatPanes a.item', 'Table').click()
+        cy.contains('#CourseStatPanes a.item', 'Attempts').click()
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          attemptsTableContentsReqular.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  if (value === null) return
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
+              })
+          })
+        })
+        cy.get('[data-cy=unify_radio_open]').click()
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          attemptsTableContentsOpen.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  if (value === null) return
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
+              })
+          })
+        })
+        cy.get('[data-cy=unify_radio_unify]').click()
+      })
+
+      it('After changing time range shows same stats', () => {
         const newYearRange = { from: '2016-2017', to: '2019-2020' }
         cy.get("div[name='fromYear']")
           .click()
@@ -302,6 +342,22 @@ describe('Course Statistics tests', () => {
         cy.contains('Show population').should('be.enabled')
       })
     })
+
+    it('If no data available, unify toggle is disabled', () => {
+      cy.get("input[placeholder='Search by a course code']").type('AYCSM14111')
+      cy.contains('AYCSM14111').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('AYCSM14111 Avoin yo: Full Stack -websovelluskehitys: React Native')
+      cy.get('[data-cy=unify_radio_reqular]').find('input').should('be.disabled')
+      cy.get('[data-cy=unify_radio_unify]').find('input').should('not.be.disabled')
+    })
+
+    it('Has right to see all the sudents, because course provider is TKT', () => {
+      cy.visit('coursestatistics?courseCodes=%5B%22TKT10004%22%5D&cs_tab=0&separate=false')
+      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
+      cy.contains('Students (154)').click()
+      cy.contains('015224224')
+    })
   })
 
   it('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
@@ -314,13 +370,17 @@ describe('Course Statistics tests', () => {
     cy.contains('Faculty statistics').should('not.exist')
     cy.contains('Show population').should('not.exist')
 
+    cy.cs('viewMode-Attempts').click()
+
     const attemptsTableContents = [
       // [time, passed, failed, passrate]
-      ['Total*', 288, 213, 75, '73.96 %'],
-      ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
-      ['2019-2020', 164, 121, 43, '73.78 %'],
-      ['2018-2019', 85, 60, 25, '70.59 %'],
-      ['2017-2018', 39, 32, 7, '82.05 %'],
+      // ['Total', 288, 213, 75, '73.96 %'],
+      // ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
+      ['Total', null, 295, 216, 79, '73.22 %'],
+      ['2020-2021', null, 5, 2, 3, '40.00 %'],
+      ['2019-2020', null, 164, 121, 43, '73.78 %'],
+      ['2018-2019', null, 85, 60, 25, '70.59 %'],
+      ['2017-2018', null, 39, 32, 7, '82.05 %'],
     ]
 
     cy.contains('#CourseStatPanes a.item', 'Attempts').click()
@@ -330,6 +390,7 @@ describe('Course Statistics tests', () => {
           .eq(trIndex)
           .within(() => {
             values.forEach((value, tdIndex) => {
+              if (value === null) return
               cy.get('td').eq(tdIndex).contains(value)
             })
           })

cypress/integration/Filters.js

@@ -17,23 +17,59 @@ const clearSingleDropdownSelection = dataCyAttribute => {
   cy.cs(dataCyAttribute).find('i.clear').click({ force: true })
 }
 
+const testRangeFilter = (parentEl, min, max, expected) => {
+  cy.cs(parentEl)
+    .cs('range-selector-min')
+    .find('input')
+    .invoke('val')
+    .then(initialMin => {
+      cy.cs(parentEl)
+        .cs('range-selector-max')
+        .find('input')
+        .invoke('val')
+        .then(initialMax => {
+          cy.cs(parentEl).cs('range-selector-min').find('input').type(`{selectall}{backspace}${min}`).blur()
+          cy.cs(parentEl).cs('range-selector-max').find('input').type(`{selectall}{backspace}${max}`).blur()
+
+          checkFilteringResult(expected)
+
+          cy.cs(parentEl).cs('range-selector-min').find('input').type(`{selectall}{backspace}${initialMin}`).blur()
+          cy.cs(parentEl).cs('range-selector-max').find('input').type(`{selectall}{backspace}${initialMax}`).blur()
+        })
+    })
+}
+
 // Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
 // to every it-function. Reason behind using test function wrapper is that Cypresses internal beforeEach / afterEach
 // functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
 const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
   return (filterName, testStepFunctionToRun) => {
-    cy.get(`[data-cy="${filterName}-header"]`).click()
-    checkFilteringResult(amountWithoutFiltering)
-    testStepFunctionToRun()
-    checkFilteringResult(amountWithoutFiltering)
-    cy.get(`[data-cy="${filterName}-header"]`).click({ force: true })
+    const card = cy.cs(`${filterName}-filter-card`)
+
+    card.invoke('attr', 'data-open').then(open => {
+      console.log('open', open)
+
+      const getHeader = () => cy.cs(`${filterName}-header`)
+
+      if (open === 'false') {
+        getHeader().click({ force: true })
+      }
+
+      checkFilteringResult(amountWithoutFiltering)
+      testStepFunctionToRun()
+      checkFilteringResult(amountWithoutFiltering)
+
+      if (open === 'false') {
+        getHeader().click({ force: true })
+      }
+    })
   }
 }
 
 describe('Population Statistics', () => {
   const pathToCSBach2018 =
     '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2018'
-  const defaultAmountOfStudents = 148
+  const defaultAmountOfStudents = 159
   const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
   before(() => {
@@ -42,10 +78,11 @@ describe('Population Statistics', () => {
 
   it('Transfer filter is on not transferred by default', () => {
     cy.contains('By default only students who have not transferred to this study programme are shown.')
-    const card = cy.cs('TransferredToProgramme-filter-card')
-    card.cs('TransferredToProgramme-header').click()
-    card.get('[data-cy="option-havenot"] input').should('be.checked')
-    card.cs('TransferredToProgramme-header').click()
+
+    runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
+      const card = cy.cs('TransferredToProgramme-filter-card')
+      card.get('[data-cy="option-havenot"] input').should('be.checked')
+    })
   })
 
   it('Programme defaults to "Active Study Right" mode', () => {
@@ -63,7 +100,10 @@ describe('Population Statistics', () => {
     })
   })
 
-  it('Study track filter works', () => {
+  // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
+  // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
+  // So fix this
+  it.skip('Study track filter works', () => {
     runTestStepWithPreAndPostParts('StudyTrack', () => {
       const card = cy.cs('StudyTrack-filter-card')
       const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
@@ -74,25 +114,25 @@ describe('Population Statistics', () => {
 
   it('Graduation filter works', () => {
     runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
-      const card = cy.cs('GraduatedFromProgramme-filter-card')
+      const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
 
       const graduated = 11
-      card.cs('option-graduated-true').click()
+      getCard().cs('option-graduated-true').click()
       checkFilteringResult(graduated)
-      card.cs('option-graduated-false').click()
+      getCard().cs('option-graduated-false').click()
       checkFilteringResult(defaultAmountOfStudents - graduated)
-      card.get('[data-cy="GraduatedFromProgramme-filter-card"] [data-cy="option-all"]').click()
+      getCard().cs('option-all').click()
     })
   })
 
   it('Transfer filter works', () => {
     runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
       const transferred = 1
-      cy.cs('option-have').click()
+      cy.cs('TransferredToProgramme-filter-card').cs('option-have').click()
       checkFilteringResult(transferred)
-      cy.cs('option-all').click()
+      cy.cs('TransferredToProgramme-filter-card').cs('option-all').click()
       checkFilteringResult(defaultAmountOfStudents + transferred)
-      cy.cs('option-havenot').click()
+      cy.cs('TransferredToProgramme-filter-card').cs('option-havenot').click()
     })
   })
 
@@ -100,37 +140,29 @@ describe('Population Statistics', () => {
     runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
       cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
       cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(0)
-      checkFilteringResult(137)
+      checkFilteringResult(148)
       clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
     })
   })
 
   it('Credit filter works', () => {
     runTestStepWithPreAndPostParts('CreditsEarned', () => {
-      cy.cs('credit-filter-min').type('50')
-      cy.cs('credit-filter-max').type('150')
-      checkFilteringResult(116)
-      cy.cs('credit-filter-min').clear()
-      cy.cs('credit-filter-max').clear()
+      testRangeFilter('CreditsEarned-filter-card', 50, 150, 118)
     })
   })
 
   it('Age filter works', () => {
     runTestStepWithPreAndPostParts('Age', () => {
-      cy.cs('ageFilter-min').type('20')
-      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(41)
-      cy.cs('ageFilter-min').find('input').clear()
-      cy.cs('ageFilter-max').find('input').clear()
+      testRangeFilter('Age-filter-card', 20, 40, 49)
     })
   })
 
   it('Gender filter works', () => {
     runTestStepWithPreAndPostParts('Gender', () => {
       cy.cs('genderFilter-dropdown').selectFromDropdown(0)
-      checkFilteringResult(36)
+      checkFilteringResult(42)
       cy.cs('genderFilter-dropdown').selectFromDropdown(1)
-      checkFilteringResult(112)
+      checkFilteringResult(117)
       cy.cs('genderFilter-dropdown').selectFromDropdown(2)
       checkFilteringResult(0)
       cy.cs('genderFilter-dropdown').selectFromDropdown(3)
@@ -144,8 +176,8 @@ describe('Population Statistics', () => {
       cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.cs('startYearAtUni-dropdown').selectFromDropdown(14)
-      checkFilteringResult(121)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(7)
+      checkFilteringResult(152)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
@@ -183,14 +215,10 @@ describe('Population Statistics', () => {
   it('Filter combinations work', () => {
     runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
       runTestStepWithPreAndPostParts('Age', () => {
-        const card = cy.cs('GraduatedFromProgramme-filter-card')
-        card.cs('option-graduated-true').click()
-        cy.cs('ageFilter-min').type('20')
-        cy.cs('ageFilter-max').type('30')
-        checkFilteringResult(11)
-        cy.cs('ageFilter-min').find('input').clear()
-        cy.cs('ageFilter-max').find('input').clear()
-        card.cs('option-all').click()
+        const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
+        getCard().cs('option-graduated-true').click()
+        testRangeFilter('Age-filter-card', 20, 30, 11)
+        getCard().cs('option-all').click()
       })
     })
   })
@@ -198,7 +226,7 @@ describe('Population Statistics', () => {
 
 describe('Course Statistics', () => {
   const pathToDSAndAlgoSpring2019 =
-    '/coursepopulation?coursecodes=%5B%22TKT20001%22%2C%2258131%22%5D&from=138&separate=true&to=138&years=Kev%C3%A4t%202019'
+    '/coursepopulation?coursecodes=%5B%22TKT20001%22%2C%2258131%22%5D&from=138&separate=true&to=138&years=Kev%C3%A4t%202019&unifyCourses=unifyStats'
   const defaultAmountOfStudents = 118
   const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
@@ -232,7 +260,10 @@ describe('Course Statistics', () => {
     })
   })
 
-  it('Study track filter works', () => {
+  // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
+  // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
+  // So fix this
+  it.skip('Study track filter works', () => {
     runTestStepWithPreAndPostParts('StudyTrack', () => {
       const card = cy.cs('StudyTrack-filter-card')
       const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
@@ -243,11 +274,7 @@ describe('Course Statistics', () => {
 
   it('Age filter works', () => {
     runTestStepWithPreAndPostParts('Age', () => {
-      cy.cs('ageFilter-min').type('20')
-      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(32)
-      cy.cs('ageFilter-min').find('input').clear()
-      cy.cs('ageFilter-max').find('input').clear()
+      testRangeFilter('Age-filter-card', 20, 40, 35)
     })
   })
 
@@ -270,8 +297,8 @@ describe('Course Statistics', () => {
       cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.cs('startYearAtUni-dropdown').selectFromDropdown(12)
-      checkFilteringResult(29)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(3)
+      checkFilteringResult(84)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
@@ -280,11 +307,7 @@ describe('Course Statistics', () => {
     runTestStepWithPreAndPostParts('Grade', () => {
       runTestStepWithPreAndPostParts('Age', () => {
         cy.cs('gradeFilter-3').click()
-        cy.cs('ageFilter-min').type('20')
-        cy.cs('ageFilter-max').type('30')
-        checkFilteringResult(1)
-        cy.cs('ageFilter-min').find('input').clear()
-        cy.cs('ageFilter-max').find('input').clear()
+        testRangeFilter('Age-filter-card', 20, 30, 1)
         cy.cs('gradeFilter-3').click()
       })
     })
@@ -308,11 +331,7 @@ describe('Custom Population Statistics', () => {
 
   it('Age filter works', () => {
     runTestStepWithPreAndPostParts('Age', () => {
-      cy.cs('ageFilter-min').type('20')
-      cy.cs('ageFilter-max').type('30')
-      checkFilteringResult(1)
-      cy.cs('ageFilter-min').find('input').clear()
-      cy.cs('ageFilter-max').find('input').clear()
+      testRangeFilter('Age-filter-card', 20, 30, 1)
     })
   })
 

cypress/integration/Population_statistics.js

@@ -3,15 +3,15 @@
 const moment = require('moment')
 const _ = require('lodash')
 
-const setPopStatsUntil = (until, includeSettings = []) => {
-  cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
-  includeSettings.forEach(setting => {
-    cy.contains('Advanced settings').parent().siblings().contains(setting).click()
-  })
-  cy.get('.adv-stats-until > .form-control').click().clear().type(until)
-  cy.contains('Fetch population with new settings').click()
-  cy.contains('Advanced settings')
-}
+// const setPopStatsUntil = (until, includeSettings = []) => {
+//   cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
+//   includeSettings.forEach(setting => {
+//     cy.contains('Advanced settings').parent().siblings().contains(setting).click()
+//   })
+//   cy.get('.adv-stats-until > .form-control').click().clear().type(until)
+//   cy.contains('Fetch population with new settings').click()
+//   cy.contains('Advanced settings')
+// }
 
 describe('Population Statistics tests', () => {
   const pathToCSBach2017 =
@@ -65,13 +65,14 @@ describe('Population Statistics tests', () => {
     it('Population statistics is usable on general level', () => {
       cy.visit(pathToCSBach2017)
       cy.cs('filtered-students')
-      setPopStatsUntil('toukokuu 2020')
+      // This advanced settings does not work correctly
+      // see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3552
+      // setPopStatsUntil('toukokuu 2020')
 
       cy.get('.card').within(() => {
         cy.contains('Tietojenkäsittelytieteen kandiohjelma')
-        cy.contains('Sample size: 149 students')
+        cy.contains('Sample size: 170 students')
         cy.contains('Excludes exchange students')
-        cy.contains("Excludes students who haven't enrolled present nor absent")
         cy.contains('Excludes students with non-degree study right')
         cy.contains('Excludes students who have transferred out of this programme')
       })
@@ -80,17 +81,17 @@ describe('Population Statistics tests', () => {
 
       cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
       // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000) // a bit hacky way, wait until ui is ready
-      cy.get('[data-cy=expand-TKT1]').click()
-      cy.get('[data-cy=coursestats-link-TKT10002]').click()
+      cy.wait(25000) // a bit hacky way, wait until ui is ready
+      cy.get('[data-cy=toggle-group-module-TKT1]').click()
+      cy.contains('td', 'TKT10002').siblings().find('i.level.up').click()
       cy.wait('@coursePage')
       cy.url().should('include', '/coursestatistics')
-      cy.contains('TKT10002, 581325 Ohjelmoinnin perusteet')
+      cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
     })
 
     it("Empty 'tags' tab has a link to the page where tags can be created", () => {
       cy.visit(pathToCSBach2017)
-      cy.contains('Students (149)').click()
+      cy.contains('Students (170)').click()
       cy.get('[data-cy=student-table-tabs]').contains('Tags').click()
       cy.contains('No tags defined. You can define them here.').find('a').click()
       cy.contains('Tags').click()
@@ -119,12 +120,11 @@ describe('Population Statistics tests', () => {
 
       cy.contains('Fetch population').click()
 
-      cy.contains('Credit accumulation (for 149 students)')
+      cy.contains('Credit accumulation (for 170 students)')
 
-      // spring + fall and include cancelled
+      // spring + fall
       cy.get('[data-cy=advanced-toggle]').click()
       cy.cs('toggle-spring').click()
-      cy.cs('toggle-cancelled').click()
       cy.contains('Fetch population').click()
 
       cy.contains('Credit accumulation (for 170 students)')
@@ -140,7 +140,7 @@ describe('Population Statistics tests', () => {
       const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
       const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))
 
-      cy.get('.credits-gained-table').should('contain', '(n=149)')
+      cy.get('.credits-gained-table').should('contain', '(n=170)')
 
       for (const [start, end] of ranges) {
         let value = 'credits'
@@ -159,7 +159,7 @@ describe('Population Statistics tests', () => {
       cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
         const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)
 
-        expect(sum).to.equal(149)
+        expect(sum).to.equal(170)
       })
 
       cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('not.exist')
@@ -182,10 +182,10 @@ describe('Population Statistics tests', () => {
       cy.contains('Credit statistics').click({ force: true })
       cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
 
-      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
-      cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
-      cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
-      cy.get("[data-cy='credit-stats-min']").should('contain', '11')
+      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 170 Students')
+      cy.get("[data-cy='credit-stats-mean']").should('contain', '114.75')
+      cy.get("[data-cy='credit-stats-stdev']").should('contain', '65.53')
+      cy.get("[data-cy='credit-stats-min']").should('contain', '5')
       cy.get("[data-cy='credit-stats-max']").should('contain', '314')
     })
   })
@@ -198,7 +198,7 @@ describe('Population Statistics tests', () => {
       const existing = '010113437'
       const nonExisting = '66666666'
       cy.visit(pathToCSBach2017)
-      cy.contains('Students (149)').click()
+      cy.contains('Students (170)').click()
       cy.contains(existing)
       cy.contains(nonExisting).should('not.exist')
       cy.contains('button', 'Check studentnumbers').click()

cypress/integration/Student_statistics.js

@@ -11,6 +11,7 @@ describe('Student Statistics tests', () => {
     firstnames: 'Matti Jaakko',
     lastname: 'Nieminen',
     studentnumber: '010654019',
+    sis_person_id: 'hy-hlo-84830887',
   }
 
   it('Student statistics search form is usable', () => {
@@ -37,9 +38,9 @@ describe('Student Statistics tests', () => {
   it('Can get student specific page by clicking student', () => {
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
-    cy.contains('Started: 01.08.2011')
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 01.08.2025')
+    cy.contains('td a', student.studentnumber).click()
+    cy.contains('Started: 01.08.2017')
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 31.07.2025')
     cy.contains(student.lastname).should('not.exist')
     cy.contains(student.firstnames).should('not.exist')
 
@@ -54,7 +55,7 @@ describe('Student Statistics tests', () => {
 
   it('Can get back to search menu', () => {
     cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
     cy.get('.remove').click()
     cy.contains('Student number').should('not.exist')
     cy.contains('Credits').should('not.exist')
@@ -62,7 +63,7 @@ describe('Student Statistics tests', () => {
 
   it('Can jump to course', () => {
     cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
     cy.contains('Introduction to Machine Learning (DATA11002)')
       .siblings()
       .within(() => {
@@ -72,6 +73,14 @@ describe('Student Statistics tests', () => {
     cy.contains('Introduction to Machine Learning')
   })
 
+  it('Has correct Sisu link', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains('td a', student.studentnumber).click()
+    cy.get('[data-cy=sisulink] > a')
+      .should('have.attr', 'href')
+      .and('include', 'https://sis-helsinki.funidata.fi/tutor/role/staff/student/hy-hlo-84830887/basic/basic-info')
+  })
+
   it('Searching with bad inputs doesnt yield results', () => {
     cy.get('.prompt').type('SWAG LITTINEN')
     cy.contains('Student number').should('not.exist')
@@ -82,7 +91,7 @@ describe('Student Statistics tests', () => {
 
   it('Can jump to population page', () => {
     cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
     cy.get('i.level.up.alternate.icon').eq(0).click()
     cy.contains('Population statistics')
     cy.contains('Tietojenkäsittelytieteen kandiohjelma')

cypress/integration/Studyprogramme_overview.js

@@ -1,172 +1,403 @@
 /// <reference types="Cypress" />
 
 const deleteTag = name => {
-  cy.contains(name).siblings().contains('Delete').click()
+  cy.contains('td', name).siblings().contains('Delete').click()
   cy.contains('Are you sure you want to delete tag')
   cy.contains('Confirm').click()
-  cy.contains(name).should('not.exist')
+  cy.contains('td', name).should('not.exist')
+}
+
+const getEmptyYears = isAcademicYear => {
+  const thisYear = new Date().getFullYear()
+  const years = []
+  for (let year = thisYear; year > 2021; year--) {
+    if (isAcademicYear) {
+      years.push(`${year} - ${year + 1}`)
+    } else {
+      years.push(year)
+    }
+  }
+  return years
 }
 
 describe('Studyprogramme overview', () => {
-  // check is used by couple of tests
-  const checkProgressAndProductivity = (user = 'basic') => {
-    // Column amount depends on user. Note that with dev user, you'll see even more
-    // stuff than with admin or basic.
-    const CSBachPopulationProgress2018ForAdmin = [
-      '162',
-      '44 (27%)',
-      '118 (72%)',
-      '161 (99%)',
-      '161',
-      '1',
-      '11',
-      '11',
-      '152',
-      '125',
-      '78',
-      '42',
-      '16',
-    ]
-    const CSBachPopulationProgress2018ForBasic = [
-      '162',
-      '44 (27%)',
-      '118 (72%)',
-      '161 (99%)',
-      '161',
-      '1',
-      '11',
-      '11',
-      '152',
-      '125',
-      '78',
-      '42',
-      '16',
-    ] // cancelled and transferred from are missing
-
-    const population = user === 'basic' ? CSBachPopulationProgress2018ForBasic : CSBachPopulationProgress2018ForAdmin
-    cy.contains('Population progress')
-    cy.contains('Yearly productivity')
-
-    cy.contains('2018-2019')
-      .siblings()
-      .each((elem, index) => {
-        cy.wrap(elem).contains(population[index])
-      })
-    const populationproductivity2019 = ['7388.0', '17', '7388.00', '0.00', '349.00']
-    cy.get('table')
-      .eq(1)
-      .contains('2019')
-      .siblings()
-      .each((elem, index) => {
-        cy.wrap(elem).contains(populationproductivity2019[index])
-      })
-  }
-
-  describe('when opening programme page with admin user', () => {
+  /* Basic information overview -tests*/
+  describe('Basic information -view works for basic user', () => {
     beforeEach(() => {
-      cy.init('/study-programme/KH50_005', 'admin')
+      cy.init('/study-programme')
+      cy.viewport(1536, 960)
     })
 
-    it('progress should not be recalculating when opened for the first time', () => {
-      cy.contains('Recalculating').should('not.exist')
-      cy.contains('Last updated').should('exist')
+    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
+    it('Basic information -tab loads', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('[data-cy=Section-StudentsOfTheStudyprogramme]')
+      cy.get('[data-cy=Section-CreditsProducedByTheStudyprogramme]')
+      cy.get('[data-cy=Section-GraduatedAndThesisWritersOfTheProgramme]')
+      cy.get('[data-cy=Section-ProgrammesBeforeOrAfter]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
     })
 
-    // Following vars are used to compare values before and after stats calculation
-    // Taken from https://docs.cypress.io/api/commands/should#Compare-text-values-of-two-elements
-    const normalizeText = s => s.replace(/\s/g, '').toLowerCase()
-    let originalProgressCalculatedText
-    let originalProductivityCalculatedText
-    const doRecalculation = () => {
-      // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
-      cy.wait(5000)
-      cy.get('.attached').contains('Admin').click()
-      cy.contains('recalculate productivity').click()
-      cy.contains('recalculate throughput').click()
-      cy.get('.attached').contains('Overview').click() // TODO: add data-cy tags for tabs
-      // Wait to "recalculating" to disappear, TODO: change this to cy.intercept
-      cy.wait(5000)
-      cy.reload()
-    }
+    it('Basic information contains correct students', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Started, Graduated, Transferred Away, Transferred to]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        [2021, 0, 0, 1, 0],
+        [2020, 12, 36, 3, 2],
+        [2019, 87, 17, 0, 1],
+        [2018, 161, 1, 0, 0],
+        [2017, 171, 0, 0, 0],
+      ]
 
-    it('renders progress and productivity tables with calculated status', () => {
-      doRecalculation()
-
-      // check contains also study programme options for admin
-      cy.contains('Study programme options')
-      cy.get('table').should('have.length', 3)
-
-      checkProgressAndProductivity('admin')
-
-      // Grab update dates to be compared later
-      cy.cs('throughputUpdateStatus')
-        .invoke('text')
-        .then(text => {
-          originalProgressCalculatedText = normalizeText(text)
-          expect(originalProgressCalculatedText).not.to.contain('recalculating')
-          expect(originalProgressCalculatedText).not.to.contain('refresh')
-        })
-
-      cy.cs('productivityUpdateStatus')
-        .invoke('text')
-        .then(text => {
-          originalProductivityCalculatedText = normalizeText(text)
-          expect(originalProductivityCalculatedText).not.to.contain('recalculating')
-          expect(originalProgressCalculatedText).not.to.contain('refresh')
-        })
+      cy.checkTableStats(tableContents, 'StudentsOfTheStudyprogramme')
     })
 
-    it('renders progress and productivity tables with calculated status after recalculating stats again', () => {
-      doRecalculation()
-      checkProgressAndProductivity('admin')
+    it('Basic information contains correct credits', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Total, Major credits, Non-major credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        [2021, 0, 0, 0, 0],
+        [2020, 4928, 548, 4213, 167],
+        [2019, 7737, 1644, 5744, 349],
+        [2018, 6730, 1820, 4638, 272],
+        [2017, 2900, 788, 1724, 388],
+      ]
 
-      // Check new calculation statuses are reported
-      const newProgressCalculatedTextElement = cy.cs('throughputUpdateStatus').invoke('text')
-      const newProductivityCalculatedTextElement = cy.cs('productivityUpdateStatus').invoke('text')
+      cy.checkTableStats(tableContents, 'CreditsProducedByTheStudyprogramme')
+    })
 
-      newProgressCalculatedTextElement.should(text => {
-        const newProgressCalculatedText = normalizeText(text)
-        expect(newProgressCalculatedText).not.to.contain('recalculating')
-        expect(newProgressCalculatedText).not.to.contain('refresh')
-        expect(newProgressCalculatedText).not.to.contain(originalProgressCalculatedText)
-      })
+    it('Basic information contains correct thesis writers and graduates', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Graduated, Wrote thesis]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 36, 35],
+        [2019, 17, 25],
+        [2018, 1, 5],
+        [2017, 0, 0],
+      ]
 
-      newProductivityCalculatedTextElement.should(text => {
-        const newProductivityCalculatedText = normalizeText(text)
-        expect(newProductivityCalculatedText).not.to.contain('recalculating')
-        expect(newProductivityCalculatedText).not.to.contain('refresh')
-        expect(newProductivityCalculatedText).not.to.contain(originalProgressCalculatedText)
-      })
+      cy.checkTableStats(tableContents, 'GraduatedAndThesisWritersOfTheProgramme')
+    })
+
+    it.skip('Special studyrights can be excluded and basic data changes accordingly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen maisteriohjelma').click()
+      cy.get('[data-cy=StudentToggle]').click()
+      const years = getEmptyYears()
+      const studentTableContents = [
+        // [Year, Started, Graduated]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 22, 0],
+        [2019, 12, 0],
+        [2018, 0, 0],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
+
+      const creditTableContents = [
+        // [Year, Total, Major students credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0]),
+        [2021, 0, 0, 0],
+        [2020, 117.5, 0, 7.5],
+        [2019, 67, 0, 0],
+        [2018, 0, 0, 0],
+        [2017, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+
+      const graduatedTableContents = [
+        // [Year, Graduated, Wrote thesis]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 0, 0],
+        [2019, 0, 0],
+        [2018, 0, 0],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(graduatedTableContents, 'GraduatedAndThesisWritersOfTheProgramme')
+      cy.get('[data-cy=StudentToggle]').click()
+    })
+
+    it('Year can be changed to academic year, and data changes accordingly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen maisteriohjelma').click()
+      cy.get('[data-cy=YearToggle]').click()
+      const isAcademicYear = true
+      const years = getEmptyYears(isAcademicYear)
+      const studentTableContents = [
+        // [Year, Started, Graduated, Transferred away, Transferred to]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0],
+        ['2020 - 2021', 5, 0, 0, 0],
+        ['2019 - 2020', 27, 0, 0, 0],
+        ['2018 - 2019', 4, 0, 0, 0],
+        ['2017 - 2018', 0, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
+
+      const creditTableContents = [
+        // [Year, Total, Major students credits, Non-major students credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0],
+        ['2020 - 2021', 0, 0, 0, 0],
+        ['2019 - 2020', 338.5, 0, 331, 7.5],
+        ['2018 - 2019', 173, 0, 173, 0],
+        ['2017 - 2018', 42, 0, 42, 0],
+      ]
+
+      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+      cy.get('[data-cy=YearToggle]').click()
+    })
+
+    it('Basic information graphs render', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+
+      cy.get('[data-cy=Graph-StudentsOfTheStudyprogramme')
+        .should('contain', 'Started')
+        .should('contain', 'Graduated')
+        .should('contain', 'Transferred away')
+
+      cy.get('[data-cy=Graph-CreditsProducedByTheStudyprogramme')
+        .should('contain', 'Major students credits')
+        .should('contain', 'Non-major students credits')
+        .should('contain', 'Transferred credits')
+        .should('contain', 788)
+        .should('contain', 1820)
+
+      cy.get('[data-cy=Graph-GraduatedAndThesisWritersOfTheProgramme')
+        .should('contain', 'Graduated students')
+        .should('contain', 'Wrote thesis')
+        .should('contain', 1)
+        .should('contain', 5)
+        .should('contain', 17)
+        .should('contain', 25)
+        .should('contain', 36)
+        .should('contain', 35)
+
+      cy.get('[data-cy=Graph-2019-AverageGraduationTimes]')
+        .should('contain', '2019')
+        .should('contain', 'n = 17')
+        .should('contain', '23 kk')
+
+      cy.get('[data-cy=Graph-ProgrammesBeforeOrAfter')
+        .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
+        .should('contain', 'Datatieteen maisteriohjelma')
+        .should('contain', 'Matematiikan ja tilastotieteen maisteriohjelma')
+        .should('contain', 1)
+        .should('contain', 4)
+        .should('contain', 13)
     })
   })
 
-  describe('when opening programme page with basic user', () => {
+  /* Studytrack overview -tests*/
+  describe('Studytrack overview works for basic user', () => {
     beforeEach(() => {
-      cy.init('/study-programme/KH50_005')
+      cy.init('/study-programme')
     })
 
-    it('renders progress and productivity tables with previosly calculated values', () => {
-      // check doesn't contain study programme options for admin
-      cy.contains('Study programme options').should('not.exist')
-      cy.get('table').should('have.length', 2)
-      checkProgressAndProductivity()
+    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
+    it('Studytracks and student populations -tab loads', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      cy.get('[data-cy=Section-StudytrackOverview]')
+      cy.get('[data-cy=Section-StudytrackProgress]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
     })
 
-    it('can open Thesis page', () => {
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.contains('Thesis Courses').click()
-      cy.contains('Add thesis course').click()
-      cy.contains('No results')
+    it('Students of the studyprogramme are shown correctly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      const tableContents = [
+        // [Year, All, Started studying, Currently enrolled, Absent, Cancelled, Graduated, Men, Women, Finnish]
+        [
+          '2020 - 2021',
+          12,
+          '100.0 %',
+          10,
+          '83.3 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          10,
+          '83.3 %',
+          2,
+          '16.7 %',
+          12,
+          '100.0 %',
+        ],
+        [
+          '2019 - 2020',
+          86,
+          '100.0 %',
+          86,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          1,
+          '1.2 %',
+          65,
+          '75.6 %',
+          21,
+          '24.4 %',
+          86,
+          '100.0 %',
+        ],
+        [
+          '2018 - 2019',
+          162,
+          '100.0 %',
+          161,
+          '99.4 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          11,
+          '6.8 %',
+          118,
+          '72.8 %',
+          44,
+          '27.2 %',
+          161,
+          '99.4 %',
+        ],
+        [
+          '2017 - 2018',
+          171,
+          '100.0 %',
+          171,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          3,
+          '1.8 %',
+          42,
+          '24.6 %',
+          135,
+          '78.9 %',
+          36,
+          '21.1 %',
+          166,
+          '97.1 %',
+        ],
+        [
+          'Total',
+          431,
+          '100.0 %',
+          431,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          3,
+          '0.7 %',
+          54,
+          '12.5 %',
+          328,
+          '76.1 %',
+          103,
+          '23.9 %',
+          425,
+          '98.6 %',
+        ],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudytrackOverview')
     })
 
-    it('can move to Population statistics page by clickin', () => {
-      cy.get('i.level.up.alternate.icon').eq(0).click()
-      cy.contains('Students (10)')
+    it('Student progress data is shown correctly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      const tableContents = [
+        ['2020 - 2021', 12, 5, 2, 2, 0, 2, 0, 1],
+        ['2019 - 2020', 86, 13, 15, 32, 11, 2, 0, 13],
+        ['2018 - 2019', 162, 7, 16, 35, 33, 27, 18, 26],
+        ['2017 - 2018', 171, 10, 22, 18, 22, 24, 27, 48],
+        ['Total', 431, 35, 55, 87, 66, 55, 45, 88],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudytrackProgress')
+    })
+
+    it('Studytrack overview graphs render', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+
+      cy.get('[data-cy=Graph-StudytrackProgress]')
+        .should('contain', 'Less than 30 credits')
+        .should('contain', '30-59 credits')
+        .should('contain', 'More than 180 credits')
+        .should('contain', '5.8%') // The percentage for less than 30 credits in 2017-2018, to check that the graph renders
+        .should('contain', '4.3%') // The percentage for less than 30 credits in 2018-2019
+        .should('contain', '15.1%') // The percentage for less than 30 credits in 2019-2020
+
+      cy.get('[data-cy=Graph-2018-AverageGraduationTimes]')
+        .should('contain', '2018 - 2019')
+        .should('contain', 'n = 11 / 162')
+        .should('contain', '22 kk')
+    })
+  })
+
+  /* Programme courses -tests*/
+  describe('Programme courses works for basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme', 'admin') // Remove admin when the tab is made visible to normal users
+    })
+
+    it('Programme courses -tab loads', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Programme courses').click()
+      cy.get('[data-cy=CoursesYearFilter]')
+      cy.get('[data-cy=CourseTabs]')
+    })
+
+    it('Programme course list contains correct courses in alphabetical order', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Programme courses').click()
+
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('Aineopintojen harjoitustyö: Tietokantasovellus')
+        cy.get('tr').eq(7).contains('DevOps with Docker')
+        cy.get('tr').eq(-1).contains('Äidinkielinen viestintä')
+      })
+    })
+
+    // TO-DO add tests for sorting and the upcoming credit statistics
+  })
+
+  /* Tag-tests*/
+  describe('Basic information -view works for basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme')
     })
 
     it('can create and delete tags for population', () => {
       const name = `tag-${new Date().getTime()}`
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('.attached').contains('Tags').click()
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
@@ -178,7 +409,7 @@ describe('Studyprogramme overview', () => {
 
     it('can create personal tags', () => {
       const name = `tag-${new Date().getTime()}`
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('.attached').contains('Tags').click()
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
@@ -194,7 +425,7 @@ describe('Studyprogramme overview', () => {
 
       const student = '010113437'
 
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('.attached').contains('Tags').click()
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
@@ -202,7 +433,7 @@ describe('Studyprogramme overview', () => {
       cy.contains(name)
 
       cy.contains('Add tags to students').click()
-      cy.get('.form > .field > .dropdown').click().get('.ui > .search').type(name).click()
+      cy.get('.form > .field > .dropdown').click().get('.ui > input.search').type(name).click()
 
       cy.get('.form > .field > .dropdown > .visible').contains(name).click()
 
@@ -211,7 +442,7 @@ describe('Studyprogramme overview', () => {
 
       cy.contains('Student statistics').click()
       cy.get('.prompt').type(student)
-      cy.contains(student).click()
+      cy.contains('a', student).click()
       cy.contains(name)
 
       cy.go('back')
@@ -221,7 +452,7 @@ describe('Studyprogramme overview', () => {
 
       cy.contains('Student statistics').click()
       cy.get('.prompt').type(student)
-      cy.contains(student).click()
+      cy.contains('a', student).click()
       cy.contains(name).should('not.exist')
     })
   })

cypress/integration/Teachers.js

@@ -16,7 +16,7 @@ describe('Teachers page tests', () => {
     cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
     cy.cs('course-providers').children('.icon').click()
     cy.get('.form > .fluid').click()
-    cy.contains('Name')
+    cy.contains('Teacher')
     cy.contains(teacher1)
     cy.contains(teacher2)
   })
@@ -32,21 +32,13 @@ describe('Teachers page tests', () => {
     cy.url().should('include', '/teachers')
     cy.get('.borderless > :nth-child(3)').click()
     cy.get('.prompt').type(teacher1.split()[0])
-    cy.contains(teacher1).click()
+    cy.contains('td', teacher1).click()
     cy.contains('Aineopintojen harjoitustyö: Tietokantasovellus')
   })
 
   it('Check leaderboad works', () => {
     cy.get('.borderless > :nth-child(2)').click({ force: true })
     cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.contains('2019-2020').click({ force: true })
-    cy.contains('Recalculate this year').click({ force: true })
-    cy.wait(5000)
-    cy.reload()
-    cy.contains('2019-2020').click({ force: true })
-    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.contains('Passed')
-    cy.contains('Ahokas Jarmo Ilkka')
+    cy.contains('2020-2021').click({ force: true })
   })
 })

cypress/integration/Users.js

@@ -39,9 +39,7 @@ describe('Users tests', () => {
 
     it("mocking normal user shows only the mocked user's programmes", () => {
       cy.contains('mocking').should('not.exist')
-      cy.contains('tr', 'basic').within(() => {
-        cy.contains('.button', 'Edit').click()
-      })
+      cy.cs('user-edit-button-basic').click()
       cy.get('i.spy').click()
       cy.contains('mocking as basic')
       cy.wait(1000)

cypress/support/commands.js

@@ -7,7 +7,7 @@ const adminUserHeaders = {
   uid: 'admin',
   displayname: 'Admin User',
   'shib-session-id': 'mock-cypress-session',
-  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users;grp-toska',
   edupersonaffiliation: 'member;employee;faculty',
   mail: 'grp-toska+mockadminuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-6666666',
@@ -62,9 +62,13 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
  * Shorthand for using "Cypress Selectors" (CS), i.e., `data-cy` attributes.
  */
 Cypress.Commands.add('cs', { prevSubject: 'optional' }, (subject, name) => {
-  const p = subject === undefined ? cy : cy.wrap(subject)
+  const selector = `[data-cy="${name}"]`
 
-  return p.get(`[data-cy='${name}']`)
+  if (subject === undefined) {
+    return cy.get(selector)
+  } else {
+    return cy.wrap(subject).find(selector)
+  }
 })
 
 /**
@@ -100,3 +104,18 @@ Cypress.Commands.add('selectStudyProgramme', name => {
   cy.cs('select-study-programme').click().children().contains(name).click()
   cy.contains('See population').click()
 })
+
+Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {
+  cy.get(`[data-cy=Table-${tableName}]>tbody`).within(() => {
+    correctStats.forEach((values, trIndex) => {
+      cy.get('tr')
+        .eq(trIndex)
+        .within(() => {
+          values.forEach((value, tdIndex) => {
+            cy.get('td').eq(tdIndex).contains(value)
+          })
+        })
+    })
+    cy.get('tr').should('have.length', correctStats.length)
+  })
+})

cypress/support/index.js

@@ -14,3 +14,13 @@
 // ***********************************************************
 
 import './commands'
+
+// This error is benign and seems to be only present in Cypress
+const resizeObserverLoopErrRe = /^[^(ResizeObserver loop limit exceeded)]/
+
+Cypress.on('uncaught:exception', err => {
+  /* returning false here prevents Cypress from failing the test */
+  if (resizeObserverLoopErrRe.test(err.message)) {
+    return false
+  }
+})

docker-compose.ci.yml

@@ -103,9 +103,35 @@ services:
     profiles:
       - updater
 
+  sis-updater-scheduler:
+    <<: *defaults
+    build:
+      target: development
+      context: updater/sis-updater-scheduler
+    container_name: sis-updater-scheduler
+    depends_on:
+      - redis
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      REDIS_URI: //redis:6379
+      SECRET_TOKEN: dev
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
+    ports:
+      - 8082:8082
+    profiles:
+      - updater
+
   sis-updater-nats:
     <<: *defaults
-    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_channels 1000 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:

docker-compose.yml

@@ -1,3 +1,5 @@
+version: '2.1'
+
 x-defaults: &defaults
   restart: unless-stopped
 
@@ -97,7 +99,7 @@ services:
 
   sis-updater-nats:
     <<: *defaults
-    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_channels 1000 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:
@@ -122,6 +124,8 @@ services:
       SIS_IMPORTER_USER: postgres
       SIS_NATS_TOKEN: dev
       SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
     ports:
       - 8082:8082
     profiles:

package.json

@@ -15,7 +15,8 @@
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
     "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "rapodiff": "docker exec backend node /usr/src/app/src/rapodiff.js"
   },
   "repository": {
     "type": "git",
@@ -27,24 +28,24 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@babel/eslint-parser": "^7.15.0",
-    "@babel/plugin-syntax-jsx": "^7.14.5",
-    "concurrently": "^6.2.1",
-    "cypress": "^8.3.1",
+    "@babel/eslint-parser": "^7.17.0",
+    "@babel/plugin-syntax-jsx": "^7.16.7",
+    "concurrently": "^6.5.1",
+    "cypress": "^8.7.0",
     "eslint": "^7.32.0",
     "eslint-config-airbnb": "^18.2.1",
-    "eslint-config-prettier": "^8.3.0",
+    "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-babel": "^5.3.1",
-    "eslint-plugin-cypress": "^2.11.3",
-    "eslint-plugin-import": "^2.24.2",
-    "eslint-plugin-jsx-a11y": "^6.4.1",
+    "eslint-plugin-cypress": "^2.12.1",
+    "eslint-plugin-import": "^2.25.4",
+    "eslint-plugin-jsx-a11y": "^6.5.1",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
     "eslint-plugin-prettier": "^3.4.1",
-    "eslint-plugin-react": "^7.24.0",
-    "eslint-plugin-react-hooks": "^4.2.0",
-    "husky": "^7.0.2",
-    "lint-staged": "^11.1.2",
-    "prettier": "^2.3.2",
+    "eslint-plugin-react": "^7.29.3",
+    "eslint-plugin-react-hooks": "^4.3.0",
+    "husky": "^7.0.4",
+    "lint-staged": "^11.2.6",
+    "prettier": "^2.5.1",
     "stylelint": "^13.13.1",
     "stylelint-config-prettier": "^8.0.2",
     "stylelint-config-standard": "^22.0.0",

services/backend/Dockerfile

@@ -17,7 +17,7 @@ ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
 ARG SENTRY_RELEASE
 ENV SENTRY_RELEASE=$SENTRY_RELEASE
 COPY --chown=node:node ./package* /usr/src/app/
-RUN npm ci --production
+RUN npm ci --production && apk add --no-cache vim curl nano
 COPY --chown=node:node . /usr/src/app/
 USER node
 CMD ["npm", "start"]

services/backend/package-lock.json

@@ -3,10 +3,15 @@
   "requires": true,
   "lockfileVersion": 1,
   "dependencies": {
+    "@colors/colors": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz",
+      "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ=="
+    },
     "@dabh/diagnostics": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz",
-      "integrity": "sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==",
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz",
+      "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==",
       "requires": {
         "colorspace": "1.1.x",
         "enabled": "2.0.x",
@@ -76,48 +81,48 @@
       }
     },
     "@sentry/tracing": {
-      "version": "6.11.0",
-      "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.11.0.tgz",
-      "integrity": "sha512-9VA1/SY++WeoMQI4K6n/sYgIdRtCu9NLWqmGqu/5kbOtESYFgAt1DqSyqGCr00ZjQiC2s7tkDkTNZb38K6KytQ==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.18.2.tgz",
+      "integrity": "sha512-hg6NLqrqJ5sUPTyWEQ2RqdnhQVnyLtx8II0IyWxQLDWD8UCe3Mu6G7mroDtakPWcP+lWz6OnKfMEfuhMcxR8fw==",
       "requires": {
-        "@sentry/hub": "6.11.0",
-        "@sentry/minimal": "6.11.0",
-        "@sentry/types": "6.11.0",
-        "@sentry/utils": "6.11.0",
+        "@sentry/hub": "6.18.2",
+        "@sentry/minimal": "6.18.2",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
         "@sentry/hub": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.11.0.tgz",
-          "integrity": "sha512-pT9hf+ZJfVFpoZopoC+yJmFNclr4NPqPcl2cgguqCHb69DklD1NxgBNWK8D6X05qjnNFDF991U6t1mxP9HrGuw==",
+          "version": "6.18.2",
+          "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.18.2.tgz",
+          "integrity": "sha512-d0AugekMkbnN12b4EXMjseJxtLPc9S20DGobCPUb4oAQT6S2oDQEj1jwP6PQ5vtgyy+GMYWxBMgqAQ4pjVYISQ==",
           "requires": {
-            "@sentry/types": "6.11.0",
-            "@sentry/utils": "6.11.0",
+            "@sentry/types": "6.18.2",
+            "@sentry/utils": "6.18.2",
             "tslib": "^1.9.3"
           }
         },
         "@sentry/minimal": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.11.0.tgz",
-          "integrity": "sha512-XkZ7qrdlGp4IM/gjGxf1Q575yIbl5RvPbg+WFeekpo16Ufvzx37Mr8c2xsZaWosISVyE6eyFpooORjUlzy8EDw==",
+          "version": "6.18.2",
+          "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.18.2.tgz",
+          "integrity": "sha512-n7KYuo34W2LxE+3dnZ47of7XHuORINCnXq66XH72eoj67tf0XeWbIhEJrYGmoLRyRfoCYYrBLWiDl/uTjLzrzQ==",
           "requires": {
-            "@sentry/hub": "6.11.0",
-            "@sentry/types": "6.11.0",
+            "@sentry/hub": "6.18.2",
+            "@sentry/types": "6.18.2",
             "tslib": "^1.9.3"
           }
         },
         "@sentry/types": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.11.0.tgz",
-          "integrity": "sha512-gm5H9eZhL6bsIy/h3T+/Fzzz2vINhHhqd92CjHle3w7uXdTdFV98i2pDpErBGNTSNzbntqOMifYEB5ENtZAvcg=="
+          "version": "6.18.2",
+          "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.18.2.tgz",
+          "integrity": "sha512-WzpJf/Q5aORTzrSwer/As1NlO90dBAQpaHV2ikDDKqOyMWEgjKb5/4gh59p9gH8JMMnLetP1AvQel0fOj5UnUw=="
         },
         "@sentry/utils": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.11.0.tgz",
-          "integrity": "sha512-IOvyFHcnbRQxa++jO+ZUzRvFHEJ1cZjrBIQaNVc0IYF0twUOB5PTP6joTcix38ldaLeapaPZ9LGfudbvYvxkdg==",
+          "version": "6.18.2",
+          "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.18.2.tgz",
+          "integrity": "sha512-EC619jesknyu4xpwud5WC/5odYLz6JUy7OSFy5405PpdGeh/m8XUvuJAx4zDx0Iz/Mlk0S1Md+ZcQwqkv39dkw==",
           "requires": {
-            "@sentry/types": "6.11.0",
+            "@sentry/types": "6.18.2",
             "tslib": "^1.9.3"
           }
         }
@@ -137,10 +142,28 @@
         "tslib": "^1.9.3"
       }
     },
+    "@types/debug": {
+      "version": "4.1.7",
+      "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz",
+      "integrity": "sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==",
+      "requires": {
+        "@types/ms": "*"
+      }
+    },
+    "@types/ms": {
+      "version": "0.7.31",
+      "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz",
+      "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA=="
+    },
     "@types/node": {
-      "version": "14.14.39",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.39.tgz",
-      "integrity": "sha512-Qipn7rfTxGEDqZiezH+wxqWYR8vcXq5LRpZrETD19Gs4o8LbklbmqotSUsMU+s5G3PJwMRDfNEYoxrcBwIxOuw=="
+      "version": "17.0.21",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz",
+      "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ=="
+    },
+    "@types/validator": {
+      "version": "13.7.1",
+      "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.7.1.tgz",
+      "integrity": "sha512-I6OUIZ5cYRk5lp14xSOAiXjWrfVoMZVjDuevBYgQDYzZIjsf2CAISpEcXOkFAtpAHbmWIDLcZObejqny/9xq5Q=="
     },
     "abbrev": {
       "version": "1.1.1",
@@ -149,12 +172,12 @@
       "dev": true
     },
     "accepts": {
-      "version": "1.3.7",
-      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz",
-      "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==",
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
+      "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==",
       "requires": {
-        "mime-types": "~2.1.24",
-        "negotiator": "0.6.2"
+        "mime-types": "~2.1.34",
+        "negotiator": "0.6.3"
       }
     },
     "agent-base": {
@@ -165,11 +188,6 @@
         "debug": "4"
       }
     },
-    "any-promise": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
-      "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8="
-    },
     "arr-diff": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
@@ -206,9 +224,9 @@
       "dev": true
     },
     "async": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/async/-/async-3.2.1.tgz",
-      "integrity": "sha512-XdD5lRO/87udXCMC9meWdYiR+Nq6ZjUfXidViUZGu2F1MO4T3XwZ1et0hb2++BgLfhyJwy44BGB/yx80ABx8hg=="
+      "version": "3.2.3",
+      "resolved": "https://registry.npmjs.org/async/-/async-3.2.3.tgz",
+      "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g=="
     },
     "async-each": {
       "version": "1.0.3",
@@ -217,9 +235,9 @@
       "dev": true
     },
     "async-lock": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.3.0.tgz",
-      "integrity": "sha512-8A7SkiisnEgME2zEedtDYPxUPzdv3x//E7n5IFktPAtMYSEAV7eNJF0rMwrVyUFj6d/8rgajLantbjcNRQYXIg=="
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.3.1.tgz",
+      "integrity": "sha512-zK7xap9UnttfbE23JmcrNIyueAn6jWshihJqA33U/hEnKprF/lVGBDsBv/bqLm2YMMl1DnpHhUY044eA0t1TUw=="
     },
     "atob": {
       "version": "2.1.2",
@@ -228,9 +246,9 @@
       "dev": true
     },
     "axios": {
-      "version": "0.21.2",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.2.tgz",
-      "integrity": "sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg==",
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
       "requires": {
         "follow-redirects": "^1.14.0"
       }
@@ -291,20 +309,20 @@
       "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
     },
     "body-parser": {
-      "version": "1.19.0",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
-      "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
+      "version": "1.19.2",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz",
+      "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==",
       "requires": {
-        "bytes": "3.1.0",
+        "bytes": "3.1.2",
         "content-type": "~1.0.4",
         "debug": "2.6.9",
         "depd": "~1.1.2",
-        "http-errors": "1.7.2",
+        "http-errors": "1.8.1",
         "iconv-lite": "0.4.24",
         "on-finished": "~2.3.0",
-        "qs": "6.7.0",
-        "raw-body": "2.4.0",
-        "type-is": "~1.6.17"
+        "qs": "6.9.7",
+        "raw-body": "2.4.3",
+        "type-is": "~1.6.18"
       },
       "dependencies": {
         "debug": {
@@ -338,9 +356,9 @@
       "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw=="
     },
     "bytes": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
-      "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg=="
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
+      "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="
     },
     "cache-base": {
       "version": "1.0.1",
@@ -514,12 +532,12 @@
       }
     },
     "color": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz",
-      "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==",
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz",
+      "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==",
       "requires": {
-        "color-convert": "^1.9.1",
-        "color-string": "^1.5.2"
+        "color-convert": "^1.9.3",
+        "color-string": "^1.6.0"
       }
     },
     "color-convert": {
@@ -536,25 +554,20 @@
       "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
     },
     "color-string": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.6.0.tgz",
-      "integrity": "sha512-c/hGS+kRWJutUBEngKKmk4iH3sD59MBkoxVapS/0wgpCz2u7XsNloxknyvBhzwEs1IbV36D9PwqLPJ2DTu3vMA==",
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.0.tgz",
+      "integrity": "sha512-9Mrz2AQLefkH1UvASKj6v6hj/7eWgjnT/cVsR8CumieLoT+g900exWeNogqtweI8dxloXN9BDQTYro1oWu/5CQ==",
       "requires": {
         "color-name": "^1.0.0",
         "simple-swizzle": "^0.2.2"
       }
     },
-    "colors": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz",
-      "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA=="
-    },
     "colorspace": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz",
-      "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==",
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz",
+      "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==",
       "requires": {
-        "color": "3.0.x",
+        "color": "^3.1.3",
         "text-hex": "1.0.x"
       }
     },
@@ -571,11 +584,18 @@
       "dev": true
     },
     "content-disposition": {
-      "version": "0.5.3",
-      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
-      "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==",
+      "version": "0.5.4",
+      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
+      "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==",
       "requires": {
-        "safe-buffer": "5.1.2"
+        "safe-buffer": "5.2.1"
+      },
+      "dependencies": {
+        "safe-buffer": {
+          "version": "5.2.1",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+          "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
+        }
       }
     },
     "content-type": {
@@ -602,7 +622,8 @@
     "core-util-is": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
-      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
+      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
+      "dev": true
     },
     "cors": {
       "version": "2.8.5",
@@ -838,16 +859,16 @@
       }
     },
     "express": {
-      "version": "4.17.1",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz",
-      "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==",
+      "version": "4.17.3",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz",
+      "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==",
       "requires": {
-        "accepts": "~1.3.7",
+        "accepts": "~1.3.8",
         "array-flatten": "1.1.1",
-        "body-parser": "1.19.0",
-        "content-disposition": "0.5.3",
+        "body-parser": "1.19.2",
+        "content-disposition": "0.5.4",
         "content-type": "~1.0.4",
-        "cookie": "0.4.0",
+        "cookie": "0.4.2",
         "cookie-signature": "1.0.6",
         "debug": "2.6.9",
         "depd": "~1.1.2",
@@ -861,13 +882,13 @@
         "on-finished": "~2.3.0",
         "parseurl": "~1.3.3",
         "path-to-regexp": "0.1.7",
-        "proxy-addr": "~2.0.5",
-        "qs": "6.7.0",
+        "proxy-addr": "~2.0.7",
+        "qs": "6.9.7",
         "range-parser": "~1.2.1",
-        "safe-buffer": "5.1.2",
-        "send": "0.17.1",
-        "serve-static": "1.14.1",
-        "setprototypeof": "1.1.1",
+        "safe-buffer": "5.2.1",
+        "send": "0.17.2",
+        "serve-static": "1.14.2",
+        "setprototypeof": "1.2.0",
         "statuses": "~1.5.0",
         "type-is": "~1.6.18",
         "utils-merge": "1.0.1",
@@ -875,9 +896,9 @@
       },
       "dependencies": {
         "cookie": {
-          "version": "0.4.0",
-          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz",
-          "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg=="
+          "version": "0.4.2",
+          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz",
+          "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA=="
         },
         "debug": {
           "version": "2.6.9",
@@ -891,6 +912,11 @@
           "version": "2.0.0",
           "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
           "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+        },
+        "safe-buffer": {
+          "version": "5.2.1",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+          "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
         }
       }
     },
@@ -951,11 +977,6 @@
         }
       }
     },
-    "fast-safe-stringify": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz",
-      "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA=="
-    },
     "fecha": {
       "version": "4.2.1",
       "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.1.tgz",
@@ -1003,9 +1024,9 @@
       "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="
     },
     "follow-redirects": {
-      "version": "1.14.3",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.3.tgz",
-      "integrity": "sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw=="
+      "version": "1.14.9",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
+      "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w=="
     },
     "for-in": {
       "version": "1.0.2",
@@ -1014,9 +1035,9 @@
       "dev": true
     },
     "forwarded": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
-      "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
+      "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="
     },
     "fragment-cache": {
       "version": "0.2.1",
@@ -1106,22 +1127,15 @@
       }
     },
     "http-errors": {
-      "version": "1.7.2",
-      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
-      "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz",
+      "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==",
       "requires": {
         "depd": "~1.1.2",
-        "inherits": "2.0.3",
-        "setprototypeof": "1.1.1",
+        "inherits": "2.0.4",
+        "setprototypeof": "1.2.0",
         "statuses": ">= 1.5.0 < 2",
-        "toidentifier": "1.0.0"
-      },
-      "dependencies": {
-        "inherits": {
-          "version": "2.0.3",
-          "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
-          "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
-        }
+        "toidentifier": "1.0.1"
       }
     },
     "https-proxy-agent": {
@@ -1160,9 +1174,9 @@
       "dev": true
     },
     "inflection": {
-      "version": "1.13.1",
-      "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.1.tgz",
-      "integrity": "sha512-dldYtl2WlN0QDkIDtg8+xFwOS2Tbmp12t1cHa5/YClU6ZQjTFm7B66UcVbh9NQB+HvT5BAd2t5+yKsBkw5pcqA=="
+      "version": "1.13.2",
+      "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.2.tgz",
+      "integrity": "sha512-cmZlljCRTBFouT8UzMzrGcVEvkv6D/wBdcdKG7J1QH5cXjtU75Dm+P27v9EKu/Y43UYyCJd1WC4zLebRrC8NBw=="
     },
     "inherits": {
       "version": "2.0.4",
@@ -1303,7 +1317,8 @@
     "isarray": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
+      "dev": true
     },
     "isexe": {
       "version": "2.0.0",
@@ -1334,14 +1349,14 @@
       "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
     },
     "logform": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz",
-      "integrity": "sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg==",
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/logform/-/logform-2.4.0.tgz",
+      "integrity": "sha512-CPSJw4ftjf517EhXZGGvTHHkYobo7ZCc0kvwUoOYcjfR2UVrI66RHj8MCrfAdEitdmFqbu2BYdYs8FHHZSb6iw==",
       "requires": {
-        "colors": "^1.2.1",
-        "fast-safe-stringify": "^2.0.4",
+        "@colors/colors": "1.5.0",
         "fecha": "^4.2.0",
         "ms": "^2.1.1",
+        "safe-stable-stringify": "^2.3.1",
         "triple-beam": "^1.3.0"
       }
     },
@@ -1400,16 +1415,16 @@
       "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
     },
     "mime-db": {
-      "version": "1.47.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz",
-      "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw=="
+      "version": "1.51.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz",
+      "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g=="
     },
     "mime-types": {
-      "version": "2.1.30",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz",
-      "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==",
+      "version": "2.1.34",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz",
+      "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==",
       "requires": {
-        "mime-db": "1.47.0"
+        "mime-db": "1.51.0"
       }
     },
     "minimatch": {
@@ -1443,9 +1458,9 @@
       "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ=="
     },
     "moment-timezone": {
-      "version": "0.5.33",
-      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz",
-      "integrity": "sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==",
+      "version": "0.5.34",
+      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.34.tgz",
+      "integrity": "sha512-3zAEHh2hKUs3EXLESx/wsgw6IQdusOT8Bxm3D9UrHPQR7zlMmzwybC8zHEM1tQ4LJwP7fcxrWr8tuBg05fFCbg==",
       "requires": {
         "moment": ">= 2.9.0"
       }
@@ -1514,9 +1529,9 @@
       }
     },
     "negotiator": {
-      "version": "0.6.2",
-      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
-      "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
+      "version": "0.6.3",
+      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
+      "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="
     },
     "nodemon": {
       "version": "1.19.4",
@@ -2205,14 +2220,14 @@
       "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
     },
     "pg": {
-      "version": "8.7.1",
-      "resolved": "https://registry.npmjs.org/pg/-/pg-8.7.1.tgz",
-      "integrity": "sha512-7bdYcv7V6U3KAtWjpQJJBww0UEsWuh4yQ/EjNf2HeO/NnvKjpvhEIe/A/TleP6wtmSKnUnghs5A9jUoK6iDdkA==",
+      "version": "8.7.3",
+      "resolved": "https://registry.npmjs.org/pg/-/pg-8.7.3.tgz",
+      "integrity": "sha512-HPmH4GH4H3AOprDJOazoIcpI49XFsHCe8xlrjHkWiapdbHK+HLtbm/GQzXYAZwmPju/kzKhjaSfMACG+8cgJcw==",
       "requires": {
         "buffer-writer": "2.0.0",
         "packet-reader": "1.0.0",
         "pg-connection-string": "^2.5.0",
-        "pg-pool": "^3.4.1",
+        "pg-pool": "^3.5.1",
         "pg-protocol": "^1.5.0",
         "pg-types": "^2.1.0",
         "pgpass": "1.x"
@@ -2229,9 +2244,9 @@
       "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="
     },
     "pg-pool": {
-      "version": "3.4.1",
-      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.4.1.tgz",
-      "integrity": "sha512-TVHxR/gf3MeJRvchgNHxsYsTCHQ+4wm3VIHSS19z8NC0+gioEhq1okDY1sm/TYbfoP6JLFx01s0ShvZ3puP/iQ=="
+      "version": "3.5.1",
+      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.1.tgz",
+      "integrity": "sha512-6iCR0wVrro6OOHFsyavV+i6KYL4lVNyYAB9RD18w66xSzN+d8b66HiwuP30Gp1SH5O9T82fckkzsRjlrhD0ioQ=="
     },
     "pg-protocol": {
       "version": "1.5.0",
@@ -2251,11 +2266,11 @@
       }
     },
     "pgpass": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.4.tgz",
-      "integrity": "sha512-YmuA56alyBq7M59vxVBfPJrGSozru8QAdoNlWuW3cz8l+UX3cWge0vTvjKhsSHSJpo3Bom8/Mm6hf0TR5GY0+w==",
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+      "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
       "requires": {
-        "split2": "^3.1.1"
+        "split2": "^4.1.0"
       }
     },
     "posix-character-classes": {
@@ -2290,14 +2305,15 @@
     "process-nextick-args": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
-      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
+      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
+      "dev": true
     },
     "proxy-addr": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz",
-      "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==",
+      "version": "2.0.7",
+      "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
+      "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==",
       "requires": {
-        "forwarded": "~0.1.2",
+        "forwarded": "0.2.0",
         "ipaddr.js": "1.9.1"
       }
     },
@@ -2314,9 +2330,9 @@
       "dev": true
     },
     "qs": {
-      "version": "6.7.0",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
-      "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ=="
+      "version": "6.9.7",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz",
+      "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw=="
     },
     "range-parser": {
       "version": "1.2.1",
@@ -2324,12 +2340,12 @@
       "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
     },
     "raw-body": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
-      "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
+      "version": "2.4.3",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz",
+      "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==",
       "requires": {
-        "bytes": "3.1.0",
-        "http-errors": "1.7.2",
+        "bytes": "3.1.2",
+        "http-errors": "1.8.1",
         "iconv-lite": "0.4.24",
         "unpipe": "1.0.0"
       }
@@ -2350,6 +2366,7 @@
       "version": "2.3.7",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
       "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
+      "dev": true,
       "requires": {
         "core-util-is": "~1.0.0",
         "inherits": "~2.0.3",
@@ -2430,12 +2447,9 @@
       "dev": true
     },
     "retry-as-promised": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-3.2.0.tgz",
-      "integrity": "sha512-CybGs60B7oYU/qSQ6kuaFmRd9sTZ6oXSc0toqePvV74Ac6/IFZSI1ReFQmtCN+uvW1Mtqdwpvt/LGOiCBAY2Mg==",
-      "requires": {
-        "any-promise": "^1.3.0"
-      }
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-5.0.0.tgz",
+      "integrity": "sha512-6S+5LvtTl2ggBumk04hBo/4Uf6fRJUwIgunGZ7CYEBCeufGFW1Pu6ucUf/UskHeWOIsUcLOGLFXPig5tR5V1nA=="
     },
     "safe-buffer": {
       "version": "5.1.2",
@@ -2451,11 +2465,39 @@
         "ret": "~0.1.10"
       }
     },
+    "safe-stable-stringify": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.3.1.tgz",
+      "integrity": "sha512-kYBSfT+troD9cDA85VDnHZ1rpHC50O0g1e6WlGHVCz/g+JS+9WKLj+XwFYyR8UbrZN8ll9HUpDAAddY58MGisg=="
+    },
     "safer-buffer": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
+    "semver": {
+      "version": "7.3.5",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
+      "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
+      "requires": {
+        "lru-cache": "^6.0.0"
+      },
+      "dependencies": {
+        "lru-cache": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+          "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+          "requires": {
+            "yallist": "^4.0.0"
+          }
+        },
+        "yallist": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
+        }
+      }
+    },
     "semver-diff": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-2.1.0.tgz",
@@ -2474,9 +2516,9 @@
       }
     },
     "send": {
-      "version": "0.17.1",
-      "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz",
-      "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==",
+      "version": "0.17.2",
+      "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz",
+      "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==",
       "requires": {
         "debug": "2.6.9",
         "depd": "~1.1.2",
@@ -2485,9 +2527,9 @@
         "escape-html": "~1.0.3",
         "etag": "~1.8.1",
         "fresh": "0.5.2",
-        "http-errors": "~1.7.2",
+        "http-errors": "1.8.1",
         "mime": "1.6.0",
-        "ms": "2.1.1",
+        "ms": "2.1.3",
         "on-finished": "~2.3.0",
         "range-parser": "~1.2.1",
         "statuses": "~1.5.0"
@@ -2507,76 +2549,66 @@
               "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
             }
           }
-        },
-        "ms": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
-          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
         }
       }
     },
     "sequelize": {
-      "version": "6.6.5",
-      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.6.5.tgz",
-      "integrity": "sha512-QyRrJrDRiwuiILqTMHUA1yWOPIL12KlfmgZ3hnzQwbMvp2vJ6fzu9bYJQB+qPMosck4mBUggY4Cjoc6Et8FBIQ==",
+      "version": "6.17.0",
+      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.17.0.tgz",
+      "integrity": "sha512-AZus+0YZDq91Zg0hzDaO5atTzHgJruI23V8nBlAhkLuI81Z53nSRdAe/4R1A6vGOZ/RfCLP9idF4tfQnoAsM5A==",
       "requires": {
-        "debug": "^4.1.1",
-        "dottie": "^2.0.0",
-        "inflection": "1.13.1",
-        "lodash": "^4.17.20",
-        "moment": "^2.26.0",
-        "moment-timezone": "^0.5.31",
-        "retry-as-promised": "^3.2.0",
-        "semver": "^7.3.2",
-        "sequelize-pool": "^6.0.0",
+        "@types/debug": "^4.1.7",
+        "@types/validator": "^13.7.1",
+        "debug": "^4.3.3",
+        "dottie": "^2.0.2",
+        "inflection": "^1.13.2",
+        "lodash": "^4.17.21",
+        "moment": "^2.29.1",
+        "moment-timezone": "^0.5.34",
+        "pg-connection-string": "^2.5.0",
+        "retry-as-promised": "^5.0.0",
+        "semver": "^7.3.5",
+        "sequelize-pool": "^7.1.0",
         "toposort-class": "^1.0.1",
-        "uuid": "^8.1.0",
-        "validator": "^13.6.0",
+        "uuid": "^8.3.2",
+        "validator": "^13.7.0",
         "wkx": "^0.5.0"
       },
       "dependencies": {
-        "lru-cache": {
-          "version": "6.0.0",
-          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
-          "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+        "debug": {
+          "version": "4.3.3",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz",
+          "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==",
           "requires": {
-            "yallist": "^4.0.0"
+            "ms": "2.1.2"
           }
         },
-        "semver": {
-          "version": "7.3.5",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
-          "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
-          "requires": {
-            "lru-cache": "^6.0.0"
-          }
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
         },
         "uuid": {
           "version": "8.3.2",
           "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
           "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
-        },
-        "yallist": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
         }
       }
     },
     "sequelize-pool": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-6.1.0.tgz",
-      "integrity": "sha512-4YwEw3ZgK/tY/so+GfnSgXkdwIJJ1I32uZJztIEgZeAO6HMgj64OzySbWLgxj+tXhZCJnzRfkY9gINw8Ft8ZMg=="
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-7.1.0.tgz",
+      "integrity": "sha512-G9c0qlIWQSK29pR/5U2JF5dDQeqqHRragoyahj/Nx4KOOQ3CPPfzxnfqFPCSB7x5UgjOgnZ61nSxz+fjDpRlJg=="
     },
     "serve-static": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",
-      "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==",
+      "version": "1.14.2",
+      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz",
+      "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==",
       "requires": {
         "encodeurl": "~1.0.2",
         "escape-html": "~1.0.3",
         "parseurl": "~1.3.3",
-        "send": "0.17.1"
+        "send": "0.17.2"
       }
     },
     "set-value": {
@@ -2618,9 +2650,9 @@
       }
     },
     "setprototypeof": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
-      "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw=="
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
+      "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
     },
     "shebang-command": {
       "version": "1.2.0",
@@ -2842,24 +2874,9 @@
       }
     },
     "split2": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz",
-      "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==",
-      "requires": {
-        "readable-stream": "^3.0.0"
-      },
-      "dependencies": {
-        "readable-stream": {
-          "version": "3.6.0",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
-          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
-          "requires": {
-            "inherits": "^2.0.3",
-            "string_decoder": "^1.1.1",
-            "util-deprecate": "^1.0.1"
-          }
-        }
-      }
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz",
+      "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ=="
     },
     "stack-trace": {
       "version": "0.0.10",
@@ -3072,9 +3089,9 @@
       }
     },
     "toidentifier": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
-      "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw=="
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
+      "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="
     },
     "toposort-class": {
       "version": "1.0.1",
@@ -3323,19 +3340,20 @@
       }
     },
     "winston": {
-      "version": "3.3.3",
-      "resolved": "https://registry.npmjs.org/winston/-/winston-3.3.3.tgz",
-      "integrity": "sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw==",
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/winston/-/winston-3.6.0.tgz",
+      "integrity": "sha512-9j8T75p+bcN6D00sF/zjFVmPp+t8KMPB1MzbbzYjeN9VWxdsYnTB40TkbNUEXAmILEfChMvAMgidlX64OG3p6w==",
       "requires": {
         "@dabh/diagnostics": "^2.0.2",
-        "async": "^3.1.0",
+        "async": "^3.2.3",
         "is-stream": "^2.0.0",
-        "logform": "^2.2.0",
+        "logform": "^2.4.0",
         "one-time": "^1.0.0",
         "readable-stream": "^3.4.0",
+        "safe-stable-stringify": "^2.3.1",
         "stack-trace": "0.0.x",
         "triple-beam": "^1.3.0",
-        "winston-transport": "^4.4.0"
+        "winston-transport": "^4.5.0"
       },
       "dependencies": {
         "is-stream": {
@@ -3356,12 +3374,25 @@
       }
     },
     "winston-transport": {
-      "version": "4.4.0",
-      "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz",
-      "integrity": "sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw==",
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.5.0.tgz",
+      "integrity": "sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q==",
       "requires": {
-        "readable-stream": "^2.3.7",
-        "triple-beam": "^1.2.0"
+        "logform": "^2.3.2",
+        "readable-stream": "^3.6.0",
+        "triple-beam": "^1.3.0"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "3.6.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
+          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
+          "requires": {
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
+          }
+        }
       }
     },
     "wkx": {

services/backend/package.json

@@ -4,35 +4,31 @@
   "scripts": {
     "start": "node --max_old_space_size=4096 index.js",
     "dev": "nodemon --max_old_space_size=4096 index.js",
-    "diff:students": "node /usr/src/app/src/sisoodi_diff/students/index.js",
-    "diff:populations": "node /usr/src/app/src/sisoodi_diff/populations.js",
-    "diff:courses": "node /usr/src/app/src/sisoodi_diff/courses/index.js",
-    "diff:transfers": "node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "node /usr/src/app/src/sisoodi_diff/trends.js"
+    "rapodiff": "node /usr/src/app/src/rapodiff.js"
   },
   "dependencies": {
     "@sentry/node": "^5.30.0",
-    "@sentry/tracing": "^6.11.0",
-    "async": "^3.2.1",
-    "async-lock": "^1.3.0",
-    "axios": "^0.21.2",
+    "@sentry/tracing": "^6.18.2",
+    "async": "^3.2.3",
+    "async-lock": "^1.3.1",
+    "axios": "^0.21.4",
     "bluebird": "^3.7.2",
     "cors": "^2.8.5",
     "cron": "^1.8.2",
-    "express": "^4.17.1",
+    "express": "^4.17.3",
     "express-async-errors": "^3.1.1",
     "lodash": "^4.17.21",
     "lru-cache": "^5.1.1",
     "moment": "^2.29.1",
-    "moment-timezone": "^0.5.33",
+    "moment-timezone": "^0.5.34",
     "morgan": "^1.10.0",
-    "pg": "^8.7.1",
+    "pg": "^8.7.3",
     "redis": "^3.1.2",
-    "sequelize": "^6.6.5",
+    "sequelize": "^6.17.0",
     "umzug": "^2.3.0",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "uuid": "^3.4.0",
-    "winston": "^3.3.3"
+    "winston": "^3.6.0"
   },
   "devDependencies": {
     "nodemon": "^1.19.4"

services/backend/src/database/connection.js

@@ -16,7 +16,7 @@ class DbConnection extends EventEmitter {
       pool: {
         max: 25,
         min: 0,
-        acquire: 10000,
+        acquire: 30000,
         idle: 300000000,
       },
       logging: false,

services/backend/src/database/migrations_kone/20220307_00_remove_thesis_courses.js

@@ -0,0 +1,6 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('thesis_courses')
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_user/20220112_00_drop_hy_groups.js

@@ -0,0 +1,9 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('user_hy_group')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS user_hy_group_id_seq')
+    await queryInterface.dropTable('hy_groups')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS hy_groups_id_seq')
+  },
+  down: async () => {},
+}

services/backend/src/events.js

@@ -1,21 +1,16 @@
 const { CronJob } = require('cron')
 const moment = require('moment')
-const {
-  refreshProtoC,
-  refreshStatus,
-  refreshStatusGraduated,
-  refreshUber,
-  refreshProtoCProgramme,
-  getStartYears,
-} = require('./services/trends')
+const { refreshStatus, refreshStatusGraduated, refreshUber, getStartYears } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
-const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studyprogramme')
+const { updateBasicView, updateStudytrackView } = require('./services/studyprogrammeUpdates')
 const { findAndSaveTeachers } = require('./services/topteachers')
-const { setProductivity, setThroughput, patchProductivity, patchThroughput } = require('./services/analyticsService')
 const { isProduction } = require('./conf-backend')
 const { getCurrentSemester } = require('./services/semesters')
 const logger = require('./util/logger')
+const {
+  dbConnections: { sequelize },
+} = require('./database/connection')
 
 const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
 
@@ -24,47 +19,23 @@ const refreshStudyrightAssociations = async () => {
   logger.info('Studyright associations refreshed!')
 }
 
-const refreshOverview = async () => {
-  logger.info('Refreshing throughput and productivity for programmes...')
-  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH') || code.includes('MH'))
+const refreshNewOverviews = async () => {
+  logger.info('Refreshing studyprogramme and studytrack overview statistics for all programmes')
+  const codes = (await getAllProgrammes())
+    .map(p => p.code)
+    .filter(code => code.includes('KH') || code.includes('MH') || /^(T)[0-9]{6}$/.test(code))
   let ready = 0
   for (const code of codes) {
-    let programmeStatsSince = new Date('2017-07-31')
-    if (code.includes('MH') || code.includes('KH')) {
-      programmeStatsSince = new Date('2017-07-31')
-    } else {
-      programmeStatsSince = new Date('2000-07-31')
-    }
     try {
-      await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-      const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
-      await setThroughput(data)
+      await updateBasicView(code)
+      await updateStudytrackView(code)
+      ready += 1
     } catch (e) {
-      logger.error({ message: `failed to recalculate throughput for ${code}`, meta: e })
-      try {
-        await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-      } catch (e) {
-        logger.error({ message: 'failed to update throughtput status to error', meta: e })
-      }
+      logger.error({ message: `Failed to update overview stats for programme ${code}`, meta: e })
     }
-    try {
-      await patchProductivity({ [code]: { status: 'RECALCULATING' } })
-      const data = await productivityStatsForStudytrack(code, programmeStatsSince)
-      await setProductivity(data)
-    } catch (e) {
-      logger.error({ message: `failed to recalculate productivity for ${code}`, meta: e })
-      try {
-        await patchProductivity({
-          [code]: { status: 'RECALCULATION ERRORED' },
-        })
-      } catch (e) {
-        logger.error({ message: 'failed to update productivity status to error', meta: e })
-      }
-    }
-    ready += 1
     logger.info(`${ready}/${codes.length} programmes done`)
   }
-  logger.info('Throughput and productivity for programmes refreshed')
+  logger.info('Studyprogramme and studytrack overview stats refreshed!')
 }
 
 const refreshTeacherLeaderboard = async () => {
@@ -73,19 +44,6 @@ const refreshTeacherLeaderboard = async () => {
   await findAndSaveTeachers(currentSemestersYearCode - 1, currentSemestersYearCode)
 }
 
-const refreshProtoCtoRedis = async () => {
-  logger.info('Refreshing CDS ProtoC')
-  const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false' }
-  const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false' }
-  const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true' }
-  const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true' }
-  await refreshProtoC(defaultQuery)
-  await refreshProtoC(onlyOld)
-  await refreshProtoC(onlyEnr)
-  await refreshProtoC(bothToggles)
-  logger.info('Refreshing CDS ProtoC doned')
-}
-
 const refreshStatusToRedis = async () => {
   const unixMillis = moment().valueOf()
   const date = new Date(Number(unixMillis))
@@ -122,32 +80,12 @@ const refreshUberToRedis = async () => {
   logger.info(`Refreshing CDS Uber data doned`)
 }
 
-const refreshProtoCProgrammeToRedis = async () => {
-  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH'))
-  for (const code of codes) {
-    logger.info(`Refreshing ProtoCProgramme code ${code}`)
-    const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
-    const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
-    const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
-    const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true', code }
-    const queries = [defaultQuery, onlyOld, onlyEnr, bothToggles]
-    for (const query of queries) {
-      try {
-        await refreshProtoCProgramme(query)
-      } catch (e) {
-        logger.error({
-          message: `Error when refreshing ProtocProgramme code ${code}, query ${Object.keys({ query })[0]}`,
-          meta: e,
-        }) // object.keys is hax to print query variable name
-      }
-    }
-    logger.info(`Refreshing ProtoCProgramme code ${code} doned`)
-  }
-  logger.info(`Refreshing protocprogramme doned`)
+const refreshProtoCMaterializedView = async () => {
+  await sequelize.query('REFRESH MATERIALIZED VIEW organization_yearly_credits')
 }
 
 const refreshStatistics = async () => {
-  const statfuncs = [refreshStudyrightAssociations, refreshOverview, refreshTeacherLeaderboard]
+  const statfuncs = [refreshStudyrightAssociations, refreshTeacherLeaderboard]
   logger.info('Refreshing statistics')
   for (const func of statfuncs) {
     await func()
@@ -156,7 +94,7 @@ const refreshStatistics = async () => {
 }
 
 const refreshTrends = async () => {
-  const trendfuncs = [refreshProtoCtoRedis, refreshStatusToRedis, refreshUberToRedis, refreshProtoCProgrammeToRedis]
+  const trendfuncs = [refreshProtoCMaterializedView, refreshStatusToRedis, refreshUberToRedis]
   logger.info('Refreshing trends')
   for (const func of trendfuncs) {
     await func()
@@ -166,9 +104,10 @@ const refreshTrends = async () => {
 
 const startCron = () => {
   if (isProduction) {
+    logger.info('Cronjob for refreshing stats started: runs at 3am every day.')
     // refresh 3am every day
     schedule('0 3 * * *', async () => {
-      for (const func of [refreshStatistics, refreshTrends]) {
+      for (const func of [refreshStatistics, refreshTrends, refreshNewOverviews]) {
         await func()
       }
     })

services/backend/src/middleware/accessLogger.js

@@ -1,5 +1,6 @@
 const morgan = require('morgan')
 const logger = require('../util/logger')
+const _ = require('lodash')
 
 // So this appears to be a hack to get neatly formatted stats like response-time etc. from morgan
 // without actually using morgan what it's used for (LOGGING REQUESTS!).
@@ -27,7 +28,8 @@ const accessLogger = morgan((tokens, req, res) => {
   ].join(' ')
 
   logger.info(message, {
-    ...meta,
+    // don't log student list which might be huge
+    ..._.omit(meta, ['studentsUserCanAccess']),
     // pass this as a custom field so we can filter by it in graylog
     isUsageStats: true,
   })

services/backend/src/middleware/auth.js

@@ -1,6 +1,8 @@
 const roles = requiredRoles => async (req, res, next) => {
   if (req.user) {
-    const { roles } = req
+    const {
+      user: { roles },
+    } = req
 
     if (requiredRoles.every(r => roles.indexOf(r) >= 0) || roles.includes('admin')) {
       return next()

services/backend/src/middleware/currentUserMiddleware.js

@@ -1,6 +1,6 @@
 const Sentry = require('@sentry/node')
 const { ApplicationError } = require('../util/customErrors')
-const { getUser, getUserDataFor, getMockedUser } = require('../services/userService')
+const { getUser, getMockedUser } = require('../services/userService')
 const { requiredGroup } = require('../conf-backend')
 const _ = require('lodash')
 
@@ -8,49 +8,45 @@ const parseIamGroups = iamGroups => iamGroups?.split(';').filter(Boolean) ?? []
 
 const hasRequiredIamGroup = iamGroups => _.intersection(iamGroups, requiredGroup).length > 0
 
-const currentUserMiddleware = async (req, _, next) => {
-  const { uid: username, 'shib-session-id': sessionId, shib_logout_url: logoutUrl } = req.headers
+const currentUserMiddleware = async (req, _res, next) => {
+  const {
+    'shib-session-id': sessionId,
+    'x-show-as-user': showAsUser,
+    displayname: name,
+    hygroupcn,
+    hypersonsisuid: sisId,
+    mail: email,
+    shib_logout_url: logoutUrl,
+    uid: username,
+  } = req.headers
 
-  if (!sessionId) {
-    throw new ApplicationError('No session id found in request headers', 403, { logoutUrl })
+  if (!sessionId || !username) {
+    throw new ApplicationError('Not enough data in request headers', 403, { logoutUrl })
   }
 
-  if (!username) {
-    throw new ApplicationError('No username found in request headers', 403, { logoutUrl })
-  }
-  const { displayname: name, mail: email, hygroupcn: iamGroups, hypersonsisuid: sisId } = req.headers
-  const parsedIamGroups = parseIamGroups(iamGroups)
+  const iamGroups = parseIamGroups(hygroupcn)
 
-  req.user = await getUser({
+  if (!hasRequiredIamGroup(iamGroups)) {
+    throw new ApplicationError('User does not have required iam group', 403, { logoutUrl })
+  }
+
+  let user = await getUser({
     username,
     name,
     email,
-    iamGroups: parsedIamGroups,
+    iamGroups,
     sisId,
   })
 
-  if (!hasRequiredIamGroup(parsedIamGroups)) {
-    throw new ApplicationError('User is not enabled', 403, { logoutUrl })
+  if (showAsUser && user.isAdmin) {
+    user = await getMockedUser({ userToMock: showAsUser, mockedBy: username })
   }
 
-  const showAsUser = req.headers['x-show-as-user']
+  Sentry.setUser({ username: user.mockedBy ?? username })
 
-  if (showAsUser) {
-    const mockedUser = await getMockedUser(username, showAsUser)
-    if (mockedUser) {
-      req.user = mockedUser
-    }
-  }
+  req.user = user
   req.logoutUrl = logoutUrl
 
-  const { userId, mockedBy } = req.user
-
-  Sentry.setUser({ username: mockedBy ?? userId })
-
-  // TODO: remove this garbo
-  const userData = await getUserDataFor(userId)
-  Object.assign(req, userData)
-
   next()
 }
 

services/backend/src/models/enrollment.js

@@ -0,0 +1,69 @@
+const { Model, DATE, INTEGER, STRING, BOOLEAN } = require('sequelize')
+const { dbConnections } = require('../database/connection')
+
+class Enrollment extends Model {}
+
+Enrollment.init(
+  {
+    id: {
+      primaryKey: true,
+      type: STRING,
+    },
+    studentnumber: {
+      type: STRING,
+      references: {
+        model: 'student',
+        key: 'studentnumber',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+    },
+    course_code: {
+      type: STRING,
+    },
+    state: {
+      type: STRING,
+    },
+    enrollment_date_time: {
+      type: DATE,
+    },
+    course_id: {
+      type: STRING,
+      references: {
+        model: 'course',
+        key: 'id',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+    },
+    semester_composite: {
+      type: STRING,
+      references: {
+        model: 'semesters',
+        key: 'composite',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+    },
+    semestercode: {
+      type: INTEGER,
+    },
+    createdAt: {
+      type: DATE,
+    },
+    updatedAt: {
+      type: DATE,
+    },
+    is_open: {
+      type: BOOLEAN,
+    },
+  },
+  {
+    underscored: false,
+    sequelize: dbConnections.sequelize,
+    modelName: 'enrollment',
+    tableName: 'enrollment',
+  }
+)
+
+module.exports = Enrollment

services/backend/src/models/index.js

@@ -10,6 +10,7 @@ const CreditType = require('./creditType')
 const Credit = require('./credit')
 const CreditTeacher = require('./creditTeacher')
 const ElementDetail = require('./elementDetail')
+const Studyplan = require('./studyplan')
 const StudyrightExtent = require('./studyrightExtent')
 const Studyright = require('./studyright')
 const StudyrightElement = require('./studyrightElement')
@@ -17,6 +18,7 @@ const Transfer = require('./transfer')
 const ProgrammeModule = require('./programmeModule')
 const ProgrammeModuleChild = require('./programmeModuleChild')
 const ExcludedCourse = require('./excludedCourse')
+const Enrollment = require('./enrollment')
 
 CourseType.hasMany(Course, { foreignKey: 'coursetypecode', sourceKey: 'coursetypecode' })
 Course.belongsTo(CourseType, { foreignKey: 'coursetypecode', targetKey: 'coursetypecode' })
@@ -62,6 +64,9 @@ Teacher.belongsToMany(Credit, { through: CreditTeacher, foreignKey: 'teacher_id'
 Organization.hasMany(Studyright, { foreignKey: 'facultyCode', sourceKey: 'code' })
 Studyright.belongsTo(Organization, { foreignKey: 'facultyCode', targetKey: 'code' })
 
+Studyplan.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
+Student.hasMany(Studyplan, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
+
 Studyright.belongsTo(Student, { foreignKey: 'studentStudentnumber', targetKey: 'studentnumber' })
 Student.hasMany(Studyright, { foreignKey: 'studentStudentnumber', sourceKey: 'studentnumber' })
 
@@ -88,6 +93,12 @@ Studyright.hasMany(Transfer, { foreignKey: 'studyrightid', sourceKey: 'studyrigh
 Transfer.belongsTo(ElementDetail, { as: 'source', foreignKey: 'sourcecode' })
 Transfer.belongsTo(ElementDetail, { as: 'target', foreignKey: 'targetcode' })
 
+Enrollment.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
+Student.hasMany(Enrollment, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
+Enrollment.belongsTo(Course, { foreignKey: 'course_id' })
+Course.hasMany(Enrollment, { foreignKey: 'course_id' })
+Enrollment.belongsTo(Semester, { foreignKey: { name: 'semester_composite', allowNull: false } })
+
 ProgrammeModule.belongsToMany(ProgrammeModule, {
   as: 'parents',
   through: ProgrammeModuleChild,
@@ -114,6 +125,7 @@ module.exports = {
   Credit,
   CreditTeacher,
   ElementDetail,
+  Studyplan,
   StudyrightExtent,
   Studyright,
   StudyrightElement,
@@ -121,4 +133,5 @@ module.exports = {
   ProgrammeModule,
   ProgrammeModuleChild,
   ExcludedCourse,
+  Enrollment,
 }

services/backend/src/models/models_kone.js

@@ -104,20 +104,6 @@ const MandatoryCourseLabels = sequelizeKone.define('mandatory_course_labels', {
   },
 })
 
-const ThesisCourse = sequelizeKone.define('thesis_courses', {
-  programmeCode: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  courseCode: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  thesisType: {
-    type: Sequelize.ENUM([ThesisTypeEnums.BACHELOR, ThesisTypeEnums.MASTER]),
-  },
-})
-
 const CustomPopulationSearch = sequelizeKone.define('custom_population_searches', {
   id: {
     primaryKey: true,
@@ -173,7 +159,6 @@ module.exports = {
   MandatoryCourse,
   TagStudent,
   Tag,
-  ThesisCourse,
   ThesisTypeEnums,
   MandatoryCourseLabels,
   CustomPopulationSearch,

services/backend/src/models/models_user.js

@@ -83,23 +83,6 @@ const User = sequelizeUser.define(
     timestamps: false,
   }
 )
-const HyGroup = sequelizeUser.define('hy_group', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true,
-  },
-
-  code: { type: Sequelize.STRING },
-
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})
 
 User.hasMany(UserElementDetails, { as: 'programme' })
 UserElementDetails.belongsTo(User)
@@ -110,16 +93,12 @@ User.belongsToMany(AccessGroup, {
 })
 AccessGroup.belongsToMany(User, { through: 'user_accessgroup' })
 
-User.belongsToMany(HyGroup, { through: 'user_hy_group', as: 'hy_group' })
-HyGroup.belongsToMany(User, { through: 'user_hy_group' })
-
 User.hasMany(UserFaculties, { as: 'faculty', foreignKey: 'userId' })
 
 module.exports = {
   User,
   UserElementDetails,
   AccessGroup,
-  HyGroup,
   UserFaculties,
   sequelizeUser,
 }

services/backend/src/models/studyplan.js

@@ -0,0 +1,48 @@
+const { Model, DATE, ARRAY, STRING, INTEGER } = require('sequelize')
+const { dbConnections } = require('../database/connection')
+
+class Studyplan extends Model {}
+
+Studyplan.init(
+  {
+    id: {
+      primaryKey: true,
+      type: INTEGER,
+      autoIncrement: true,
+      allowNull: false,
+    },
+    studentnumber: {
+      type: STRING,
+      references: {
+        model: 'student',
+        key: 'studentnumber',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+      unique: 'source', // This and programme_code create a composite key for this table.
+      allowNull: false,
+    },
+    programme_code: {
+      type: STRING,
+      allowNull: false,
+      unique: 'source',
+    },
+    included_courses: {
+      type: ARRAY(STRING),
+    },
+    createdAt: {
+      type: DATE,
+    },
+    updatedAt: {
+      type: DATE,
+    },
+  },
+  {
+    underscored: false,
+    sequelize: dbConnections.sequelize,
+    modelName: 'studyplan',
+    tableName: 'studyplan',
+  }
+)
+
+module.exports = Studyplan

services/backend/src/models/studyright.js

@@ -9,9 +9,6 @@ Studyright.init(
       primaryKey: true,
       type: STRING,
     },
-    canceldate: {
-      type: DATE,
-    },
     startdate: {
       type: DATE,
     },
@@ -27,6 +24,9 @@ Studyright.init(
     graduated: {
       type: INTEGER,
     },
+    active: {
+      type: INTEGER,
+    },
     // irtisanomisperuste
     studentStudentnumber: {
       type: STRING,

services/backend/src/rapodiff.js

@@ -0,0 +1,363 @@
+/* eslint no-console: 0 */
+
+/*
+  depends on nodeproxy running in importer.cs.helsinki.fi, and env RAPO_NODEPROXY (see importer for a proper value)
+  needs also acces to importer db api, uses env IMPORTER_API
+
+  known differences in APIs
+    - rapo does not show students that have graduated (it however takes some days for rapo to notice a graduation)
+    - rapo does not handle the starting year of transfers properly (shows in old staring population)
+    - rapo show some students incorrectly passive, see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3521
+*/
+
+const axios = require('axios').default
+
+const _ = require('lodash')
+
+const { optimizedStatisticsOf } = require('./services/populations')
+const { Student, Transfer, StudyrightElement, Studyright } = require('./models')
+
+const token = process.env.RAPO_NODEPROXY
+
+const getFromRapo = async (urlStart, urlEnd) => {
+  const { data: rapoStuffLasna } = await axios.get(urlStart)
+  const { data: rapoStuffLasnaMinus } = await axios.get(urlEnd)
+
+  const rapoStudents = _.difference(
+    rapoStuffLasna.map(s => s.opiskelijanumero),
+    rapoStuffLasnaMinus.map(s => s.opiskelijanumero)
+  )
+
+  return rapoStudents
+}
+
+const known_problems_in_oodikone = {
+  KH30_001: ['013896191'],
+  KH40_003: ['013200457'],
+  KH55_001: ['013195834', '015119708', '015138336'],
+}
+
+const known_problems_in_rapo = {
+  KH55_001: ['015160142', '015135575', '015119708', '015138336'],
+}
+
+const getFromOodikone = async (programme, year) => {
+  const isNotAmongKnownProblems = student => {
+    const hasProblems = known_problems_in_oodikone[programme] ? known_problems_in_oodikone[programme] : []
+    return !hasProblems.includes(student.studentNumber)
+  }
+
+  const theStudyrights = student => {
+    const theOne = s => s.studyright_elements.find(e => e.code === programme)
+    // in some rare cases student may have more than one studyright for same programme
+    return student.studyrights.filter(theOne)
+  }
+
+  const notGraduated = student => {
+    const graduation = theStudyrights(student).filter(r => r.graduated === 1)
+    return graduation.length === 0
+  }
+
+  const notTransferred = student => {
+    const transfer = student.transfers.find(t => t.targetcode === programme)
+    return transfer === undefined
+  }
+
+  const months = 12
+
+  const oodikoneQuery = {
+    semesters: ['FALL', 'SPRING'],
+    months,
+    studyRights: { programme },
+    year,
+  }
+
+  const oodikoneStuff = (await optimizedStatisticsOf(oodikoneQuery)).students
+
+  return oodikoneStuff
+    .filter(notGraduated)
+    .filter(notTransferred)
+    .filter(isNotAmongKnownProblems)
+    .map(s => s.studentNumber)
+}
+
+const isAcuallyActive = async student => {
+  const currentSemester = 144
+  const tokenImporterApi = process.env.IMPORTER_API
+
+  const url = `https://importer.cs.helsinki.fi/api/importer/students/${student}/rapo_semester_enrollments?token=${tokenImporterApi}`
+  const { data } = await axios.get(url)
+  const terms = data.reduce((all, s) => all.concat(s.terms), []).filter(t => t.semester_code === currentSemester)
+  const active = terms.filter(t => t.type !== 'NEGLECTED')
+  const negl = terms.filter(t => t.type === 'NEGLECTED')
+  return active.length > 0 && negl.length > 0
+}
+
+const hasTransferred = async (student, programme) => {
+  const data = await Student.findByPk(student, {
+    include: [
+      {
+        model: Transfer,
+      },
+    ],
+  })
+
+  const transferTo = data.transfers.find(t => t.targetcode === programme)
+  return transferTo !== undefined
+}
+
+const graduatedVeryRecently = async (student, programme) => {
+  const data = await Student.findByPk(student, {
+    include: [
+      {
+        model: Studyright,
+        include: [
+          {
+            model: StudyrightElement,
+            required: true,
+            where: {
+              code: programme,
+            },
+          },
+        ],
+        where: {
+          graduated: 1,
+        },
+      },
+    ],
+  })
+
+  if (!data || data.studyrights.length === 0) {
+    return false
+  }
+
+  const graduated = new Date(data.studyrights[0].enddate)
+  const weekAgo = new Date(new Date().getTime() - 7 * 24 * 60 * 60 * 1000)
+  return graduated > weekAgo
+}
+
+const programme_diff_year = async (programme, facultyCode, year, brief = false) => {
+  const oodikoneStudents = await getFromOodikone(programme, year)
+
+  const studyrightStartDate = `${year}-08-01`
+  const studyrightEndDate = `${year + 1}-08-01`
+
+  const rapoUrlStart = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+    )
+  const rapoUrlEnd = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+    )
+
+  const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
+  const rapoStudentsPoissa = await getFromRapo(rapoUrlStart('Poissa'), rapoUrlEnd('Poissa'))
+  const rapoStudentPoissaLaki = await getFromRapo(
+    rapoUrlStart('Poissa ei kuluta opintoaikaa'),
+    rapoUrlEnd('Poissa ei kuluta opintoaikaa')
+  )
+
+  const isNotAmongKnownProblems = student => {
+    const hasProblems = known_problems_in_rapo[programme] ? known_problems_in_rapo[programme] : []
+    return !hasProblems.includes(student.studentNumber)
+  }
+
+  const rapoStudents = rapoStudentsLasna
+    .concat(rapoStudentsPoissa)
+    .concat(rapoStudentPoissaLaki)
+    .filter(isNotAmongKnownProblems)
+
+  const onlyInOodikone = _.difference(oodikoneStudents, rapoStudents)
+  const onlyInRapo = _.difference(rapoStudents, oodikoneStudents)
+
+  const wrongMissingInRapo = []
+  for (let student of onlyInOodikone) {
+    const status = await isAcuallyActive(student)
+    if (status) {
+      wrongMissingInRapo.push(student)
+    }
+  }
+
+  // these are in wrong RAPO-population
+  const transferred = []
+  for (let student of onlyInRapo) {
+    const isTransfer = await hasTransferred(student, programme)
+    if (isTransfer) {
+      transferred.push(student)
+    } else {
+      const isRecentGraduation = await graduatedVeryRecently(student, programme)
+      if (isRecentGraduation) {
+        transferred.push(student)
+      }
+    }
+  }
+
+  const checkOodikoneOnly = _.difference(onlyInOodikone, wrongMissingInRapo)
+  const checkRapoOnly = _.difference(onlyInRapo, transferred)
+
+  if (!brief) {
+    console.log('vuosi', year)
+    console.log('')
+    console.log('oodikone', oodikoneStudents.length, ' transferred', transferred.length)
+    console.log('rapo', rapoStudents.length, ' activeness wrong rapo', wrongMissingInRapo.length)
+    console.log('only rapo', checkRapoOnly.length)
+    console.log('only oodikone', checkOodikoneOnly.length)
+
+    console.log('')
+
+    if (checkRapoOnly.length > 0) {
+      console.log('Only in Rapo:')
+      console.log(checkRapoOnly.join('\n'))
+    }
+
+    if (checkOodikoneOnly.length > 0) {
+      console.log('Only in Oodikone:')
+      console.log(checkOodikoneOnly.join('\n'))
+    }
+  }
+
+  if (checkRapoOnly.length + checkOodikoneOnly.length) {
+    console.log('******************************', year)
+    if (checkRapoOnly.length > 0) {
+      console.log('Only in Rapo:')
+      console.log(checkRapoOnly.join('\n'))
+    }
+
+    if (checkOodikoneOnly.length > 0) {
+      console.log('Only in Oodikone:')
+      console.log(checkOodikoneOnly.join('\n'))
+    }
+  }
+
+  if (!brief) {
+    console.log('')
+  }
+}
+
+const programme_diff = async (programme, faculty, brief) => {
+  console.log()
+  console.log(programme)
+  console.log('--------', '\n')
+
+  await programme_diff_year(programme, faculty, 2017, brief)
+  await programme_diff_year(programme, faculty, 2018, brief)
+  await programme_diff_year(programme, faculty, 2019, brief)
+  await programme_diff_year(programme, faculty, 2020, brief)
+  await programme_diff_year(programme, faculty, 2021, brief)
+}
+
+const faculty_diff = async (faculty, programmes, brief) => {
+  console.log('\n')
+  console.log(faculty)
+  console.log('==')
+  for (let programme of programmes) {
+    await programme_diff(programme, faculty, brief)
+  }
+}
+
+// eslint-disable-next-line no-unused-vars
+const msc_programme_diff_year = async (programme, facultyCode, year, brief = false) => {
+  const oodikoneStudents = await getFromOodikone(programme, year)
+
+  const studyrightStartDate = `${year}-08-01`
+  const studyrightEndDate = `${year + 1}-08-01`
+
+  const rapoUrlStart = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+    )
+  const rapoUrlEnd = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+    )
+
+  const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
+  const rapoStudentsPoissa = await getFromRapo(rapoUrlStart('Poissa'), rapoUrlEnd('Poissa'))
+  const rapoStudentPoissaLaki = await getFromRapo(
+    rapoUrlStart('Poissa ei kuluta opintoaikaa'),
+    rapoUrlEnd('Poissa ei kuluta opintoaikaa')
+  )
+
+  const rapoStudents = rapoStudentsLasna.concat(rapoStudentsPoissa).concat(rapoStudentPoissaLaki)
+
+  const onlyInOodikone = _.difference(oodikoneStudents, rapoStudents)
+  const onlyInRapo = _.difference(rapoStudents, oodikoneStudents)
+
+  const wrongInRapo = []
+  for (let student of onlyInOodikone) {
+    const status = await isAcuallyActive(student)
+    if (status) {
+      //wrongInRapo.push(student)
+    }
+  }
+
+  // in wrong RAPO population
+  const transferred = []
+  for (let student of onlyInRapo) {
+    const status = await hasTransferred(student, programme)
+    if (status) {
+      transferred.push(student)
+    }
+  }
+
+  const checkOodikoneOnly = _.difference(onlyInOodikone, wrongInRapo)
+  const checkRapoOnly = _.difference(onlyInRapo, transferred)
+  const both = _.intersection(oodikoneStudents, rapoStudents)
+
+  console.log('vuosi', year)
+
+  if (!brief) {
+    console.log('')
+    console.log('oodikone', oodikoneStudents.length, ' transferred', transferred.length)
+    console.log('rapo', rapoStudents.length, ' activeness wrong rapo', wrongInRapo.length)
+    console.log('both', both.length)
+    console.log('only rapo', checkRapoOnly.length)
+    console.log('only oodikone', checkOodikoneOnly.length)
+
+    console.log('')
+
+    if (checkRapoOnly.length > 0) {
+      console.log('Only in Rapo:')
+      console.log(checkRapoOnly.join('\n'))
+    }
+
+    if (checkOodikoneOnly.length > 0) {
+      console.log('Only in Oodikone:')
+      console.log(checkOodikoneOnly.join('\n'))
+    }
+  }
+
+  if (checkRapoOnly.length + checkOodikoneOnly.length) {
+    console.log('******************************************************')
+  }
+
+  if (!brief) {
+    console.log('')
+  }
+}
+
+const bsc_priogrammes_of_faculties = {
+  H10: ['KH10_001'],
+  H20: ['KH20_001'],
+  H30: ['KH30_001', 'KH30_002'],
+  H40: ['KH40_001', 'KH40_002', 'KH40_003', 'KH40_004', 'KH40_005', 'KH40_006'],
+  H50: ['KH50_001', 'KH50_002', 'KH50_003', 'KH50_004', 'KH50_005', 'KH50_006', 'KH50_007', 'KH50_008'],
+  H55: ['KH55_001'],
+  H60: ['KH60_001'],
+  H70: ['KH70_001', 'KH70_002', 'KH70_003', 'KH70_004'],
+  H74: ['KH74_001'],
+  H80: ['KH80_001', 'KH80_002', 'KH80_003', 'KH80_004'],
+  H90: ['KH90_001'],
+}
+
+const main = async () => {
+  for (let faculty of Object.keys(bsc_priogrammes_of_faculties)) {
+    await faculty_diff(faculty, bsc_priogrammes_of_faculties[faculty], true)
+  }
+
+  //await programme_diff_year('KH55_001', 'H55', 2021, true)
+  //await programme_diff('KH60_001', 'H60', true)
+}
+
+main().then(() => process.exit(0))

services/backend/src/routes/courses.js

@@ -16,7 +16,9 @@ router.get('/v2/coursesmulti', async (req, res) => {
 })
 
 router.get('/v3/courseyearlystats', async (req, res) => {
-  const { rights, roles } = req
+  const {
+    user: { rights, roles },
+  } = req
 
   const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
 

services/backend/src/routes/elementdetails.js

@@ -1,23 +1,6 @@
 const router = require('express').Router()
 const MandatoryCourses = require('../services/mandatoryCourses')
-const {
-  productivityStatsForStudytrack,
-  throughputStatsForStudytrack,
-  optionData,
-} = require('../services/studyprogramme')
-const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
-const {
-  getProductivity,
-  setProductivity,
-  getThroughput,
-  setThroughput,
-  patchProductivity,
-  patchThroughput,
-} = require('../services/analyticsService')
 const { getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
-const logger = require('../util/logger')
-
-const programmeStatsSince = new Date('2017-07-31')
 
 router.get('/elementdetails/all', async (req, res) => {
   const elementdetails = await getAllElementDetails()
@@ -29,22 +12,6 @@ router.get('/studyprogrammes', async (req, res) => {
   res.json(studyrights)
 })
 
-router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
-  const code = req.params.id
-
-  let level
-  if (code.includes('MH')) {
-    level = 'MSC'
-  } else if (code.includes('KH')) {
-    level = 'BSC'
-  } else {
-    return res.json([])
-  }
-
-  const data = await optionData(new Date('2017-07-31'), new Date(), code, level)
-  res.json(data)
-})
-
 router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
   if (req.params.id) {
     const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
@@ -54,138 +21,4 @@ router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
   }
 })
 
-router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    let data = null
-    try {
-      data = await getProductivity(code)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} productivity`)
-    }
-    if (!data) {
-      try {
-        const stats = await productivityStatsForStudytrack(code, programmeStatsSince)
-        data = await setProductivity(stats)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      }
-    }
-    return res.json(data)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-  res.status(200).end()
-
-  try {
-    if (code.includes('MH') || code.includes('KH')) {
-      const data = await productivityStatsForStudytrack(code, new Date('2017-07-31'))
-      await setProductivity(data)
-    } else {
-      const data = await productivityStatsForStudytrack(code, new Date('2000-07-31'))
-      await setProductivity(data)
-    }
-  } catch (e) {
-    try {
-      await patchProductivity({
-        [code]: { status: 'RECALCULATION ERRORED' },
-      })
-    } catch (e) {
-      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      return
-    }
-
-    logger.error(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-  }
-  logger.info(`Produdtiviy stats recalculation for studyprogramme ${code} done`)
-})
-
-router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    const thesisPromise = findProgrammeTheses(req.params.id)
-
-    let data = null
-    try {
-      data = await getThroughput(code)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} throughput`)
-    }
-    if (!data) {
-      try {
-        const result = await throughputStatsForStudytrack(req.params.id, programmeStatsSince.getFullYear())
-        data = await setThroughput(result)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      }
-    }
-    return res.json({ ...data, thesis: await thesisPromise })
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-  res.status(200).end()
-
-  try {
-    if (code.includes('MH') || code.includes('KH')) {
-      const data = await throughputStatsForStudytrack(code, 2017)
-      await setThroughput(data)
-    } else {
-      const data = await throughputStatsForStudytrack(code, 2000)
-      await setThroughput(data)
-    }
-  } catch (e) {
-    try {
-      await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-    } catch (e) {
-      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      return
-    }
-    logger.error(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
-  }
-  logger.info(`Throughput stats recalculation for studyprogramme ${code} done`)
-})
-
-router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {
-  const { id } = req.params
-  if (id) {
-    const thesis = await findProgrammeTheses(id)
-    res.json(thesis)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.post('/v2/studyprogrammes/:id/thesis', async (req, res) => {
-  const { id } = req.params
-  const { course, thesisType } = req.body
-  if (id && course && thesisType) {
-    const thesis = await createThesisCourse(id, course, thesisType)
-    res.status(201).json(thesis)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
-  const { id, course } = req.params
-  if (id && course) {
-    const deleted = await deleteThesisCourse(id, course)
-    res.status(204).json(deleted)
-  } else {
-    res.status(422).end()
-  }
-})
-
 module.exports = router

services/backend/src/routes/feedback.js

@@ -1,28 +1,19 @@
 const router = require('express').Router()
-const { getUserDataFor } = require('../services/userService')
 const { sendFeedbackToToska } = require('../services/mailservice')
 const logger = require('../util/logger')
 
-router.post('/email', async (req, res) => {
-  const { content } = req.body
+router.post('/email', async req => {
   const {
-    user: { userId },
+    body: { content },
+    user,
   } = req
 
-  const { email, full_name } = await getUserDataFor(userId)
-
-  const result = await sendFeedbackToToska({
+  await sendFeedbackToToska({
     feedbackContent: content,
-    userId,
-    userEmail: email,
-    userFullName: full_name,
+    user,
   })
-  if (result.error) {
-    return res.status(400).json(result).end()
-  }
-  logger.info(`${userId} succesfully sent some feedback mail to toska`)
 
-  res.status(200).json('success').end()
+  logger.info(`${user.userId} succesfully sent feedback to toska`)
 })
 
 module.exports = router

services/backend/src/routes/login.js

@@ -1,22 +1,19 @@
 const router = require('express').Router()
 const { ApplicationError } = require('../util/customErrors')
+const _ = require('lodash')
 
 router.get('/login', async (req, res) => {
-  const { user: userFromReq, logoutUrl } = req
+  const { user, logoutUrl } = req
 
-  if (!userFromReq) {
+  if (!user) {
     throw new ApplicationError('User not found', 404)
   }
-  const roles = userFromReq.roles.map(role => role.group_code)
-  const user = {
-    ...userFromReq,
-    roles,
-    isAdmin: roles.includes('admin'),
-  }
 
   res.send({
-    user,
+    // don't send possibly huge list of students to frontend
+    user: _.omit(user, ['studentsUserCanAccess']),
     logoutUrl,
   })
 })
+
 module.exports = router

services/backend/src/routes/population.js

@@ -1,15 +1,16 @@
 const crypto = require('crypto')
 const Sentry = require('@sentry/node')
 const router = require('express').Router()
+const _ = require('lodash')
 const Population = require('../services/populations')
 const Student = require('../services/students')
 const StudyrightService = require('../services/studyrights')
-const UserService = require('../services/userService')
 const TagService = require('../services/tags')
 const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
-const { getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')
-const { logger } = require('../util/logger')
+const { mapToProviders } = require('../util/utils')
+
+const { ApplicationError } = require('../util/customErrors')
 
 const filterPersonalTags = (population, userId) => {
   return {
@@ -80,20 +81,13 @@ router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) =>
   if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
     return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
   }
-  let studentnumberlist
-  const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
+
   const {
-    user: { userId },
-    roles,
+    user: { isAdmin, studentsUserCanAccess },
   } = req
 
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = studentnumbers
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-  }
+  const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
+  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
 
   const result = await Population.bottlenecksOf(
     {
@@ -120,19 +114,12 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
     res.status(400).json({ error: 'The body should have a tag defined' })
     return
   }
-  let studentnumberlist
-  const studentnumbers = await Student.findByTag(tag)
   const {
-    user: { userId },
-    roles,
+    user: { isAdmin, studentsUserCanAccess },
   } = req
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = studentnumbers
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-  }
+  const studentnumbers = await Student.findByTag(tag)
+  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
+
   const result = await Population.bottlenecksOf(
     {
       year: 1900,
@@ -154,27 +141,17 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
 })
 
 router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
-  if (!req.body.studentnumberlist) {
-    res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
-    return
-  }
-  let studentnumberlist
   const {
-    user: { userId, sisPersonId },
-    roles,
+    user: { isAdmin, studentsUserCanAccess },
+    body: { studentnumberlist: studentnumbersInReq },
   } = req
 
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = req.body.studentnumberlist
-  } else if (req.body.usingStudyGuidanceGroups) {
-    const studentsUserCanAccess = await getAllStudentsUserHasInGroups(sisPersonId)
-    studentnumberlist = req.body.studentnumberlist.filter(s => studentsUserCanAccess.has(s))
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
+  if (!studentnumbersInReq) {
+    throw new ApplicationError('The body should have a studentnumberlist defined', 400)
   }
 
+  const studentnumberlist = isAdmin ? studentnumbersInReq : _.intersection(studentnumbersInReq, studentsUserCanAccess)
+
   const result = await Population.bottlenecksOf(
     {
       year: 1900,
@@ -204,13 +181,13 @@ router.get('/v3/populationstatistics', async (req, res) => {
   let studyRights = null
   try {
     studyRights = JSON.parse(studyRightsJSON)
-    const { rights, roles } = req
+    const {
+      user: { rights, isAdmin },
+    } = req
 
-    if (!roles || !roles.includes('admin')) {
-      if (!rights.includes(studyRights.programme)) {
-        res.status(403).json([])
-        return
-      }
+    if (!isAdmin && !rights.includes(studyRights.programme)) {
+      res.status(403).json([])
+      return
     }
   } catch (e) {
     res.status(400).json({ error: 'The query had invalid studyRights' })
@@ -259,27 +236,21 @@ router.get('/v3/populationstatistics', async (req, res) => {
 })
 
 router.get('/v3/populationstatisticsbytag', async (req, res) => {
-  const { tag, studyRights: studyRightsJSON, months, year } = req.query
-  const { user } = req
+  const {
+    user: { id, isAdmin, studentsUserCanAccess },
+    query: { tag, studyRights: studyRightsJSON, months, year },
+  } = req
 
   if (!tag) return res.status(400).json({ error: 'The query should have a tag defined' })
   const foundTag = await TagService.findTagById(tag)
   if (!foundTag) return res.status(404).json({ error: 'Tag not found' })
 
   const semesters = ['FALL', 'SPRING']
-  let studentnumberlist
+
   const studentnumbers = await Student.findByTag(tag)
-  const {
-    user: { userId },
-    roles,
-  } = req
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = studentnumbers
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-  }
+
+  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
+
   const studyRights = JSON.parse(studyRightsJSON)
   const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
   const yearDifference = Number(year) - Number(newStartYear)
@@ -301,12 +272,14 @@ router.get('/v3/populationstatisticsbytag', async (req, res) => {
     return
   }
 
-  res.json(filterPersonalTags(result, user.id))
+  res.json(filterPersonalTags(result, id))
 })
 
 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
-  const { coursecodes, from, to, separate: sep } = req.query
-  const { user, roles } = req
+  const { coursecodes, from, to, separate: sep, unifyCourses } = req.query
+  const {
+    user: { id, isAdmin, studentsUserCanAccess: allStudentsUserCanAccess, rights },
+  } = req
   const separate = sep ? JSON.parse(sep) : false
 
   if (!coursecodes || !from || !to) {
@@ -322,7 +295,13 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
   }
 
   const semesters = ['FALL', 'SPRING']
-  const studentnumbers = await Student.findByCourseAndSemesters(JSON.parse(coursecodes), from, to, separate)
+  const studentnumbers = await Student.findByCourseAndSemesters(
+    JSON.parse(coursecodes),
+    from,
+    to,
+    separate,
+    unifyCourses
+  )
   const result = await Population.optimizedStatisticsOf(
     {
       // Useless, because studentnumbers are already filtered above by from & to.
@@ -334,15 +313,15 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
     },
     studentnumbers
   )
-
-  let studentsUserCanAccess
-  if (roles && roles.includes('admin')) {
-    studentsUserCanAccess = new Set(studentnumbers)
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(user.userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentsUserCanAccess = new Set(await Student.filterStudentnumbersByAccessrights(studentnumbers, codes))
+  let courseproviders = []
+  if (!isAdmin) {
+    courseproviders = await CourseService.getCourseProvidersForCourses(JSON.parse(coursecodes))
   }
+  const rightsMappedToProviders = mapToProviders(rights)
+  const found = courseproviders.some(r => rightsMappedToProviders.includes(r))
+
+  const studentsUserCanAccess =
+    isAdmin || found ? new Set(studentnumbers) : new Set(_.intersection(studentnumbers, allStudentsUserCanAccess))
 
   const randomHash = crypto.randomBytes(12).toString('hex')
   const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
@@ -368,24 +347,16 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
     return
   }
 
-  res.json(filterPersonalTags(result, user.id))
+  res.json(filterPersonalTags(result, id))
 })
 
 router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
-  const { studentnumberlist, usingStudyGuidanceGroups } = req.body
-  const { roles, user } = req
+  const { studentnumberlist } = req.body
+  const {
+    user: { isAdmin, id, studentsUserCanAccess },
+  } = req
 
-  if (usingStudyGuidanceGroups) {
-    if (!user.sisPersonId) {
-      logger.error(`User ${user.userId} tried to get person groups but personId was ${user.sisPersonId} in header`)
-      return res.status(400).json({ error: 'Not possible to get groups without personId' })
-    }
-  }
-
-  const studentsUserCanAccess = usingStudyGuidanceGroups
-    ? await getAllStudentsUserHasInGroups(user.sisPersonId)
-    : await UserService.getStudentsUserCanAccess(studentnumberlist, roles, user.userId)
-  const filteredStudentNumbers = studentnumberlist.filter(s => studentsUserCanAccess.has(s))
+  const filteredStudentNumbers = isAdmin ? studentnumberlist : _.intersection(studentnumberlist, studentsUserCanAccess)
 
   const result = await Population.optimizedStatisticsOf(
     {
@@ -400,12 +371,14 @@ router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
     Sentry.captureException(new Error(result.error))
     return
   }
-  res.status(200).json(filterPersonalTags(result, user.id))
+  res.status(200).json(filterPersonalTags(result, id))
 })
 
 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
-  const { rights, roles } = req
-  if (roles && roles.includes('admin')) {
+  const {
+    user: { rights, roles },
+  } = req
+  if (roles?.includes('admin')) {
     const studyrights = await StudyrightService.getAssociations()
     res.json(studyrights)
   } else {
@@ -421,8 +394,23 @@ router.get('/v3/populationstatistics/studyprogrammes/unfiltered', async (req, re
 
 router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
   const { courseCodes } = req.query
-  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(courseCodes))
-  return res.json(maxYearsToCreatePopulationFrom)
+  const maxYearsToCreatePopulationFromOpen = await CourseService.maxYearsToCreatePopulationFrom(
+    JSON.parse(courseCodes),
+    'openStats'
+  )
+  const maxYearsToCreatePopulationFromUni = await CourseService.maxYearsToCreatePopulationFrom(
+    JSON.parse(courseCodes),
+    'reqularStats'
+  )
+  const maxYearsToCreatePopulationFromBoth = await CourseService.maxYearsToCreatePopulationFrom(
+    JSON.parse(courseCodes),
+    'unifyStats'
+  )
+  return res.json({
+    openCourses: maxYearsToCreatePopulationFromOpen,
+    uniCourses: maxYearsToCreatePopulationFromUni,
+    unifyCourses: maxYearsToCreatePopulationFromBoth,
+  })
 })
 
 module.exports = router

services/backend/src/routes/students.js

@@ -1,8 +1,6 @@
 const router = require('express').Router()
 const Student = require('../services/students')
-const userService = require('../services/userService')
-const Unit = require('../services/units')
-const { getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')
+const { ApplicationError } = require('../util/customErrors')
 
 const filterStudentTags = (student, userId) => {
   return {
@@ -13,8 +11,7 @@ const filterStudentTags = (student, userId) => {
 
 router.get('/students', async (req, res) => {
   const {
-    roles,
-    user: { userId, sisPersonId },
+    user: { isAdmin, studentsUserCanAccess },
     query: { searchTerm },
   } = req
 
@@ -26,66 +23,30 @@ router.get('/students', async (req, res) => {
       .slice(0, 2)
       .find(t => t.length > 3)
   ) {
-    return res.status(400).json({ error: 'at least one search term must be longer than 3 characters' })
+    throw new ApplicationError('at least one search term must be longer than 3 characters', 400)
   }
 
-  if (roles && roles.includes('admin')) {
-    let results = []
-    if (trimmedSearchTerm) {
-      results = await Student.bySearchTerm(trimmedSearchTerm)
-    }
-    return res.status(200).json(results).end()
-  } else {
-    const unitsUserCanAccess = await userService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    let extraStudents = []
-    if (roles.includes('studyGuidanceGroups')) {
-      const matchingOnlyByTerm = await Student.bySearchTerm(trimmedSearchTerm)
-      const studentsUserCanAccess = await getAllStudentsUserHasInGroups(sisPersonId)
-      extraStudents.push(...matchingOnlyByTerm.filter(s => studentsUserCanAccess.has(s.studentNumber)))
-    }
-    const matchingStudents = await Student.bySearchTermAndElements(trimmedSearchTerm, codes)
-    res.json([...matchingStudents, ...extraStudents])
+  let results = []
+  if (trimmedSearchTerm) {
+    results = isAdmin
+      ? await Student.bySearchTerm(trimmedSearchTerm)
+      : await Student.bySearchTermAndStudentNumbers(trimmedSearchTerm, studentsUserCanAccess)
   }
+  res.json(results)
 })
 
 router.get('/students/:id', async (req, res) => {
   const { id: studentId } = req.params
-  const { roles, user } = req
+  const {
+    user: { id, isAdmin, studentsUserCanAccess },
+  } = req
 
-  if (roles?.includes('admin')) {
-    const results = await Student.withId(studentId)
-    return results.error
-      ? res.status(400).json({ error: 'error finding student' }).end()
-      : res.status(200).json(filterStudentTags(results, user.id)).end()
+  if (!isAdmin && !studentsUserCanAccess.includes(studentId)) {
+    throw new ApplicationError('Error finding student', 400)
   }
 
-  const uid = req.user.userId
   const student = await Student.withId(studentId)
-  if (student.error) {
-    return res.status(400).json({ error: 'error finding student' }).end()
-  }
-  const units = await userService.getUnitsFromElementDetails(uid)
-
-  const rights = await Promise.all(
-    units.map(async unit => {
-      const jtn = await Unit.hasStudent(unit.id, student.studentNumber)
-      return jtn
-    })
-  )
-
-  if (roles?.includes('studyGuidanceGroups')) {
-    const studentsUserCanAccess = await getAllStudentsUserHasInGroups(user.sisPersonId)
-    if (studentsUserCanAccess.has(studentId)) {
-      return res.status(200).json(filterStudentTags(student, user.id)).end()
-    }
-  }
-
-  if (rights.some(right => right !== null)) {
-    res.status(200).json(filterStudentTags(student, user.id)).end()
-  } else {
-    res.status(400).json({ error: 'error finding student' }).end()
-  }
+  res.json(filterStudentTags(student, id))
 })
 
 module.exports = router

services/backend/src/routes/studyProgramme.js

@@ -1,11 +1,9 @@
 const router = require('express').Router()
-const {
-  getBasicStatsForStudytrack,
-  getCreditStatsForStudytrack,
-  getGraduationStatsForStudytrack,
-} = require('../services/studyprogrammeStats')
+const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics')
+const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
+const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
 const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
-
+const { getStudyprogrammeCoursesForStudytrack } = require('../services/studyprogrammeCourses')
 const {
   getBasicStats,
   setBasicStats,
@@ -16,16 +14,18 @@ const {
   getStudytrackStats,
   setStudytrackStats,
 } = require('../services/analyticsService')
+const { updateBasicView, updateStudytrackView } = require('../services/studyprogrammeUpdates')
 const logger = require('../util/logger')
 
 router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
 
   if (code) {
     let data = null
     try {
-      data = await getBasicStats(code, yearType)
+      data = await getBasicStats(code, yearType, specialGroups)
     } catch (e) {
       logger.error(`Failed to get code ${code} basic stats: ${e}`)
     }
@@ -33,9 +33,12 @@ router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
       try {
         const result = await getBasicStatsForStudytrack({
           studyprogramme: req.params.id,
-          yearType,
+          settings: {
+            isAcademicYear: yearType === 'ACADEMIC_YEAR',
+            includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
+          },
         })
-        data = await setBasicStats(result, yearType)
+        data = await setBasicStats(result, yearType, specialGroups)
       } catch (e) {
         logger.error(`Failed to update code ${code} basic stats: ${e}`)
       }
@@ -49,11 +52,13 @@ router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
 router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
 
   if (code) {
     let data = null
     try {
-      data = await getCreditStats(code, yearType)
+      data = await getCreditStats(code, yearType, specialGroups)
+      data = null
     } catch (e) {
       logger.error(`Failed to get code ${code} credit stats`)
     }
@@ -61,9 +66,12 @@ router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
       try {
         const result = await getCreditStatsForStudytrack({
           studyprogramme: req.params.id,
-          yearType,
+          settings: {
+            isAcademicYear: yearType === 'ACADEMIC_YEAR',
+            includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
+          },
         })
-        data = await setCreditStats(result, yearType)
+        data = await setCreditStats(result, yearType, specialGroups)
       } catch (e) {
         logger.error(`Failed to update code ${code} credit stats`)
       }
@@ -77,11 +85,12 @@ router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
 router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
 
   if (code) {
     let data = null
     try {
-      data = await getGraduationStats(code, yearType)
+      data = await getGraduationStats(code, yearType, specialGroups)
     } catch (e) {
       logger.error(`Failed to get code ${code} graduation stats`)
     }
@@ -89,9 +98,12 @@ router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
       try {
         const result = await getGraduationStatsForStudytrack({
           studyprogramme: req.params.id,
-          yearType,
+          settings: {
+            isAcademicYear: yearType === 'ACADEMIC_YEAR',
+            includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
+          },
         })
-        data = await setGraduationStats(result, yearType)
+        data = await setGraduationStats(result, yearType, specialGroups)
       } catch (e) {
         logger.error(`Failed to update code ${code} graduation stats`)
       }
@@ -102,20 +114,61 @@ router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
   }
 })
 
-router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
+router.get('/v2/studyprogrammes/:id/coursestats', async (req, res) => {
   const code = req.params.id
+  const showByYear = req.query?.academicyear
+  const date = new Date()
+  const endOfToday = new Date()
+  endOfToday.setHours(23, 59, 59, 999)
+
+  if (isNaN(date.getTime()) || date.getTime() > endOfToday.getTime()) {
+    return res.status(400).json({ error: 'Invalid date' })
+  }
+  // End of day
+  date.setHours(23, 59, 59, 999)
 
   if (code) {
     let data = null
     try {
-      data = await getStudytrackStats(code)
+      data = null
+    } catch (e) {
+      logger.error(`Failed to get code ${code} graduation stats`)
+    }
+    if (!data) {
+      try {
+        data = await getStudyprogrammeCoursesForStudytrack(date.getTime(), code, showByYear)
+      } catch (e) {
+        logger.error(`Failed to get code ${code} programme courses stats: ${e}`)
+      }
+      return res.json(data)
+    } else {
+      res.status(422).end()
+    }
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
+  const code = req.params.id
+  const graduated = req.query?.graduated
+  const specialGroups = req.query?.special_groups
+
+  if (code) {
+    let data = null
+    try {
+      data = await getStudytrackStats(code, graduated, specialGroups)
     } catch (e) {
       logger.error(`Failed to get code ${code} studytrack stats: ${e}`)
     }
     if (!data) {
       try {
-        const result = await getStudytrackStatsForStudyprogramme({ studyprogramme: req.params.id })
-        data = await setStudytrackStats(result)
+        const result = await getStudytrackStatsForStudyprogramme({
+          studyprogramme: code,
+          settings: {
+            graduated: graduated === 'GRADUATED_INCLUDED',
+            specialGroups: specialGroups === 'SPECIAL_INCLUDED',
+          },
+        })
+        data = await setStudytrackStats(result, graduated, specialGroups)
       } catch (e) {
         logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
       }
@@ -126,4 +179,34 @@ router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
   }
 })
 
+router.get('/v2/studyprogrammes/:id/update_basicview', async (req, res) => {
+  const code = req.params.id
+  if (code) {
+    let result = null
+    try {
+      result = await updateBasicView(code)
+    } catch (e) {
+      logger.error(`Failed to update code ${code} basic stats: ${e}`)
+    }
+    return res.json(result)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/update_studytrackview', async (req, res) => {
+  const code = req.params.id
+  if (code) {
+    let result = null
+    try {
+      result = await updateStudytrackView(code)
+    } catch (e) {
+      logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
+    }
+    return res.json(result)
+  } else {
+    res.status(422).end()
+  }
+})
+
 module.exports = router

services/backend/src/routes/tags.js

@@ -12,44 +12,50 @@ const filterRelevantStudentTags = (studentTags, userId) => {
   return studentTags.filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
 }
 
-router.get('/tags', async (req, res) => {
+router.get('/tags', async (_req, res) => {
   const tags = await Tags.findTags()
   res.status(200).json(tags)
 })
 
 router.get('/tags/:studytrack', async (req, res) => {
   const { studytrack } = req.params
-  const { rights, roles, user } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const tags = await Tags.findTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantTags(tags, user.id))
+  res.status(200).json(filterRelevantTags(tags, id))
 })
 
 router.post('/tags', async (req, res) => {
   const {
     tag: { studytrack, tagname, year, personal_user_id },
   } = req.body
-  const { rights, roles, user } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
   const tags = await Tags.findTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantTags(tags, user.id))
+  res.status(200).json(filterRelevantTags(tags, id))
 })
 
 router.delete('/tags', async (req, res) => {
   const { tag } = req.body
-  const { rights, roles, user } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(tag.studytrack) && !(roles && roles.includes('admin')) && !(tag.personal_user_id === user.id))
+  if (!rights.includes(tag.studytrack) && !roles?.includes('admin') && !(tag.personal_user_id === id))
     return res.status(403).end()
 
   await Tags.deleteTag(tag)
   const t = await Tags.findTagsByStudytrack(tag.studytrack)
-  res.status(200).json(filterRelevantTags(t, user.id))
+  res.status(200).json(filterRelevantTags(t, id))
 })
 
 router.get('/studenttags', async (req, res) => {
@@ -59,12 +65,14 @@ router.get('/studenttags', async (req, res) => {
 
 router.get('/studenttags/:studytrack', async (req, res) => {
   const { studytrack } = req.params
-  const { rights, roles, user } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, user.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.get('/studenttags/:studentnumber', async (req, res) => {
@@ -76,20 +84,24 @@ router.get('/studenttags/:studentnumber', async (req, res) => {
 
 router.post('/studenttags/:studentnumber', async (req, res) => {
   const { tag, studytrack } = req.body
-  const { rights, roles, user } = req
+  const {
+    user: { roles, rights, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   await TagStudent.createStudentTag(tag)
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, user.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.post('/studenttags', async (req, res) => {
   const { tags, studytrack } = req.body
-  const { rights, roles, user } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const existingtags = await Tags.findTagsByStudytrack(studytrack)
   const existingTagids = existingtags.map(t => t.tag_id)
@@ -106,35 +118,39 @@ router.post('/studenttags', async (req, res) => {
 
   await TagStudent.createMultipleStudentTags(tags)
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, user.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.delete('/studenttags/delete_one', async (req, res) => {
   const { tag_id, studentnumber, studytrack } = req.body
-  const { rights, roles, user } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
   if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
 
   await TagStudent.deleteStudentTag(studentnumber, tag_id)
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, user.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.delete('/studenttags/delete_many', async (req, res) => {
   const { tagId, studentnumbers, studytrack } = req.body
-  const { rights, roles, user } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
   if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
 
   await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, user.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 module.exports = router

services/backend/src/routes/teachers.js

@@ -42,7 +42,9 @@ router.get('/top/categories', async (req, res) => {
 })
 
 router.get('/stats', async (req, res) => {
-  const { rights, roles } = req
+  const {
+    user: { rights, roles },
+  } = req
 
   const { providers, semesterStart, semesterEnd } = req.query
   if (!providers || !semesterStart) {

services/backend/src/services/analyticsService.js

@@ -1,183 +1,104 @@
 const { redisClient } = require('../services/redis')
 const moment = require('moment')
 
-const createRedisKeyForProductivity = id => `PRODUCTIVITY_${id}`
-const createRedisKeyForThroughput = id => `THROUGHPUT_${id}`
-const createRedisKeyForBasicStats = (id, yearType) => `BASIC_STATS_${id}_${yearType}`
-const createRedisKeyForCreditStats = (id, yearType) => `CREDIT_STATS_${id}_${yearType}`
-const createRedisKeyForGraduationStats = (id, yearType) => `GRADUATION_STATS_${id}_${yearType}`
-const createRedisKeyForStudytrackStats = id => `STUDYTRACK_STATS_${id}`
+// Only new bachelor, masters and doctoral programmes get their data updated in redis every night, use redis for them
+const isUpdatedNewProgramme = code => code.includes('KH') || code.includes('MH') || /^(T)[0-9]{6}$/.test(code)
+const createRedisKeyForBasicStats = (id, yearType, specialGroups) => `BASIC_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForCreditStats = (id, yearType, specialGroups) => `CREDIT_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForGraduationStats = (id, yearType, specialGroups) =>
+  `GRADUATION_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForStudytrackStats = (id, graduated, specialGroups) =>
+  `STUDYTRACK_STATS_${id}_${graduated}_${specialGroups}`
 
-const getProductivity = async id => {
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return {
-    [id]: JSON.parse(dataFromRedis),
-  }
-}
-
-const setProductivity = async data => {
-  const { id } = data
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: dataToRedis,
-  }
-}
-
-const patchProductivity = async data => {
-  const { id } = data
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataFromRedis = JSON.parse(await redisClient.getAsync(redisKey))
-  const patchedData = {
-    ...dataFromRedis,
-    ...data,
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(patchedData))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: patchedData,
-  }
-}
-
-const getThroughput = async id => {
-  const redisKey = createRedisKeyForThroughput(id)
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return {
-    [id]: JSON.parse(dataFromRedis),
-  }
-}
-
-const setThroughput = async data => {
-  const { id, data: dataToSet } = data
-  const redisKey = createRedisKeyForThroughput(id)
-  const dataToRedis = {
-    data: dataToSet.years,
-    totals: dataToSet.totals,
-    stTotals: dataToSet.stTotals,
-    status: 'DONE',
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: dataToRedis,
-  }
-}
-
-const patchThroughput = async data => {
-  const [id, dataToPatch] = Object.entries(data)[0]
-  const redisKey = createRedisKeyForThroughput(id)
-  const { status } = dataToPatch
-  const dataFromRedis = JSON.parse(await redisClient.getAsync(redisKey))
-  const patchedData = {
-    ...dataFromRedis,
-    status,
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(patchedData))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: patchedData,
-  }
-}
-
-const getBasicStats = async (id, yearType) => {
-  const redisKey = createRedisKeyForBasicStats(id, yearType)
+const getBasicStats = async (id, yearType, specialGroups) => {
+  if (!isUpdatedNewProgramme(id)) return null
+  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
 }
 
-const setBasicStats = async (data, yearType) => {
+const setBasicStats = async (data, yearType, specialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForBasicStats(id, yearType)
+  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
   const dataToRedis = {
     ...data,
     status: 'DONE',
     lastUpdated: moment().format(),
   }
+  if (!isUpdatedNewProgramme(id)) return dataToRedis
   const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
   if (setOperationStatus !== 'OK') return null
   return dataToRedis
 }
 
-const getCreditStats = async (id, yearType) => {
-  const redisKey = createRedisKeyForCreditStats(id, yearType)
+const getCreditStats = async (id, yearType, specialGroups) => {
+  if (!isUpdatedNewProgramme(id)) return null
+  const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
 }
 
-const setCreditStats = async (data, yearType) => {
+const setCreditStats = async (data, yearType, specialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForCreditStats(id, yearType)
+  const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
   const dataToRedis = {
     ...data,
     status: 'DONE',
     lastUpdated: moment().format(),
   }
+  if (!isUpdatedNewProgramme(id)) return dataToRedis
   const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
   if (setOperationStatus !== 'OK') return null
   return dataToRedis
 }
 
-const getGraduationStats = async (id, yearType) => {
-  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+const getGraduationStats = async (id, yearType, specialGroups) => {
+  if (!isUpdatedNewProgramme(id)) return null
+  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
 }
 
-const setGraduationStats = async (data, yearType) => {
+const setGraduationStats = async (data, yearType, specialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
   const dataToRedis = {
     ...data,
     status: 'DONE',
     lastUpdated: moment().format(),
   }
+  if (!isUpdatedNewProgramme(id)) return dataToRedis
   const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
   if (setOperationStatus !== 'OK') return null
   return dataToRedis
 }
 
-const getStudytrackStats = async id => {
-  const redisKey = createRedisKeyForStudytrackStats(id)
+const getStudytrackStats = async (id, graduated, specialGroups) => {
+  if (!isUpdatedNewProgramme(id)) return null
+  const redisKey = createRedisKeyForStudytrackStats(id, graduated, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
 }
 
-const setStudytrackStats = async data => {
+const setStudytrackStats = async (data, graduated, specialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForStudytrackStats(id)
+  const redisKey = createRedisKeyForStudytrackStats(id, graduated, specialGroups)
   const dataToRedis = {
     ...data,
     status: 'DONE',
     lastUpdated: moment().format(),
   }
+  if (!isUpdatedNewProgramme(id)) return dataToRedis
   const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
   if (setOperationStatus !== 'OK') return null
   return dataToRedis
 }
 
 module.exports = {
-  getProductivity,
-  setProductivity,
-  patchProductivity,
-  getThroughput,
-  setThroughput,
-  patchThroughput,
   getBasicStats,
   setBasicStats,
   getCreditStats,

services/backend/src/services/cache.js

@@ -1,12 +0,0 @@
-const LRU = require('lru-cache')
-
-// Max 25 users can be stored in the cache
-const userDataCache = new LRU({
-  max: 25,
-  length: () => 1,
-  maxAge: 1000 * 60 * 60,
-})
-
-module.exports = {
-  userDataCache,
-}

services/backend/src/services/course_stats_counter.js

@@ -40,18 +40,22 @@ class CourseStatsCounter {
       triedOfPopulation: undefined,
       passingSemesters: this.initializePassingSemesters(),
     }
+    this.enrollments = {
+      enrolledNoGrade: new Set(),
+      semesters: this.initializePassingSemesters({}),
+    }
     this.grades = {}
   }
 
-  initializePassingSemesters() {
+  initializePassingSemesters(initialValue = 0) {
     const passingSemesters = {
-      BEFORE: 0,
-      LATER: 0,
+      BEFORE: initialValue,
+      LATER: initialValue,
     }
 
     for (let i = 0; i < 7; i++) {
-      passingSemesters[fall[i]] = 0
-      passingSemesters[spring[i]] = 0
+      passingSemesters[fall[i]] = initialValue
+      passingSemesters[spring[i]] = initialValue
     }
 
     return passingSemesters
@@ -91,6 +95,13 @@ class CourseStatsCounter {
     }
   }
 
+  markEnrollment(studentnumber, state, semester) {
+    if (!this.enrollments[state]) this.enrollments[state] = new Set()
+    this.enrollments[state].add(studentnumber)
+    if (!this.enrollments.semesters[semester][state]) this.enrollments.semesters[semester][state] = new Set()
+    this.enrollments.semesters[semester][state].add(studentnumber)
+  }
+
   failedBefore(studentnumber) {
     return this.students.failed[studentnumber] !== undefined
   }
@@ -173,6 +184,10 @@ class CourseStatsCounter {
   getFinalStats() {
     const stats = this.stats
     const students = this.students
+    const allStudents = Object.keys(students.all)
+    const totalEnrolledNoGrade = [...(this.enrollments.ENROLLED || [])].filter(
+      student => !allStudents.includes(student)
+    ).length
 
     stats.students = lengthOf(students.all)
     stats.passed = lengthOf(students.passed)
@@ -185,6 +200,9 @@ class CourseStatsCounter {
     stats.triedOfPopulation = percentageOf(stats.students, this.studentsInTotal)
     stats.perStudent = stats.attempts / (stats.passed + stats.failed)
     stats.passingSemestersCumulative = this.getPassingSemestersCumulative()
+    stats.totalStudents = stats.students + totalEnrolledNoGrade
+    stats.totalEnrolledNoGrade = totalEnrolledNoGrade
+    stats.percentageWithEnrollments = percentageOf(stats.passed, stats.totalStudents)
 
     return {
       stats,

services/backend/src/services/course_yearly_stats_counter.js

@@ -8,12 +8,23 @@ class CourseYearlyStatsCounter {
   }
 
   initProgramme(code, name) {
-    this.programmes[code] = { name, students: {}, passed: {}, credits: {} }
+    this.programmes[code] = {
+      name,
+      students: {},
+      passed: {},
+      credits: {},
+    }
   }
 
   initFacultyYear(code) {
     const year = `${1949 + Number(code)}-${1950 + Number(code)}`
-    this.facultyStats[code] = { year, allStudents: [], allPassed: [], faculties: {}, allCredits: 0 }
+    this.facultyStats[code] = {
+      year,
+      allStudents: [],
+      allPassed: [],
+      faculties: {},
+      allCredits: 0,
+    }
   }
 
   initFaculty(yearcode, faculty_code, organization) {
@@ -43,8 +54,11 @@ class CourseYearlyStatsCounter {
           passedEventually: [],
           neverPassed: [],
         },
+        grades: {},
         studentnumbers: [],
       },
+      enrollments: [],
+      allEnrollments: [],
       yearcode,
     }
   }
@@ -97,11 +111,41 @@ class CourseYearlyStatsCounter {
     })
   }
 
+  markStudyProgrammesEnrollment(studentnumber, programmes, yearcode, state, enrollment_date_time) {
+    programmes.forEach(({ code, name, faculty_code, organization }) => {
+      this.markStudyProgrammeEnrollment(
+        code,
+        name,
+        studentnumber,
+        yearcode,
+        state,
+        enrollment_date_time,
+        faculty_code,
+        organization
+      )
+    })
+  }
+
   markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode) {
     if (!this.groups[groupcode]) {
       this.initGroup(groupcode, groupname, coursecode, yearcode)
     }
     this.markCreditToAttempts(studentnumber, passed, grade, groupcode)
+    this.markBestEffortGrade(studentnumber, passed, grade, groupcode)
+  }
+
+  markEnrollmentToGroup(studentnumber, state, enrollment_date_time, groupcode, groupname, coursecode, yearcode) {
+    if (!this.groups[groupcode]) this.initGroup(groupcode, groupname, coursecode, yearcode)
+
+    const enrollment = { studentnumber, state, enrollment_date_time }
+    this.groups[groupcode].allEnrollments.push(enrollment)
+    const oldEnrollment = this.groups[groupcode].enrollments.find(e => e.studentnumber === studentnumber)
+    if (!oldEnrollment) return this.groups[groupcode].enrollments.push({ studentnumber, state, enrollment_date_time })
+    if (oldEnrollment.state === 'ENROLLED' || oldEnrollment.state === 'CONFIRMED') return
+    if (state !== 'ENROLLED' || state !== 'CONFIRMED') return
+    this.groups[groupcode].enrollments = this.groups[groupcode].enrollments
+      .filter(e => e.studentnumber !== studentnumber)
+      .concat([enrollment])
   }
 
   markCreditToAttempts(studentnumber, passed, grade, groupcode) {
@@ -118,6 +162,19 @@ class CourseYearlyStatsCounter {
     }
   }
 
+  markBestEffortGrade(studentnumber, passed, grade, groupcode) {
+    const current = this.groups[groupcode].students.grades[studentnumber]
+
+    if (!current) {
+      this.groups[groupcode].students.grades[studentnumber] = { grade, passed }
+      return
+    }
+
+    if (current.passed && !passed) return
+    if (current.passed && Number(current.grade) >= Number(grade)) return
+    this.groups[groupcode].students.grades[studentnumber] = { grade, passed }
+  }
+
   markCreditToStudentCategories(studentnumber, passed, attainment_date, groupcode) {
     if (!this.students.has(studentnumber)) {
       if (passed) {
@@ -183,8 +240,17 @@ class CourseYearlyStatsCounter {
     }
 
     const groupStatistics = Object.values(this.groups).map(({ ...rest }) => {
+      const { students } = rest
+      const grades = {}
+      Object.keys(students.grades).forEach(student => {
+        const { grade, passed } = students.grades[student]
+        const parsed = passed ? grade : '0'
+        if (!grades[parsed]) grades[parsed] = []
+        grades[parsed].push(student)
+      })
       const normalStats = {
         ...rest,
+        students: { ...students, grades },
       }
 
       if (anonymizationSalt && normalStats.students.studentnumbers.length < 6) {

services/backend/src/services/courses.js

@@ -8,12 +8,14 @@ const {
   Studyright,
   Semester,
   Organization,
+  Enrollment,
 } = require('../models')
 
 const { parseCredit } = require('./parseCredits')
 const Op = Sequelize.Op
 const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
 const { sortMainCode, getSortRank } = require('../util/utils')
+const { parseEnrollment } = require('./parseEnrollments')
 
 const byNameOrCode = (searchTerm, language) =>
   Course.findAll({
@@ -59,10 +61,12 @@ const findOneByCode = code => {
   })
 }
 
-const creditsForCourses = async (codes, anonymizationSalt, unifyOpenUniCourses) => {
+const creditsForCourses = async (codes, anonymizationSalt, unification) => {
   let is_open = false
 
-  if (unifyOpenUniCourses) {
+  if (unification === 'open') is_open = true
+
+  if (unification === 'unify') {
     is_open = {
       [Op.in]: [false, true],
     }
@@ -122,6 +126,71 @@ const creditsForCourses = async (codes, anonymizationSalt, unifyOpenUniCourses)
   return parsedCredits
 }
 
+const enrollmentsForCourses = async (codes, anonymizationSalt, unification) => {
+  let is_open = false
+
+  if (unification === 'open') is_open = true
+
+  if (unification === 'unify') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
+  const enrollments = await Enrollment.findAll({
+    include: [
+      {
+        model: Student,
+        attributes: ['studentnumber'],
+        include: {
+          model: StudyrightElement,
+          attributes: ['code', 'startdate'],
+          include: [
+            {
+              model: ElementDetail,
+              attributes: ['name', 'type'],
+              where: {
+                type: {
+                  [Op.eq]: 20,
+                },
+              },
+            },
+            {
+              model: Studyright,
+              attributes: ['prioritycode', 'faculty_code'],
+              where: {
+                prioritycode: {
+                  [Op.eq]: 1,
+                },
+              },
+              include: {
+                model: Organization,
+              },
+            },
+          ],
+        },
+      },
+      {
+        model: Semester,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+      },
+    ],
+    where: {
+      course_code: {
+        [Op.in]: codes,
+      },
+      studentnumber: {
+        [Op.ne]: null,
+      },
+      enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
+      state: ['ENROLLED', 'CONFIRMED', 'REJECTED', 'ABORTED_BY_STUDENT', 'ABORTED_BY_TEACHER'],
+      [Op.or]: [{ is_open }, { is_open: null }],
+    },
+  })
+
+  return enrollments.map(enrollment => parseEnrollment(enrollment, anonymizationSalt))
+}
+
 const bySearchTerm = async (term, language) => {
   const formatCourse = course => ({
     name: course.name[language],
@@ -162,23 +231,20 @@ const allCodeAlternatives = async code => {
   return sortMainCode([...course.substitutions, code])
 }
 
-const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anonymizationSalt) => {
+const yearlyStatsOfNew = async (coursecode, separate, unification, anonymizationSalt) => {
   const courseForSubs = await Course.findOne({
     where: { code: coursecode },
   })
 
   let codes = courseForSubs.substitutions ? sortMainCode([...courseForSubs.substitutions, coursecode]) : [coursecode]
 
-  if (isOpenUniCourseCode(coursecode) && !unifyOpenUniCourses) {
-    codes = [coursecode]
+  if (unification === 'reqular') {
+    codes = codes.filter(course => !isOpenUniCourseCode(course))
   }
 
-  if (!isOpenUniCourseCode(coursecode) && !unifyOpenUniCourses) {
-    codes = codes.filter(code => !code.match(/^[A][0-9]|^AY/))
-  }
-
-  const [credits, course] = await Promise.all([
-    creditsForCourses(codes, anonymizationSalt, unifyOpenUniCourses),
+  const [credits, enrollments, course] = await Promise.all([
+    creditsForCourses(codes, anonymizationSalt, unification),
+    enrollmentsForCourses(codes, anonymizationSalt, unification),
     Course.findOne({
       where: {
         code: coursecode,
@@ -233,7 +299,27 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
     counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode)
     counter.markCreditToStudentCategories(studentnumber, passed, attainment_date, groupcode)
   }
+
+  enrollments.forEach(enrollment => {
+    const { studentnumber, semestercode, semestername, yearcode, yearname, coursecode, state, enrollment_date_time } =
+      enrollment
+
+    const groupcode = separate ? semestercode : yearcode
+    const groupname = separate ? semestername : yearname
+
+    counter.markEnrollmentToGroup(
+      studentnumber,
+      state,
+      enrollment_date_time,
+      groupcode,
+      groupname,
+      coursecode,
+      yearcode
+    )
+  })
+
   const statistics = counter.getFinalStatistics(anonymizationSalt)
+
   return {
     ...statistics,
     coursecode,
@@ -242,7 +328,17 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
   }
 }
 
-const maxYearsToCreatePopulationFrom = async coursecodes => {
+const maxYearsToCreatePopulationFrom = async (coursecodes, unifyCourses) => {
+  let is_open = false
+
+  if (unifyCourses === 'openStats') is_open = true
+
+  if (unifyCourses === 'unifyStats') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
   const maxAttainmentDate = new Date(
     Math.max(
       ...(
@@ -257,6 +353,7 @@ const maxYearsToCreatePopulationFrom = async coursecodes => {
       ).map(c => new Date(c.max_attainment_date).getTime())
     )
   )
+
   const attainmentThreshold = new Date(maxAttainmentDate.getFullYear(), 0, 1)
   attainmentThreshold.setFullYear(attainmentThreshold.getFullYear() - 6)
 
@@ -268,6 +365,7 @@ const maxYearsToCreatePopulationFrom = async coursecodes => {
       attainment_date: {
         [Op.gt]: attainmentThreshold,
       },
+      is_open,
     },
     order: [['attainment_date', 'ASC']],
   })
@@ -291,11 +389,18 @@ const maxYearsToCreatePopulationFrom = async coursecodes => {
   return maxYearsToCreatePopulationFrom
 }
 
-const courseYearlyStats = async (coursecodes, separate, unifyOpenUniCourses, anonymizationSalt) => {
-  const stats = await Promise.all(
-    coursecodes.map(code => yearlyStatsOfNew(code, separate, unifyOpenUniCourses, anonymizationSalt))
+const courseYearlyStats = async (coursecodes, separate, anonymizationSalt) => {
+  const statsReqular = await Promise.all(
+    coursecodes.map(async code => {
+      const unifyStats = await yearlyStatsOfNew(code, separate, 'unify', anonymizationSalt)
+      const reqularStats = await yearlyStatsOfNew(code, separate, 'reqular', anonymizationSalt)
+      const openStats = await yearlyStatsOfNew(code, separate, 'open', anonymizationSalt)
+
+      return { unifyStats, reqularStats, openStats }
+    })
   )
-  return stats
+
+  return statsReqular
 }
 
 const nameLikeTerm = name => {
@@ -381,6 +486,23 @@ const byNameAndOrCodeLike = async (name, code) => {
   return { courses }
 }
 
+const getCourseProvidersForCourses = async codes => {
+  return (
+    await Organization.findAll({
+      attributes: ['code'],
+      include: {
+        model: Course,
+        where: {
+          code: {
+            [Op.in]: codes,
+          },
+        },
+      },
+      raw: true,
+    })
+  ).map(({ code }) => code)
+}
+
 module.exports = {
   byCode,
   byName,
@@ -393,4 +515,5 @@ module.exports = {
   unifyOpenUniversity,
   allCodeAlternatives,
   findOneByCode,
+  getCourseProvidersForCourses,
 }

services/backend/src/services/mailservice.js

@@ -1,9 +1,9 @@
 const axios = require('axios')
 const { isProduction, pateToken } = require('../conf-backend')
-const logger = require('../util/logger')
+const { ApplicationError } = require('../util/customErrors')
 
 const pateClient = axios.create({
-  baseURL: 'https://pate.toska.cs.helsinki.fi',
+  baseURL: 'https://importer.cs.helsinki.fi/api/pate/',
   params: {
     token: pateToken,
   },
@@ -20,33 +20,22 @@ const baseSettings = {
 }
 
 const sendEmail = async (options = {}) => {
-  if (!isProduction) {
-    const errorMessage = 'Skipped sending email in non-production environment'
-    logger.error({ message: errorMessage, meta: options })
-    return { error: errorMessage }
-  }
-
-  if (!pateToken) {
-    const errorMessage = 'Skipped sending email since pate token was not available'
-    logger.error({ message: errorMessage, meta: options })
-    return { error: errorMessage }
-  }
-
-  const { data } = await pateClient.post('/', options)
-
-  return data
+  if (!isProduction) throw new ApplicationError('Email sending is disabled in development mode.')
+  if (!pateToken) throw new ApplicationError('Email sending failed because pate token is missing.')
+  return await pateClient.post('/', options)
 }
 
-const sendFeedbackToToska = ({ feedbackContent, userId, userEmail, userFullName }) => {
-  const userDetails = `Sent by ${userFullName}, userid: ${userId}, email: ${userEmail}`
+const sendFeedbackToToska = async ({ feedbackContent, user }) => {
+  const { name, userId, email } = user
+  const userDetails = `Sent by ${name}, userid: ${userId}, email: ${email}`
   const text = [feedbackContent, userDetails].join('<br />')
-  return sendEmail({
+  await sendEmail({
     template,
     emails: [
       {
         to: 'Toska <grp-toska@helsinki.fi>',
-        replyTo: userEmail,
-        subject: `Oodikone feedback from ${userFullName}`,
+        replyTo: email,
+        subject: `Oodikone feedback from ${name}`,
         text,
       },
     ],

services/backend/src/services/newStudyprogramme.js

@@ -1,338 +0,0 @@
-const sequelize = require('sequelize')
-const { Op } = sequelize
-const {
-  Credit,
-  Course,
-  Organization,
-  Studyright,
-  StudyrightElement,
-  ElementDetail,
-  Transfer,
-  Student,
-  SemesterEnrollment,
-  Semester,
-} = require('../models')
-const { formatStudyright, formatStudent } = require('./studyprogrammeHelpers')
-
-const whereStudents = studentnumbers => {
-  return studentnumbers ? studentnumbers : { [Op.not]: null }
-}
-
-const studytrackStudents = async studentnumbers =>
-  (
-    await Student.findAll({
-      where: {
-        studentnumber: {
-          [Op.in]: studentnumbers,
-        },
-      },
-    })
-  ).map(formatStudent)
-
-const enrolledStudyrights = async studytrack => {
-  const studyrights = await Studyright.findAll({
-    attributes: ['studyrightid', 'studystartdate', 'enddate', 'prioritycode'],
-    include: [
-      {
-        model: StudyrightElement,
-        required: true,
-        where: { code: studytrack },
-      },
-      {
-        model: Student,
-        attributes: ['studentnumber'],
-        required: true,
-      },
-    ],
-  })
-
-  const studentnumbers = studyrights.map(s => s.student.studentnumber)
-
-  const students = await Student.findAll({
-    attributes: ['studentnumber'],
-    include: [
-      {
-        model: Studyright,
-        include: [
-          {
-            model: StudyrightElement,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        ],
-        attributes: ['studyrightid'],
-      },
-      {
-        model: SemesterEnrollment,
-        attributes: ['semestercode'],
-        include: [
-          {
-            model: Semester,
-            required: true,
-          },
-        ],
-        where: {
-          enrollmenttype: 1,
-        },
-      },
-    ],
-    where: {
-      studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-    },
-  })
-
-  let enrollments = new Set()
-
-  students.forEach(student => {
-    student.semester_enrollments.forEach(e => {
-      const enrollmentStartDate = e.semester.startdate
-      const enrollmentEndDate = e.semester.enddate
-      const correctStudyRight = studyrights.find(
-        s =>
-          student.studentnumber === s.student.studentnumber &&
-          s.studystartdate <= enrollmentStartDate && // Studying in the programme should have started on the same day or earlier than the enrollment
-          s.enddate >= enrollmentEndDate && // Studying in the programme should have ended on the same day or later than the enrollment
-          (!s.canceldate || s.canceldate > enrollmentStartDate) && // Studyright should not be canceled OR the possible canceldate should be after the enrollmentStartDate
-          (s.prioritycode === 1 || s.prioritycode === 30) // Studyright should be primary or the student should have graduated
-      )
-      if (correctStudyRight) {
-        enrollments.add({
-          studentnumber: student.studentnumber,
-          startdate: e.semester.startdate,
-        })
-      }
-    })
-  })
-
-  return enrollments
-}
-
-const startedStudyrights = async (studytrack, since, studentnumbers) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        studystartdate: {
-          [Op.gte]: since,
-        },
-        student_studentnumber: whereStudents(studentnumbers),
-      },
-    })
-  ).map(formatStudyright)
-
-const graduatedStudyRights = async (studytrack, since, studentnumbers) =>
-  await Studyright.findAll({
-    include: {
-      model: StudyrightElement,
-      required: true,
-      include: {
-        model: ElementDetail,
-        required: true,
-        where: {
-          code: studytrack,
-        },
-      },
-    },
-    where: {
-      graduated: 1,
-      enddate: {
-        [Op.gte]: since,
-      },
-      student_studentnumber: whereStudents(studentnumbers),
-    },
-  })
-
-const cancelledStudyRights = async (studytrack, since) => {
-  return await Studyright.findAll({
-    include: {
-      model: StudyrightElement,
-      required: true,
-      where: {
-        code: {
-          [Op.eq]: studytrack,
-        },
-      },
-    },
-    where: {
-      canceldate: {
-        [Op.gte]: since,
-      },
-    },
-  })
-}
-
-const transfersAway = async (studytrack, since) => {
-  return await Transfer.findAll({
-    where: {
-      transferdate: {
-        [Op.gte]: since,
-      },
-      sourcecode: studytrack,
-    },
-    distinct: true,
-    col: 'studentnumber',
-  })
-}
-
-const transfersTo = async (studytrack, since) => {
-  return await Transfer.findAll({
-    where: {
-      transferdate: {
-        [Op.gte]: since,
-      },
-      targetcode: studytrack,
-    },
-    distinct: true,
-    col: 'studentnumber',
-  })
-}
-
-const getProgrammesStudents = async studyprogramme =>
-  (
-    await Studyright.findAll({
-      attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode'],
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          where: {
-            code: {
-              [Op.in]: [studyprogramme],
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-    })
-  ).map(formatStudyright)
-
-const getCreditsForStudyProgramme = async (provider, since) =>
-  await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
-      where: {
-        is_study_module: false,
-      },
-      include: {
-        model: Organization,
-        required: true,
-        where: {
-          code: provider,
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.notIn]: [10, 9, 7],
-      },
-      isStudyModule: {
-        [Op.not]: true,
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-    },
-  })
-
-const getTransferredCredits = async (provider, since) =>
-  await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'credittypecode', 'student_studentnumber'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
-      where: {
-        is_study_module: false,
-      },
-      include: {
-        model: Organization,
-        required: true,
-        where: {
-          code: provider,
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.eq]: [9],
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-    },
-  })
-
-const getThesisCredits = async (provider, since, thesisType) =>
-  await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
-      where: {
-        course_unit_type: {
-          [Op.in]: thesisType,
-        },
-      },
-      include: {
-        model: Organization,
-        required: true,
-        where: {
-          code: provider,
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.notIn]: [10, 9, 7],
-      },
-      isStudyModule: {
-        [Op.not]: true,
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-    },
-  })
-
-module.exports = {
-  studytrackStudents,
-  enrolledStudyrights,
-  startedStudyrights,
-  graduatedStudyRights,
-  cancelledStudyRights,
-  transfersAway,
-  transfersTo,
-  getProgrammesStudents,
-  getCreditsForStudyProgramme,
-  getTransferredCredits,
-  getThesisCredits,
-}

services/backend/src/services/parseCredits.js

@@ -65,4 +65,4 @@ const parseCredit = (credit, anonymizationSalt) => {
   }
 }
 
-module.exports = { parseCredit }
+module.exports = { parseCredit, formatStudyrightElement }

services/backend/src/services/parseEnrollments.js

@@ -0,0 +1,48 @@
+const crypto = require('crypto')
+const { formatStudyrightElement } = require('./parseCredits')
+
+const parseEnrollment = (enrollment, anonymizationSalt) => {
+  const { student, semester, state, enrollment_date_time, course_code } = enrollment
+  const { studentnumber, studyright_elements } = student
+  const { yearcode, yearname, semestercode, name: semestername } = semester
+
+  if (anonymizationSalt) {
+    const anonymizedStudentNumber = crypto
+      .createHash('sha256')
+      .update(`${studentnumber}${anonymizationSalt}`)
+      .digest('hex')
+
+    const anonymizedStudent = {
+      studentnumber: anonymizedStudentNumber,
+    }
+
+    return {
+      obfuscated: true,
+      student: anonymizedStudent,
+      yearcode,
+      yearname,
+      semestercode,
+      semestername,
+      coursecode: course_code,
+      state,
+      enrollment_date_time,
+      studentnumber: anonymizedStudentNumber,
+      programmes: studyright_elements.map(formatStudyrightElement),
+    }
+  }
+
+  return {
+    student,
+    yearcode,
+    yearname,
+    semestercode,
+    semestername,
+    coursecode: course_code,
+    state,
+    enrollment_date_time,
+    studentnumber,
+    programmes: studyright_elements.map(formatStudyrightElement),
+  }
+}
+
+module.exports = { parseEnrollment }

services/backend/src/services/populations.js

@@ -10,11 +10,13 @@ const {
   Studyright,
   StudyrightExtent,
   ElementDetail,
+  Studyplan,
   CourseType,
   SemesterEnrollment,
   Semester,
   Transfer,
   StudyrightElement,
+  Enrollment,
 } = require('../models')
 const {
   dbConnections: { sequelize },
@@ -47,6 +49,7 @@ const formatStudentForPopulationStatistics = (
     abbreviatedname,
     email,
     studyrights,
+    studyplans,
     semester_enrollments,
     transfers,
     updatedAt,
@@ -58,6 +61,7 @@ const formatStudentForPopulationStatistics = (
     tags,
     option,
     birthdate,
+    sis_person_id,
   },
   credits,
   startDate,
@@ -105,6 +109,8 @@ const formatStudentForPopulationStatistics = (
     starting: moment(started).isBetween(startDateMoment, endDateMoment, null, '[]'),
     option,
     birthdate,
+    studyplans,
+    sis_person_id,
   }
 }
 
@@ -199,6 +205,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
         'updatedAt',
         'gender_code',
         'birthdate',
+        'sis_person_id',
         /* 'gender_fi',
         'gender_sv',
         'gender_en' */
@@ -217,7 +224,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
             'enddate',
             'extentcode',
             'graduated',
-            'canceldate',
+            'active',
             'prioritycode',
             'faculty_code',
             'studystartdate',
@@ -250,6 +257,11 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
             },
           },
         },
+        {
+          model: Studyplan,
+          attributes: ['included_courses', 'programme_code'],
+          separate: true,
+        },
       ],
       where: {
         studentnumber: {
@@ -361,10 +373,11 @@ const studentnumbersWithAllStudyrightElements = async (
   startDate,
   endDate,
   exchangeStudents,
-  cancelledStudents,
   nondegreeStudents,
-  transferredStudents,
-  tag
+  transferredOutStudents,
+  tag,
+  transferredToStudents,
+  graduatedStudents
 ) => {
   // eslint-disable-line
 
@@ -383,9 +396,6 @@ const studentnumbersWithAllStudyrightElements = async (
   if (!nondegreeStudents) {
     filteredExtents.push(33, 99, 14, 13)
   }
-  if (!cancelledStudents) {
-    studyrightWhere.canceldate = null
-  }
 
   let studentWhere = {}
   if (tag) {
@@ -403,7 +413,7 @@ const studentnumbersWithAllStudyrightElements = async (
   }
 
   const students = await Studyright.findAll({
-    attributes: ['student_studentnumber', 'graduated'],
+    attributes: ['student_studentnumber', 'graduated', 'enddate'],
     include: {
       model: StudyrightElement,
       attributes: [],
@@ -444,7 +454,7 @@ const studentnumbersWithAllStudyrightElements = async (
     raw: true,
   })
 
-  const studentnumbers = [...new Set(students.map(s => s.student_studentnumber))]
+  let studentnumbers = [...new Set(students.map(s => s.student_studentnumber))]
 
   // bit hacky solution, but this is used to filter out studentnumbers who have since changed studytracks
   const rights = await Studyright.findAll({
@@ -491,17 +501,21 @@ const studentnumbersWithAllStudyrightElements = async (
     return acc
   }, {})
 
+  // Take the newest studytrack primarily by latest starting date in the track, secondarily by the latest enddate
   const filteredStudentnumbers = studentnumbers.filter(studentnumber => {
-    const newestStudytrack = sortBy(formattedStudytracks[studentnumber], ['enddate', 'startdate']).reverse()[0]
+    const newestStudytrack = sortBy(formattedStudytracks[studentnumber], ['startdate', 'enddate']).reverse()[0]
     if (!newestStudytrack) return false
     return studyRights.includes(newestStudytrack.code)
   })
 
-  const studentnumberlist = filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers
+  // Use the filtered list, if the search includes studytracks
+  // Then the studyrights length is > 1, which means that there is [studyright, studytrack].
+  // When searching only for studyprogramme, there is [studyright]
+  let studentnumberlist = studyRights.length > 1 ? filteredStudentnumbers : studentnumbers
 
   // fetch students that have transferred out of the programme and filter out these studentnumbers
-  if (!transferredStudents) {
-    const transferredOutStudents = (
+  if (!transferredOutStudents) {
+    const transfersOut = (
       await Transfer.findAll({
         attributes: ['studentnumber'],
         where: {
@@ -519,8 +533,63 @@ const studentnumbersWithAllStudyrightElements = async (
       })
     ).map(s => s.studentnumber)
 
-    const notTransferredStudents = studentnumberlist.filter(sn => !transferredOutStudents.includes(sn))
-    return notTransferredStudents
+    studentnumberlist = studentnumberlist.filter(sn => !transfersOut.includes(sn))
+  }
+
+  // fetch students that have transferred to the programme and filter out these studentnumbers
+  if (transferredToStudents) {
+    const transfersTo = (
+      await Transfer.findAll({
+        attributes: ['studentnumber'],
+        where: {
+          targetcode: {
+            [Op.in]: studyRights,
+          },
+          transferdate: {
+            [Op.gt]: startDate,
+          },
+          studentnumber: {
+            [Op.in]: studentnumberlist,
+          },
+        },
+        raw: true,
+      })
+    ).map(s => s.studentnumber)
+    studentnumberlist = studentnumberlist.filter(sn => !transfersTo.includes(sn))
+  }
+
+  // fetch students that have graduated from the programme and filter out these studentnumbers
+  if (graduatedStudents) {
+    const graduated = (
+      await Student.findAll({
+        attributes: ['studentnumber'],
+        include: [
+          {
+            model: Studyright,
+            include: [
+              {
+                model: StudyrightElement,
+                required: true,
+                where: {
+                  code: {
+                    [Op.in]: studyRights,
+                  },
+                },
+              },
+            ],
+            where: {
+              graduated: 1,
+            },
+          },
+        ],
+        where: {
+          studentnumber: {
+            [Op.in]: studentnumbers,
+          },
+        },
+      })
+    ).map(s => s.studentnumber)
+    studentnumberlist = studentnumberlist.filter(sn => !graduated.includes(sn))
   }
 
   return studentnumberlist
@@ -535,13 +604,11 @@ const parseQueryParams = query => {
     ? `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd[semesters.find(s => s === 'SPRING')]}`
     : `${year}-${semesterEnd[semesters.find(s => s === 'FALL')]}`
   const exchangeStudents = studentStatuses && studentStatuses.includes('EXCHANGE')
-  const cancelledStudents = studentStatuses && studentStatuses.includes('CANCELLED')
   const nondegreeStudents = studentStatuses && studentStatuses.includes('NONDEGREE')
   const transferredStudents = studentStatuses && studentStatuses.includes('TRANSFERRED')
 
   return {
     exchangeStudents,
-    cancelledStudents,
     nondegreeStudents,
     transferredStudents,
     // if someone passes something falsy like null as the studyright, remove it so it doesn't break
@@ -743,22 +810,13 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
   if (
     formattedQueryParams.studentStatuses &&
     !formattedQueryParams.studentStatuses.every(
-      status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
+      status => status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
     )
   ) {
-    return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE or TRANSFERRED' }
+    return { error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' }
   }
-  const {
-    studyRights,
-    startDate,
-    months,
-    endDate,
-    exchangeStudents,
-    cancelledStudents,
-    nondegreeStudents,
-    transferredStudents,
-    tag,
-  } = parseQueryParams(formattedQueryParams)
+  const { studyRights, startDate, months, endDate, exchangeStudents, nondegreeStudents, transferredStudents, tag } =
+    parseQueryParams(formattedQueryParams)
 
   // db startdate is formatted to utc so need to change it when querying
   const formattedStartDate = new Date(moment.tz(startDate, 'Europe/Helsinki').format()).toUTCString()
@@ -770,7 +828,6 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
         formattedStartDate,
         endDate,
         exchangeStudents,
-        cancelledStudents,
         nondegreeStudents,
         transferredStudents
       )
@@ -799,12 +856,20 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
     formattedQueryParams,
     optionData
   )
+
   return formattedStudents
 }
 
-const findCourses = async (studentnumbers, beforeDate) => {
+const getSubstitutions = async codes => {
+  const courses = await Course.findAll({ where: { code: codes }, attributes: ['code', 'substitutions'], raw: true })
+  return [...new Set(courses.map(({ code, substitutions }) => [code, ...substitutions]).flat())]
+}
+
+const findCourses = async (studentnumbers, beforeDate, courses = []) => {
+  const codesFilter = courses.length ? { code: await getSubstitutions(courses) } : {}
   const res = await Course.findAll({
     attributes: ['code', 'name', 'coursetypecode', 'substitutions'],
+    where: { ...codesFilter },
     include: [
       {
         required: true,
@@ -824,6 +889,19 @@ const findCourses = async (studentnumbers, beforeDate) => {
         model: CourseType,
         required: true,
       },
+      {
+        required: false,
+        model: Enrollment,
+        attributes: ['course_code', 'studentnumber', 'state', 'enrollment_date_time', 'course_code'],
+        where: {
+          studentnumber: {
+            [Op.in]: studentnumbers,
+          },
+          enrollment_date_time: {
+            [Op.lt]: beforeDate,
+          },
+        },
+      },
     ],
   })
   return res
@@ -841,38 +919,29 @@ const parseCreditInfo = credit => ({
   date: credit.attainment_date,
 })
 
-// const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
-
-/* const unifyOpenUniversity = code => {
-  const regexresult = isOpenUniCourseCode(code)
-  if (!regexresult) return code
-  return regexresult[1]
-}
- */
 const findMainCourse = async code => {
   const course = await Course.findOne({
     where: { code: code },
   })
-  const sortedCourses = sortMainCode([...course.substitutions, code])
-  const result = await Course.findOne({
-    attributes: ['code', 'name'],
-    where: { code: sortedCourses[0] },
-  })
+
+  let result = []
+
+  if (course.substitutions) {
+    const sortedCourses = sortMainCode([...course.substitutions, code])
+    result = await Course.findOne({
+      attributes: ['code', 'name'],
+      where: { code: sortedCourses[0] },
+    })
+  } else {
+    return course
+  }
 
   return result
 }
 
 const bottlenecksOf = async (query, studentnumberlist) => {
   const isValidRequest = async (query, params) => {
-    const {
-      studyRights,
-      startDate,
-      endDate,
-      exchangeStudents,
-      cancelledStudents,
-      nondegreeStudents,
-      transferredStudents,
-    } = params
+    const { studyRights, startDate, endDate, exchangeStudents, nondegreeStudents, transferredStudents } = params
 
     if (!query.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
       return { error: 'Semester should be either SPRING OR FALL' }
@@ -880,10 +949,10 @@ const bottlenecksOf = async (query, studentnumberlist) => {
     if (
       query.studentStatuses &&
       !query.studentStatuses.every(
-        status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
+        status => status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
       )
     ) {
-      return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE or TRANSFERRED' }
+      return { error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' }
     }
     if (query.selectedStudents) {
       const allStudents = await studentnumbersWithAllStudyrightElements(
@@ -891,7 +960,6 @@ const bottlenecksOf = async (query, studentnumberlist) => {
         startDate,
         endDate,
         exchangeStudents,
-        cancelledStudents,
         nondegreeStudents,
         transferredStudents,
         query.tag
@@ -905,19 +973,10 @@ const bottlenecksOf = async (query, studentnumberlist) => {
     return null
   }
 
-  const getStudentsAndCourses = async (selectedStudents, studentnumberlist) => {
+  const getStudentsAndCourses = async (selectedStudents, studentnumberlist, courseCodes) => {
     if (!studentnumberlist) {
-      const {
-        months,
-        studyRights,
-        startDate,
-        endDate,
-        exchangeStudents,
-        cancelledStudents,
-        nondegreeStudents,
-        transferredStudents,
-        tag,
-      } = params
+      const { months, studyRights, startDate, endDate, exchangeStudents, nondegreeStudents, transferredStudents, tag } =
+        params
       const studentnumbers =
         selectedStudents ||
         (await studentnumbersWithAllStudyrightElements(
@@ -925,7 +984,6 @@ const bottlenecksOf = async (query, studentnumberlist) => {
           startDate,
           endDate,
           exchangeStudents,
-          cancelledStudents,
           nondegreeStudents,
           transferredStudents,
           tag
@@ -936,7 +994,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
         return numbers
       }, {})
 
-      const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months))
+      const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months), courseCodes)
       return [allstudents, courses]
     } else {
       const allstudents = studentnumberlist.reduce((numbers, num) => {
@@ -949,7 +1007,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
   }
   const params = parseQueryParams(query)
   const [[allstudents, courses], error] = await Promise.all([
-    getStudentsAndCourses(query.selectedStudents, studentnumberlist),
+    getStudentsAndCourses(query.selectedStudents, studentnumberlist, query.courses),
     isValidRequest(query, params),
   ])
   if (error) return error
@@ -970,11 +1028,6 @@ const bottlenecksOf = async (query, studentnumberlist) => {
     const sortedMaincourse = await findMainCourse(course.code)
     if (sortedMaincourse) maincourse = sortedMaincourse
 
-    /*  if (isOpenUniCourseCode) {
-      const nonOpen = unifyOpenUniversity(course.code)
-      const response = await findOneByCode(nonOpen)
- */
-
     if (!stats[maincourse.code]) {
       stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudentslength)
     }
@@ -990,6 +1043,10 @@ const bottlenecksOf = async (query, studentnumberlist) => {
       const semester = getPassingSemester(startYear, date)
       coursestats.markCredit(studentnumber, grade, passingGrade, failingGrade, improvedGrade, semester)
     })
+    course.enrollments.forEach(({ studentnumber, state, enrollment_date_time }) => {
+      const semester = getPassingSemester(startYear, enrollment_date_time)
+      coursestats.markEnrollment(studentnumber, state, semester)
+    })
     stats[maincourse.code] = coursestats
   }
 

services/backend/src/services/semesters.js

@@ -1,3 +1,4 @@
+const moment = require('moment')
 const sequelize = require('sequelize')
 const { Semester } = require('../models')
 const Op = sequelize.Op
@@ -28,8 +29,8 @@ const getSemestersAndYears = async () => {
         acc.years[yearcode] = {
           yearcode,
           yearname,
-          startdate: Math.min(acc.years[yearcode].startdate, startdate),
-          enddate: Math.max(acc.years[yearcode].enddate, enddate),
+          startdate: moment.min(moment(acc.years[yearcode].startdate), moment(startdate)),
+          enddate: moment.max(moment(acc.years[yearcode].enddate), moment(enddate)),
         }
       }
       return acc

services/backend/src/services/students.js

@@ -12,6 +12,7 @@ const {
   ElementDetail,
   SemesterEnrollment,
   Semester,
+  Studyplan,
 } = require('../models')
 const { TagStudent, Tag } = require('../models/models_kone')
 const Op = Sequelize.Op
@@ -57,6 +58,10 @@ const byId = async id => {
         {
           model: SemesterEnrollment,
         },
+        {
+          model: Studyplan,
+          attributes: ['included_courses', 'programme_code'],
+        },
       ],
     }),
     TagStudent.findAll({
@@ -99,28 +104,44 @@ const byId = async id => {
   return student
 }
 
-const findByCourseAndSemesters = async (coursecodes, from, to, separate) =>
+const getUnifyStatus = unifyCourses => {
+  switch (unifyCourses) {
+    case 'unifyStats':
+      return [true, false]
+
+    case 'openStats':
+      return [true]
+
+    case 'reqularStats':
+      return [false]
+
+    default:
+      return [true, false]
+  }
+}
+
+const findByCourseAndSemesters = async (coursecodes, from, to, separate, unifyCourses = 'unifyStats') =>
   (
     await sequelize.query(
       `
-  SELECT
-    studentnumber
-  FROM student
-  INNER JOIN credit ON
-    student.studentnumber=credit.student_studentnumber
+    SELECT
+      studentnumber
+      FROM student
+      INNER JOIN credit ON
+      student.studentnumber=credit.student_studentnumber
   WHERE
-    course_code IN (:coursecodes) AND
-    attainment_date
+  course_code IN (:coursecodes) AND
+  attainment_date
   BETWEEN
-    (select startdate FROM semesters where ${
-      separate ? 'semestercode' : 'yearcode'
-    }=:minYearCode ORDER BY semestercode LIMIT 1) AND
-    (select enddate FROM semesters where ${
-      separate ? 'semestercode' : 'yearcode'
-    }=:maxYearCode ORDER BY semestercode DESC LIMIT 1);
-`,
+  (select startdate FROM semesters where ${
+    separate ? 'semestercode' : 'yearcode'
+  }=:minYearCode ORDER BY semestercode LIMIT 1) AND
+  (select enddate FROM semesters where ${separate ? 'semestercode' : 'yearcode'}=:maxYearCode AND 
+  is_open IN (:isOpen)
+  ORDER BY semestercode DESC LIMIT 1);
+  `,
       {
-        replacements: { coursecodes, minYearCode: from, maxYearCode: to },
+        replacements: { coursecodes, minYearCode: from, maxYearCode: to, isOpen: getUnifyStatus(unifyCourses) },
         type: sequelize.QueryTypes.SELECT,
         raw: true,
       }
@@ -140,7 +161,7 @@ const findByTag = async tag => {
   ).map(st => st.studentnumber)
 }
 
-const formatStudent = ({
+const formatStudent = async ({
   firstnames,
   lastname,
   studentnumber,
@@ -153,11 +174,13 @@ const formatStudent = ({
   studyrights,
   semester_enrollments,
   transfers,
+  studyplans,
   updatedAt,
   createdAt,
   tags,
+  sis_person_id,
 }) => {
-  const toCourse = ({ id, grade, credits, credittypecode, attainment_date, course, isStudyModule }) => {
+  const toCourse = ({ id, grade, credits, credittypecode, is_open, attainment_date, course, isStudyModule }) => {
     try {
       course = course.get()
     } catch (e) {
@@ -183,11 +206,13 @@ const formatStudent = ({
       credits,
       credittypecode,
       isStudyModuleCredit: isStudyModule,
+      isOpenCourse: is_open,
     }
   }
 
   studyrights = studyrights || []
   semester_enrollments = semester_enrollments || []
+  studyplans = studyplans || []
   const semesterenrollments = semester_enrollments.map(
     ({ semestercode, enrollmenttype, name, yearname, startYear }) => ({
       yearname,
@@ -211,6 +236,13 @@ const formatStudent = ({
     .map(toCourse)
     .filter(c => c.course.name !== 'missing')
 
+  const hopsCourses = studyplans.map(sp => sp.included_courses).flat()
+
+  const allCourses = await sequelize.query(
+    'SELECT DISTINCT ON(course_code) course_code, credits FROM credit WHERE course_code IN (:hopsCourses)',
+    { replacements: { hopsCourses: hopsCourses.concat('id-to-fill-array') } }
+  )
+
   return {
     firstnames,
     lastname,
@@ -225,14 +257,17 @@ const formatStudent = ({
     email,
     semesterenrollments,
     updatedAt: updatedAt || createdAt,
+    studyplans,
+    allCourses: allCourses[0],
     tags,
+    sis_person_id: sis_person_id,
   }
 }
 
 const withId = async id => {
   try {
     const result = await byId(id)
-    return formatStudent(result)
+    return await formatStudent(result)
   } catch (e) {
     logger.error({ message: 'error when fetching single student', meta: e })
     return {
@@ -285,59 +320,74 @@ const bySearchTerm = async searchterm => {
       [Op.or]: [nameLike(terms), studentnumberLike(terms)],
     },
   })
-  return matches.map(formatStudent)
+  return await Promise.all(matches.map(async s => await formatStudent(s)))
 }
 
-const bySearchTermAndElements = async (searchterm, codes) => {
+const bySearchTermAndStudentNumbers = async (searchterm, studentnumbers) => {
   const terms = splitByEmptySpace(searchterm)
   const matches = await Student.findAll({
-    include: {
-      model: StudyrightElement,
-      required: true,
-      where: {
-        code: {
-          [Op.in]: codes,
+    where: {
+      [Op.and]: {
+        [Op.or]: [nameLike(terms), studentnumberLike(terms)],
+        studentnumber: {
+          [Op.in]: studentnumbers,
         },
       },
     },
-    where: {
-      [Op.or]: [nameLike(terms), studentnumberLike(terms)],
-    },
   })
-  return matches.map(formatStudent)
+  return await Promise.all(matches.map(async s => await formatStudent(s)))
 }
 
-const filterStudentnumbersByAccessrights = async (studentnumbers, codes) => {
-  const students = await Student.findAll({
-    attributes: ['studentnumber'],
-    include: {
-      attributes: [],
-      model: StudyrightElement,
-      required: true,
-      where: {
-        code: {
-          [Op.in]: codes,
+const filterStudentnumbersByAccessrights = async (studentnumbers, codes) =>
+  (
+    await Student.findAll({
+      attributes: ['studentnumber'],
+      include: {
+        attributes: [],
+        model: StudyrightElement,
+        required: true,
+        where: {
+          code: {
+            [Op.in]: codes,
+          },
         },
       },
-    },
-    where: {
-      studentnumber: {
-        [Op.in]: studentnumbers,
+      where: {
+        studentnumber: {
+          [Op.in]: studentnumbers,
+        },
       },
-    },
-    raw: true,
-  })
-  return students.map(student => student.studentnumber)
-}
+      raw: true,
+    })
+  ).map(({ studentnumber }) => studentnumber)
+
+const getStudentnumbersByElementdetails = async codes =>
+  (
+    await Student.findAll({
+      attributes: ['studentnumber'],
+      include: {
+        attributes: [],
+        model: StudyrightElement,
+        required: true,
+        where: {
+          code: {
+            [Op.in]: codes,
+          },
+        },
+      },
+      raw: true,
+    })
+  ).map(({ studentnumber }) => studentnumber)
 
 module.exports = {
   withId,
   bySearchTerm,
   createStudent,
   updateStudent,
-  bySearchTermAndElements,
+  bySearchTermAndStudentNumbers,
   filterStudentnumbersByAccessrights,
   findByCourseAndSemesters,
   findByTag,
   splitByEmptySpace,
+  getStudentnumbersByElementdetails,
 }

services/backend/src/services/studyGuidanceGroups.js

@@ -2,42 +2,42 @@ const { getImporterClient } = require('../util/importerClient')
 const importerClient = getImporterClient()
 const { StudyGuidanceGroupTag } = require('../models/models_kone')
 const logger = require('../util/logger')
+const _ = require('lodash')
 
-const getAllStudentsUserHasInGroups = async sisPersonId => {
+const getGroupsFromImporter = async sisPersonId => {
   if (!importerClient) {
-    return new Set()
+    return []
   }
+  const answerTimeout = new Promise(resolve => setTimeout(resolve, 2000))
   try {
-    const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
-    return new Set(
-      Object.values(data)
-        .map(group => group.members)
-        .flat()
-        .map(member => member.personStudentNumber)
-    )
+    const result = await Promise.race([importerClient.get(`/person-groups/person/${sisPersonId}`), answerTimeout])
+    if (!result) return []
+    return Object.values(result.data)
   } catch (error) {
     logger.error("Couldn't fetch users studyguidance groups")
-    return new Set()
+    return []
   }
 }
 
 const getAllGroupsAndStudents = async sisPersonId => {
-  if (!importerClient) {
-    return []
-  }
-  try {
-    const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
-    const tagsByGroupId = (await StudyGuidanceGroupTag.findAll()).reduce((acc, curr) => {
-      const { studyProgramme, year } = curr
-      return { ...acc, [curr.studyGuidanceGroupId]: { studyProgramme, year } }
-    }, {})
-    return Object.values(data).map(group => ({ ...group, tags: tagsByGroupId[group.id] }))
-  } catch (error) {
-    logger.error("Couldn't fetch users studyguidance groups")
-    return []
-  }
+  const tagsByGroupId = (await StudyGuidanceGroupTag.findAll()).reduce((acc, curr) => {
+    const { studyProgramme, year, studyGuidanceGroupId } = curr
+    return { ...acc, [studyGuidanceGroupId]: { studyProgramme, year } }
+  }, {})
+  return (await getGroupsFromImporter(sisPersonId)).map(group => ({
+    ...group,
+    tags: tagsByGroupId[group.id],
+  }))
 }
 
+const getAllStudentsUserHasInGroups = async sisPersonId =>
+  _.uniqBy(
+    (await getGroupsFromImporter(sisPersonId))
+      .map(group => group.members)
+      .flat()
+      .map(member => member.personStudentNumber)
+  )
+
 const changeGroupTags = async ({ groupId, tags }) => {
   const { studyProgramme, year } = tags
   const tagToUpdate = studyProgramme ? { studyProgramme } : { year }
@@ -56,8 +56,11 @@ const changeGroupTags = async ({ groupId, tags }) => {
   return result
 }
 
+const checkStudyGuidanceGroupsAccess = async hyPersonSisuId => (await getGroupsFromImporter(hyPersonSisuId)).length > 0
+
 module.exports = {
   getAllStudentsUserHasInGroups,
   getAllGroupsAndStudents,
   changeGroupTags,
+  checkStudyGuidanceGroupsAccess,
 }

services/backend/src/services/studyprogrammeBasics.js

@@ -0,0 +1,149 @@
+const { indexOf } = require('lodash')
+const {
+  getCorrectStudentnumbers,
+  getStatsBasis,
+  defineYear,
+  getYearsArray,
+  getStartDate,
+  tableTitles,
+  alltimeStartDate,
+  alltimeEndDate,
+} = require('./studyprogrammeHelpers')
+const { graduatedStudyRights, startedStudyrights, transfersAway, transfersTo } = require('./studyprogramme')
+const { getYearStartAndEndDates } = require('../util/semester')
+
+const getStartedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+      const studyrights = await startedStudyrights(studyprogramme, since, studentnumbersOfTheYear)
+
+      studyrights.forEach(({ studystartdate }) => {
+        const startYear = defineYear(studystartdate, isAcademicYear)
+        graphStats[indexOf(years, startYear)] += 1
+        tableStats[startYear] += 1
+      })
+    })
+  )
+  return { graphStats, tableStats }
+}
+
+const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+    codes: [studyprogramme],
+    startDate: alltimeStartDate,
+    endDate: alltimeEndDate,
+    includeAllSpecials,
+  })
+
+  const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
+  studyrights.forEach(({ enddate }) => {
+    const graduationYear = defineYear(enddate, isAcademicYear)
+    graphStats[indexOf(years, graduationYear)] += 1
+    tableStats[graduationYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredAwayStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const studyrights = await transfersAway(studyprogramme, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredToStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const studyrights = await transfersTo(studyprogramme, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getBasicStatsForStudytrack = async ({ studyprogramme, settings }) => {
+  const { includeAllSpecials, isAcademicYear } = settings
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const started = await getStartedStats(queryParameters)
+  const graduated = await getGraduatedStats(queryParameters)
+  const transferredAway = await getTransferredAwayStats(queryParameters)
+  const transferredTo = await getTransferredToStats(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const titles = tableTitles['basics'][includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED']
+  const tableStats = reversedYears.map(year =>
+    includeAllSpecials
+      ? [
+          year,
+          started.tableStats[year],
+          graduated.tableStats[year],
+          transferredAway.tableStats[year],
+          transferredTo.tableStats[year],
+        ]
+      : [year, started.tableStats[year], graduated.tableStats[year]]
+  )
+
+  return {
+    id: studyprogramme,
+    years,
+    graphStats: includeAllSpecials
+      ? [
+          {
+            name: 'Started',
+            data: started.graphStats,
+          },
+          {
+            name: 'Graduated',
+            data: graduated.graphStats,
+          },
+          {
+            name: 'Transferred away',
+            data: transferredAway.graphStats,
+          },
+          {
+            name: 'Transferred To',
+            data: transferredTo.graphStats,
+          },
+        ]
+      : [
+          {
+            name: 'Started',
+            data: started.graphStats,
+          },
+          {
+            name: 'Graduated',
+            data: graduated.graphStats,
+          },
+        ],
+    tableStats,
+    titles,
+  }
+}
+
+module.exports = {
+  getBasicStatsForStudytrack,
+}

services/backend/src/services/studyprogrammeCourses.js

@@ -0,0 +1,115 @@
+const _ = require('lodash')
+
+const {
+  getStudentsForProgrammeCourses,
+  getCurrentStudyYearStartDate,
+  getOwnStudentsForProgrammeCourses,
+  getStudentsWithoutStudyrightForProgrammeCourses,
+  getOtherStudentsForProgrammeCourses,
+} = require('./studyprogramme')
+const { mapToProviders } = require('../util/utils')
+
+const getCurrentYearStartDate = () => {
+  return new Date(new Date().getFullYear(), 0, 1)
+}
+
+const makeYearlyPromises = (years, providerCode, academicYear, type, studyprogramme) => {
+  return years.map(
+    year =>
+      new Promise(async res => {
+        const from = academicYear == 'true' ? new Date(year, 7, 1, 0, 0, 0) : new Date(year, 0, 1, 0, 0, 0)
+        const to = academicYear == 'true' ? new Date(year + 1, 6, 31, 23, 59, 59) : new Date(year, 11, 31, 23, 59, 59)
+
+        let result = null
+
+        switch (type) {
+          case 'allStudents':
+            result = await getStudentsForProgrammeCourses(from, to, providerCode)
+            break
+          case 'ownStudents':
+            result = await getOwnStudentsForProgrammeCourses(from, to, providerCode, studyprogramme)
+            break
+          case 'withoutStudyright':
+            result = await getStudentsWithoutStudyrightForProgrammeCourses(from, to, providerCode)
+            break
+          case 'otherStudents':
+            result = await getOtherStudentsForProgrammeCourses(from, to, providerCode, studyprogramme)
+            break
+          default:
+            result = await getStudentsForProgrammeCourses(from, to, providerCode)
+        }
+
+        res(
+          result.map(c => {
+            c['year'] = year
+            return c
+          })
+        )
+      })
+  )
+}
+
+const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme, academicYear) => {
+  const providerCode = mapToProviders([studyprogramme])[0]
+  const startDate = academicYear == 'true' ? await getCurrentStudyYearStartDate(unixMillis) : getCurrentYearStartDate()
+  const startYear = startDate.getFullYear()
+  const yearRange = _.range(2017, startYear + 1)
+
+  const yearlyStudentByCoursePromises = makeYearlyPromises(yearRange, providerCode, academicYear, 'allStudents')
+  const yearlyProgrammeStudentsPromises = makeYearlyPromises(
+    yearRange,
+    providerCode,
+    academicYear,
+    'ownStudents',
+    studyprogramme
+  )
+  const yearlyStudentsWithoutStudyrightPromises = makeYearlyPromises(
+    yearRange,
+    providerCode,
+    academicYear,
+    'withoutStudyright'
+  )
+  const yearlyOtherProgrammeStudentsPromises = makeYearlyPromises(
+    yearRange,
+    providerCode,
+    academicYear,
+    'otherStudents',
+    studyprogramme
+  )
+
+  const [
+    yearlyStudentByCourse,
+    yearlyProgrammeStudents,
+    yearlyStudentsWithoutStudyright,
+    yearlyOtherProgrammeStudents,
+  ] = await Promise.all([
+    Promise.all(yearlyStudentByCoursePromises),
+    Promise.all(yearlyProgrammeStudentsPromises),
+    Promise.all(yearlyStudentsWithoutStudyrightPromises),
+    Promise.all(yearlyOtherProgrammeStudentsPromises),
+  ])
+
+  const res = [
+    ...yearlyStudentByCourse.flat(),
+    ...yearlyProgrammeStudents.flat(),
+    ...yearlyStudentsWithoutStudyright.flat(),
+    ...yearlyOtherProgrammeStudents.flat(),
+  ].reduce((acc, curr) => {
+    if (!acc[curr.code + curr.year]) {
+      acc[curr.code + curr.year] = {
+        code: curr.code,
+        name: curr.name,
+        year: curr.year,
+        totalAll: curr.totalAll | 0,
+        totalOwn: curr.totalOwn | 0,
+        totalWithout: curr.totalWithout | 0,
+        totalOthers: curr.totalOthers | 0,
+      }
+    }
+    acc[curr.code + curr.year] = _.merge(acc[curr.code + curr.year], curr)
+    return acc
+  }, {})
+  return Object.values(res)
+}
+
+module.exports = { getStudyprogrammeCoursesForStudytrack }

services/backend/src/services/studyprogrammeCredits.js

@@ -0,0 +1,143 @@
+const { indexOf } = require('lodash')
+
+const { mapToProviders } = require('../util/utils')
+const {
+  getStatsBasis,
+  defineYear,
+  isMajorStudentCredit,
+  getYearsArray,
+  getStartDate,
+  isSpecialGroupCredit,
+  tableTitles,
+} = require('./studyprogrammeHelpers')
+const {
+  getProgrammesStudyrights,
+  getCreditsForStudyProgramme,
+  getTransferredCredits,
+  allTransfers,
+} = require('./studyprogramme')
+
+// Fetches all the credits for the studyprogramme and divides them into major-students and non-major students credits
+// Division is done on the basis that whether the student had a primary studyright to the programme on the attainment_date
+// If special groups are excluded, the transfer students are filtered away from the major-students as well
+const getRegularCreditStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  let studyrights = await getProgrammesStudyrights(studyprogramme, since)
+  const transfers = (await allTransfers(studyprogramme, since)).map(t => t.studyrightid)
+
+  if (!includeAllSpecials) {
+    studyrights.filter(s => !transfers.includes(s.studyrightid))
+  }
+
+  const credits = await getCreditsForStudyProgramme(providercode, since)
+
+  let majors = getStatsBasis(years)
+  let nonMajors = getStatsBasis(years)
+
+  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
+    const studyright = studyrights.find(studyright => studyright.studentnumber == student_studentnumber)
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+
+    if (!includeAllSpecials && isSpecialGroupCredit(studyright, attainment_date, transfers)) {
+      return
+    }
+
+    if (isMajorStudentCredit(studyright, attainment_date)) {
+      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      majors.tableStats[attainmentYear] += credits || 0
+    } else {
+      nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      nonMajors.tableStats[attainmentYear] += credits || 0
+    }
+  })
+
+  if (majors.graphStats.every(year => year === 0) && nonMajors.graphStats.every(year => year === 0)) {
+    return {
+      majors: { graphStats: [], tableStats: {} },
+      nonMajors: { graphStats: [], tableStats: {} },
+    }
+  }
+
+  return { majors, nonMajors }
+}
+
+// Fetches all the credits with the type "transferred / hyväksiluettu" and divides them by year
+const getTransferredCreditStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const credits = await getTransferredCredits(providercode, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  credits.forEach(({ attainment_date, credits }) => {
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+    graphStats[indexOf(years, attainmentYear)] += credits || 0
+    tableStats[attainmentYear] += credits || 0
+  })
+
+  if (graphStats.every(year => year === 0)) {
+    return { graphStats: [], tableStats: {} }
+  }
+
+  return { graphStats, tableStats }
+}
+
+// Fetches all credits for the studytrack and combines them into the statistics for the table
+// and graph in the studyprogramme overview
+const getCreditStatsForStudytrack = async ({ studyprogramme, settings }) => {
+  const { isAcademicYear, includeAllSpecials } = settings
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const { majors, nonMajors } = await getRegularCreditStats(queryParameters)
+  const transferred = await getTransferredCreditStats(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const titles = tableTitles['credits'][includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED']
+  const dataFound = [majors, nonMajors, transferred].some(d => d.graphStats.length)
+
+  if (!dataFound) return null
+
+  const tableStats = reversedYears.map(year =>
+    includeAllSpecials
+      ? [
+          year,
+          majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
+          majors.tableStats[year],
+          nonMajors.tableStats[year],
+          transferred.tableStats[year],
+        ]
+      : [
+          year,
+          majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
+          majors.tableStats[year],
+          transferred.tableStats[year],
+        ]
+  )
+
+  const graphStats = [
+    {
+      name: 'Major students credits',
+      data: majors.graphStats,
+    },
+    {
+      name: 'Non-major students credits',
+      data: nonMajors.graphStats,
+    },
+    {
+      name: 'Transferred credits',
+      data: transferred.graphStats,
+    },
+  ]
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    graphStats,
+    titles,
+  }
+}
+
+module.exports = {
+  getCreditStatsForStudytrack,
+}

services/backend/src/services/studyprogrammeGraduations.js

@@ -0,0 +1,255 @@
+const moment = require('moment')
+const { indexOf, uniqBy } = require('lodash')
+
+const { mapToProviders } = require('../util/utils')
+const {
+  getCorrectStudentnumbers,
+  getStatsBasis,
+  defineYear,
+  getYearsObject,
+  getYearsArray,
+  getMedian,
+  getMean,
+  getStartDate,
+  getThesisType,
+  alltimeStartDate,
+  alltimeEndDate,
+} = require('./studyprogrammeHelpers')
+const {
+  graduatedStudyRights,
+  getThesisCredits,
+  followingStudyrights,
+  previousStudyrights,
+  allStudyrights,
+  allTransfers,
+} = require('./studyprogramme')
+const { getAllProgrammes } = require('./studyrights')
+
+const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  const studentnumbers = await getCorrectStudentnumbers({
+    codes: [studyprogramme],
+    startDate: alltimeStartDate,
+    endDate: alltimeEndDate,
+    includeAllSpecials,
+  })
+
+  const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbers)
+  studyrights.forEach(({ enddate }) => {
+    const graduationYear = defineYear(enddate, isAcademicYear)
+    graphStats[indexOf(years, graduationYear)] += 1
+    tableStats[graduationYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getThesisStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const thesisType = getThesisType(studyprogramme)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  const studentnumbers = await getCorrectStudentnumbers({
+    codes: [studyprogramme],
+    startDate: alltimeStartDate,
+    endDate: alltimeEndDate,
+    includeAllSpecials,
+  })
+
+  const credits = await getThesisCredits(providercode, since, thesisType, studentnumbers)
+
+  credits?.forEach(({ attainment_date }) => {
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+    graphStats[indexOf(years, attainmentYear)] += 1
+    tableStats[attainmentYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getGraduationTimeStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  let graduationAmounts = getYearsObject({ years })
+  let graduationTimes = getYearsObject({ years, emptyArrays: true })
+
+  const studentnumbers = await getCorrectStudentnumbers({
+    codes: [studyprogramme],
+    startDate: alltimeStartDate,
+    endDate: alltimeEndDate,
+    includeAllSpecials,
+  })
+
+  const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbers)
+
+  studyrights.forEach(({ enddate, studystartdate }) => {
+    const graduationYear = defineYear(enddate, isAcademicYear)
+    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+    graduationAmounts[graduationYear] += 1
+    graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
+  })
+  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
+  const comparisonValue = studyprogramme.includes('KH') ? 72 : 48
+
+  const medians = getYearsObject({ years, emptyArrays: true })
+  const means = getYearsObject({ years, emptyArrays: true })
+
+  // HighCharts graph require the data to have this format (ie. actual value, "empty value")
+  years.forEach(year => {
+    const median = getMedian(graduationTimes[year])
+    const mean = getMean(graduationTimes[year])
+    medians[year] = [
+      ['', median],
+      ['', comparisonValue - median],
+    ]
+    means[year] = [
+      ['', mean],
+      ['', comparisonValue - mean],
+    ]
+  })
+  return { medians, means, graduationAmounts }
+}
+
+// Creates (studyrightid, transfer)-map out of programmes transfers
+const getTransferStudyrightMap = async (studyprogramme, since) => {
+  const transfers = await allTransfers(studyprogramme, since)
+  const studyrightTransferMap = new Map()
+  transfers.forEach(t => studyrightTransferMap.set(t.studyrightid, t))
+  return studyrightTransferMap
+}
+
+const formatStats = (stats, years) => {
+  const tableStats = Object.values(stats)
+    .filter(p => years.map(year => p[year]).find(started => started !== 0)) // Filter out programmes with no-one started between the selected years
+    .map(p => [p.code, p.name['fi'], ...years.map(year => p[year])])
+
+  const graphStats = Object.values(stats)
+    .filter(p => years.map(year => p[year]).find(started => started !== 0)) // Filter out programmes with no-one started between the selected years
+    .map(p => ({ name: p.name['fi'], code: p.code, data: years.map(year => p[year]) }))
+
+  return { tableStats, graphStats }
+}
+
+const getProgrammesBeforeStarting = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const all = await allStudyrights(studyprogramme)
+  const allStudents = all.map(s => s.studentnumber)
+
+  // Get studyrights for bachelor studyprogrammes. This excludes studytracks.
+  const programmes = await getAllProgrammes()
+  let studyrightsBeforeMasters = await previousStudyrights(programmes, allStudents)
+  const studyprogrammeCodes = uniqBy(studyrightsBeforeMasters, 'code').map(s => ({ code: s.code, name: s.name }))
+  const stats = {}
+  studyprogrammeCodes.forEach(c => (stats[c.code] = { ...c, ...getYearsObject({ years }) }))
+
+  // Map transfers to the Master's programme
+  const transfers = await getTransferStudyrightMap(studyprogramme, since)
+
+  studyrightsBeforeMasters.forEach(({ studentnumber, code, givendate }) => {
+    const masterStudyright = all.find(s => s.studentnumber === studentnumber)
+    const transfer = transfers.get(masterStudyright.studyrightid)
+
+    // If special studyrights are excluded, transfers to the master's programme should be excluded
+    if (!includeAllSpecials && transfers.get(masterStudyright.studyrightid)) {
+      return
+    }
+
+    // If all studyrights are included, and transfer exists, define the start year in the programme, based on when the student transferred to it
+    // Otherwise, the year is defined based on when the student started in the chosen master's programme
+    const programmeStartDate = includeAllSpecials && transfer ? transfer.transferdate : masterStudyright?.studystartdate
+    const startYear = defineYear(programmeStartDate, isAcademicYear)
+
+    // The master studyright should have been given on the same date as the bachelor's
+    if (years.includes(startYear) && code && masterStudyright.givendate.getTime() === givendate.getTime()) {
+      stats[code][startYear] += 1
+    }
+  })
+
+  const graphAndTableStats = formatStats(stats, years)
+  return graphAndTableStats
+}
+
+const getProgrammesAfterGraduation = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const graduated = await graduatedStudyRights(studyprogramme, since)
+  const graduatedStudents = graduated.map(s => s.studentnumber)
+
+  // Get studyrights for masters studyprogramme. This excludes studytracks.
+  const programmes = await getAllProgrammes()
+  let studyrightsAfterThisProgramme = await followingStudyrights(since, programmes, graduatedStudents)
+  const studyprogrammeCodes = uniqBy(studyrightsAfterThisProgramme, 'code').map(s => ({ code: s.code, name: s.name }))
+  const stats = {}
+  studyprogrammeCodes.forEach(c => (stats[c.code] = { ...c, ...getYearsObject({ years }) }))
+
+  const transfers = await getTransferStudyrightMap(studyprogramme, since)
+
+  studyrightsAfterThisProgramme.forEach(({ studentnumber, studystartdate, code, givendate }) => {
+    const bachelorStudyright = graduated.find(s => s.studentnumber === studentnumber)
+
+    // If special studyrights are excluded, transfers to the bachelor programme should be excluded
+    if (!includeAllSpecials && transfers.get(bachelorStudyright.studyrightid)) {
+      return
+    }
+
+    // If all studyrights are included, and transfer exists, define the start year in the programme, based on when the student transferred to it
+    // Otherwise, the year is defined based on when the student started in the chosen master's programme
+    const startYear = defineYear(studystartdate, isAcademicYear)
+
+    // The master studyright should have been given on the same date as the bachelor's
+    if (years.includes(startYear) && code && bachelorStudyright.givendate.getTime() === givendate.getTime()) {
+      stats[code][startYear] += 1
+    }
+  })
+
+  const graphAndTableStats = formatStats(stats, years)
+  return graphAndTableStats
+}
+
+const getProgrammesBeforeOrAfter = async (studyprogramme, queryParameters) => {
+  if (studyprogramme.includes('KH')) return await getProgrammesAfterGraduation(queryParameters)
+  if (studyprogramme.includes('MH')) return await getProgrammesBeforeStarting(queryParameters)
+  return null
+}
+
+const getGraduationStatsForStudytrack = async ({ studyprogramme, settings }) => {
+  const { isAcademicYear, includeAllSpecials } = settings
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const thesis = await getThesisStats(queryParameters)
+
+  const graduated = await getGraduatedStats(queryParameters)
+  const graduationTimeStats = await getGraduationTimeStats(queryParameters)
+  const programmesBeforeOrAfter = await getProgrammesBeforeOrAfter(studyprogramme, queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+
+  const titles = ['', 'Graduated', 'Wrote thesis']
+  const programmesBeforeOrAfterTitles = ['Code', 'Programme', ...years]
+  const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    titles,
+    graphStats: [
+      {
+        name: 'Graduated students',
+        data: graduated.graphStats,
+      },
+      {
+        name: 'Wrote thesis',
+        data: thesis.graphStats,
+      },
+    ],
+    graduationMedianTime: graduationTimeStats.medians,
+    graduationMeanTime: graduationTimeStats.means,
+    graduationAmounts: graduationTimeStats.graduationAmounts,
+    programmesBeforeOrAfterTableStats: programmesBeforeOrAfter?.tableStats,
+    programmesBeforeOrAfterGraphStats: programmesBeforeOrAfter?.graphStats,
+    programmesBeforeOrAfterTitles,
+  }
+}
+
+module.exports = {
+  getGraduationStatsForStudytrack,
+}

services/backend/src/services/studyprogrammeHelpers.js

@@ -1,16 +1,67 @@
 const { mean } = require('lodash')
+const { studentnumbersWithAllStudyrightElements } = require('./populations')
 
 // Helper functions
+
+const getCorrectStudentnumbers = async ({ codes, startDate, endDate, includeAllSpecials, includeGraduated = true }) => {
+  let studentnumbers = []
+  const exchangeStudents = includeAllSpecials
+  const nondegreeStudents = includeAllSpecials
+  const transferredOutStudents = includeAllSpecials
+  const transferredToStudents = !includeAllSpecials
+  const graduatedStudents = !includeGraduated
+
+  studentnumbers = await studentnumbersWithAllStudyrightElements(
+    codes,
+    startDate,
+    endDate,
+    exchangeStudents,
+    nondegreeStudents,
+    transferredOutStudents,
+    null,
+    transferredToStudents,
+    graduatedStudents
+  )
+
+  return studentnumbers
+}
+
+const defineStartDate = (studyright_elements, studystartdate) => {
+  if (studyright_elements?.length && studyright_elements[0].startdate > studystartdate) {
+    return studyright_elements[0].startdate
+  }
+  return studystartdate
+}
+
 const formatStudyright = studyright => {
-  const { studyrightid, studystartdate, enddate, graduated, prioritycode, extentcode, student } = studyright
-  return {
+  const {
     studyrightid,
     studystartdate,
     enddate,
+    givendate,
     graduated,
+    active,
+    prioritycode,
+    extentcode,
+    student,
+    studyright_elements,
+  } = studyright
+
+  return {
+    studyrightid,
+    studystartdate: defineStartDate(studyright_elements, studystartdate),
+    enddate,
+    givendate,
+    graduated,
+    active,
     prioritycode,
     extentcode,
     studentnumber: student.studentnumber,
+    code: studyright_elements?.length ? studyright_elements[0].code : null,
+    name:
+      studyright_elements?.length && studyright_elements[0].element_detail && studyright_elements[0].element_detail.name
+        ? studyright_elements[0].element_detail.name
+        : null,
   }
 }
 
@@ -24,8 +75,20 @@ const formatStudent = student => {
   }
 }
 
-const getYearsArray = (since, isAcademicYear) => {
+const formatTransfer = transfer => {
+  const { sourcecode, targetcode, transferdate, studyrightid } = transfer
+  return {
+    sourcecode,
+    targetcode,
+    transferdate,
+    studyrightid,
+  }
+}
+
+const getYearsArray = (since, isAcademicYear, yearsCombined) => {
   const years = []
+  const allYears = 'Total'
+  if (yearsCombined) years.push(allYears)
   for (let i = since; i <= new Date().getFullYear(); i++) {
     const year = isAcademicYear ? `${i} - ${i + 1}` : i
     years.push(year)
@@ -33,7 +96,7 @@ const getYearsArray = (since, isAcademicYear) => {
   return years
 }
 
-const getYearsObject = (years, emptyArrays = false) => {
+const getYearsObject = ({ years, emptyArrays = false }) => {
   let yearsObject = {}
   for (const year of years) {
     yearsObject = { ...yearsObject, [year]: emptyArrays ? [] : 0 }
@@ -41,19 +104,36 @@ const getYearsObject = (years, emptyArrays = false) => {
   return yearsObject
 }
 
+const getAcademicYearsObject = ({ years, emptyArrays = false }) => {
+  let yearsObject = {}
+  for (const year of years) {
+    yearsObject = { ...yearsObject, [`${year} - ${year + 1}`]: emptyArrays ? [] : 0 }
+  }
+  return yearsObject
+}
+
 const getStatsBasis = years => {
   return {
     graphStats: new Array(years.length).fill(0),
-    tableStats: getYearsObject(years),
+    tableStats: getYearsObject({ years }),
   }
 }
 
 const isMajorStudentCredit = (studyright, attainment_date) =>
   studyright &&
   (studyright.prioritycode === 1 || studyright.prioritycode === 30) && // Is studyright state = MAIN or state = GRADUATED
+  studyright.studystartdate && // The student has started studying in the programme
   studyright.studystartdate <= attainment_date && // Has the credit been attained after studying in the programme started
-  studyright.enddate >= attainment_date && // Has the credit been attained before the studyright ended
-  (!studyright.canceldate || studyright.canceldate >= attainment_date) // If the studyright was cancelled, was the credit attained before it was cancelled
+  studyright.enddate >= attainment_date // Has the credit been attained before the studyright ended
+
+const isSpecialGroupCredit = (studyright, attainment_date, transfers) => {
+  if (!studyright) return true // If there is no studyright matching the credit, is not a major student credit
+  if (studyright.studystartdate > attainment_date) return true // Credits before the studyright started are not major student credits
+  if (studyright.enddate && attainment_date > studyright.enddate) return true // Credits after studyright are not major student credits
+  if ([7, 9, 16, 34, 33, 99, 14, 13].includes(studyright.extentcode)) return true // Excludes non-degree studyrights and exchange students
+  if (transfers.includes(studyright.studyrightid)) return true // Excludes both transfers in and out of the programme
+  return false
+}
 
 const getMedian = values => {
   if (values.length === 0) return 0
@@ -79,10 +159,13 @@ const defineYear = (date, isAcademicYear) => {
 const getStartDate = (studyprogramme, isAcademicYear) => {
   if ((studyprogramme.includes('KH') || studyprogramme.includes('MH')) && isAcademicYear) return new Date('2017-08-01')
   if (studyprogramme.includes('KH') || studyprogramme.includes('MH')) return new Date('2017-01-01')
-  if (isAcademicYear) return new Date('2000-08-01')
-  return new Date('2000-01-01')
+  if (isAcademicYear) return new Date('2017-08-01')
+  return new Date('2017-01-01')
 }
 
+const alltimeStartDate = new Date('1900-01-01')
+const alltimeEndDate = new Date()
+
 // There are 9 course_unit_types
 // 1. urn:code:course-unit-type:regular
 // 2. urn:code:course-unit-type:bachelors-thesis
@@ -102,7 +185,7 @@ const getThesisType = studyprogramme => {
     return ['urn:code:course-unit-type:bachelors-thesis']
   if (/^(T)[0-9]{6}$/.test(studyprogramme))
     return ['urn:code:course-unit-type:doctors-thesis', 'urn:code:course-unit-type:licentiate-thesis']
-  return 'thesis-type-not-defined'
+  return ['urn:code:course-unit-type:doctors-thesis', 'urn:code:course-unit-type:licentiate-thesis']
 }
 
 const getPercentage = (value, total) => {
@@ -114,7 +197,7 @@ const getPercentage = (value, total) => {
 
 const getEmptyArray = length => new Array(length).fill(0)
 
-const getCreditGraphStats = years => ({
+const getBachelorCreditGraphStats = years => ({
   lte30: {
     name: 'Less than 30 credits',
     data: getEmptyArray(years.length),
@@ -135,27 +218,189 @@ const getCreditGraphStats = years => ({
     name: '120-149 credits',
     data: getEmptyArray(years.length),
   },
-  mte150: {
-    name: 'More than 150 credits',
+  lte180: {
+    name: '150-179 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte180: {
+    name: 'More than 180 credits',
     data: getEmptyArray(years.length),
   },
 })
 
-const creditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'mte150']
+const getMasterCreditGraphStats = years => ({
+  lte200: {
+    name: 'Less than 200 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte220: {
+    name: '200-219 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte240: {
+    name: '220-239 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte260: {
+    name: '240-259 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte280: {
+    name: '260-279 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte300: {
+    name: '280-299 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte300: {
+    name: 'More than 300 credits',
+    data: getEmptyArray(years.length),
+  },
+})
+
+const getDoctoralCreditGraphStats = years => ({
+  lte50: {
+    name: 'Less than 50 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte100: {
+    name: '50-99 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte150: {
+    name: '100-149 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte200: {
+    name: '150-199 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte250: {
+    name: '200-249 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte300: {
+    name: '250-299 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte300: {
+    name: 'More than 300 credits',
+    data: getEmptyArray(years.length),
+  },
+})
+
+const getCreditGraphStats = (studyprogramme, years) => {
+  if (studyprogramme.includes('KH')) return getBachelorCreditGraphStats(years)
+  if (studyprogramme.includes('MH')) return getMasterCreditGraphStats(years)
+  return getDoctoralCreditGraphStats(years)
+}
+
+const bachelorCreditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'lte180', 'mte180']
+const masterCreditThresholds = ['lte200', 'lte220', 'lte240', 'lte260', 'lte280', 'lte300', 'mte300']
+const doctoralCreditThresholds = ['lte50', 'lte100', 'lte150', 'lte200', 'lte250', 'lte300', 'mte300']
+const bachelorCreditAmounts = [30, 60, 90, 120, 150, 180, 180]
+const masterCreditAmounts = [200, 220, 240, 260, 280, 300, 300]
+const doctoralCreditAmounts = [50, 100, 150, 200, 250, 300, 300]
+
+const getCreditThresholds = studyprogramme => {
+  if (studyprogramme.includes('KH')) {
+    return { creditThresholdKeys: bachelorCreditThresholds, creditThresholdAmounts: bachelorCreditAmounts }
+  }
+  if (studyprogramme.includes('MH')) {
+    return { creditThresholdKeys: masterCreditThresholds, creditThresholdAmounts: masterCreditAmounts }
+  }
+  return { creditThresholdKeys: doctoralCreditThresholds, creditThresholdAmounts: doctoralCreditAmounts }
+}
+
+const tableTitles = {
+  basics: {
+    SPECIAL_EXCLUDED: ['', 'Started studying', 'Graduated'],
+    SPECIAL_INCLUDED: ['', 'Started studying', 'Graduated', 'Transferred away', 'Transferred to'],
+  },
+  credits: {
+    SPECIAL_EXCLUDED: ['', 'Total', 'Major students credits', 'Transferred credits'],
+    SPECIAL_INCLUDED: ['', 'Total', 'Major students credits', 'Non major students credits', 'Transferred credits'],
+  },
+  creditProgress: {
+    bachelor: [
+      '',
+      'All',
+      '< 30 credits',
+      '30-59 credits',
+      '60-89 credits',
+      '90-119 credits',
+      '120-149 credits',
+      '150-179 credits',
+      '> 180 credits',
+    ],
+    master: [
+      '',
+      'All',
+      '< 200 credits',
+      '200-219 credits',
+      '220-239 credits',
+      '240-259 credits',
+      '260-279 credits',
+      '280-299 credits',
+      '> 300 credits',
+    ],
+    doctoral: [
+      '',
+      'All',
+      '< 50 credits',
+      '50-99 credits',
+      '100-149 credits',
+      '150-199 credits',
+      '200-249 credits',
+      '250-299 credits',
+      '> 300 credits',
+    ],
+  },
+  studytracks: [
+    '',
+    'All',
+    'Started studying',
+    'Currently enrolled',
+    'Absent',
+    'Inactive',
+    'Graduated',
+    'Men',
+    'Women',
+    'Finnish',
+  ],
+}
+
+const getCreditProgressTableTitles = studyprogramme => {
+  if (studyprogramme.includes('KH')) return tableTitles.creditProgress.bachelor
+  if (studyprogramme.includes('MH')) return tableTitles.creditProgress.master
+  return tableTitles.creditProgress.doctoral
+}
 
 module.exports = {
+  getCorrectStudentnumbers,
   formatStudyright,
   formatStudent,
+  formatTransfer,
   getYearsArray,
   getYearsObject,
+  getAcademicYearsObject,
   getStatsBasis,
   isMajorStudentCredit,
+  isSpecialGroupCredit,
   getMedian,
   getMean,
   defineYear,
   getStartDate,
+  alltimeStartDate,
+  alltimeEndDate,
   getThesisType,
   getPercentage,
+  getBachelorCreditGraphStats,
+  getMasterCreditGraphStats,
+  getDoctoralCreditGraphStats,
   getCreditGraphStats,
-  creditThresholds,
+  getCreditThresholds,
+  tableTitles,
+  getCreditProgressTableTitles,
 }

services/backend/src/services/studyprogrammeStats.js

@@ -1,315 +0,0 @@
-const moment = require('moment')
-const { indexOf } = require('lodash')
-
-const { mapToProviders } = require('../util/utils')
-const {
-  getStatsBasis,
-  defineYear,
-  isMajorStudentCredit,
-  getYearsObject,
-  getYearsArray,
-  getMedian,
-  getMean,
-  getStartDate,
-  getThesisType,
-} = require('./studyprogrammeHelpers')
-const {
-  graduatedStudyRights,
-  startedStudyrights,
-  cancelledStudyRights,
-  transfersAway,
-  transfersTo,
-  getProgrammesStudents,
-  getCreditsForStudyProgramme,
-  getTransferredCredits,
-  getThesisCredits,
-} = require('./newStudyprogramme')
-
-// const getEnrolledStats = async (studytrack, since, years, isAcademicYear) => {
-//   const students = await enrolledStudyrights(studytrack)
-//   const { graphStats, tableStats } = getStatsBasis(years)
-
-//   students.forEach(({ startdate }) => {
-//     if (startdate.getMonth() === 7) {
-//       const enrolledYear = defineYear(startdate, isAcademicYear)
-
-//       graphStats[indexOf(years, enrolledYear)] += 1
-//       tableStats[enrolledYear] += 1
-//     }
-//   })
-
-//   return { graphStats, tableStats }
-// }
-
-const getGraduatedStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await graduatedStudyRights(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ enddate }) => {
-    const graduationYear = defineYear(enddate, isAcademicYear)
-    graphStats[indexOf(years, graduationYear)] += 1
-    tableStats[graduationYear] += 1
-  })
-  return { graphStats, tableStats }
-}
-
-const getStartedStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await startedStudyrights(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ studystartdate }) => {
-    const startYear = defineYear(studystartdate, isAcademicYear)
-    graphStats[indexOf(years, startYear)] += 1
-    tableStats[startYear] += 1
-  })
-  return { graphStats, tableStats }
-}
-
-const getCancelledStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await cancelledStudyRights(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ canceldate }) => {
-    const cancelYear = defineYear(canceldate, isAcademicYear)
-    graphStats[indexOf(years, cancelYear)] += 1
-    tableStats[cancelYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getTransferredAwayStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await transfersAway(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ transferdate }) => {
-    const transferYear = defineYear(transferdate, isAcademicYear)
-    graphStats[indexOf(years, transferYear)] += 1
-    tableStats[transferYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getTransferredToStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await transfersTo(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ transferdate }) => {
-    const transferYear = defineYear(transferdate, isAcademicYear)
-    graphStats[indexOf(years, transferYear)] += 1
-    tableStats[transferYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getRegularCreditStats = async (studytrack, since, years, isAcademicYear) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const studyrights = await getProgrammesStudents(studytrack)
-  const credits = await getCreditsForStudyProgramme(providercode, since)
-
-  let majors = getStatsBasis(years)
-  let nonMajors = getStatsBasis(years)
-
-  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
-    const studyright = studyrights.find(studyright => studyright.studentnumber == student_studentnumber)
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-
-    if (isMajorStudentCredit(studyright, attainment_date)) {
-      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
-      majors.tableStats[attainmentYear] += credits || 0
-    } else {
-      nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
-      nonMajors.tableStats[attainmentYear] += credits || 0
-    }
-  })
-
-  return { majors, nonMajors }
-}
-
-const getTransferredCreditStats = async (studytrack, since, years, isAcademicYear) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const credits = await getTransferredCredits(providercode, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  credits.forEach(({ attainment_date, credits }) => {
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-    graphStats[indexOf(years, attainmentYear)] += credits || 0
-    tableStats[attainmentYear] += credits || 0
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getThesisStats = async (studytrack, since, years, isAcademicYear) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const thesisType = getThesisType(studytrack)
-  const credits = await getThesisCredits(providercode, since, thesisType)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  credits.forEach(({ attainment_date }) => {
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-    graphStats[indexOf(years, attainmentYear)] += 1
-    tableStats[attainmentYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getGraduationTimeStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await graduatedStudyRights(studytrack, since)
-  let graduationAmounts = getYearsObject(years)
-  let graduationTimes = getYearsObject(years, true)
-
-  studyrights.forEach(({ enddate, studystartdate }) => {
-    const graduationYear = defineYear(enddate, isAcademicYear)
-    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
-    graduationAmounts[graduationYear] += 1
-    graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
-  })
-
-  const medians = getYearsObject(years, true)
-  const means = getYearsObject(years, true)
-
-  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
-  const comparisonValue = studytrack.includes('KH') ? 72 : 48
-
-  // HighCharts graph require the data to have this format (ie. actual value, "empty value")
-  years.forEach(year => {
-    const median = getMedian(graduationTimes[year])
-    const mean = getMean(graduationTimes[year])
-    medians[year] = [
-      ['', median],
-      ['', comparisonValue - median],
-    ]
-    means[year] = [
-      ['', mean],
-      ['', comparisonValue - mean],
-    ]
-  })
-  return { medians, means, graduationAmounts }
-}
-
-const getBasicStatsForStudytrack = async ({ studyprogramme, yearType }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-  const started = await getStartedStats(studyprogramme, since, years, isAcademicYear)
-  const graduated = await getGraduatedStats(studyprogramme, since, years, isAcademicYear)
-  const cancelled = await getCancelledStats(studyprogramme, since, years, isAcademicYear)
-  const transferredAway = await getTransferredAwayStats(studyprogramme, since, years, isAcademicYear)
-  const transferredTo = await getTransferredToStats(studyprogramme, since, years, isAcademicYear)
-
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const tableStats = reversedYears.map(year => [
-    year,
-    started.tableStats[year],
-    graduated.tableStats[year],
-    cancelled.tableStats[year],
-    transferredAway.tableStats[year],
-    transferredTo.tableStats[year],
-  ])
-
-  return {
-    id: studyprogramme,
-    years,
-    graphStats: [
-      {
-        name: 'Started',
-        data: started.graphStats,
-      },
-      {
-        name: 'Graduated',
-        data: graduated.graphStats,
-      },
-      {
-        name: 'Cancelled',
-        data: cancelled.graphStats,
-      },
-      {
-        name: 'Transferred away',
-        data: transferredAway.graphStats,
-      },
-      {
-        name: 'Transferred To',
-        data: transferredTo.graphStats,
-      },
-    ],
-    tableStats,
-  }
-}
-
-const getCreditStatsForStudytrack = async ({ studyprogramme, yearType }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-  const { majors, nonMajors } = await getRegularCreditStats(studyprogramme, since, years, isAcademicYear)
-  const transferred = await getTransferredCreditStats(studyprogramme, since, years, isAcademicYear)
-
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const tableStats = reversedYears.map(year => [
-    year,
-    majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
-    majors.tableStats[year],
-    nonMajors.tableStats[year],
-    transferred.tableStats[year],
-  ])
-
-  return {
-    id: studyprogramme,
-    years,
-    tableStats,
-    graphStats: [
-      {
-        name: 'Major students credits',
-        data: majors.graphStats,
-      },
-      {
-        name: 'Non-major students credits',
-        data: nonMajors.graphStats,
-      },
-      {
-        name: 'Transferred credits',
-        data: transferred.graphStats,
-      },
-    ],
-  }
-}
-
-const getGraduationStatsForStudytrack = async ({ studyprogramme, yearType }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-  const thesis = await getThesisStats(studyprogramme, since, years, isAcademicYear)
-  const graduated = await getGraduatedStats(studyprogramme, since, years, isAcademicYear)
-  const graduationTimeStats = await getGraduationTimeStats(studyprogramme, since, years, isAcademicYear)
-
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
-
-  return {
-    id: studyprogramme,
-    years,
-    tableStats,
-    graphStats: [
-      {
-        name: 'Graduated students',
-        data: graduated.graphStats,
-      },
-      {
-        name: 'Wrote thesis',
-        data: thesis.graphStats,
-      },
-    ],
-    graduationMedianTime: graduationTimeStats.medians,
-    graduationMeanTime: graduationTimeStats.means,
-    graduationAmounts: graduationTimeStats.graduationAmounts,
-  }
-}
-
-module.exports = {
-  getBasicStatsForStudytrack,
-  getCreditStatsForStudytrack,
-  getGraduationStatsForStudytrack,
-}

services/backend/src/services/studyprogrammeUpdates.js

@@ -0,0 +1,115 @@
+const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics')
+const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
+const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
+const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
+const {
+  setBasicStats,
+  setCreditStats,
+  setGraduationStats,
+  setStudytrackStats,
+} = require('../services/analyticsService')
+const logger = require('../util/logger')
+
+const updateBasicView = async code => {
+  const specialCalendar = {
+    yearType: 'CALENDAR_YEAR',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const specialAcademic = {
+    yearType: 'ACADEMIC_YEAR',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const specialExcludedCalendar = {
+    yearType: 'CALENDAR_YEAR',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
+
+  const specialExcludedAcademic = {
+    yearType: 'ACADEMIC_YEAR',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
+
+  const options = [specialCalendar, specialAcademic, specialExcludedCalendar, specialExcludedAcademic]
+
+  for (const option of options) {
+    try {
+      const basicStats = await getBasicStatsForStudytrack({
+        studyprogramme: code,
+        settings: {
+          isAcademicYear: option.yearType === 'ACADEMIC_YEAR',
+          includeAllSpecials: option.specialGroups === 'SPECIAL_INCLUDED',
+        },
+      })
+      await setBasicStats(basicStats, option.yearType, option.specialGroups)
+
+      const creditStats = await getCreditStatsForStudytrack({
+        studyprogramme: code,
+        settings: {
+          isAcademicYear: option.yearType === 'ACADEMIC_YEAR',
+          includeAllSpecials: option.specialGroups === 'SPECIAL_INCLUDED',
+        },
+      })
+      await setCreditStats(creditStats, option.yearType, option.specialGroups)
+
+      const graduationStats = await getGraduationStatsForStudytrack({
+        studyprogramme: code,
+        settings: {
+          isAcademicYear: option.yearType === 'ACADEMIC_YEAR',
+          includeAllSpecials: option.specialGroups === 'SPECIAL_INCLUDED',
+        },
+      })
+      await setGraduationStats(graduationStats, option.yearType, option.specialGroups)
+    } catch (e) {
+      logger.error(e)
+    }
+    return 'OK'
+  }
+}
+
+const updateStudytrackView = async code => {
+  const specialGraduated = {
+    graduated: 'GRADUATED_INCLUDED',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const specialExcludedGraduated = {
+    graduated: 'GRADUATED_INCLUDED',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
+  const specialGraduatedExcluded = {
+    graduated: 'GRADUATED_EXCLUDED',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const specialExcludedGraduatedExcluded = {
+    graduated: 'GRADUATED_EXCLUDED',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
+
+  const options = [
+    specialGraduated,
+    specialExcludedGraduated,
+    specialGraduatedExcluded,
+    specialExcludedGraduatedExcluded,
+  ]
+
+  for (const option of options) {
+    try {
+      const stats = await getStudytrackStatsForStudyprogramme({
+        studyprogramme: code,
+        settings: {
+          specialGroups: option.specialGroups === 'SPECIAL_INCLUDED',
+          graduated: option.graduated === 'GRADUATED_INCLUDED',
+        },
+      })
+      await setStudytrackStats(stats, option.graduated, option.specialGroups)
+    } catch (e) {
+      logger.error(e)
+    }
+  }
+
+  return 'OK'
+}
+
+module.exports = {
+  updateBasicView,
+  updateStudytrackView,
+}

services/backend/src/services/studytrackStats.js

@@ -1,100 +1,173 @@
 const { indexOf } = require('lodash')
+const moment = require('moment')
 
 const { getAssociations } = require('./studyrights')
-const { studentnumbersWithAllStudyrightElements } = require('./populations')
 const {
+  getMedian,
+  getMean,
   getStartDate,
   getYearsArray,
   getPercentage,
-  getCreditGraphStats,
   getYearsObject,
-  creditThresholds,
+  getCorrectStudentnumbers,
+  getCreditThresholds,
+  getCreditGraphStats,
+  tableTitles,
+  getCreditProgressTableTitles,
 } = require('./studyprogrammeHelpers')
-const { studytrackStudents, startedStudyrights, graduatedStudyRights } = require('./newStudyprogramme')
+const {
+  studytrackStudents,
+  allStudyrights,
+  startedStudyrights,
+  inactiveStudyrights,
+  enrolledStudents,
+  absentStudents,
+  graduatedStudyRights,
+} = require('./studyprogramme')
 const { getAcademicYearDates } = require('../util/semester')
 
-const getStudentData = students => {
-  const data = { female: 0, male: 0, finnish: 0, lte30: 0, lte60: 0, lte90: 0, lte120: 0, lte150: 0, mte150: 0 }
+const getStudentData = (students, thresholdKeys, thresholdAmounts) => {
+  let data = { female: 0, male: 0, finnish: 0 }
+  thresholdKeys.forEach(t => (data[t] = 0))
+
   students.forEach(({ gender_code, home_country_en, creditcount }) => {
     data.male += gender_code === '1' ? 1 : 0
     data.female += gender_code === '2' ? 1 : 0
     data.finnish += home_country_en === 'Finland' ? 1 : 0
-    data.lte30 += creditcount < 30 ? 1 : 0
-    data.lte60 += creditcount >= 30 && creditcount < 60 ? 1 : 0
-    data.lte90 += creditcount >= 60 && creditcount < 90 ? 1 : 0
-    data.lte120 += creditcount >= 90 && creditcount < 120 ? 1 : 0
-    data.lte150 += creditcount >= 120 && creditcount < 150 ? 1 : 0
-    data.mte150 += creditcount >= 150 ? 1 : 0
+    data[thresholdKeys[0]] += creditcount < thresholdAmounts[0] ? 1 : 0
+    data[thresholdKeys[1]] += creditcount >= thresholdAmounts[0] && creditcount < thresholdAmounts[1] ? 1 : 0
+    data[thresholdKeys[2]] += creditcount >= thresholdAmounts[1] && creditcount < thresholdAmounts[2] ? 1 : 0
+    data[thresholdKeys[3]] += creditcount >= thresholdAmounts[2] && creditcount < thresholdAmounts[3] ? 1 : 0
+    data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
+    data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
+    data[thresholdKeys[6]] += creditcount >= thresholdAmounts[6] ? 1 : 0
   })
   return data
 }
 
+const getGraduationTimeStats = async ({
+  year,
+  graduated,
+  studyprogramme,
+  track,
+  graduationMeanTime,
+  graduationMedianTime,
+  graduationAmounts,
+}) => {
+  // Count how long each student took to graduate
+  let graduationTimes = []
+  graduated.forEach(({ enddate, studystartdate }) => {
+    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+    graduationAmounts[track][year] += 1
+    graduationTimes = [...graduationTimes, timeToGraduation]
+  })
+
+  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
+  const comparisonValue = studyprogramme.includes('KH') ? 72 : 48
+
+  // HighCharts graph requires the data to have this format (ie. actual value, "empty value")
+  const median = getMedian(graduationTimes)
+  const mean = getMean(graduationTimes)
+  graduationMedianTime[track][year] = [
+    ['', median],
+    ['', comparisonValue - median],
+  ]
+  graduationMeanTime[track][year] = [
+    ['', mean],
+    ['', comparisonValue - mean],
+  ]
+}
+
 // Goes through the programme and all its studytracks for the said year and adds the wanted stats to the data objects
-const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studytrackNames, year, years, data }) => {
-  const { mainStatsByYear, mainStatsByTrack, creditGraphStats, creditTableStats, emptyTracks } = data
-  const { startDate, endDate } = getAcademicYearDates(year)
+const getStudytrackDataForTheYear = async ({
+  studyprogramme,
+  since,
+  settings,
+  studytracks,
+  studytrackNames,
+  year,
+  years,
+  data,
+  creditThresholdKeys,
+  creditThresholdAmounts,
+}) => {
+  const { specialGroups: includeAllSpecials, graduated: includeGraduated } = settings
+  const { startDate, endDate } = getAcademicYearDates(year, since)
+
+  const {
+    mainStatsByYear,
+    mainStatsByTrack,
+    creditGraphStats,
+    creditTableStats,
+    graduationMeanTime,
+    graduationMedianTime,
+    graduationAmounts,
+    totalAmounts,
+    emptyTracks,
+  } = data
 
   await Promise.all(
     studytracks.map(async track => {
       const codes = studyprogramme === track ? [studyprogramme] : [studyprogramme, track]
-      const studentnumbers = await studentnumbersWithAllStudyrightElements(
+
+      const studentnumbers = await getCorrectStudentnumbers({
         codes,
         startDate,
         endDate,
-        true,
-        true,
-        true,
-        true
-      )
+        includeAllSpecials,
+        includeGraduated,
+      })
 
-      // All the stats are counted for the students who actually started studying
+      // Get all the studyrights and students for the calculations
+      const all = await allStudyrights(track, studentnumbers)
+      const students = await studytrackStudents(studentnumbers)
+      const studentData = getStudentData(students, creditThresholdKeys, creditThresholdAmounts)
       const started = await startedStudyrights(track, startDate, studentnumbers)
-      const startedStudentnumbers = [...new Set(started.map(s => s.studentnumber))]
-      const students = await studytrackStudents(startedStudentnumbers)
-      const studentData = getStudentData(students)
+      const enrolled = await enrolledStudents(track, startDate, studentnumbers)
+      const absent = await absentStudents(track, studentnumbers)
+      const inactive = await inactiveStudyrights(track, studentnumbers)
       const graduated = await graduatedStudyRights(track, startDate, studentnumbers)
 
       // If the track has no stats for that year, it should be removed from the table and dropdown options
-      if (started.length === 0) {
+      if (all.length === 0) {
         emptyTracks.has(track) ? emptyTracks.set(track, emptyTracks.get(track) + 1) : emptyTracks.set(track, 1)
         return
       }
 
       // Count stats for creditgraph for the year per track
-      creditThresholds.forEach(
+      creditThresholdKeys.forEach(
         threshold => (creditGraphStats[track][threshold].data[indexOf(years, year)] = studentData[threshold])
       )
 
       // Count stats for the credit progress table for the year per track
       creditTableStats[track] = [
         ...creditTableStats[track],
-        [
-          `${year} - ${year + 1}`,
-          started.length,
-          studentData.lte30,
-          studentData.lte60,
-          studentData.lte90,
-          studentData.lte120,
-          studentData.lte150,
-          studentData.mte150,
-        ],
+        [year, all.length, ...creditThresholdKeys.map(threshold => studentData[threshold])],
       ]
 
       // Count stats for the main studytrack table grouped by tracks
       mainStatsByTrack[track] = [
         ...mainStatsByTrack[track],
         [
-          `${year} - ${year + 1}`,
+          year,
+          all.length,
+          getPercentage(all.length, all.length),
           started.length,
-          getPercentage(started.length, started.length),
-          studentData.male,
-          getPercentage(studentData.male, started.length),
-          studentData.female,
-          getPercentage(studentData.female, started.length),
-          studentData.finnish,
-          getPercentage(studentData.finnish, started.length),
+          getPercentage(started.length, all.length),
+          enrolled.length,
+          getPercentage(enrolled.length, all.length),
+          absent.length,
+          getPercentage(absent.length, all.length),
+          inactive.length,
+          getPercentage(inactive.length, all.length),
           graduated.length,
-          getPercentage(graduated.length, started.length),
+          getPercentage(graduated.length, all.length),
+          studentData.male,
+          getPercentage(studentData.male, all.length),
+          studentData.female,
+          getPercentage(studentData.female, all.length),
+          studentData.finnish,
+          getPercentage(studentData.finnish, all.length),
         ],
       ]
 
@@ -102,19 +175,39 @@ const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studyt
       mainStatsByYear[year] = [
         ...mainStatsByYear[year],
         [
-          studytrackNames[track]?.name['fi'] || `${year} - ${year + 1}`,
+          studytrackNames[track]?.name['fi'] ? `${studytrackNames[track]?.name['fi']}, ${track}` : year,
+          all.length,
+          getPercentage(all.length, all.length),
           started.length,
-          getPercentage(started.length, started.length),
-          studentData.male,
-          getPercentage(studentData.male, started.length),
-          studentData.female,
-          getPercentage(studentData.female, started.length),
-          studentData.finnish,
-          getPercentage(studentData.finnish, started.length),
+          getPercentage(started.length, all.length),
+          enrolled.length,
+          getPercentage(enrolled.length, all.length),
+          absent.length,
+          getPercentage(absent.length, all.length),
+          inactive.length,
+          getPercentage(inactive.length, all.length),
           graduated.length,
-          getPercentage(graduated.length, started.length),
+          getPercentage(graduated.length, all.length),
+          studentData.male,
+          getPercentage(studentData.male, all.length),
+          studentData.female,
+          getPercentage(studentData.female, all.length),
+          studentData.finnish,
+          getPercentage(studentData.finnish, all.length),
         ],
       ]
+
+      // Count stats for the graduation time charts grouped by year
+      totalAmounts[track][year] = await all.length
+      await getGraduationTimeStats({
+        year,
+        graduated,
+        studyprogramme,
+        track,
+        graduationMedianTime,
+        graduationMeanTime,
+        graduationAmounts,
+      })
     })
   )
 }
@@ -133,27 +226,46 @@ const getStudytrackOptions = (studyprogramme, studytrackNames, studytracks, empt
 }
 
 // Creates empty objects for each statistic type, which are then updated with the studytrack data
-const getEmptyStatsObjects = (years, studytracks) => {
-  const mainStatsByYear = getYearsObject(years, true)
+const getEmptyStatsObjects = (years, studytracks, studyprogramme) => {
+  const mainStatsByYear = getYearsObject({ years, emptyArrays: true })
   const mainStatsByTrack = {}
   const creditGraphStats = {}
   const creditTableStats = {}
+  const graduationMedianTime = {}
+  const graduationMeanTime = {}
+  const graduationAmounts = {}
+  const totalAmounts = {}
   const emptyTracks = new Map()
 
   studytracks.forEach(async track => {
     mainStatsByTrack[track] = []
-    creditGraphStats[track] = await getCreditGraphStats(years)
+    creditGraphStats[track] = getCreditGraphStats(studyprogramme, years)
     creditTableStats[track] = []
+    graduationMedianTime[track] = getYearsObject({ years, emptyArrays: true })
+    graduationMeanTime[track] = getYearsObject({ years, emptyArrays: true })
+    graduationAmounts[track] = getYearsObject({ years })
+    totalAmounts[track] = getYearsObject({ years })
   })
 
-  return { mainStatsByYear, mainStatsByTrack, creditGraphStats, creditTableStats, emptyTracks }
+  return {
+    mainStatsByYear,
+    mainStatsByTrack,
+    creditGraphStats,
+    creditTableStats,
+    graduationMedianTime,
+    graduationMeanTime,
+    graduationAmounts,
+    totalAmounts,
+    emptyTracks,
+  }
 }
 
 // Combines all the data for the Populations and Studytracks -view
-const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {
+const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, settings }) => {
   const isAcademicYear = true
+  const includeYearsCombined = true
   const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear())
+  const years = getYearsArray(since.getFullYear(), isAcademicYear, includeYearsCombined)
 
   const associations = await getAssociations()
   const studytracks = associations.programmes[studyprogramme]
@@ -162,17 +274,22 @@ const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {
 
   const studytrackNames = associations.studyTracks
 
-  const data = getEmptyStatsObjects(years, studytracks)
+  const data = getEmptyStatsObjects(years, studytracks, studyprogramme)
+  const { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(studyprogramme)
 
   await Promise.all(
     years.map(async year => {
       return await getStudytrackDataForTheYear({
         studyprogramme,
+        since,
+        settings,
         studytracks,
         studytrackNames,
         year,
         years,
         data,
+        creditThresholdKeys,
+        creditThresholdAmounts,
       })
     })
   )
@@ -181,12 +298,12 @@ const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {
 
   return {
     id: studyprogramme,
-    years: getYearsArray(since.getFullYear(), isAcademicYear),
-    mainStatsByTrack: data.mainStatsByTrack,
-    mainStatsByYear: data.mainStatsByYear,
-    creditTableStats: data.creditTableStats,
-    creditGraphStats: data.creditGraphStats,
+    years: getYearsArray(since.getFullYear(), isAcademicYear, includeYearsCombined),
+    ...data,
     studytrackOptions,
+    includeGraduated: settings.graduated,
+    populationTitles: tableTitles['studytracks'],
+    creditTableTitles: getCreditProgressTableTitles(studyprogramme),
   }
 }
 

services/backend/src/services/thesis.js

@@ -1,29 +0,0 @@
-const { ThesisCourse } = require('../models/models_kone')
-
-const createThesisCourse = (programme, course, type) =>
-  ThesisCourse.create({
-    programmeCode: programme,
-    courseCode: course,
-    thesisType: type,
-  })
-
-const deleteThesisCourse = (programme, course) =>
-  ThesisCourse.destroy({
-    where: {
-      programmeCode: programme,
-      courseCode: course,
-    },
-  })
-
-const findProgrammeTheses = programme =>
-  ThesisCourse.findAll({
-    where: {
-      programmeCode: programme,
-    },
-  })
-
-module.exports = {
-  createThesisCourse,
-  deleteThesisCourse,
-  findProgrammeTheses,
-}

services/backend/src/services/topteachers.js

@@ -107,7 +107,9 @@ const findTopTeachers = async yearcode => {
     .filter(isRegularCourse)
     .map(credit => {
       const { credits, course, credittypecode } = credit
-      const teachers = credit.teachers.map(({ id, name }) => ({ id, name }))
+      const teachers = credit.teachers
+        .filter(({ id }) => !id.includes('hy-hlo-org')) // Remove faculties from the leaderboards
+        .map(({ id, name }) => ({ id, name }))
       const passed = Credit.passed(credit) || Credit.improved(credit)
       const failed = Credit.failed(credit)
       const transferred = credittypecode === 9

services/backend/src/services/trends/protoC.js

@@ -2,15 +2,28 @@ const _ = require('lodash')
 const { getRedisCDS, saveToRedis, getTargetStudentCounts } = require('./shared')
 const { getAssociations } = require('../studyrights')
 
-const REDIS_KEY_PROTOC = 'PROTOC_DATA_V2'
+const REDIS_KEY_PROTOC = 'PROTOC_DATA_V3.3'
 const REDIS_KEY_PROTOC_PROGRAMME = 'PROTOC_PROGRAMME_DATA_V2'
 
+const tryParseInt = (number, defaultValue) => {
+  try {
+    return parseInt(number, 10)
+  } catch (e) {
+    return defaultValue
+  }
+}
+
 const calculateProtoC = async query => {
   const associations = await getAssociations()
 
+  const startYear = tryParseInt(query.startYear, null)
+  const endYear = tryParseInt(query.endYear, null)
+
   const data = await getTargetStudentCounts({
     includeOldAttainments: query.include_old_attainments === 'true',
     excludeNonEnrolled: query.exclude_non_enrolled === 'true',
+    startYear,
+    endYear,
   })
 
   const programmeData = data.filter(d => d.programmeType !== 30)
@@ -97,10 +110,16 @@ const calculateProtoCProgramme = async query => {
   const codes = associations.programmes[query.code]
     ? [...associations.programmes[query.code].studytracks, query.code]
     : []
+
+  const startYear = tryParseInt(query.startYear, null)
+  const endYear = tryParseInt(query.endYear, null)
+
   const data = await getTargetStudentCounts({
-    codes: codes,
+    codes,
     includeOldAttainments: query.include_old_attainments === 'true',
     excludeNonEnrolled: query.exclude_non_enrolled === 'true',
+    startYear,
+    endYear,
   })
 
   const programmeData = data.find(d => d.programmeType === 20)
@@ -157,10 +176,10 @@ const calculateProtoCProgramme = async query => {
 }
 
 const getProtoC = async (query, doRefresh = false) => {
-  const { include_old_attainments, exclude_non_enrolled } = query
+  const { include_old_attainments, exclude_non_enrolled, startYear, endYear } = query
 
   // redis keys for different queries
-  const KEY = `${REDIS_KEY_PROTOC}_OLD_${include_old_attainments.toUpperCase()}_ENR_${exclude_non_enrolled.toUpperCase()}`
+  const KEY = `${REDIS_KEY_PROTOC}_OLD_${include_old_attainments.toUpperCase()}_ENR_${exclude_non_enrolled.toUpperCase()}_FROM_${startYear}_TO_${endYear}`
   const protoC = await getRedisCDS(KEY)
   if (!protoC || doRefresh) {
     const data = await calculateProtoC(query)
@@ -172,8 +191,8 @@ const getProtoC = async (query, doRefresh = false) => {
 
 // used for studytrack view
 const getProtoCProgramme = async (query, doRefresh = false) => {
-  const { include_old_attainments, exclude_non_enrolled, code } = query
-  const KEY = `${REDIS_KEY_PROTOC_PROGRAMME}_CODE_${code}_OLD_${include_old_attainments.toUpperCase()}_ENR_${exclude_non_enrolled.toUpperCase()}`
+  const { include_old_attainments, exclude_non_enrolled, code, startYear, endYear } = query
+  const KEY = `${REDIS_KEY_PROTOC_PROGRAMME}_CODE_${code}_OLD_${include_old_attainments.toUpperCase()}_ENR_${exclude_non_enrolled.toUpperCase()}_FROM_${startYear}_TO_${endYear}`
   const protoCProgramme = await getRedisCDS(KEY)
 
   if (!protoCProgramme || doRefresh) {

services/backend/src/services/trends/shared.js

@@ -16,7 +16,16 @@ const saveToRedis = async (data, REDIS_KEY, expire = false) => {
   }
 }
 
-const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNonEnrolled }) => {
+const getTargetStudentCounts = async ({
+  codes,
+  includeOldAttainments,
+  excludeNonEnrolled,
+  startYear = 2017,
+  endYear = 2020,
+}) => {
+  // only pick studyrights during which the student has enrolled at least once, whether it's
+  // a present or non-present enrollment
+
   return await sequelize.query(
     `
     SELECT
@@ -25,39 +34,42 @@ const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNon
         ss.programme_code "programmeCode",
         ss.programme_name "programmeName",
         ss.programme_type "programmeType",
-        COUNT(ss.studentnumber) "programmeTotalStudents",
-        SUM(
-             public.is_in_target(
-                 CURRENT_TIMESTAMP,
-                 ss.studystartdate,
-                 public.next_date_occurrence(ss.studystartdate), -- e.g if studystartdate = 2017-07-31 and it's now 2020-02-02, this returns 2020-07-31. However, if it's now 2020-11-01 (we've passed the date already), it returns 2021-07-31.
-                 ss.credits,
-                 CASE
-                     -- HAX: instead of trying to guess which category the student is in if
-                     --      the studyright is currently cancelled, just make the target HUGE
-                     --      so we can subtract these from the non-goal students
-                     WHEN ss.currently_cancelled = 1 THEN 999999999
-                     -- credit target: 60 credits per year since starting
-                     ELSE 60 * ceil(EXTRACT(EPOCH FROM (now() - ss.studystartdate) / 365) / 86400)
-                 END
-             )
+        0 AS "students3y",
+        COUNT(ss.studentnumber) FILTER (
+          WHERE
+            public.is_in_target(
+                CURRENT_TIMESTAMP,
+                ss.studystartdate,
+                public.next_date_occurrence(ss.studystartdate), -- e.g if studystartdate = 2017-07-31 and it's now 2020-02-02, this returns 2020-07-31. However, if it's now 2020-11-01 (we've passed the date already), it returns 2021-07-31.
+                ss.credits,
+                CASE
+                    -- HAX: instead of trying to guess which category the student is in if
+                    --      the studyright is currently cancelled, just make the target HUGE
+                    --      so we can subtract these from the non-goal students
+                    WHEN ss.currently_cancelled = 1 THEN 999999999
+                    -- credit target: 60 credits per year since starting
+                    ELSE 60 * ceil(EXTRACT(EPOCH FROM (now() - ss.studystartdate) / 365) / 86400)
+                END
+            ) = 1
         ) "students3y",
-        SUM(
-             public.is_in_target(
-                 CURRENT_TIMESTAMP,
-                 ss.studystartdate,
-                 public.next_date_occurrence(ss.studystartdate),
-                 ss.credits,
-                 CASE
-                     -- HAX: instead of trying to guess which category the student is in if
-                     --      the studyright is currently cancelled, just make the target HUGE
-                     --      so we can subtract these from the non-goal students
-                     WHEN ss.currently_cancelled = 1 THEN 999999999
-                     -- credit target: 45 credits per year since starting
-                     ELSE 45 * ceil(EXTRACT(EPOCH FROM (now() - ss.studystartdate) / 365) / 86400)
-                 END
-             )
+        COUNT(ss.studentnumber) FILTER (
+          WHERE
+            public.is_in_target(
+                CURRENT_TIMESTAMP,
+                ss.studystartdate,
+                public.next_date_occurrence(ss.studystartdate),
+                ss.credits,
+                CASE
+                    -- HAX: instead of trying to guess which category the student is in if
+                    --      the studyright is currently cancelled, just make the target HUGE
+                    --      so we can subtract these from the non-goal students
+                    WHEN ss.currently_cancelled = 1 THEN 999999999
+                    -- credit target: 45 credits per year since starting
+                    ELSE 45 * ceil(EXTRACT(EPOCH FROM (now() - ss.studystartdate) / 365) / 86400)
+                END
+            ) = 1
         ) "students4y",
+        COUNT(ss.studentnumber) "programmeTotalStudents",
         SUM(ss.currently_cancelled) "currentlyCancelled"
     FROM (
         SELECT
@@ -68,92 +80,16 @@ const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNon
             programme_type,
             studystartdate,
             studentnumber,
-            SUM(credits) credits,
+            SUM(credits) FILTER (WHERE NOT credits.is_old OR :include_old_attainments) credits,
             CASE
                 WHEN SUM(currently_cancelled) > 0 THEN 1
                 ELSE 0
             END currently_cancelled
-        FROM (
-            SELECT
-                org.code org_code,
-                org.name->>'fi' org_name,
-                element_details.code programme_code,
-                element_details.name->>'fi' programme_name,
-                element_details.type programme_type,
-                studyright.studystartdate studystartdate,
-                studyright.student_studentnumber studentnumber,
-                credit.credits credits,
-                CASE
-                    WHEN studyright.canceldate IS NOT NULL AND studyright.graduated != 1 THEN 1
-                    ELSE 0
-                END currently_cancelled
-            FROM
-                organization org
-                ${
-                  excludeNonEnrolled
-                    ? `
-                -- only pick studyrights during which the student has enrolled at least once, whether it's
-                -- a present or non-present enrollment
-                INNER JOIN (
-                  SELECT DISTINCT -- remove duplicate join rows from se with DISTINCT
-                      studyright.*
-                  FROM
-                      studyright
-                      INNER JOIN (
-                          SELECT
-                              studentnumber,
-                              startdate
-                          FROM
-                              semester_enrollments
-                              INNER JOIN semesters
-                                  ON semester_enrollments.semestercode = semesters.semestercode
-                      ) se
-                          ON se.studentnumber = studyright.student_studentnumber
-                  WHERE
-                      se.startdate IS NOT NULL
-                      AND se.startdate >= studyright.studystartdate
-                ) studyright
-                    ON org.code = studyright.faculty_code`
-                    : `
-                INNER JOIN studyright
-                      ON org.code = studyright.faculty_code`
-                }
-                
-                INNER JOIN (
-                    -- HACK: fix updater writing duplicates where enddate has changed
-                    SELECT DISTINCT ON (studyrightid, startdate, code, studentnumber)
-                        *
-                    FROM studyright_elements
-                ) s_elements
-                    ON studyright.studyrightid = s_elements.studyrightid
-                INNER JOIN element_details
-                    ON s_elements.code = element_details.code
-                LEFT JOIN transfers
-                    ON studyright.studyrightid = transfers.studyrightid
-                LEFT JOIN LATERAL (
-                    SELECT
-                        student_studentnumber,
-                        attainment_date,
-                        credits
-                    FROM credit
-                    WHERE credit.credittypecode IN (4, 9) -- Completed or Transferred
-                        AND (credit."isStudyModule" = false OR credit."isStudyModule" IS NULL)
-                        ${
-                          includeOldAttainments
-                            ? ''
-                            : "AND credit.attainment_date >= studyright.studystartdate -- only include credits attained during studyright's time"
-                        }
-                ) credit
-                    ON credit.student_studentnumber = studyright.student_studentnumber
-            WHERE
-                studyright.extentcode = 1 -- Bachelor's
-                AND org.code NOT IN ('01', 'H02955')
-                AND studyright.prioritycode IN (1, 30) -- Primary or Graduated
-                AND studyright.studystartdate IN ('2017-08-01 00:00:00+00', '2018-08-01 00:00:00+00', '2019-08-01 00:00:00+00')
-                AND element_details.type IN (20,30) -- programme
-                ${!!codes && codes.length > 0 ? 'AND element_details.code IN (:codes)' : ''}
-                AND transfers.studyrightid IS NULL -- Not transferred within faculty
-        ) s
+        FROM organization_yearly_credits credits
+        WHERE
+          (NOT :exclude_non_enrolled OR credits.enrollment_exists) AND
+          (:ignore_codes OR credits.programme_code IN (:codes)) AND
+          credits.startyear BETWEEN :start_year AND :end_year
         GROUP BY (1,2), (3,4) ,5 , 6, 7
     ) ss
     GROUP BY (1, 2), (3, 4), 5;
@@ -161,7 +97,12 @@ const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNon
     {
       type: sequelize.QueryTypes.SELECT,
       replacements: {
-        codes: codes,
+        codes: !codes || codes.length === 0 ? ['DUMMY'] : codes,
+        ignore_codes: !codes || codes.length === 0,
+        exclude_non_enrolled: excludeNonEnrolled,
+        start_year: startYear,
+        end_year: endYear,
+        include_old_attainments: includeOldAttainments,
       },
     }
   )

services/backend/src/services/trends/statusAttainments.js

@@ -284,6 +284,7 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
     from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
     to: new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS),
   })
+
   const yearlyAccCreditsPromises = makeYearlyCreditsPromises(
     startYear,
     yearRange,

services/backend/src/services/trends/statusGraduated.js

@@ -1,4 +1,5 @@
 const _ = require('lodash')
+const Sentry = require('@sentry/node')
 const {
   dbConnections: { sequelize },
 } = require('../../database/connection')
@@ -93,10 +94,15 @@ const calculateStatusGraduated = async (unixMillis, showByYear) => {
       // init faculty
       if (!acc[curr.faculty_code]) {
         const org = orgs.find(o => o.code === curr.faculty_code)
-        acc[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        if (org) {
+          acc[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        } else {
+          Sentry.captureException(new Error(`Failed to find org for faculty code: ${curr.faculty_code}`))
+        }
       }
-      if (!acc[curr.faculty_code]['yearly'][data.year])
+      if (!acc[curr.faculty_code]['yearly'][data.year]) {
         acc[curr.faculty_code]['yearly'][data.year] = { acc: 0, total: 0 }
+      }
       // init programme
       if (!acc[curr.faculty_code]['drill'][curr.code]) {
         const element = elements.find(e => e.code === curr.code)
@@ -128,7 +134,11 @@ const calculateStatusGraduated = async (unixMillis, showByYear) => {
       // if no accumulated init faculty
       if (!mankeled[curr.faculty_code]) {
         const org = orgs.find(o => o.code === curr.faculty_code)
-        mankeled[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        if (org) {
+          mankeled[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        } else {
+          Sentry.captureException(new Error(`Failed to find org for faculty code: ${curr.faculty_code}`))
+        }
       }
       // if no accumulated init programme
       if (!mankeled[curr.faculty_code]['drill'][curr.code]) {

services/backend/src/services/units.js

@@ -1,41 +0,0 @@
-const Sequelize = require('sequelize')
-const { ElementDetail, StudyrightElement } = require('../models')
-const Op = Sequelize.Op
-
-const hasStudent = async (code, studentnumber) =>
-  StudyrightElement.findOne({
-    where: {
-      studentnumber,
-      code,
-    },
-  })
-
-const parseUnitFromElement = element => ({
-  id: element.code,
-  name: element.name.fi,
-  enabled: true,
-  type: element.type,
-})
-
-const getUnitsFromElementDetails = async () => {
-  const elementdetails = await ElementDetail.findAll({
-    where: {
-      type: {
-        [Op.or]: ['20'],
-      },
-    },
-  })
-  return elementdetails.map(parseUnitFromElement)
-}
-
-const getUnitFromElementDetail = async id => {
-  const element = await ElementDetail.findByPk(id)
-  return parseUnitFromElement(element)
-}
-
-module.exports = {
-  hasStudent,
-  getUnitsFromElementDetails,
-  getUnitFromElementDetail,
-  parseUnitFromElement,
-}

services/backend/src/services/userService.js

@@ -1,173 +1,280 @@
-const UnitService = require('./units')
-const elementDetailService = require('./elementdetails')
-const { filterStudentnumbersByAccessrights } = require('./students')
-const { userDataCache } = require('./cache')
-const { getImporterClient } = require('../util/importerClient')
-const logger = require('../util/logger')
+const Sequelize = require('sequelize')
+const { Op } = Sequelize
+const LRU = require('lru-cache')
+const { getStudentnumbersByElementdetails } = require('./students')
 const { facultiesAndProgrammesForTrends } = require('../services/organisations')
 const { sendNotificationAboutNewUser } = require('../services/mailservice')
-const User = require('./users/users')
-const AccessGroup = require('./users/accessgroups')
+const { checkStudyGuidanceGroupsAccess, getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')
 const _ = require('lodash')
+const { User, UserElementDetails, AccessGroup, UserFaculties, sequelizeUser } = require('../models/models_user')
 
-// Private helpers
-const enrichProgrammesFromFaculties = faculties =>
-  facultiesAndProgrammesForTrends.filter(f => faculties.includes(f.faculty_code)).map(f => f.programme_code)
+const courseStatisticsGroup = 'grp-oodikone-basic-users'
+const hyOneGroup = 'hy-one'
+const toskaGroup = 'grp-toska'
 
-const enrichWithProgrammes = toEnrich => {
-  if (Array.isArray(toEnrich)) {
-    return toEnrich.map(obj => enrichWithProgrammes(obj))
-  }
-  if (toEnrich.rights) {
-    return {
-      ...toEnrich,
-      rights: _.union(toEnrich.rights, enrichProgrammesFromFaculties(toEnrich.faculties)),
-    }
-  }
+// Max 25 users can be stored in the cache
+const userDataCache = new LRU({
+  max: 25,
+  length: () => 1,
+  maxAge: 1000 * 60 * 60,
+})
 
-  if (toEnrich.elementdetails) {
-    return {
-      ...toEnrich,
-      elementdetails: _.union(
-        toEnrich.elementdetails,
-        enrichProgrammesFromFaculties(toEnrich.faculty.map(f => f.faculty_code))
-      ),
-    }
+// Stuff to work with old userservice based user-db (separate microservice, now deprecated)
+const userIncludes = [
+  {
+    separate: true,
+    model: UserElementDetails,
+    as: 'programme',
+  },
+  {
+    model: AccessGroup,
+    as: 'accessgroup',
+    attributes: ['id', 'group_code', 'group_info'],
+  },
+  {
+    separate: true,
+    model: UserFaculties,
+    as: 'faculty',
+  },
+]
+
+const accessGroupsByCodes = codes =>
+  AccessGroup.findAll({
+    where: {
+      group_code: {
+        [Op.in]: codes,
+      },
+    },
+  })
+
+const byUsername = async username =>
+  await User.findOne({
+    where: {
+      username,
+    },
+    include: userIncludes,
+  })
+
+const byId = async id =>
+  await User.findOne({
+    where: {
+      id,
+    },
+    include: userIncludes,
+  })
+
+// Crud things from old userservice
+
+// some fields that frontend needs
+const formatUserForAdminView = user => ({
+  ...user.get(),
+  elementdetails: user.programme.map(p => p.elementDetailCode),
+  is_enabled: true, // this is probably not needed: is here mainly because old userservice created users for every logged in user, even if they hadn't correct iamgroups
+})
+
+const addProgrammes = async (id, codes) => {
+  for (const code of codes) {
+    await UserElementDetails.upsert({ userId: id, elementDetailCode: code })
   }
-  return toEnrich
-}
-const checkStudyGuidanceGroupsAccess = async hyPersonSisuId => {
-  if (!hyPersonSisuId) {
-    logger.error('Not possible to get groups without personId header')
-    return false
-  }
-  const importerClient = getImporterClient()
-  if (!importerClient) return false
-  const answerTimeout = new Promise(resolve => setTimeout(resolve, 2000))
-  const result = await Promise.race([importerClient.get(`/person-groups/person/${hyPersonSisuId}`), answerTimeout])
-  if (!result) return false
-  const { data } = result
-  return data && Object.values(data).length > 0
+  const user = await byId(id)
+  userDataCache.del(user.username)
 }
 
-const byUsernameData = async username => {
-  // OK, enriched
-  const user = await User.byUsernameMinified(username)
-  const roles = user.accessgroup.map(({ group_code }) => group_code)
-  const rights = User.getUserProgrammes(user)
-  const faculties = user.faculty.map(({ faculty_code }) => faculty_code)
-  const data = {
-    email: user.email,
-    full_name: user.full_name,
-    roles,
-    rights,
-    faculties,
+const removeProgrammes = async (id, codes) => {
+  for (const code of codes) {
+    await UserElementDetails.destroy({
+      where: { userId: id, elementDetailCode: code },
+    })
   }
-  return enrichWithProgrammes(data)
+  const user = await byId(id)
+  userDataCache.del(user.username)
 }
 
-// Exported helpers
-const getUserDataFor = async username => {
-  // no need to enrich
-  let userData = userDataCache.get(username)
-  if (!userData) {
-    userData = await byUsernameData(username)
-    userDataCache.set(username, userData)
-  }
+const modifyRights = async (username, rights) => {
+  const rightsToAdd = Object.entries(rights)
+    .map(([code, val]) => {
+      if (val === true) {
+        return code
+      }
+    })
+    .filter(code => code)
+  const rightsToRemove = Object.entries(rights)
+    .map(([code, val]) => {
+      if (val === false) {
+        return code
+      }
+    })
+    .filter(code => code)
 
-  return {
-    ...userData,
-    faculties: new Set(userData.faculties),
-  }
+  const user = await byUsername(username)
+  const accessGroupsToAdd = await accessGroupsByCodes(rightsToAdd)
+  const accessGroupsToRemove = await accessGroupsByCodes(rightsToRemove)
+
+  await user.addAccessgroup(accessGroupsToAdd)
+  await user.removeAccessgroup(accessGroupsToRemove)
+  userDataCache.del(username)
 }
 
-const getUnitsFromElementDetails = async username => {
-  // No need to enrich
-  let userData = userDataCache.get(username)
-  if (!userData) {
-    userData = await byUsernameData(username)
-    userDataCache.set(username, userData)
-  }
-  const elementDetails = await elementDetailService.byCodes(userData.rights)
-  return elementDetails.map(element => UnitService.parseUnitFromElement(element))
-}
-
-const getStudentsUserCanAccess = async (studentnumbers, roles, userId) => {
-  // No need to enrich
-  let studentsUserCanAccess
-  if (roles?.includes('admin')) {
-    studentsUserCanAccess = new Set(studentnumbers)
-  } else {
-    const unitsUserCanAccess = await getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentsUserCanAccess = new Set(await filterStudentnumbersByAccessrights(studentnumbers, codes))
-  }
-  return studentsUserCanAccess
-}
-
-// FOR ROUTES 1-to-1 mapping
 const modifyAccess = async body => {
   const { username, accessgroups } = body
-  await User.modifyRights(username, accessgroups)
-  const user = await User.byUsername(username)
-  return enrichWithProgrammes(User.getUserData(user))
+  await modifyRights(username, accessgroups)
+  userDataCache.del(username)
+  return formatUserForAdminView(await byUsername(username))
 }
 
-const getAccessGroups = async () => enrichWithProgrammes(await AccessGroup.findAll())
-
-const enableElementDetails = async (uid, codes) => {
-  await User.addProgrammes(uid, codes)
-  const user = await User.byId(uid)
-  return enrichWithProgrammes(User.getUserData(user))
+const enableElementDetails = async (id, codes) => {
+  await addProgrammes(id, codes)
+  const user = await byId(id)
+  userDataCache.del(user.username)
+  return formatUserForAdminView(user)
 }
 
-const findAll = async () => enrichWithProgrammes((await User.findAll()).map(User.getUserData))
-
-const setFaculties = async (uid, faculties) => {
-  await User.setFaculties(uid, faculties)
-  const user = await User.byId(uid)
-  return enrichWithProgrammes(User.getUserData(user))
+const setFaculties = async (id, faculties) => {
+  await sequelizeUser.transaction(async transaction => {
+    await UserFaculties.destroy({ where: { userId: id }, transaction })
+    for (const faculty of faculties) {
+      await UserFaculties.create({ userId: id, faculty_code: faculty }, { transaction })
+    }
+  })
+  const user = await byId(id)
+  userDataCache.del(user.username)
+  return formatUserForAdminView(user)
 }
 
-const removeElementDetails = async (uid, codes) => {
-  await User.removeProgrammes(uid, codes)
-  const user = await User.byId(uid)
-  return enrichWithProgrammes(User.getUserData(user))
+const removeElementDetails = async (id, codes) => {
+  await removeProgrammes(id, codes)
+  const user = await byId(id)
+  userDataCache.del(user.userId)
+  return formatUserForAdminView(user)
 }
 
 const updateUser = async (username, fields) => {
-  userDataCache.del(username)
-  const uid = username
-  const user = await User.byUsername(uid)
+  const user = await byUsername(username)
   if (!user) {
-    throw new Error(`User ${uid} not found`)
+    throw new Error(`User ${username} not found`)
   }
-  await User.updateUser(user, fields)
-  const returnedUser = await User.byUsername(username)
-  return enrichWithProgrammes(User.getUserData(returnedUser))
+  await User.upsert({ id: user.id, ...fields })
+  user.update(fields)
+
+  userDataCache.del(username)
+
+  return formatUserForAdminView(await byUsername(username))
 }
 
-const getLoginDataWithoutToken = async (uid, full_name, hyGroups, email, hyPersonSisuId) => {
-  const hasStudyGuidanceGroupAccess = await checkStudyGuidanceGroupsAccess(hyPersonSisuId)
-  const result = await User.loginWithoutToken(
-    uid,
-    full_name,
-    hyGroups,
-    email,
-    hyPersonSisuId,
-    hasStudyGuidanceGroupAccess
+const getAccessGroups = async () => await AccessGroup.findAll()
+
+// newer logic
+const enrichProgrammesFromFaculties = faculties =>
+  facultiesAndProgrammesForTrends.filter(f => faculties.includes(f.faculty_code)).map(f => f.programme_code)
+
+const findAll = async () =>
+  (
+    await User.findAll({
+      include: userIncludes,
+    })
+  ).map(user => ({
+    ...user.get(),
+    is_enabled: true,
+    elementdetails: _.uniqBy([
+      ...user.programme.map(p => p.elementDetailCode),
+      ...enrichProgrammesFromFaculties(user.faculty.map(f => f.faculty_code)),
+    ]),
+  }))
+
+const formatUser = async userFromDb => {
+  const [accessGroups, programmes, faculties] = [
+    ['accessgroup', 'group_code'],
+    ['programme', 'elementDetailCode'],
+    ['faculty', 'faculty_code'],
+  ].map(([field, code]) => userFromDb[field].map(entry => entry[code]))
+
+  const rights = _.uniqBy([...programmes, ...enrichProgrammesFromFaculties(faculties)])
+
+  const {
+    dataValues: { id, username: userId, full_name: name, email, language, sisu_person_id: sisPersonId },
+  } = userFromDb
+
+  const studentsUserCanAccess = _.uniqBy(
+    (await Promise.all([getStudentnumbersByElementdetails(rights), getAllStudentsUserHasInGroups(sisPersonId)])).flat()
   )
-  return result
+
+  // attribute naming is a bit confusing, but it's used so widely in oodikone, that it's not probably worth changing
+  return {
+    id, // our internal id, not the one from sis, (e.g. 101)
+    userId, // acually username (e.g. mluukkai)
+    name,
+    language,
+    sisPersonId, //
+    email,
+    is_enabled: true, // this is probably not needed: is here mainly because old userservice created users for every logged in user, even if they hadn't correct iamgroups
+    rights,
+    roles: accessGroups,
+    studentsUserCanAccess, // studentnumbers used in various parts of backend. For admin this is usually empty, since no programmes / faculties are set.
+    isAdmin: accessGroups.includes('admin'),
+  }
 }
 
-const getMockedUser = async (uid, asUser) => await User.superlogin(uid, asUser)
+const getMockedUser = async ({ userToMock, mockedBy }) => {
+  if (userDataCache.has(userToMock)) {
+    return userDataCache.get(userToMock)
+  }
+  const toReturn = {
+    ...(await formatUser(await byUsername(userToMock))),
+    mockedBy,
+  }
+  userDataCache.set(userToMock, toReturn)
+  return toReturn
+}
 
 const getUser = async ({ username, name, email, iamGroups, sisId }) => {
-  const { payload: user, isNew } = await getLoginDataWithoutToken(username, name, iamGroups, email, sisId)
-  if (isNew) {
-    await sendNotificationAboutNewUser({ userId: username, userFullName: name })
+  if (userDataCache.has(username)) {
+    return userDataCache.get(username)
   }
-  return user
+
+  const isNewUser = !(await User.findOne({ where: { username } }))
+  const language = 'fi'
+
+  await User.upsert({
+    full_name: name,
+    username,
+    email,
+    language,
+    sisu_person_id: sisId,
+    last_login: new Date(),
+  })
+
+  const userFromDb = await byUsername(username)
+  const formattedUser = await formatUser(userFromDb)
+  const { roles: currentAccessGroups } = formattedUser
+
+  // Modify accessgroups based on current groups and iamGroups
+  let newAccessGroups = []
+  if (await checkStudyGuidanceGroupsAccess(sisId)) newAccessGroups.push('studyGuidanceGroups')
+  if (iamGroups.includes(courseStatisticsGroup)) newAccessGroups.push('courseStatistics')
+  if (iamGroups.includes(hyOneGroup) || currentAccessGroups.includes('teachers')) newAccessGroups.push('teachers')
+  if (iamGroups.includes(toskaGroup) || currentAccessGroups.includes('admin')) newAccessGroups.push('admin')
+  if (_.difference(newAccessGroups, currentAccessGroups).length > 0) {
+    userFromDb.setAccessgroup(
+      (
+        await AccessGroup.findAll({
+          where: {
+            group_code: {
+              [Op.in]: newAccessGroups,
+            },
+          },
+        })
+      ).map(({ id }) => id)
+    )
+  }
+
+  if (isNewUser) await sendNotificationAboutNewUser({ userId: username, userFullName: name })
+
+  const toReturn = {
+    ...formattedUser,
+    roles: newAccessGroups,
+  }
+  userDataCache.set(username, toReturn)
+  return toReturn
 }
 
 module.exports = {
@@ -177,11 +284,7 @@ module.exports = {
   findAll,
   modifyAccess,
   getAccessGroups,
-  getUnitsFromElementDetails,
   setFaculties,
-  getUserDataFor,
-  getStudentsUserCanAccess,
-  getLoginDataWithoutToken,
   getUser,
   getMockedUser,
 }

services/backend/src/services/users/accessgroups.js

@@ -1,17 +0,0 @@
-const { AccessGroup } = require('../../models/models_user')
-const { Op } = require('sequelize')
-
-const byId = id => AccessGroup.findByPk(id)
-
-const findAll = () => AccessGroup.findAll()
-
-const byCodes = codes =>
-  AccessGroup.findAll({
-    where: {
-      group_code: {
-        [Op.in]: codes,
-      },
-    },
-  })
-
-module.exports = { byId, byCodes, findAll }

services/backend/src/services/users/hygroups.js

@@ -1,24 +0,0 @@
-const { HyGroup } = require('../../models/models_user')
-const { Op } = require('sequelize')
-
-const byId = id => HyGroup.findByPk(id)
-
-const findAll = () => HyGroup.findAll()
-
-const byCodes = codes =>
-  HyGroup.findAll({
-    where: {
-      code: {
-        [Op.in]: codes,
-      },
-    },
-  })
-
-const create = code => HyGroup.create({ code })
-
-const addGroups = async (uid, codes) => {
-  const user = await byId(uid)
-  const groups = await byCodes(codes)
-  await user.addHygroups(groups)
-}
-module.exports = { byId, byCodes, findAll, create, addGroups }

services/backend/src/services/users/users.js

@@ -1,339 +0,0 @@
-const Sequelize = require('sequelize')
-const _ = require('lodash')
-const moment = require('moment')
-const {
-  User,
-  UserElementDetails,
-  AccessGroup,
-  HyGroup,
-  UserFaculties,
-  sequelizeUser,
-} = require('../../models/models_user')
-const AccessService = require('./accessgroups')
-const HyGroupService = require('./hygroups')
-const Op = Sequelize.Op
-
-// move these to conf
-const courseStatisticsGroup = 'grp-oodikone-basic-users'
-const hyOneGroup = 'hy-one'
-const requiredGroup =
-  process.env.NODE_ENV === 'staging'
-    ? ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
-    : ['grp-oodikone-users', 'grp-oodikone-basic-users']
-
-// help for refactoring away from token
-const generateTokenData = async (uid, mockedBy = null) => {
-  const user = await byUsername(uid)
-  const userData = getUserData(user)
-  const programmes = userData.elementdetails
-  const payload = {
-    id: user.id,
-    userId: uid, // username
-    name: user.full_name,
-    enabled: userData.is_enabled,
-    language: user.language,
-    mockedBy,
-    rights: programmes,
-    roles: user.accessgroup,
-    createdAt: moment().toISOString(),
-    sisPersonId: user.sisu_person_id,
-  }
-  return payload
-}
-
-const createMissingGroups = async (group, service) => {
-  const savedGroups = await service.findAll()
-  group.forEach(async code => {
-    if (!savedGroups.map(sg => sg.code).includes(code)) {
-      await service.create(code)
-    }
-  })
-}
-
-const updateGroups = async (user, hyGroups) => {
-  let hyGroupsToBeUpdated = (await user.getHy_group()).map(hg => hg.code)
-  let hyGroupsToAdd = []
-  let hyGroupsToDelete = []
-
-  hyGroups.forEach(async hyGroup => {
-    if (!hyGroupsToBeUpdated.includes(hyGroup)) {
-      hyGroupsToAdd = hyGroupsToAdd.concat(hyGroup)
-    }
-  })
-  hyGroupsToBeUpdated.forEach(async hyGroup => {
-    if (!hyGroups.includes(hyGroup)) {
-      hyGroupsToDelete = hyGroupsToDelete.concat(hyGroup)
-    }
-  })
-
-  await user.addHy_group(await HyGroupService.byCodes(hyGroupsToAdd))
-  await user.removeHy_group(await HyGroupService.byCodes(hyGroupsToDelete))
-}
-
-const loginBase = async (uid, full_name, hyGroups, mail, hyPersonSisuId, hasStudyGuidanceGroupAccess) => {
-  let user = await byUsername(uid)
-  let isNew = false
-  const lastLogin = new Date()
-  await createMissingGroups(hyGroups, HyGroupService)
-
-  if (!user) {
-    user = await createUser(uid, full_name, mail, hyPersonSisuId, lastLogin)
-
-    const userHyGroups = await HyGroupService.byCodes(hyGroups)
-    await user.addHy_group(userHyGroups)
-
-    isNew = true
-  } else {
-    user = await updateUser(user, { full_name, email: mail, sisu_person_id: hyPersonSisuId, last_login: lastLogin })
-    await updateGroups(user, hyGroups)
-  }
-
-  await determineAccessToCourseStats(user, hyGroups)
-  await determineAccessToTeachersForOne(user, hyGroups)
-  await determineAccessToStudyGuidanceGroups(user, hasStudyGuidanceGroupAccess)
-
-  return isNew
-}
-
-const loginWithoutToken = async (
-  uid,
-  full_name,
-  hyGroups,
-  affiliations,
-  mail,
-  hyPersonSisuId,
-  hasStudyGuidanceGroupAccess
-) => {
-  const isNew = await loginBase(
-    uid,
-    full_name,
-    hyGroups,
-    affiliations,
-    mail,
-    hyPersonSisuId,
-    hasStudyGuidanceGroupAccess
-  )
-  const payload = await generateTokenData(uid)
-  return { payload, isNew }
-}
-
-const superlogin = async (uid, asUser) => {
-  const user = await byUsername(uid)
-  if (user && user.accessgroup.map(r => r.group_code).includes('admin')) {
-    const token = await generateTokenData(asUser, uid)
-    return token
-  }
-  return undefined
-}
-
-const userIncludes = [
-  {
-    separate: true,
-    model: UserElementDetails,
-    as: 'programme',
-  },
-  {
-    model: AccessGroup,
-    as: 'accessgroup',
-    attributes: ['id', 'group_code', 'group_info'],
-  },
-  {
-    required: false,
-    model: HyGroup,
-    as: 'hy_group',
-    where: { code: requiredGroup },
-  },
-  {
-    separate: true,
-    model: UserFaculties,
-    as: 'faculty',
-  },
-]
-
-const getUserData = user => {
-  if (user == null) return null
-  const newuser = user.get()
-  newuser.elementdetails = getUserProgrammes(newuser)
-  const hyGroups = newuser.hy_group.map(e => e.code)
-  newuser.is_enabled = requiredGroup === null || _.intersection(hyGroups, requiredGroup).length > 0
-  newuser.hy_group = null
-  return newuser
-}
-
-const byUsernameMinified = async username => {
-  const userMinified = await User.findOne({
-    where: {
-      username,
-    },
-    include: [
-      {
-        separate: true,
-        model: UserElementDetails,
-        as: 'programme',
-        attributes: ['elementDetailCode'],
-      },
-      {
-        model: AccessGroup,
-        as: 'accessgroup',
-        attributes: ['group_code'],
-      },
-      {
-        separate: true,
-        model: UserFaculties,
-        as: 'faculty',
-        attributes: ['faculty_code'],
-      },
-    ],
-  })
-  return userMinified
-}
-
-const byUsername = async username => {
-  const user = await User.findOne({
-    where: {
-      username: {
-        [Op.eq]: username,
-      },
-    },
-    include: userIncludes,
-  })
-  return user
-}
-
-const createUser = async (username, fullname, email, hyPersonSisuId, lastLogin) => {
-  return User.create({
-    username: username,
-    full_name: fullname,
-    email,
-    sisu_person_id: hyPersonSisuId,
-    last_login: lastLogin,
-  })
-}
-
-const updateUser = async (userObject, values) => {
-  await User.upsert({ id: userObject.id, ...values })
-  return userObject.update(values)
-}
-
-const byId = async id => {
-  const user = await User.findOne({
-    where: {
-      id: {
-        [Op.eq]: id,
-      },
-    },
-    include: userIncludes,
-  })
-  return user
-}
-
-const getUserProgrammes = user => user.programme.map(p => p.elementDetailCode)
-
-const getUserAccessGroups = async username => {
-  const user = await byUsername(username)
-  return await user.getAccessgroup()
-}
-
-const findAll = async () => {
-  const users = await User.findAll({
-    include: userIncludes,
-  })
-  return users
-}
-
-const addProgrammes = async (uid, codes) => {
-  for (const code of codes) {
-    await UserElementDetails.upsert({ userId: uid, elementDetailCode: code })
-  }
-}
-
-const removeProgrammes = async (uid, codes) => {
-  for (const code of codes) {
-    await UserElementDetails.destroy({
-      where: { userId: uid, elementDetailCode: code },
-    })
-  }
-}
-
-const setFaculties = async (uid, faculties) => {
-  await sequelizeUser.transaction(async transaction => {
-    await UserFaculties.destroy({ where: { userId: uid }, transaction })
-    for (const faculty of faculties) {
-      await UserFaculties.create({ userId: uid, faculty_code: faculty }, { transaction })
-    }
-  })
-}
-
-const modifyRights = async (username, rights) => {
-  const rightsToAdd = Object.entries(rights)
-    .map(([code, val]) => {
-      if (val === true) {
-        return code
-      }
-    })
-    .filter(code => code)
-  const rightsToRemove = Object.entries(rights)
-    .map(([code, val]) => {
-      if (val === false) {
-        return code
-      }
-    })
-    .filter(code => code)
-
-  const user = await byUsername(username)
-  const accessGroupsToAdd = await AccessService.byCodes(rightsToAdd)
-  const accessGroupsToRemove = await AccessService.byCodes(rightsToRemove)
-
-  await user.addAccessgroup(accessGroupsToAdd)
-  await user.removeAccessgroup(accessGroupsToRemove)
-}
-const getRoles = async user => {
-  const foundUser = byUsername(user)
-  return foundUser.accessgroup
-}
-
-const determineAccessToCourseStats = async (user, hyGroups) => {
-  const accessGroups = (user && user.accessgroup) || []
-  const alreadyAccess = accessGroups.some(({ group_code }) => group_code === 'courseStatistics')
-  if (hyGroups.includes(courseStatisticsGroup) && !alreadyAccess) {
-    await modifyRights(user.username, { courseStatistics: true })
-  } else if (!hyGroups.includes(courseStatisticsGroup) && alreadyAccess) {
-    await modifyRights(user.username, { courseStatistics: false })
-  }
-}
-
-const determineAccessToTeachersForOne = async (user, hyGroups) => {
-  const accessGroups = (user && user.accessgroup) || []
-  const alreadyAccess = accessGroups.some(({ group_code }) => group_code === 'teachers')
-  if (hyGroups.includes(hyOneGroup) && !alreadyAccess) {
-    await modifyRights(user.username, { teachers: true })
-  }
-}
-
-const determineAccessToStudyGuidanceGroups = async (user, hasStudyGuidanceGroupAccess) => {
-  const accessGroups = (user && user.accessgroup) || []
-  const alreadyAccess = accessGroups.some(({ group_code }) => group_code === 'studyGuidanceGroups')
-  if (hasStudyGuidanceGroupAccess && !alreadyAccess) {
-    await modifyRights(user.username, { studyGuidanceGroups: true })
-  } else if (!hasStudyGuidanceGroupAccess && alreadyAccess) {
-    await modifyRights(user.username, { studyGuidanceGroups: false })
-  }
-}
-
-module.exports = {
-  byUsername,
-  byUsernameMinified,
-  updateUser,
-  findAll,
-  byId,
-  getUserProgrammes,
-  addProgrammes,
-  removeProgrammes,
-  superlogin,
-  modifyRights,
-  getUserAccessGroups,
-  getRoles,
-  getUserData,
-  setFaculties,
-  loginWithoutToken,
-}

services/backend/src/util/semester.js

@@ -29,16 +29,43 @@ const getPassingSemester = (startYear, date) => {
   return yearCount < 6 ? `${yearCount}-${semester}` : 'LATER'
 }
 
-const getAcademicYearDates = year => {
+const getAcademicYearDates = (year, since) => {
+  if (year === 'Total') {
+    return {
+      startDate: `${since.getFullYear()}-${semesterStart['FALL']}`,
+      endDate: `${moment(new Date()).add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`,
+    }
+  }
+  const startYear = typeof year === 'number' ? year : Number(year.slice(0, 4))
+  const endYear =
+    typeof year === 'number' ? moment(year, 'YYYY').add(1, 'years').format('YYYY') : Number(year.slice(7, 11))
   return {
-    startDate: `${year}-${semesterStart['FALL']}`,
-    endDate: `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`,
+    startDate: `${startYear}-${semesterStart['FALL']}`,
+    endDate: `${endYear}-${semesterEnd['SPRING']}`,
+  }
+}
+
+const getYearStartAndEndDates = (year, isAcademicYear) => {
+  if (isAcademicYear) {
+    const firstYear = year.slice(0, 4)
+    return {
+      startDate: new Date(`${firstYear}-${semesterStart['FALL']}`).toISOString(),
+      endDate: new Date(
+        `${moment(firstYear, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`
+      ).toISOString(),
+    }
+  }
+
+  return {
+    startDate: new Date(`${year}-01-01`).toISOString(),
+    endDate: new Date(`${year}-12-31`).toISOString(),
   }
 }
 
 module.exports = {
   getPassingSemester,
   getAcademicYearDates,
+  getYearStartAndEndDates,
   semesterEnd,
   semesterStart,
 }

services/backend/src/util/utils.js

@@ -1,5 +1,5 @@
-const mapToProviders = elementDetails =>
-  elementDetails.map(r => {
+const mapToProviders = elementDetails => {
+  return elementDetails.map(r => {
     const isNumber = str => !Number.isNaN(Number(str))
     if (r.includes('_')) {
       const [left, right] = r.split('_')
@@ -8,17 +8,23 @@ const mapToProviders = elementDetails =>
       const providercode = `${prefix}0-${suffix}`
       return providercode
     }
+    if (/^(T)[0-9]{6}$/.test(r)) {
+      const numbers = r.substring(1)
+      return `7${numbers}`
+    }
     return r
   })
+}
 
 // sort substitutions so that main code is first
-const newLetterBasedCode = /^[A-Za-z]{3,}/ // new letter based codes come first
+const newLetterBasedCode = /^[A-Za-z]/ // new letter based codes come first
 const oldNumericCode = /^\d/ // old numeric codes come second
 const openUniCode = /^AY?(.+?)(?:en|fi|sv)?$/ // open university codes come last
 const openUniCodeA = /A\d/ // open university with just A come last
 const digi = /DIGI-A?(.+?)(?:en|fi|sv)?$/ // digi-a goes on top courses goes third
+const bscsCode = /BSCS??/
 
-const codeRegexes = [openUniCodeA, openUniCode, oldNumericCode, newLetterBasedCode, digi]
+const codeRegexes = [openUniCodeA, openUniCode, bscsCode, oldNumericCode, newLetterBasedCode, digi]
 
 const getSortRank = code => {
   for (let i = 0; i < codeRegexes.length; i++) {

services/frontend/package-lock.json

@@ -1908,26 +1908,19 @@
       }
     },
     "@popperjs/core": {
-      "version": "2.9.2",
-      "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.9.2.tgz",
-      "integrity": "sha512-VZMYa7+fXHdwIq1TDhSXoVmSPEGM/aa+6Aiq3nVVJ9bXr24zScr+NlKFKC3iPljA7ho/GAZr+d2jOf5GIRC30Q=="
+      "version": "2.11.2",
+      "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.2.tgz",
+      "integrity": "sha512-92FRmppjjqz29VMJ2dn+xdyXZBrMlE42AV6Kq6BwjWV7CNUW1hs2FtxSNLQE+gJhaZ6AAmYuO9y8dshhcBl7vA=="
     },
     "@reduxjs/toolkit": {
-      "version": "1.6.2",
-      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.6.2.tgz",
-      "integrity": "sha512-HbfI/hOVrAcMGAYsMWxw3UJyIoAS9JTdwddsjlr5w3S50tXhWb+EMyhIw+IAvCVCLETkzdjgH91RjDSYZekVBA==",
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.8.0.tgz",
+      "integrity": "sha512-cdfHWfcvLyhBUDicoFwG1u32JqvwKDxLxDd7zSmSoFw/RhYLOygIRtmaMjPRUUHmVmmAGAvquLLsKKU/677kSQ==",
       "requires": {
-        "immer": "^9.0.6",
-        "redux": "^4.1.0",
-        "redux-thunk": "^2.3.0",
-        "reselect": "^4.0.0"
-      },
-      "dependencies": {
-        "immer": {
-          "version": "9.0.6",
-          "resolved": "https://registry.npmjs.org/immer/-/immer-9.0.6.tgz",
-          "integrity": "sha512-G95ivKpy+EvVAnAab4fVa4YGYn24J1SpEktnJX7JJ45Bd7xqME/SCplFzYFmTbrkwZbQ4xJK1xMTUYBkN6pWsQ=="
-        }
+        "immer": "^9.0.7",
+        "redux": "^4.1.2",
+        "redux-thunk": "^2.4.1",
+        "reselect": "^4.1.5"
       }
     },
     "@rollup/plugin-node-resolve": {
@@ -1978,13 +1971,13 @@
       }
     },
     "@sentry/browser": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-6.9.0.tgz",
-      "integrity": "sha512-4JnEPcwoNs6JqeEd4wscBq+hxpotEJ0DJ4eOIsaNZIMyqEHXBHTXCk/gfrSsiZFrkHM4PgvUHOxaC0HcZ92oBA==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-6.18.2.tgz",
+      "integrity": "sha512-EsqKSNboi2gOiMuEwQranLucxrARi00y2vgUnaPXcqTKTlVlHDetoWHvq8/r29idA1JHGka5tDrwrmWccWIkrg==",
       "requires": {
-        "@sentry/core": "6.9.0",
-        "@sentry/types": "6.9.0",
-        "@sentry/utils": "6.9.0",
+        "@sentry/core": "6.18.2",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
@@ -1996,14 +1989,14 @@
       }
     },
     "@sentry/core": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/core/-/core-6.9.0.tgz",
-      "integrity": "sha512-oFX2qQcMLujCeIuCQGlhpTUIOXiU5n6V2lqDnvMXUV8gKpplBPalwdlR9bgbSi+VO8u7LjHR1IKM0RAPWgNHWw==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/core/-/core-6.18.2.tgz",
+      "integrity": "sha512-r5ad/gq5S/JHc9sd5CUhZQT9ojQ+f+thk/AoGeGawX/8HURZYAgIqD565d6FK0VsZEDkdRMl58z1Qon20h3y1g==",
       "requires": {
-        "@sentry/hub": "6.9.0",
-        "@sentry/minimal": "6.9.0",
-        "@sentry/types": "6.9.0",
-        "@sentry/utils": "6.9.0",
+        "@sentry/hub": "6.18.2",
+        "@sentry/minimal": "6.18.2",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
@@ -2015,12 +2008,12 @@
       }
     },
     "@sentry/hub": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.9.0.tgz",
-      "integrity": "sha512-5mors7ojbo7G85ZmoVPQBgFBMONAJwyZfV0LNLy14GenoaVNuxTPyvAQiJb1FYq+x6YZ3CvqGX6r74KRKQU87w==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.18.2.tgz",
+      "integrity": "sha512-d0AugekMkbnN12b4EXMjseJxtLPc9S20DGobCPUb4oAQT6S2oDQEj1jwP6PQ5vtgyy+GMYWxBMgqAQ4pjVYISQ==",
       "requires": {
-        "@sentry/types": "6.9.0",
-        "@sentry/utils": "6.9.0",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
@@ -2032,12 +2025,12 @@
       }
     },
     "@sentry/minimal": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.9.0.tgz",
-      "integrity": "sha512-GBZ6wG2Rc1wInYEl2BZTZc/t57O1Da876ifLsSPpEQAEnGWbqZWb8RLjZskH09ZIL/K4XCIDDi5ySzN8kFUWJw==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.18.2.tgz",
+      "integrity": "sha512-n7KYuo34W2LxE+3dnZ47of7XHuORINCnXq66XH72eoj67tf0XeWbIhEJrYGmoLRyRfoCYYrBLWiDl/uTjLzrzQ==",
       "requires": {
-        "@sentry/hub": "6.9.0",
-        "@sentry/types": "6.9.0",
+        "@sentry/hub": "6.18.2",
+        "@sentry/types": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
@@ -2049,51 +2042,17 @@
       }
     },
     "@sentry/tracing": {
-      "version": "6.11.0",
-      "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.11.0.tgz",
-      "integrity": "sha512-9VA1/SY++WeoMQI4K6n/sYgIdRtCu9NLWqmGqu/5kbOtESYFgAt1DqSyqGCr00ZjQiC2s7tkDkTNZb38K6KytQ==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.18.2.tgz",
+      "integrity": "sha512-hg6NLqrqJ5sUPTyWEQ2RqdnhQVnyLtx8II0IyWxQLDWD8UCe3Mu6G7mroDtakPWcP+lWz6OnKfMEfuhMcxR8fw==",
       "requires": {
-        "@sentry/hub": "6.11.0",
-        "@sentry/minimal": "6.11.0",
-        "@sentry/types": "6.11.0",
-        "@sentry/utils": "6.11.0",
+        "@sentry/hub": "6.18.2",
+        "@sentry/minimal": "6.18.2",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
-        "@sentry/hub": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.11.0.tgz",
-          "integrity": "sha512-pT9hf+ZJfVFpoZopoC+yJmFNclr4NPqPcl2cgguqCHb69DklD1NxgBNWK8D6X05qjnNFDF991U6t1mxP9HrGuw==",
-          "requires": {
-            "@sentry/types": "6.11.0",
-            "@sentry/utils": "6.11.0",
-            "tslib": "^1.9.3"
-          }
-        },
-        "@sentry/minimal": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.11.0.tgz",
-          "integrity": "sha512-XkZ7qrdlGp4IM/gjGxf1Q575yIbl5RvPbg+WFeekpo16Ufvzx37Mr8c2xsZaWosISVyE6eyFpooORjUlzy8EDw==",
-          "requires": {
-            "@sentry/hub": "6.11.0",
-            "@sentry/types": "6.11.0",
-            "tslib": "^1.9.3"
-          }
-        },
-        "@sentry/types": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.11.0.tgz",
-          "integrity": "sha512-gm5H9eZhL6bsIy/h3T+/Fzzz2vINhHhqd92CjHle3w7uXdTdFV98i2pDpErBGNTSNzbntqOMifYEB5ENtZAvcg=="
-        },
-        "@sentry/utils": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.11.0.tgz",
-          "integrity": "sha512-IOvyFHcnbRQxa++jO+ZUzRvFHEJ1cZjrBIQaNVc0IYF0twUOB5PTP6joTcix38ldaLeapaPZ9LGfudbvYvxkdg==",
-          "requires": {
-            "@sentry/types": "6.11.0",
-            "tslib": "^1.9.3"
-          }
-        },
         "tslib": {
           "version": "1.14.1",
           "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
@@ -2102,16 +2061,16 @@
       }
     },
     "@sentry/types": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.9.0.tgz",
-      "integrity": "sha512-v52HJqLoLapEnqS2NdVtUXPvT+aezQgNXQkp8hiQ3RUdTm5cffwBVG7wlbpE6OsOOIZxd6p1zKylFkwCypiIIA=="
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.18.2.tgz",
+      "integrity": "sha512-WzpJf/Q5aORTzrSwer/As1NlO90dBAQpaHV2ikDDKqOyMWEgjKb5/4gh59p9gH8JMMnLetP1AvQel0fOj5UnUw=="
     },
     "@sentry/utils": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.9.0.tgz",
-      "integrity": "sha512-PimDr6KAi4cCp5hQZ8Az2/pDcdfhTu7WAU30Dd9MZwknpHSTmD4G6QvkdrB5er6kMMnNQOC7rMo6w/Do3m6X3w==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.18.2.tgz",
+      "integrity": "sha512-EC619jesknyu4xpwud5WC/5odYLz6JUy7OSFy5405PpdGeh/m8XUvuJAx4zDx0Iz/Mlk0S1Md+ZcQwqkv39dkw==",
       "requires": {
-        "@sentry/types": "6.9.0",
+        "@sentry/types": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
@@ -2330,9 +2289,9 @@
       }
     },
     "@types/hast": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.2.tgz",
-      "integrity": "sha512-Op5W7jYgZI7AWKY5wQ0/QNMzQM7dGQPyW1rXKNiymVCy5iTfdPuGu4HhYNOM2sIv8gUfIuIdcYlXmAepwaowow==",
+      "version": "2.3.4",
+      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.4.tgz",
+      "integrity": "sha512-wLEm0QvaoawEDoTRwzTXp4b4jpwiJDvR5KMnFnVodm3scufTlBOWRD6N1OBf9TZMhjlNsSfcO5V+7AF4+Vy+9g==",
       "requires": {
         "@types/unist": "*"
       }
@@ -2378,9 +2337,9 @@
       "integrity": "sha512-YSBPTLTVm2e2OoQIDYx8HaeWJ5tTToLH67kXR7zYNGupXMEHa2++G8k+DczX2cFVgalypqtyZIcU19AFcmOpmg=="
     },
     "@types/mdast": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.4.tgz",
-      "integrity": "sha512-gIdhbLDFlspL53xzol2hVzrXAbzt71erJHoOwQZWssjaiouOotf03lNtMmFm9VfFkvnLWccSVjUAZGQ5Kqw+jA==",
+      "version": "3.0.10",
+      "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.10.tgz",
+      "integrity": "sha512-W864tg/Osz1+9f4lrGTZpCSO5/z4608eUp19tbozkq2HJK6i3z1kT0H9tlADXuYIb1YYOBByU4Jsqkk75q48qA==",
       "requires": {
         "@types/unist": "*"
       }
@@ -2485,9 +2444,9 @@
       }
     },
     "@types/unist": {
-      "version": "2.0.5",
-      "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.5.tgz",
-      "integrity": "sha512-wnra4Vw9dopnuybR6HBywJ/URYpYrKLoepBTEtgfJup8Ahoi2zJECPP2cwiXp7btTvOT2CULv87aQRA4eZSP6g=="
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.6.tgz",
+      "integrity": "sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ=="
     },
     "@types/webpack": {
       "version": "4.41.30",
@@ -3160,9 +3119,9 @@
       "integrity": "sha512-99FZt8qS/xukgxU/8daV8WT7wAakqBzt6lF3XCweO6pwcf50/NgxxBj6ZC7/ejR+F4PWeFpkb9lAMH3y2quBXA=="
     },
     "axios": {
-      "version": "0.21.2",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.2.tgz",
-      "integrity": "sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg==",
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
       "requires": {
         "follow-redirects": "^1.14.0"
       }
@@ -4087,13 +4046,35 @@
       "integrity": "sha512-/4YgnZS8y1UXXmC02xD5rRrBEu6T5ub+mQHLNRj0fzTRbgdBYhsNo2V5EqwgqrExjxsjtF/OpAKAMkKsxbD5XQ=="
     },
     "cfb": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.0.tgz",
-      "integrity": "sha512-sXMvHsKCICVR3Naq+J556K+ExBo9n50iKl6LGarlnvuA2035uMlGA/qVrc0wQtow5P1vJEw9UyrKLCbtIKz+TQ==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.1.tgz",
+      "integrity": "sha512-wT2ScPAFGSVy7CY+aauMezZBnNrfnaLSrxHUHdea+Td/86vrk6ZquggV+ssBR88zNs0OnBkL2+lf9q0K+zVGzQ==",
       "requires": {
-        "adler-32": "~1.2.0",
+        "adler-32": "~1.3.0",
         "crc-32": "~1.2.0",
-        "printj": "~1.1.2"
+        "printj": "~1.3.0"
+      },
+      "dependencies": {
+        "adler-32": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.0.tgz",
+          "integrity": "sha512-f5nltvjl+PRUh6YNfUstRaXwJxtfnKEWhAWWlmKvh+Y3J2+98a0KKVYDEhz6NdKGqswLhjNGznxfSsZGOvOd9g==",
+          "requires": {
+            "printj": "~1.2.2"
+          },
+          "dependencies": {
+            "printj": {
+              "version": "1.2.3",
+              "resolved": "https://registry.npmjs.org/printj/-/printj-1.2.3.tgz",
+              "integrity": "sha512-sanczS6xOJOg7IKDvi4sGOUOe7c1tsEzjwlLFH/zgwx/uyImVM9/rgBkc8AfiQa/Vg54nRd8mkm9yI7WV/O+WA=="
+            }
+          }
+        },
+        "printj": {
+          "version": "1.3.1",
+          "resolved": "https://registry.npmjs.org/printj/-/printj-1.3.1.tgz",
+          "integrity": "sha512-GA3TdL8szPK4AQ2YnOe/b+Y1jUFwmmGMMK/qbY7VcE3Z7FU8JstbKiKRzO6CIiAKPhTO8m01NoQ0V5f3jc4OGg=="
+        }
       }
     },
     "chalk": {
@@ -4152,6 +4133,11 @@
       "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
       "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="
     },
+    "chroma-js": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/chroma-js/-/chroma-js-2.4.2.tgz",
+      "integrity": "sha512-U9eDw6+wt7V8z5NncY2jJfZa+hUH8XEj8FQHgFJTrUFnJfXYf4Ml4adI2vXZOjqRDpFWtYVWypDfZwnJ+HIR4A=="
+    },
     "chrome-trace-event": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz",
@@ -4299,20 +4285,9 @@
       }
     },
     "codepage": {
-      "version": "1.14.0",
-      "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.14.0.tgz",
-      "integrity": "sha1-jL4lSBMjVZ19MHVxsP/5HnodL5k=",
-      "requires": {
-        "commander": "~2.14.1",
-        "exit-on-epipe": "~1.0.1"
-      },
-      "dependencies": {
-        "commander": {
-          "version": "2.14.1",
-          "resolved": "https://registry.npmjs.org/commander/-/commander-2.14.1.tgz",
-          "integrity": "sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw=="
-        }
-      }
+      "version": "1.15.0",
+      "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz",
+      "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA=="
     },
     "collect-v8-coverage": {
       "version": "1.0.1",
@@ -4617,12 +4592,19 @@
       }
     },
     "crc-32": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.0.tgz",
-      "integrity": "sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.1.tgz",
+      "integrity": "sha512-Dn/xm/1vFFgs3nfrpEVScHoIslO9NZRITWGz/1E/St6u4xw99vfZzVkW0OSnzx2h9egej9xwMCEut6sqwokM/w==",
       "requires": {
         "exit-on-epipe": "~1.0.1",
-        "printj": "~1.1.0"
+        "printj": "~1.3.1"
+      },
+      "dependencies": {
+        "printj": {
+          "version": "1.3.1",
+          "resolved": "https://registry.npmjs.org/printj/-/printj-1.3.1.tgz",
+          "integrity": "sha512-GA3TdL8szPK4AQ2YnOe/b+Y1jUFwmmGMMK/qbY7VcE3Z7FU8JstbKiKRzO6CIiAKPhTO8m01NoQ0V5f3jc4OGg=="
+        }
       }
     },
     "create-ecdh": {
@@ -6657,11 +6639,6 @@
         "bser": "2.1.1"
       }
     },
-    "fflate": {
-      "version": "0.3.11",
-      "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.3.11.tgz",
-      "integrity": "sha512-Rr5QlUeGN1mbOHlaqcSYMKVpPbgLy0AWT/W0EHxA6NGI12yO1jpoui2zBBvU2G824ltM6Ut8BFgfHSBGfkmS0A=="
-    },
     "figgy-pudding": {
       "version": "3.5.2",
       "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz",
@@ -6837,9 +6814,9 @@
       }
     },
     "follow-redirects": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz",
-      "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg=="
+      "version": "1.14.8",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz",
+      "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA=="
     },
     "for-in": {
       "version": "1.0.2",
@@ -7355,9 +7332,9 @@
       "integrity": "sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ=="
     },
     "highcharts": {
-      "version": "9.1.2",
-      "resolved": "https://registry.npmjs.org/highcharts/-/highcharts-9.1.2.tgz",
-      "integrity": "sha512-RwCnJnxAUCGG4R/WDG+QP/4VJNtOpkQTkXcuf58DjDvm9ZILplSMdZvrbMRC9Y9ecuJQCjjZsiifvyS5E2IBvQ=="
+      "version": "9.3.3",
+      "resolved": "https://registry.npmjs.org/highcharts/-/highcharts-9.3.3.tgz",
+      "integrity": "sha512-QeOvm6cifeZYYdTLm4IxZsXcOE9c4xqfs0z0OJJ0z7hhA9WG0rmcVAyuIp5HBl/znjA/ayYHmpYjBYD/9PG4Fg=="
     },
     "highcharts-custom-events": {
       "version": "3.0.10",
@@ -7741,9 +7718,9 @@
       "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg=="
     },
     "immer": {
-      "version": "9.0.7",
-      "resolved": "https://registry.npmjs.org/immer/-/immer-9.0.7.tgz",
-      "integrity": "sha512-KGllzpbamZDvOIxnmJ0jI840g7Oikx58lBPWV0hUh7dtAyZpFqqrBZdKka5GlTwMTZ1Tjc/bKKW4VSFAt6BqMA=="
+      "version": "9.0.12",
+      "resolved": "https://registry.npmjs.org/immer/-/immer-9.0.12.tgz",
+      "integrity": "sha512-lk7UNmSbAukB5B6dh9fnh5D0bJTOFKxVg2cyJWTYrWRfhLrLMBquONcUs3aFq507hNoIZEDDh8lb8UtOizSMhA=="
     },
     "import-cwd": {
       "version": "2.1.0",
@@ -10412,9 +10389,9 @@
       "optional": true
     },
     "nanoid": {
-      "version": "3.1.23",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.23.tgz",
-      "integrity": "sha512-FiB0kzdP0FFVGDKlRLEQ1BgDzU87dy5NnzjeW9YZNt+/c3+q82EQDUwniSAUxp/F0gFNI1ZhKU1FqYsMuqZVnw=="
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.2.0.tgz",
+      "integrity": "sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA=="
     },
     "nanomatch": {
       "version": "1.2.13",
@@ -12351,13 +12328,13 @@
       }
     },
     "prop-types": {
-      "version": "15.7.2",
-      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz",
-      "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==",
+      "version": "15.8.1",
+      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
+      "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
       "requires": {
         "loose-envify": "^1.4.0",
         "object-assign": "^4.1.1",
-        "react-is": "^16.8.1"
+        "react-is": "^16.13.1"
       },
       "dependencies": {
         "react-is": {
@@ -12455,9 +12432,9 @@
       "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc="
     },
     "qs": {
-      "version": "6.10.1",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.1.tgz",
-      "integrity": "sha512-M528Hph6wsSVOBiYUnGf+K/7w0hNshs/duGsNXPUCLH5XAqjEtiPGwNONLV0tBH8NoGb0mvD5JubnUTrujKDTg==",
+      "version": "6.10.3",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz",
+      "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==",
       "requires": {
         "side-channel": "^1.0.4"
       }
@@ -12565,9 +12542,9 @@
       }
     },
     "react-datetime": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/react-datetime/-/react-datetime-3.0.4.tgz",
-      "integrity": "sha512-v6MVwCve+DRaLN2f22LTO5TlrPpkUXumPkp1zfrbhaFtSYGl2grZ2JtwJfLxRj/T4ACyePAV4srCR6cMSiQ/Iw==",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/react-datetime/-/react-datetime-3.1.1.tgz",
+      "integrity": "sha512-gHCTjAniCcMb6jdXpz+MpVe/uCeaHNDOofg+l41nLlJI3uBLBMV40CQbGB2TCTUpCzGT1mCs4vQzKGMjXO/WWQ==",
       "requires": {
         "prop-types": "^15.5.7"
       }
@@ -12734,15 +12711,15 @@
       "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w=="
     },
     "react-markdown": {
-      "version": "6.0.2",
-      "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-6.0.2.tgz",
-      "integrity": "sha512-Et2AjXAsbmPP1nLQQRqmVgcqzfwcz8uQJ8VAdADs8Nk/aaUA0YeU9RDLuCtD+GwajCnm/+Iiu2KPmXzmD/M3vA==",
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-6.0.3.tgz",
+      "integrity": "sha512-kQbpWiMoBHnj9myLlmZG9T1JdoT/OEyHK7hqM6CqFT14MAkgWiWBUYijLyBmxbntaN6dCDicPcUhWhci1QYodg==",
       "requires": {
         "@types/hast": "^2.0.0",
         "@types/unist": "^2.0.3",
         "comma-separated-tokens": "^1.0.0",
         "prop-types": "^15.7.2",
-        "property-information": "^5.0.0",
+        "property-information": "^5.3.0",
         "react-is": "^17.0.0",
         "remark-parse": "^9.0.0",
         "remark-rehype": "^8.0.0",
@@ -12772,6 +12749,11 @@
         }
       }
     },
+    "react-range": {
+      "version": "1.8.12",
+      "resolved": "https://registry.npmjs.org/react-range/-/react-range-1.8.12.tgz",
+      "integrity": "sha512-vY2eCNDvKlXtEpl8xx5AQVIBeE4ZZ46dLVL+82f4FcS9XrPqz4+XagPHmq3IPqYuVQ2H4b8UArXzvVAsd/B03Q=="
+    },
     "react-redux": {
       "version": "7.2.6",
       "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-7.2.6.tgz",
@@ -12800,12 +12782,17 @@
       "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.8.3.tgz",
       "integrity": "sha512-X8jZHc7nCMjaCqoU+V2I0cOhNW+QMBwSUkeXnTi8IPe6zaRWfn60ZzvFDZqWPfmSJfjub7dDW1SP0jaHWLu/hg=="
     },
+    "react-reverse-portal": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/react-reverse-portal/-/react-reverse-portal-2.1.0.tgz",
+      "integrity": "sha512-ck1igNI4tUFozJ89X7qqFtWy1wSL7WEd/5iZbjBjyf+6vjZ3leg4JJxdVPbFt9+zwMLcQSPsIkw9xGu9E+VqCQ=="
+    },
     "react-router": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/react-router/-/react-router-5.2.0.tgz",
-      "integrity": "sha512-smz1DUuFHRKdcJC0jobGo8cVbhO3x50tCL4icacOlcwDOEQPq4TMqwx3sY1TP+DvtTgz4nm3thuo7A+BK2U0Dw==",
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/react-router/-/react-router-5.2.1.tgz",
+      "integrity": "sha512-lIboRiOtDLFdg1VTemMwud9vRVuOCZmUIT/7lUoZiSpPODiiH1UQlfXy+vPLC/7IWdFYnhRwAyNqA/+I7wnvKQ==",
       "requires": {
-        "@babel/runtime": "^7.1.2",
+        "@babel/runtime": "^7.12.13",
         "history": "^4.9.0",
         "hoist-non-react-statics": "^3.1.0",
         "loose-envify": "^1.3.1",
@@ -12817,6 +12804,14 @@
         "tiny-warning": "^1.0.0"
       },
       "dependencies": {
+        "@babel/runtime": {
+          "version": "7.17.2",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.2.tgz",
+          "integrity": "sha512-hzeyJyMA1YGdJTuWU0e/j4wKXrU4OMFvY2MSlaI9B7VQb0r5cxTE3EAIS2Q7Tn2RIcDkRvTA/v2JsAEhxe99uw==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        },
         "isarray": {
           "version": "0.0.1",
           "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
@@ -12838,17 +12833,27 @@
       }
     },
     "react-router-dom": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-5.2.0.tgz",
-      "integrity": "sha512-gxAmfylo2QUjcwxI63RhQ5G85Qqt4voZpUXSEqCwykV0baaOTQDR1f0PmY8AELqIyVc0NEZUj0Gov5lNGcXgsA==",
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-5.3.0.tgz",
+      "integrity": "sha512-ObVBLjUZsphUUMVycibxgMdh5jJ1e3o+KpAZBVeHcNQZ4W+uUGGWsokurzlF4YOldQYRQL4y6yFRWM4m3svmuQ==",
       "requires": {
-        "@babel/runtime": "^7.1.2",
+        "@babel/runtime": "^7.12.13",
         "history": "^4.9.0",
         "loose-envify": "^1.3.1",
         "prop-types": "^15.6.2",
-        "react-router": "5.2.0",
+        "react-router": "5.2.1",
         "tiny-invariant": "^1.0.2",
         "tiny-warning": "^1.0.0"
+      },
+      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.17.2",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.2.tgz",
+          "integrity": "sha512-hzeyJyMA1YGdJTuWU0e/j4wKXrU4OMFvY2MSlaI9B7VQb0r5cxTE3EAIS2Q7Tn2RIcDkRvTA/v2JsAEhxe99uw==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        }
       }
     },
     "react-scripts": {
@@ -13086,9 +13091,9 @@
       }
     },
     "redux-thunk": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.4.0.tgz",
-      "integrity": "sha512-/y6ZKQNU/0u8Bm7ROLq9Pt/7lU93cT0IucYMrubo89ENjxPa7i8pqLKu6V4X7/TvYovQ6x01unTeyeZ9lgXiTA=="
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.4.1.tgz",
+      "integrity": "sha512-OOYGNY5Jy2TWvTL1KgAlVy6dcx3siPJ1wTq741EPyUKfn6W6nChdICjZwCd0p8AZBs5kWpZlbkXW2nE/zjUa+Q=="
     },
     "regenerate": {
       "version": "1.4.2",
@@ -13261,9 +13266,9 @@
       "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
     },
     "reselect": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/reselect/-/reselect-4.1.2.tgz",
-      "integrity": "sha512-wg60ebcPOtxcptIUfrr7Jt3h4BR86cCW3R7y4qt65lnNb4yz4QgrXcbSioVsIOYguyz42+XTHIyJ5TEruzkFgQ=="
+      "version": "4.1.5",
+      "resolved": "https://registry.npmjs.org/reselect/-/reselect-4.1.5.tgz",
+      "integrity": "sha512-uVdlz8J7OO+ASpBYoz1Zypgx0KasCY20H+N8JD13oUMtPvSHQuscrHop4KbXrbsBcdB9Ds7lVK7eRkBIfO43vQ=="
     },
     "resolve": {
       "version": "1.18.1",
@@ -13837,9 +13842,9 @@
       }
     },
     "semantic-ui-react": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/semantic-ui-react/-/semantic-ui-react-2.0.3.tgz",
-      "integrity": "sha512-a0hGN6XXw64sRSKwWqMCKSI/AGLohxNeWuErS39eswvBbUnLjBij8ZoEdiqDiz/PuWpwYIRjgmQVrut+7h3b2g==",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/semantic-ui-react/-/semantic-ui-react-2.1.2.tgz",
+      "integrity": "sha512-NG7qMUi2Ab2gPWzOnGyBO9Ao2tjf9MZjjMMPbmg8uEUcEFm+z1QPqCKhW+eYMiZekH8L9azw9M+hzoTIB1lZgQ==",
       "requires": {
         "@babel/runtime": "^7.10.5",
         "@fluentui/react-component-event-listener": "~0.51.6",
@@ -13848,8 +13853,8 @@
         "@semantic-ui-react/event-stack": "^3.1.2",
         "clsx": "^1.1.1",
         "keyboard-key": "^1.1.0",
-        "lodash": "^4.17.19",
-        "lodash-es": "^4.17.15",
+        "lodash": "^4.17.21",
+        "lodash-es": "^4.17.21",
         "prop-types": "^15.7.2",
         "react-is": "^16.8.6 || ^17.0.0",
         "react-popper": "^2.2.4",
@@ -15343,9 +15348,9 @@
       }
     },
     "unfuck-spa-shibboleth-session": {
-      "version": "3.1.3",
-      "resolved": "https://registry.npmjs.org/unfuck-spa-shibboleth-session/-/unfuck-spa-shibboleth-session-3.1.3.tgz",
-      "integrity": "sha512-ZchOl94XvUz0eTLfZtx1iuqoKcNFN7WePB9Qz+6vPAkHO+F+H1zBatJo0lLgEdZGm1wqCl7gArrRuilcnsUp/A==",
+      "version": "3.1.4",
+      "resolved": "https://registry.npmjs.org/unfuck-spa-shibboleth-session/-/unfuck-spa-shibboleth-session-3.1.4.tgz",
+      "integrity": "sha512-eh20O3yTDRlRYmV8DQhELvnwFS78I8mdMUjetdvvIpO/tGSO5mzDooCrF9ssOUxGtCA6VtlcoVBs3FjFSm1Wng==",
       "requires": {
         "axios": "^0.21.1"
       }
@@ -15375,9 +15380,9 @@
       "integrity": "sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg=="
     },
     "unified": {
-      "version": "9.2.1",
-      "resolved": "https://registry.npmjs.org/unified/-/unified-9.2.1.tgz",
-      "integrity": "sha512-juWjuI8Z4xFg8pJbnEZ41b5xjGUWGHqXALmBZ3FC3WX0PIx1CZBIIJ6mXbYMcf6Yw4Fi0rFUTA1cdz/BglbOhA==",
+      "version": "9.2.2",
+      "resolved": "https://registry.npmjs.org/unified/-/unified-9.2.2.tgz",
+      "integrity": "sha512-Sg7j110mtefBD+qunSLO1lqOEKdrwBFBrR6Qd8f4uwkhWNlbkaqwHse6e7QvD3AP/MNoJdEDLaf8OxYyoWgorQ==",
       "requires": {
         "bail": "^1.0.0",
         "extend": "^3.0.0",
@@ -15616,9 +15621,9 @@
       }
     },
     "url-parse": {
-      "version": "1.5.3",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz",
-      "integrity": "sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ==",
+      "version": "1.5.10",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
+      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
       "requires": {
         "querystringify": "^2.1.1",
         "requires-port": "^1.0.0"
@@ -15629,6 +15634,11 @@
       "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz",
       "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ=="
     },
+    "use-context-selector": {
+      "version": "1.3.9",
+      "resolved": "https://registry.npmjs.org/use-context-selector/-/use-context-selector-1.3.9.tgz",
+      "integrity": "sha512-YgzRyeFjoJXwFn2qLVAuIbV6EQ8DOuzu3SS/eiCxyAyvBhcn02jYSz8c5v22QQU3LW6Ez/Iyo62kKvS7Kdqt3A=="
+    },
     "util": {
       "version": "0.11.1",
       "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz",
@@ -17189,27 +17199,17 @@
       "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg=="
     },
     "xlsx": {
-      "version": "0.17.0",
-      "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.17.0.tgz",
-      "integrity": "sha512-bZ36FSACiAyjoldey1+7it50PMlDp1pcAJrZKcVZHzKd8BC/z6TQ/QAN8onuqcepifqSznR6uKnjPhaGt6ig9A==",
+      "version": "0.17.5",
+      "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.17.5.tgz",
+      "integrity": "sha512-lXNU0TuYsvElzvtI6O7WIVb9Zar1XYw7Xb3VAx2wn8N/n0whBYrCnHMxtFyIiUU1Wjf09WzmLALDfBO5PqTb1g==",
       "requires": {
         "adler-32": "~1.2.0",
         "cfb": "^1.1.4",
-        "codepage": "~1.14.0",
-        "commander": "~2.17.1",
+        "codepage": "~1.15.0",
         "crc-32": "~1.2.0",
-        "exit-on-epipe": "~1.0.1",
-        "fflate": "^0.3.8",
         "ssf": "~0.11.2",
         "wmf": "~1.0.1",
         "word": "~0.3.0"
-      },
-      "dependencies": {
-        "commander": {
-          "version": "2.17.1",
-          "resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz",
-          "integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg=="
-        }
       }
     },
     "xml-name-validator": {

services/frontend/package.json

@@ -6,37 +6,41 @@
     "build": "DISABLE_ESLINT_PLUGIN=true react-scripts build"
   },
   "dependencies": {
-    "@reduxjs/toolkit": "^1.6.2",
-    "@sentry/browser": "^6.9.0",
-    "@sentry/tracing": "^6.11.0",
-    "axios": "^0.21.2",
+    "@reduxjs/toolkit": "^1.8.0",
+    "@sentry/browser": "^6.18.2",
+    "@sentry/tracing": "^6.18.2",
+    "axios": "^0.21.4",
+    "chroma-js": "^2.4.2",
     "formik": "^2.2.9",
-    "highcharts": "^9.1.2",
+    "highcharts": "^9.3.3",
     "highcharts-custom-events": "^3.0.10",
     "highcharts-exporting": "^0.1.7",
     "history": "^4.10.1",
-    "immer": "^9.0.7",
+    "immer": "^9.0.12",
     "lodash": "^4.17.21",
     "moment": "^2.29.1",
     "path-to-regexp": "^0.1.7",
     "piwik-react-router": "^0.12.1",
-    "prop-types": "^15.7.2",
-    "qs": "^6.10.1",
+    "prop-types": "^15.8.1",
+    "qs": "^6.10.3",
     "query-string": "^6.14.1",
     "react": "^16.14.0",
-    "react-datetime": "^3.0.4",
+    "react-datetime": "^3.1.1",
     "react-dom": "^16.14.0",
     "react-highcharts": "^16.1.0",
-    "react-markdown": "^6.0.2",
+    "react-markdown": "^6.0.3",
+    "react-range": "^1.8.12",
     "react-redux": "^7.2.6",
-    "react-router-dom": "^5.2.0",
+    "react-reverse-portal": "^2.1.0",
+    "react-router-dom": "^5.3.0",
     "react-scripts": "4.0.3",
     "react-scroll-to-component": "^1.0.2",
     "semantic-ui-css": "^2.4.1",
-    "semantic-ui-react": "^2.0.3",
-    "unfuck-spa-shibboleth-session": "^3.1.3",
+    "semantic-ui-react": "^2.1.2",
+    "unfuck-spa-shibboleth-session": "^3.1.4",
+    "use-context-selector": "^1.3.9",
     "uuid": "^3.4.0",
-    "xlsx": "^0.17.0"
+    "xlsx": "^0.17.5"
   },
   "proxy": "http://backend:8080",
   "browserslist": {

services/frontend/src/apiConnection/index.js

@@ -10,7 +10,7 @@ export const getHeaders = () => {
     uid: 'mluukkai',
     displayName: 'Matti Luukkainen',
     'shib-session-id': 'mock-session',
-    hyGroupCn: 'grp-oodikone-users;grp-oodikone-basic-users',
+    hyGroupCn: 'grp-oodikone-users;grp-oodikone-basic-users;grp-toska',
     mail: 'grp-toska+mockmluukkai@helsinki.fi',
     hyPersonSisuId: 'hy-hlo-1441871',
     shib_logout_url: 'https://helsinki.fi/shibboleth-sp/Logout',
@@ -107,7 +107,7 @@ export const handleRequest = store => next => async action => {
 // All tags used for invalidating cache must be defined here
 export const RTKApi = createApi({
   reducerPath: 'api',
-  tagTypes: ['StudyGuidanceGroups'],
+  tagTypes: ['StudyGuidanceGroups', 'Semester'],
   baseQuery: fetchBaseQuery({
     baseUrl: apiBasePath,
     prepareHeaders: headers => {

services/frontend/src/common/InfoToolTips/coolDataScience.js

@@ -6,9 +6,9 @@ export default {
   
   **4 vuoden tahdissa** olevat opiskelijat 45op × ((tämä päivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet,
   
-  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu).
+  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - passiivinen).
 
-  **peruutettu**, jos opinto-oikeus merkattu perutuksi
+  **passiivinen**, jos opinto-oikeus on päättynyt tai opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle, eikä opiskelija ole vielä valmistunut. 
     
   **Include only at least once enrolled students** suodattaa pois opiskelijat jotka eivät ole ollenkaan ilmoittautunut. 
   Vakiona opiskelijat jotka eivät ole ilmoittautunut millekkään lukuvuodelle (läsnä tai poissa olevaksi)
@@ -24,9 +24,9 @@ export default {
   
   **4 vuoden tahdissa** olevat opiskelijat 45op × ((tämä päivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet,
   
-  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu).
+  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - passiivinen).
 
-  **peruutettu**, jos opinto-oikeus merkattu perutuksi
+  **passiivinen**, jos opinto-oikeus on päättynyt tai opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle, eikä opiskelija ole vielä valmistunut. 
   
   **Include only at least once enrolled students** suodattaa pois opiskelijat,
   jotka eivät ole ollenkaan ilmoittautunut. Vakiona opiskelijat, jotka eivät ole ilmoittautunut millekkään lukuvuodelle (läsnä tai poissa olevaksi) on sisällytetty laskuihin.
@@ -41,11 +41,11 @@ export default {
   
   **4 vuoden tahdissa** olevat opiskelijat 45op × ((tarkastuspäivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet,
 
-  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu). 
+  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - passiivinen). 
   
   **Tarkastuspäivät** ovat 31.7, 30.11 ja 1.4 joka vuodelle.
 
-  **peruutettu**, jos opinto-oikeus merkattu perutuksi
+  **passiivinen**, jos opinto-oikeus on päättynyt tai opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle, eikä opiskelija ole vielä valmistunut. 
 
   **Include attainments attained before the studyright start** laskee mukaan opiskelijan ennen opinto-oikeuden alkua saadut opintopisteet.
   Vakiona opintopisteet ennen opinto-oikeutta suodatetaan pois.

services/frontend/src/common/InfoToolTips/courseStatistics.js

@@ -1,7 +1,7 @@
 export default {
-  0: {
-    Attempts: `
-      Table - Attempts
+  Tables: {
+    ATTEMPTS: `
+      **Table - Attempts**
 
       Taulukko näyttää suorituskerrat lukuvuositasolla.
       Samalla opiskelijalla voi olla useampia suorituskertoja.
@@ -13,8 +13,8 @@ export default {
       Arvosanajakaumaa katsoessasi voit muuttaa luvut prosenteiksi, klikkaamalla painiketta "Relative".
       Mahdollinen sarake "Other passed" sisältää sekä arvosanan "Hyv." saaneet että kurssin hyväksilukeneet.
     `,
-    Students: `
-      Table - Students
+    STUDENT: `
+      **Table - Students**
 
       Taulukko näyttää yksittäiset opiskelijat, jotka ovat kurssia suorittaneet.
       Jokainen opiskelija on jaoteltu luokkiin: "läpäissyt ensiyrittämällä", "läpäissyt lopulta" tai "ei koskaan läpäissyt kurssia".
@@ -26,17 +26,17 @@ export default {
       Total-rivillä näkyy niiden opiskelijoiden kokonaismäärä, jotka ovat kurssista joskus arvosanan saaneet.
     `,
   },
-  1: {
-    Attempts: `
-      Pass rate chart - Attempts
+  PassRate: {
+    ATTEMPTS: `
+      **Pass rate chart - Attempts**
 
       Graafi näyttää suorituskerrat lukuvuositasolla jaoteltuina hyväksyttyihin ja hylättyihin suorituksiin.
       Hyväksytyt suoritukset sisältävät arvosanat 1-5 sekä arvosanan Hyv.
       Hylätyt suoritukset sisältävät arvosanat Hyl. ja 0, sekä suoritukset, joille opettaja on antanut merkinnän
       "Luop" (Luopunut suorituksesta) tai "Eisa" (Ei saapunut tenttiin).
     `,
-    Students: `
-      Pass rate chart - Students
+    STUDENT: `
+      **Pass rate chart - Students**
 
       Graafi näyttää yksittäiset opiskelijat, jotka ovat kurssia suorittaneet.
       Jokainen opiskelija on jaoteltu luokkiin: "läpäissyt ensiyrittämällä", "läpäissyt lopulta" tai "ei koskaan läpäissyt kurssia".
@@ -46,18 +46,9 @@ export default {
       myöhemmin keväällä 2019, tulisi merkityksi lukuvuodelle 2017-2018 pylvääseen "passed eventually".        
     `,
   },
-  2: {
-    Attempts: `
-      Grade distribution
-  
-      Graafi näyttää suorituskertojen arvosanajakauman eri lukuvuosille. Mahdollinen kategoria
-      "Other passed" sisältää sekä arvosanan "Hyv." että kurssin hyväksiluvut.
-    `,
-    Students: `
-      Grade distribution
+  GradeDistribution: `
+    **Grade distribution**
 
-      Graafi näyttää suorituskertojen arvosanajakauman eri lukuvuosille. Mahdollinen kategoria
-      "Other passed" sisältää sekä arvosanan "Hyv." että kurssin hyväksiluvut.
-    `,
-  },
+    Graafi näyttää suorituskertojen arvosanajakauman eri lukuvuosille. Mahdollinen kategoria
+  `,
 }

services/frontend/src/common/InfoToolTips/filters.js

@@ -4,9 +4,15 @@ export default {
     short:
       'Suodata opiskelijoita sen mukaan ovatko he ilmoittautuneet läsnä- tai poissaoleviksi kyseiselle lukukaudelle',
   },
+  studyrightStatus: {
+    label: 'Rajaa opiskelijoita opinto-oikeuden statuksen mukaan',
+    short:
+      'Opinto-oikeus on aktiivinen, mikäli opiskelija on ilmoittautunut läsnä- tai poissaolevaksi tälle lukukaudelle ja opinto-oikeus ei ole vielä päättynyt. \n Opinto-oikeus on passiivinen, jos opiskelija ei ole ilmoittautunut läsnä- tai poissaolevaksi tälle lukukaudelle tai opinto-oikeus on vanhentunut. Valmistuneet opiskelijat on suodatettu pois molemmista kategorioista.',
+  },
   transferred: {
     label: 'Include and exclude students from this program',
-    short: 'Mahdollisuus suodattaa opiskelijat, jotka ovat siirtyneet tutkinto-ohjelmaan',
+    short:
+      'Mahdollisuus suodattaa opiskelijat, jotka ovat siirtyneet nykyiseen 2017 alkaneeseen tutkinto-ohjelmaan vanhasta tutkinto-ohjelmasta',
   },
   courseCredits: {
     label: null,

services/frontend/src/common/InfoToolTips/index.jsx

@@ -1,5 +1,4 @@
 import PopulationStatistics from './populationStatistics'
-import PopulationOverview from './populationOverview'
 import CoolDataScience from './coolDataScience'
 import CourseStatistics from './courseStatistics'
 import CoursePopulation from './coursePopulation'
@@ -7,7 +6,6 @@ import Studyprogramme from './studyprogramme'
 
 export default {
   PopulationStatistics,
-  PopulationOverview,
   CoolDataScience,
   CourseStatistics,
   CoursePopulation,

services/frontend/src/common/InfoToolTips/populationOverview.js

@@ -1,24 +0,0 @@
-export default {
-  PopulationProgress: `Tämä koontisivu sisältää koulutusohjelman kaikkien opiskelijoiden perustiedot lukuvuosittain. Luvut perustuvat opinto-oikeuksiin kyseisessä ohjelmassa.
-
-  Population progress:
-  Vuosittain ohjelmassa aloittaneet, ohjelman keskeyttäneet / peruneet sekä ohjelmaan siirtyneet opiskelijat. 
-  Huomattavaa on, että sama opiskelija voi olla kahdessa luokassa, jos hän on esimerkiksi aloittanut opinnot jo, mutta keskeyttänyt ne lukuvuoden aikana.
-  Tarkemmat tiedot saat esille klikkaamalla nuolta kunkin lukuvuoden kohdalla.`,
-  YearlyProductivity: `
-  Yearly productivity:
-  Koulutusohjelman kunakin kalenterivuonna tuottamat opintopisteet, valmistuneiden määrän, ohjelman pääaineopiskelijoiden
-  opintopisteet sekä muiden kuin pääaineopiskelijoiden opintopisteet. Mukana on myös kuhunkin koulutusohjelmaan hyväksiluetut opintopisteet.
-  
-  Vanhojen koulutusohjelmien lakkauttaminen ja opiskelijoiden siirtäminen uusiin ohjelmiin voi aiheuttaa tavallista suuremmat opintopistemäärät lukuvuodelle 2020-2021.`,
-  Year: `Lukuvuosi`,
-  Students: `Opiskelijat, jotka ovat koulutusohjelmassa aloittaneet, opinto-oikeutensa peruneet sekä koulutusohjelmaan tai siitä pois siirtyneet.`,
-  Started: `Koulutusohjelmassa kunakin lukuvuonna aloittaneet opiskelijat. EI sisällä ohjelmaan siirtyneitä opiskelijoita.`,
-  Cancelled: `Koulutusohjelmaan oikeuttavan opinto-oikeutensa kunakin lukuvuonna peruttaneet`,
-  Graduated: `Tiettynä lukuvuonna aloittaneista opiskelijoista on valmistunut`,
-  Transferred: `Koulutusohjelmaan kyseisenä lukuvuonna siirtyneet opiskelijat. 
-  
-  Vanhojen koulutusohjelmien lakkauttaminen aiheuttaa tavallista suuremmat siirtoluvut lukuvuodelle 2020-2021.`,
-  Credits: `Kunakin vuonna aloittaneista opiskelijoista ne, jotka ovat ylittäneet tietyn opintopisterajapyykin`,
-  Thesis: `Loppututkielman kirjoittaneiden määrä`,
-}

services/frontend/src/common/InfoToolTips/populationStatistics.js

@@ -1,7 +1,7 @@
 export default {
   Advanced: `**Semesters:**  Jos haluaa tarkastella vain syyslukukaudella tai kevätlukukaudella ilmoittautuneita, haluttu ajanjakso valitaan tästä.
 
-  **Include:** Koulutusohjelmapopulaatiosta on oletusarvoisesti suodatettu pois vaihto-opiskelijat, tutkinnonsuoritusoikeudesta luopuneet ja erillisopinto-oikeudella opiskelevat. Näitä ryhmiä voi sisällyttää mukaan lisäämällä ruksin haluamaansa kohtaan.`,
+  **Include:** Koulutusohjelmapopulaatiosta on oletusarvoisesti suodatettu pois vaihto-opiskelijat ja erillisopinto-oikeudella opiskelevat. Näitä ryhmiä voi sisällyttää mukaan lisäämällä ruksin haluamaansa kohtaan.`,
   QueryCard: `**Valitun koulutusohjelmavuosikurssin perustiedot.**
   
   **Sample size** luku sisältää läsnä- ja poissaolevaksi ilmoittautuneet opiskelijat, muttei "excludes"-ryhmiä, jotka voi määritellä "advanced settings" -kohdasta.
@@ -41,6 +41,12 @@ export default {
   + **Min ja Max** kertovat pienimmän ja suurimman opintopistemäärän, jotka jollain opiskelijoilla kyseisessä populaatiossa on.
 
   Mikäli valintatapatieto kyseiselle populaatiolle on saatavissa, näytetään myös samat statistiikat per valintatapa.  
+
+  **Distribution Development**
+
+  Pylväsdiagrammi kertoo opiskelijoiden jakauman opintopistehaarukoihin kalenterivuosittain, lukuvuosittain tai lukukausittain eriteltynä. Valmistuneita opiskelijoita ei sisällytetä valmistumisen jälkeisiin pylväisiin.
+
+  **Cumulative**-valinnan ollessa käytössä kunkin opiskelijan kaikki aiemmat opintopisteet lasketaan mukaan opintopistemäärään. Muutoin vain pylvästä vastaavalla aikavälillä kirjatut opintopisteet otetaan huomioon.
   `,
   CoursesOfPopulation: `**Courses of population**
 
@@ -60,6 +66,7 @@ export default {
   **Taulukot**
   
   - **pass/fail** - oletusnäkymä, joka sisältää kurssisuoritukset, hylätyt sekä yritykset. Taulukon tiedot voi järjestää sarakkeita (n, after retry, percentage, many times, per student, passed ja attempted) klikkaamalla.
+  - **pass/fail w/ enrollments** - näkymä, joka sisältää suoritustiedot ilmoittautumisten kanssa. Huom.! ilmoittautumistiedot ovat saatavilla vain Sisun käyttöönoton jälkeiseltä ajalta.
   - **grades** - arvosanajakauma.
   - **when passed** - kurssisuorituksen ajankohta. Huom.! *Passed*-sarakkeen kokonaissumma ei välttämättä täsmää puolivuosittaisten lukumäärien summan kanssa johtuen tuplakirjauksista Oodissa.
   `,
@@ -71,6 +78,10 @@ export default {
   - **Study programme:** haluttu koulutusohjelma.
   - **Select tag (Optional):** itsetehty tai valmis tägi.
   - **Study track (Optional):** opintosuunta.`,
+  TagAndTrackMovedIntoFilters: `
+  Tägien ja opintosuuntien valinnat ovat siirtyneet tästä omiksi filtereikseen.
+  Voit tehdä valinnat siirryttyäsi tarkastelemaan ohjelman tilastoja.
+  `,
   Students: `**Students**
 
   Taulukko näyttää oletusarvoisesti vain opiskelijanumerot, mutta harmaata liukukytkintä klikkaamalla saa näkyviin opiskelijoiden nimet.

services/frontend/src/common/InfoToolTips/studyprogramme.js

@@ -1,43 +1,79 @@
 export default {
+  YearToggle: `
+  Academic year: Näyttää tilastot akateemisille vuosille jaoteltuna, esimerkiksi 1.8.2019-31.7.2020
+  Calendar year: Näyttää tilastot kalenterivuosille jaoteltuna, esimerkiksi 1.1.2019-31.12.2019 
+  `,
+  StudentToggle: `
+  All studyrights: Valinnassa ovat mukana vaihto-opiskelijat, erillisoikeudella opiskelevat ja ohjelmaan siirtyneet opiskelijat sekä ohjelmasta pois siirtyneet opiskelijat.
+  
+  Special studyrights excluded: Valinnasta ja kaikista luvuista on poissuljettu kaikki edellä mainitut erikoisryhmät.
+  `,
+  GraduatedToggle: `
+  Graduated included: Valinnassa ovat mukana ohjelmasta jo valmistuneet opiskelijat.
+  
+  Graduated excluded: Valinnasta on poistettu ohjelmasta jo valmistuneet opiskelijat.
+  `,
   StudentsOfTheStudyprogramme: `
-  **Started**: Sisältää kyseisenä vuonna aloitetut opinto-oikeudet riippumatta siitä, minä vuonna kyseinen opiskeluoikeus on myönnetty. Opiskelija on myös voinut keskeyttää opintonsa tai siirtyä pois ohjelmasta.\n
+  **All**: Sisältää kaikki kyseisenä vuonna alkaneet opinto-oikeudet riippumatta siitä, onko opintoja aloitettu tai onko niitä aloitettu ko. vuonna. Mukana myös ohjelmaan siirtyneet opiskelijat.
+
+  **Started studying**: Sisältää kyseisenä vuonna aloittaneet opiskelijat (aloitetut opinto-oikeudet) riippumatta siitä, minä vuonna kyseinen opiskeluoikeus on myönnetty. Opiskelija on myös voinut myöhemmin keskeyttää opintonsa tai siirtyä pois ohjelmasta.\n
   **Graduated**: Sisältää kyseisenä vuonna valmistuneet opiskelijat riippumatta siitä, minä vuonna opiskeluoikeus ohjelmaan on myönnetty. Jaottelu on tehty opinto-oikeuden päättymispäivän (=valmistumispäivän) perusteella.
-  **Cancelled**: Sisältää kyseisenä vuonna perutut opiskeluoikeudet riippumatta siitä, onko kyseinen opiskelija ikinä edes alkanut opiskelemaan ohjelmassa tai opiskelun aloitusvuodesta.
+    
+  **Transferred away**: Sisältää kyseisenä vuonna pois ohjelmasta siirtyneet opiskelijat (opinto-oikeudet). Sama opiskelija voi esiintyä tilastoissa useana vuonna, mikäli hän on siirtynyt ohjelmaan ja siitä pois useasti.
   
-  **Transferred away**: Sisältää kyseisenä vuonna pois ohjelmasta siirtyneet opinto-oikeudet. Sama opiskelija voi esiintyä tilastoissa useana vuonna. Kyseiset opiskelijat ovat saattaneet aloittaa ohjelmassa tai eivät koskaan ole aloittaneet ohjelmassa. 
-  
-  **Transferred to**: Sisältää kyseisenä vuonna ohjelmaan siirtyneet opinto-oikeudet.
+  **Transferred to**: Sisältää kyseisenä vuonna ohjelmaan siirtyneet opiskelijat (opinto-oikeudet).
 
   HUOM! Ohjelmaan siirtyneiden määrä on erityisen suuri vuonna 2020, jolloin vanhoista koulutusohjelmista luovuttiin ja opiskelijat siirtyivät uusiin ohjelmiin.
   `,
   CreditsProducedByTheStudyprogramme: `
-  Sisältää opintopisteet suoritusvuosittain (Sisussa näkyvä kurssin suorituspäivä)
+  Sisältää opintopisteet suoritusvuosittain. Suoritukset on jaoteltu Sisussa näkyvän kurssin suorituspäivän mukaan.
 
-  **Major students' credits**: Sisältää ne opintopisteet, joiden suorittajalla on suoritushetkellä ollut ensisijainen opinto-oikeus kyseiseen koulutusohjelmaan.
+  **Major students' credits**: Sisältää ne opintopisteet, joiden suorittajalla on suoritushetkellä ollut ensisijainen opinto-oikeus kyseiseen koulutusohjelmaan. Sisältää myös ohjelmaan siirtyneet ensisijaiset opiskelijat.
 
-  **Non-major students' credits**: Sisältää koulutusohjelman tuottamat opintopisteet, joiden suorittajalla EI ole ollut suoritushetkellä ensisijaista opinto-oikeutta kyseiseen koulutusohjelmaan.
+  **Non-major students' credits**: Sisältää koulutusohjelman tuottamat opintopisteet, joiden suorittajalla EI ole ollut suoritushetkellä ensisijaista opinto-oikeutta kyseiseen koulutusohjelmaan. Sisältää
+  esimerkiksi vaihto-opiskelijat ja erillis-opinto-oikeudella opiskelevat. Sisältää myös opintopisteet, jotka opiskelija on suorittanut koulutusohjelmaan ennen opintojen aloittamista koulutusohjelmassa tai opintojen päätyttyä.
   
-  **Transferred credits**: Sisältää opintopisteet, jotka on suoritettu kyseisenä vuonna, ja hyväksiluettu tähän koulutusohjelmaan.
+  **Transferred credits**: Sisältää opintopisteet, jotka on suoritettu kyseisenä vuonna, ja hyväksiluettu tähän koulutusohjelmaan. Mukana ovat kaikenlaisilla opinto-oikeuksilla hyväksiluetut opintopisteet.
   `,
   GraduatedAndThesisWritersOfTheProgramme: `
-  **Wrote thesis**: Sisältää kyseisenä vuonna kandidaatintutkielman hyväksytysti suorittaneet opiskelijat. Mukana ei ole mahdollisia arvosanan korotuksia.
+  **Wrote thesis**: Sisältää kyseisenä vuonna kandidaatintutkielman tai pro gradu-tutkielman hyväksytysti suorittaneet opiskelijat. Mukana ei ole mahdollisia arvosanan korotuksia.
 
   **Graduated**: Sisältää kyseisenä vuonna valmistuneet opiskelijat riippumatta siitä, minä vuonna opiskeluoikeus ohjelmaan on myönnetty. Jaottelu on tehty opinto-oikeuden päättymispäivän (=valmistumispäivän) perusteella.
   `,
   AverageGraduationTimes: `
-  **Median time**: Näyttää kyseisenä vuonna valmistuneiden opintojen keston **mediaanin** kuukausina.
+  **Median time**: Näyttää kyseisenä vuonna valmistuneiden opiskelijoiden opintojen keston **mediaanin** kuukausina. Luku sisältää mahdolliset poissaolokuukaudet.
 
-  **Mean time**: Näyttää kyseisenä vuonna valmistuneiden opintojen keston **keskiarvon** kuukausina.
+  **Mean time**: Näyttää kyseisenä vuonna valmistuneiden opiskelijoiden opintojen keston **keskiarvon** kuukausina. Luku sisältää mahdolliset poissaolokuukaudet.
   
-  Mukana ovat **vain valmistuneet opiskelijat**. Kandiohjelmissa tavoiteaika on 36 kk, maisteriohjelmissa 24 kk`,
-  StudytrackOverview: `
-  Taulukko näyttää kyseisenä lukuvuonna aloittaneet pääaineopiskelijat. Taulukko näyttää myös näistä opiskelijoista miesten, naisten ja suomalaisten osuudet sekä valmistuneiden määrän. 
+  n = **valmistuneet opiskelijat / kaikki ohjelmassa kyseisenä vuonna alkaneet opinto-oikeudet**. Kandiohjelmissa tavoiteaika on 36 kk, maisteriohjelmissa 24 kk`,
+  ProgrammesBeforeOrAfter: `
+  **Mikäli valittuna on kandiohjelma:** Sisältää maisteriohjelmat, joissa tämän kandiohjelman opiskelijat ovat aloittaneet valmistuttuaan kandeiksi. Vuosijaottelu on tehty maisteriohjelmassa aloittamisen perusteella.
+
+  Mukana ovat **vain valmistuneet opiskelijat ja vain maisteriohjelmat.**
+
+  **Mikäli valittuna on maisteriohjelma:** Sisältää kandiohjelmat, joissa tämän ohjelman opiskelijat ovat opiskelleet ennen tuloaan maisteriohjelmaan. Vuosijaottelu on tehty maisteriohjelmassa aloittamisen perusteella.
+  `,
+  StudytrackOverview: `  
+  Kuvaa koulutusohjelman kyseiseen ohjelmaan opinto-oikeudella varustettujen opiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
+  
+  HUOM!
+  **All** Sisältää kaikki opiskelijat, joiden opinto-oikeudella on aloituspäivä kyseisenä vuonna, riippumatta siitä, onko opiskelija aloittanut opiskelemaan ko. vuonna.
+  **Started studying** Sisältää kaikki opiskelijat, jotka ovat aloittaneet opiskelemaan koulutusohjelmassa kyseisenä lukuvuonna.
+  **Inactive** Sisältää kaikki opinto-oikeudet, jotka ovat alkaneet kyseisenä vuonna, mutta jotka ovat sittemmin vanhentuneet ilman, että opiskelija on valmistunut ohjelmasta. Sisältää myös opinto-oikeudet,
+  joissa opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle.
+  
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, ohjelmaan ja siitä pois siirtyneet opiskelijat sekä erillisopinto-oikeudella opiskelevat.
+  Taulukko näyttää myös näistä opiskelijoista miesten, naisten ja suomalaisten osuudet sekä valmistuneiden määrän. 
 
   Yläosan valikosta on mahdollista valita tarkasteluun yhden opintosuunnan opiskelijat. Sekä opiskelijoiden yleistiedot että edistyminen opintopisteittäin
   kuvaavat tällöin kyseisen opintosuunnan opiskelijoita.
   `,
   StudytrackProgress: `
-  Lähtökohtaisesti kuvaa koulutusohjelman kaikkien pääaineoopiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
+  Kuvaa koulutusohjelman kyseiseen ohjelmaan opinto-oikeudella varustettujen opiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
+  
+  HUOM! Mukana ovat myös muuhun kuin valittuun koulutusohjelmaan tai opintosuuntaan tehdyt opintopisteet.
+
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, erillisopinto-oikeudella opiskelevat ja ohjelmaan siirtyneet.   
 
   Mikäli yläosan valikosta valittuna on jokin tietty opintosuunta, myös nämä graafit ja taulukot kuvaavat kyseisen opintosuunnan opiskelijoita.    
   `,

services/frontend/src/common/hooks.js

@@ -271,6 +271,55 @@ export const useLanguage = () => {
   return language
 }
 
+export const useDebounce = (value, timeout, onChange) => {
+  const [innerValue, setInnerValue] = useState(value)
+  const [dirty, setDirty] = useState(false)
+
+  const timeoutRef = useRef(null)
+
+  useEffect(() => {
+    if (!isEqual(value, innerValue)) {
+      setInnerValue(value)
+      setDirty(false)
+
+      if (timeoutRef.current) {
+        clearTimeout(timeoutRef.current)
+        timeoutRef.current = null
+      }
+    }
+  }, [value])
+
+  const setValue = useCallback(
+    value => {
+      setInnerValue(value)
+      setDirty(true)
+
+      if (timeoutRef.current) {
+        clearTimeout(timeoutRef.current)
+        timeoutRef.current = null
+      }
+
+      timeoutRef.current = setTimeout(() => {
+        onChange(value)
+        setDirty(false)
+      }, timeout)
+    },
+    [innerValue, setInnerValue, timeoutRef, onChange, timeout]
+  )
+
+  const flush = useCallback(() => {
+    if (timeoutRef.current) {
+      clearTimeout(timeoutRef.current)
+      timeoutRef.current = null
+    }
+
+    onChange(innerValue)
+    setDirty(false)
+  }, [timeoutRef, onChange, innerValue])
+
+  return [innerValue, setValue, flush, dirty]
+}
+
 // From: https://www.joshwcomeau.com/snippets/react-hooks/use-toggle/
 export const useToggle = (initialValue = false) => {
   const [value, setValue] = useState(initialValue)

services/frontend/src/common/index.js

@@ -101,6 +101,52 @@ export const getTextIn = (texts, language) => {
   return null
 }
 
+export const getTextInWithOpen = (texts, language, isOpenCourse) => {
+  const prefixes = ['Avoin', 'Öppna', 'Open']
+  if (texts) {
+    const lanText = texts[language] || texts.fi || texts.en || texts.sv || Object.values(texts)[0]
+    const splitText = lanText.split(':')
+    if (prefixes.some(word => splitText[0].startsWith(word))) {
+      splitText.shift()
+    }
+    const newText = [...splitText].join()
+
+    if (isOpenCourse) {
+      switch (language) {
+        case 'fi':
+          return `${prefixes[0]} yo: ${newText}`
+
+        case 'sv':
+          return `${prefixes[1]} uni: ${newText}`
+
+        case 'en':
+          return `${prefixes[2]} uni: ${newText}`
+
+        default:
+          return `${prefixes[0]} yo: ${newText}`
+      }
+    }
+    return newText
+  }
+  return null
+}
+
+export const getUnifyTextIn = unifyCourses => {
+  switch (unifyCourses) {
+    case 'reqularStats':
+      return '(normal)'
+
+    case 'openStats':
+      return '(open)'
+
+    case 'unifyStats':
+      return '(open and normal)'
+
+    default:
+      return ''
+  }
+}
+
 export const cancelablePromise = promise => {
   let hasCanceled = false
 
@@ -160,7 +206,12 @@ export const getNewestProgramme = (studyrights, studentNumber, studentToTargetCo
         elementDetails[srE.code] &&
         elementDetails[srE.code].type === 20 &&
         (studentToTargetCourseDateMap && studentNumber
-          ? moment(studentToTargetCourseDateMap[studentNumber]).isBetween(moment(srE.startdate), moment(srE.enddate))
+          ? moment(studentToTargetCourseDateMap[studentNumber]).isBetween(
+              srE.startdate,
+              srE.enddate || moment(),
+              'day',
+              '[]'
+            )
           : true)
     )
     if (studyrightElements.length > 0) {

services/frontend/src/components/CoolDataScience/ProtoC.jsx

@@ -3,20 +3,26 @@ import React, { useState, useEffect, useCallback, useMemo } from 'react'
 import { connect } from 'react-redux'
 import Highcharts from 'highcharts'
 import ReactHighcharts from 'react-highcharts'
-import { Segment, Loader, Dimmer, Checkbox, Button, Message, Icon } from 'semantic-ui-react'
+import { Menu, Segment, Loader, Dimmer, Checkbox, Button, Message, Icon, Dropdown } from 'semantic-ui-react'
 import _ from 'lodash'
 import HighchartsCustomEvents from 'highcharts-custom-events'
 
 import TSA from '../../common/tsa'
 import InfoToolTips from '../../common/InfoToolTips'
-import { getProtoC, getProtoCProgramme } from '../../redux/coolDataScience'
+import { useGetProtoCQuery, getProtoC, getProtoCProgramme, useGetProtoCProgrammeQuery } from '../../redux/coolDataScience'
 import ReactMarkdown from 'react-markdown'
+import moment from 'moment'
+import RangeSelector from 'components/RangeSelector'
 
 HighchartsCustomEvents(Highcharts)
 
 const ANALYTICS_CATEGORY = 'Trends'
 const sendAnalytics = (action, name, value) => TSA.Matomo.sendEvent(ANALYTICS_CATEGORY, action, name, value)
 
+const upToYear = moment().isBefore({ day: 1, month: 7 })
+  ? moment().year() - 2
+  : moment().year() - 1
+
 const defaultConfig = (pointer = true) => {
   return {
     chart: {
@@ -55,6 +61,9 @@ const defaultConfig = (pointer = true) => {
       reversed: false
     },
     plotOptions: {
+      series: {
+        animation: false
+      },
       area: {
         cursor: pointer ? 'pointer' : undefined,
         fillOpacity: 0.5,
@@ -70,9 +79,8 @@ const defaultConfig = (pointer = true) => {
             function round(x) {
               return Math.round(x * 100) / 100
             }
-            return `${point.index + 1}, ${point.category}, ${point.y}, ${round(point.percentage)}%, ${
-              point.series.name
-            }`
+            return `${point.index + 1}, ${point.category}, ${point.y}, ${round(point.percentage)}%, ${point.series.name
+              }`
           }
         }
       }
@@ -80,7 +88,7 @@ const defaultConfig = (pointer = true) => {
   }
 }
 
-const makeClickableChartConfig = (sortedData, onPointClicked, org) => {
+const makeClickableChartConfig = (sortedData, onPointClicked, org, [startYear, endYear]) => {
   const addPointClickHandler = serie => {
     serie.point = {
       events: {
@@ -124,7 +132,7 @@ const makeClickableChartConfig = (sortedData, onPointClicked, org) => {
   const series = [
     {
       color: '#7f8c8d',
-      name: 'tällä hetkellä peruutettu',
+      name: 'tällä hetkellä passiivinen',
       data: sortedData.map(data => ({
         y: data.currentlyCancelled,
         // pass % of total as z so we can display it in the tooltip
@@ -266,16 +274,15 @@ const makeClickableChartConfig = (sortedData, onPointClicked, org) => {
     },
     yAxis: {
       title: {
-        text: ` ${
-          org ? `2017-2019 aloittaneet uudet kandiopiskelijat<br/>${org.name}` : '% tiedekunnan opiskelijoista'
-        }`
+        text: ` ${org ? `${startYear}-${endYear} aloittaneet uudet kandiopiskelijat<br/>${org.name}` : '% tiedekunnan opiskelijoista'
+          }`
       }
     },
     series
   })
 }
 
-const makeNonClickableChartConfig = programme => {
+const makeNonClickableChartConfig = (programme, [startYear, endYear]) => {
   const addMouseOverHandler = serie => {
     serie.point = {
       events: {
@@ -311,7 +318,7 @@ const makeNonClickableChartConfig = programme => {
   const series = [
     {
       color: '#7f8c8d',
-      name: 'tällä hetkellä peruutettu',
+      name: 'tällä hetkellä passiivinen',
       data: programme.studytracks.map(p => ({
         y: p.currentlyCancelled,
         // pass % of total as z so we can display it in the tooltip
@@ -347,7 +354,7 @@ const makeNonClickableChartConfig = programme => {
 
   return Highcharts.merge(defaultConfig(false), {
     title: {
-      text: `2017-2019 aloittaneet uudet kandiopiskelijat<br/>${programme.name}`
+      text: `${startYear}-${endYear} aloittaneet uudet kandiopiskelijat<br/>${programme.name}`
     },
     xAxis: {
       categories: programme.studytracks.map(data => data.name),
@@ -450,24 +457,29 @@ const sorters = {
   'ei tahdissa': (a, b) =>
     (countNotInTarget(a) + a.currentlyCancelled) / a.totalStudents -
     (countNotInTarget(b) + b.currentlyCancelled) / b.totalStudents,
-  peruutettu: (a, b) => a.currentlyCancelled / a.totalStudents - b.currentlyCancelled / b.totalStudents
+  passiivinen: (a, b) => a.currentlyCancelled / a.totalStudents - b.currentlyCancelled / b.totalStudents
 }
 
-const OrgChart = React.memo(({ orgs, onOrgClicked }) => {
-  return <ReactHighcharts highcharts={Highcharts} config={makeClickableChartConfig(orgs, onOrgClicked)} />
-})
-
-const ProgrammeChart = React.memo(({ programmes, onProgrammeClicked, org }) => {
+const OrgChart = React.memo(({ orgs, onOrgClicked, yearRange }) => {
   return (
-    <ReactHighcharts highcharts={Highcharts} config={makeClickableChartConfig(programmes, onProgrammeClicked, org)} />
+    <ReactHighcharts
+      highcharts={Highcharts}
+      config={makeClickableChartConfig(orgs, onOrgClicked, undefined, yearRange)}
+    />
   )
 })
 
-const StudytrackChart = React.memo(({ programme }) => {
-  return <ReactHighcharts highcharts={Highcharts} config={makeNonClickableChartConfig(programme)} />
+const ProgrammeChart = React.memo(({ programmes, onProgrammeClicked, org, yearRange }) => {
+  return (
+    <ReactHighcharts highcharts={Highcharts} config={makeClickableChartConfig(programmes, onProgrammeClicked, org, yearRange)} />
+  )
 })
 
-const ProgrammeDrilldown = ({ org, sorter, sortDir, onProgrammeClicked }) => {
+const StudytrackChart = React.memo(({ programme, yearRange }) => {
+  return <ReactHighcharts highcharts={Highcharts} config={makeNonClickableChartConfig(programme, yearRange)} />
+})
+
+const ProgrammeDrilldown = ({ org, sorter, sortDir, onProgrammeClicked, yearRange }) => {
   const orgSortedProgrammes = useMemo(() => {
     return { ...org, programmes: [...org.programmes].sort((a, b) => sorters[sorter](a, b) * sortDir) }
   }, [org, sorter, sortDir])
@@ -476,12 +488,13 @@ const ProgrammeDrilldown = ({ org, sorter, sortDir, onProgrammeClicked }) => {
     <ProgrammeChart
       programmes={orgSortedProgrammes.programmes}
       onProgrammeClicked={onProgrammeClicked}
+      yearRange={yearRange}
       org={orgSortedProgrammes}
     />
   )
 }
 
-const StudytrackDrilldown = ({ programme, sorter, sortDir }) => {
+const StudytrackDrilldown = ({ programme, sorter, sortDir, yearRange }) => {
   if (!programme.studytracks || programme.studytracks.length < 1)
     return (
       <Segment>
@@ -491,49 +504,54 @@ const StudytrackDrilldown = ({ programme, sorter, sortDir }) => {
   const programmeSortedStudytracks = useMemo(() => {
     return { ...programme, studytracks: [...programme.studytracks].sort((a, b) => sorters[sorter](a, b) * sortDir) }
   }, [programme, sorter, sortDir])
-  return <StudytrackChart programme={programmeSortedStudytracks} />
+  return <StudytrackChart programme={programmeSortedStudytracks} yearRange={yearRange} />
 }
 
 const ProtoC = ({
   programme,
-  getProtoCDispatch,
-  getProtoCProgrammeDispatch,
-  protoC,
-  protoCProgramme,
-  loadingProtoC,
-  loadingProtoCProgramme
 }) => {
+  const [firstLoad, setFirstLoad] = useState(true)
   const [sorter, setSorter] = useState('3v tahti')
   const [sortDir, setSortDir] = useState(1)
-  const [drilldownOrg, setDrilldownOrg] = useState(null)
-  const [drilldownProgramme, setDrilldownProgramme] = useState(null)
+  const [drilldownOrgCode, setDrilldownOrgCode] = useState(null)
+  const [drilldownProgrammeCode, setDrilldownProgrammeCode] = useState(null)
 
   const [includeOldAttainments, setIncludeOldAttainments] = useState(false)
   const [excludeNonEnrolled, setExcludeNonEnrolled] = useState(false)
 
-  useEffect(() => {
-    const load = async () => {
-      if (!!programme) {
-        getProtoCProgrammeDispatch({
-          includeOldAttainments: includeOldAttainments.toString(),
-          excludeNonEnrolled: excludeNonEnrolled.toString(),
-          code: programme
-        })
-      } else {
-        getProtoCDispatch({
-          includeOldAttainments: includeOldAttainments.toString(),
-          excludeNonEnrolled: excludeNonEnrolled.toString()
-        })
-      }
-    }
-    load()
-  }, [includeOldAttainments, excludeNonEnrolled])
+  const [yearRange, setYearRange] = useState([2017, upToYear])
+  const [startYear, endYear] = yearRange
 
-  useEffect(() => {
-    if (!!programme) {
-      setDrilldownProgramme(protoCProgramme)
-    }
-  }, [protoCProgramme])
+  const { data: protoC, isLoading: loadingProtoC, isFetching: fetchingProtoC } = useGetProtoCQuery({
+    excludeNonEnrolled,
+    includeOldAttainments,
+    startYear,
+    endYear,
+  })
+
+  const { date: protoCProgramme, isLoading: loadingProtoCProgramme, isFetching: fetchingProtoCProgramme } = useGetProtoCProgrammeQuery({
+    includeOldAttainments: includeOldAttainments.toString(),
+    excludeNonEnrolled: excludeNonEnrolled.toString(),
+    code: programme,
+    startYear,
+    endYear,
+  })
+
+  const drilldownOrg = useMemo(
+    () => protoC
+      ? Object.values(protoC).find(org => org.code === drilldownOrgCode)
+      : null,
+    [drilldownOrgCode, protoC],
+  )
+
+  const drilldownProgramme = useMemo(
+    () => protoC
+      ? Object.values(protoC)
+        .find(org => org.code === drilldownOrgCode)
+        ?.programmes?.find?.(prog => prog.code === drilldownProgrammeCode)
+      : null,
+    [drilldownOrgCode, drilldownProgrammeCode, protoC],
+  )
 
   const handleOldAttainmentToggled = useCallback(() => {
     setIncludeOldAttainments(previous => !previous)
@@ -546,13 +564,13 @@ const ProtoC = ({
   }, [])
 
   const handleOrgClicked = useCallback(org => {
-    setDrilldownOrg(org)
-    setDrilldownProgramme(null)
+    setDrilldownOrgCode(org.code)
+    setDrilldownProgrammeCode(null)
     sendAnalytics('C Org drilldown clicked', 'ProtoC')
   }, [])
 
   const handleProgrammeClicked = useCallback(programme => {
-    setDrilldownProgramme(programme)
+    setDrilldownProgrammeCode(programme.code)
     sendAnalytics('C Programme drilldown clicked', 'ProtoC')
   }, [])
 
@@ -577,14 +595,39 @@ const ProtoC = ({
       return a > b ? 1 : -1
     })
 
+  const RangeButtons = () => (
+    <div align="center" style={{ marginTop: '15px' }}>
+      <Menu compact>
+        <Menu.Item>
+          Started between
+        </Menu.Item>
+        <Dropdown
+          className="link item"
+          value={startYear}
+          onChange={(_, { value }) => setYearRange([value, endYear])}
+          options={_.range(2017, upToYear + 1).map((year) => ({ key: year, value: year, text: year, disabled: year > endYear }))}
+        />
+        <Menu.Item>
+          and
+        </Menu.Item>
+        <Dropdown
+          className="link item"
+          value={endYear}
+          onChange={(_, { value }) => setYearRange([startYear, value])}
+          options={_.range(2017, upToYear + 1).map((year) => ({ key: year, value: year, text: year, disabled: year < startYear }))}
+        />
+      </Menu>
+    </div>
+  )
+
   const SorterButtons = () => (
     <div align="center" style={{ marginTop: '15px' }}>
-      <Button.Group>
-        <Button style={{ cursor: 'default' }} active color="black">
+      <Menu compact>
+        <Menu.Item style={{ cursor: 'default' }} active color="black">
           Sort by:
-        </Button>
+        </Menu.Item>
         {sorterNames.map(sorterName => (
-          <Button
+          <Menu.Item
             basic={sorter !== sorterName}
             color={sorter === sorterName ? 'blue' : 'black'}
             key={sorterName}
@@ -595,7 +638,7 @@ const ProtoC = ({
             content={sorterName}
           />
         ))}
-      </Button.Group>
+      </Menu>
     </div>
   )
   // legend and checkboxes, custom legend because highcharts is pita
@@ -606,7 +649,7 @@ const ProtoC = ({
           <Icon style={{ marginLeft: '10px', color: '#6ab04c' }} name="circle" size="small" /> 3v tahdissa
           <Icon style={{ marginLeft: '10px', color: '#f9ca24' }} name="circle" size="small" /> 4v tahdissa
           <Icon style={{ marginLeft: '10px', color: '#ff7979' }} name="circle" size="small" /> ei tahdissa
-          <Icon style={{ marginLeft: '10px', color: '#7f8c8d' }} name="circle" size="small" /> tällä hetkellä peruutettu
+          <Icon style={{ marginLeft: '10px', color: '#7f8c8d' }} name="circle" size="small" /> tällä hetkellä passiivinen
         </span>
       </div>
       <div align="center">
@@ -637,9 +680,9 @@ const ProtoC = ({
       <Segment>
         <SorterButtons />
         <Segment placeholder={loadingProtoC || loadingProtoCProgramme} vertical>
-          <Dimmer inverted active={loadingProtoC || loadingProtoCProgramme} />
-          <Loader active={loadingProtoC || loadingProtoCProgramme} />
-          <StudytrackDrilldown programme={drilldownProgramme} sorter={sorter} sortDir={sortDir} />
+          <Dimmer inverted active={fetchingProtoC || fetchingProtoCProgramme} />
+          <Loader active={fetchingProtoC || fetchingProtoCProgramme} />
+          <StudytrackDrilldown yearRange={yearRange} programme={drilldownProgramme} sorter={sorter} sortDir={sortDir} />
         </Segment>
         <RenderBelowGraph />
       </Segment>
@@ -649,26 +692,37 @@ const ProtoC = ({
   return (
     <Segment>
       <div align="center">
-        <h2>Kandiohjelmat: Suhteellinen tavoiteaikaerittely, 2017-2019 aloittaneet</h2>
+        <h2>Kandiohjelmat: Suhteellinen tavoiteaikaerittely, {startYear}-{endYear} aloittaneet</h2>
       </div>
       <DrilldownMessage />
-      <SorterButtons />
+      <div style={{ display: 'flex', gapX: 15, gapY: 0, justifyContent: 'center', flexFlow: 'wrap' }}>
+        <RangeButtons />
+        <SorterButtons />
+      </div>
       <Segment placeholder={loadingProtoC || loadingProtoCProgramme} vertical>
-        <Dimmer inverted active={loadingProtoC || loadingProtoCProgramme} />
-        <Loader active={loadingProtoC || loadingProtoCProgramme} />
-        {!(loadingProtoC || loadingProtoCProgramme) && protoC && (
-          <OrgChart orgs={sortedOrgs} onOrgClicked={handleOrgClicked} />
-        )}
-        {!(loadingProtoC || loadingProtoCProgramme) && protoC && drilldownOrg && (
+        <Dimmer inverted active={fetchingProtoC || fetchingProtoCProgramme} />
+        <Loader active={fetchingProtoC || fetchingProtoCProgramme} />
+        <OrgChart
+          orgs={sortedOrgs}
+          onOrgClicked={handleOrgClicked}
+          yearRange={yearRange}
+        />
+        {drilldownOrg && (
           <ProgrammeDrilldown
             org={drilldownOrg}
             sorter={sorter}
             sortDir={sortDir}
+            yearRange={yearRange}
             onProgrammeClicked={handleProgrammeClicked}
           />
         )}
         {!(loadingProtoC || loadingProtoCProgramme) && protoC && drilldownProgramme && (
-          <StudytrackDrilldown programme={drilldownProgramme} sorter={sorter} sortDir={sortDir} />
+          <StudytrackDrilldown
+            programme={drilldownProgramme}
+            sorter={sorter}
+            sortDir={sortDir}
+            yearRange={yearRange}
+          />
         )}
       </Segment>
       <RenderBelowGraph />
@@ -679,14 +733,4 @@ const ProtoC = ({
   )
 }
 
-const mapStateToProps = ({ coolDataScience }) => ({
-  protoC: coolDataScience.data.protoC || {},
-  protoCProgramme: coolDataScience.data.protoCProgramme || {},
-  loadingProtoC: coolDataScience.pending.protoC,
-  loadingProtoCProgramme: coolDataScience.pending.protoCProgramme
-})
-
-export default connect(
-  mapStateToProps,
-  { getProtoCDispatch: getProtoC, getProtoCProgrammeDispatch: getProtoCProgramme }
-)(ProtoC)
+export default ProtoC

services/frontend/src/components/CoolDataScience/ProtoC2.jsx

@@ -1,21 +1,23 @@
-import React, { useState, useEffect, useCallback, useMemo } from 'react'
-import PropTypes, { func, bool, shape, string } from 'prop-types'
-import { connect } from 'react-redux'
+import React, { useState, useCallback, useMemo } from 'react'
 import Highcharts from 'highcharts'
 import ReactHighcharts from 'react-highcharts'
-import { Segment, Loader, Dimmer, Checkbox, Button, Message } from 'semantic-ui-react'
+import _ from 'lodash'
+import { Dropdown, Menu, Segment, Loader, Dimmer, Checkbox, Message } from 'semantic-ui-react'
 import ReactMarkdown from 'react-markdown'
 import HighchartsCustomEvents from 'highcharts-custom-events'
 
+import moment from 'moment'
 import TSA from '../../common/tsa'
 import InfoToolTips from '../../common/InfoToolTips'
-import { getProtoC, getProtoCProgramme } from '../../redux/coolDataScience'
+import { useGetProtoCQuery, useGetProtoCProgrammeQuery } from '../../redux/coolDataScience'
 
 HighchartsCustomEvents(Highcharts)
 
 const ANALYTICS_CATEGORY = 'Trends'
 const sendAnalytics = (action, name, value) => TSA.Matomo.sendEvent(ANALYTICS_CATEGORY, action, name, value)
 
+const upToYear = moment().isBefore({ day: 1, month: 7 }) ? moment().year() - 2 : moment().year() - 1
+
 const defaultConfig = () => {
   return {
     chart: {
@@ -111,7 +113,7 @@ const makeConfig = (data, sorter, type = 'column', clickHandler) => {
   const orgSeries = [
     {
       color: '#7f8c8d',
-      name: 'tällä hetkellä peruutettu',
+      name: 'tällä hetkellä passiivinen',
       data: data.map(entry => ({
         y: entry.currentlyCancelled,
         // pass % of total as z so we can display it in the tooltip
@@ -168,7 +170,7 @@ const makeConfig = (data, sorter, type = 'column', clickHandler) => {
             const { chart } = this
             chart.myLabel.destroy()
             const clickedLabel = data.find(entry => entry.name === this.value)
-            if (clickedLabel) clickHandler(clickedLabel.programmes || clickedLabel.studytracks)
+            if (clickedLabel) clickHandler(clickedLabel)
             sendAnalytics('Drilldown clicked', 'ProtoC2')
           },
           mouseover() {
@@ -248,7 +250,7 @@ const makeConfig = (data, sorter, type = 'column', clickHandler) => {
                 // clicked on top-level, drill down
                 sendAnalytics('Org drilldown clicked', 'ProtoC2')
                 const datapoint = data.find(entry => entry.code === point.custom.code)
-                clickHandler(datapoint.programmes || datapoint.studytracks)
+                clickHandler(datapoint)
               }
             },
           },
@@ -264,7 +266,7 @@ const sorters = {
   '4v tahti': (a, b) => a.students4y - b.students4y,
   '3v tahti': (a, b) => a.students3y - b.students3y,
   'ei tahdissa': (a, b) => countNotInTarget(a) + a.currentlyCancelled - (countNotInTarget(b) + b.currentlyCancelled),
-  peruutettu: (a, b) => a.currentlyCancelled - b.currentlyCancelled,
+  passiivinen: (a, b) => a.currentlyCancelled - b.currentlyCancelled,
 }
 
 const ClickableChart = React.memo(({ data, sorter, isSideways, clickHandler }) => {
@@ -290,78 +292,60 @@ NonClickableChart.defaultProps = {
   isSideways: false,
 }
 
-NonClickableChart.propTypes = {
-  data: PropTypes.arrayOf(
-    PropTypes.shape({
-      name: PropTypes.string,
-      code: PropTypes.string,
-      students3y: PropTypes.number,
-      students4y: PropTypes.number,
-      totalStudents: PropTypes.number,
-      currentlyCancelled: PropTypes.number,
-    })
-  ).isRequired,
-  sorter: PropTypes.func.isRequired,
-  isSideways: PropTypes.bool,
-}
-
 ClickableChart.defaultProps = {
   isSideways: false,
 }
 
-ClickableChart.propTypes = {
-  data: PropTypes.arrayOf(
-    PropTypes.shape({
-      name: PropTypes.string,
-      code: PropTypes.string,
-      students3y: PropTypes.number,
-      students4y: PropTypes.number,
-      totalStudents: PropTypes.number,
-      currentlyCancelled: PropTypes.number,
-    })
-  ).isRequired,
-  sorter: PropTypes.func.isRequired,
-  clickHandler: func.isRequired,
-  isSideways: PropTypes.bool,
-}
-
-const ProtoC = ({
-  getProtoCDispatch,
-  getProtoCProgrammeDispatch,
-  data,
-  isLoading,
-  programme = 'KH50_005',
-  protoCProgrammeData,
-  protoCProgrammeLoading,
-}) => {
+const ProtoC = ({ programme = 'KH50_005' }) => {
   const [sorter, setSorter] = useState('3v tahti')
   const [sortDir, setSortDir] = useState(1)
-  const [drilldownOrg, setDrilldownOrg] = useState(null)
-  const [drilldownProgramme, setDrilldownProgramme] = useState(null)
+  const [drilldownOrgCode, setDrilldownOrgCode] = useState(null)
+  const [drilldownProgrammeCode, setDrilldownProgrammeCode] = useState(null)
 
   const [includeOldAttainments, setIncludeOldAttainments] = useState(false)
   const [excludeNonEnrolled, setExcludeNonEnrolled] = useState(false)
 
-  useEffect(() => {
-    if (programme) {
-      getProtoCProgrammeDispatch({
-        includeOldAttainments: includeOldAttainments.toString(),
-        excludeNonEnrolled: excludeNonEnrolled.toString(),
-        code: programme,
-      })
-    } else {
-      getProtoCDispatch({
-        includeOldAttainments: includeOldAttainments.toString(),
-        excludeNonEnrolled: excludeNonEnrolled.toString(),
-      })
-    }
-  }, [includeOldAttainments, excludeNonEnrolled])
+  const [yearRange, setYearRange] = useState([2017, upToYear])
+  const [startYear, endYear] = yearRange
 
-  useEffect(() => {
-    if (programme) {
-      setDrilldownProgramme(protoCProgrammeData.studytracks)
+  const { data, isLoading, isFetching } = useGetProtoCQuery({
+    excludeNonEnrolled,
+    includeOldAttainments,
+    startYear,
+    endYear,
+  })
+
+  const {
+    data: protoCProgrammeData,
+    isLoading: protoCProgrammeLoading,
+    isFetching: protoCProgrammeFetching,
+  } = useGetProtoCProgrammeQuery(
+    {
+      includeOldAttainments: includeOldAttainments.toString(),
+      excludeNonEnrolled: excludeNonEnrolled.toString(),
+      code: programme,
+      startYear,
+      endYear,
+    },
+    {
+      skip: !programme,
     }
-  }, [protoCProgrammeData])
+  )
+
+  const drilldownOrg = useMemo(
+    () => (data ? Object.values(data).find(org => org.code === drilldownOrgCode) : null),
+    [drilldownOrgCode, data]
+  )
+
+  const drilldownProgramme = useMemo(
+    () =>
+      data
+        ? Object.values(data)
+            .find(org => org.code === drilldownOrgCode)
+            ?.programmes?.find?.(prog => prog.code === drilldownProgrammeCode)
+        : null,
+    [drilldownOrgCode, drilldownProgrammeCode, data]
+  )
 
   const handleOldAttainmentToggled = useCallback(() => {
     setIncludeOldAttainments(previous => !previous)
@@ -380,19 +364,21 @@ const ProtoC = ({
   }, [data, currentSorter])
 
   const sortedProgrammes = useMemo(() => {
-    return (drilldownOrg || []).sort(currentSorter)
+    return [...(drilldownOrg?.programmes || [])].sort(currentSorter)
   }, [drilldownOrg, currentSorter])
 
   const sortedStudytracks = useMemo(() => {
-    return (drilldownProgramme || []).sort(currentSorter)
-  }, [drilldownProgramme, currentSorter])
+    const studytracks = programme ? protoCProgrammeData?.studytracks : drilldownProgramme?.studytracks
+
+    return [...(studytracks || [])].sort(currentSorter)
+  }, [programme, drilldownProgramme, protoCProgrammeData, currentSorter])
 
   const drilldownOrgClick = useCallback(org => {
-    setDrilldownOrg(org)
+    setDrilldownOrgCode(org.code)
   })
 
   const drilldownProgrammeClick = useCallback(programme => {
-    setDrilldownProgramme(programme)
+    setDrilldownProgrammeCode(programme.code)
   })
 
   const { CoolDataScience } = InfoToolTips
@@ -423,15 +409,46 @@ const ProtoC = ({
     return (
       <Segment>
         <div align="center">
-          <h2>Prototyyppi: Tavoiteaikaerittely, 2017-2019 aloittaneet</h2>
+          <h2>
+            Prototyyppi: Tavoiteaikaerittely, {startYear}-{endYear} aloittaneet
+          </h2>
         </div>
-        <div align="center" style={{ marginTop: '10px' }}>
-          <Button.Group>
-            <Button style={{ cursor: 'default' }} active color="black">
+        <div
+          align="center"
+          style={{ marginTop: '10px', display: 'flex', gap: '10px 20px', flexFlow: 'wrap', justifyContent: 'center' }}
+        >
+          <Menu compact>
+            <Menu.Item>Started between</Menu.Item>
+            <Dropdown
+              className="link item"
+              value={startYear}
+              onChange={(_, { value }) => setYearRange([value, endYear])}
+              options={_.range(2017, upToYear + 1).map(year => ({
+                key: year,
+                value: year,
+                text: year,
+                disabled: year > endYear,
+              }))}
+            />
+            <Menu.Item>and</Menu.Item>
+            <Dropdown
+              className="link item"
+              value={endYear}
+              onChange={(_, { value }) => setYearRange([startYear, value])}
+              options={_.range(2017, upToYear + 1).map(year => ({
+                key: year,
+                value: year,
+                text: year,
+                disabled: year < startYear,
+              }))}
+            />
+          </Menu>
+          <Menu compact>
+            <Menu.Item style={{ cursor: 'default' }} active color="black">
               Sort by:
-            </Button>
+            </Menu.Item>
             {sorterNames.map(sorterName => (
-              <Button
+              <Menu.Item
                 basic={sorter !== sorterName}
                 color={sorter === sorterName ? 'blue' : 'black'}
                 key={sorterName}
@@ -442,11 +459,11 @@ const ProtoC = ({
                 content={sorterName}
               />
             ))}
-          </Button.Group>
+          </Menu>
         </div>
         <Segment placeholder={isLoading} vertical>
-          <Dimmer inverted active={isLoading} />
-          <Loader active={isLoading} />
+          <Dimmer inverted active />
+          <Loader active />
           {!protoCProgrammeLoading && protoCProgrammeData && (
             <>
               <NonClickableChart data={drilldownProgramme} sorter={currentSorter} isSideways />
@@ -476,19 +493,54 @@ const ProtoC = ({
     )
   }
 
+  const [studytracksLoading, studytracksFetching] = !programme
+    ? [isLoading, isFetching]
+    : [protoCProgrammeLoading, protoCProgrammeFetching]
+
   return (
     <Segment>
       <div align="center">
-        <h2>Kandiohjelmat: Tavoiteaikaerittely, 2017-2019 aloittaneet</h2>
+        <h2>
+          Kandiohjelmat: Tavoiteaikaerittely, {startYear}-{endYear} aloittaneet
+        </h2>
       </div>
       <DrilldownMessage />
-      <div align="center" style={{ marginTop: '10px' }}>
-        <Button.Group>
-          <Button style={{ cursor: 'default' }} active color="black">
+      <div
+        align="center"
+        style={{ marginTop: '10px', display: 'flex', gap: '10px 20px', flexFlow: 'wrap', justifyContent: 'center' }}
+      >
+        <Menu compact>
+          <Menu.Item>Started between</Menu.Item>
+          <Dropdown
+            className="link item"
+            value={startYear}
+            onChange={(_, { value }) => setYearRange([value, endYear])}
+            options={_.range(2017, upToYear + 1).map(year => ({
+              key: year,
+              value: year,
+              text: year,
+              disabled: year > endYear,
+            }))}
+          />
+          <Menu.Item>and</Menu.Item>
+          <Dropdown
+            className="link item"
+            value={endYear}
+            onChange={(_, { value }) => setYearRange([startYear, value])}
+            options={_.range(2017, upToYear + 1).map(year => ({
+              key: year,
+              value: year,
+              text: year,
+              disabled: year < startYear,
+            }))}
+          />
+        </Menu>
+        <Menu compact>
+          <Menu.Item style={{ cursor: 'default' }} active color="black">
             Sort by:
-          </Button>
+          </Menu.Item>
           {sorterNames.map(sorterName => (
-            <Button
+            <Menu.Item
               basic={sorter !== sorterName}
               color={sorter === sorterName ? 'blue' : 'black'}
               key={sorterName}
@@ -499,11 +551,11 @@ const ProtoC = ({
               content={sorterName}
             />
           ))}
-        </Button.Group>
+        </Menu>
       </div>
       <Segment placeholder={isLoading} vertical>
-        <Dimmer inverted active={isLoading} />
-        <Loader active={isLoading} />
+        <Dimmer inverted active={isFetching} />
+        <Loader active={isFetching} />
         {!isLoading && data && (
           <>
             <ClickableChart data={sortedOrgs} sorter={currentSorter} isSideways clickHandler={drilldownOrgClick} />
@@ -515,9 +567,17 @@ const ProtoC = ({
                 clickHandler={drilldownProgrammeClick}
               />
             )}
-            {drilldownProgramme && <NonClickableChart data={sortedStudytracks} sorter={currentSorter} isSideways />}
           </>
         )}
+      </Segment>
+      {drilldownProgramme && (
+        <Segment placeholder={studytracksLoading} vertical>
+          <Dimmer inverted active={studytracksFetching} />
+          <Loader active={studytracksFetching} />
+          <NonClickableChart data={sortedStudytracks} sorter={currentSorter} isSideways />
+        </Segment>
+      )}
+      <Segment>
         <div align="center">
           <Checkbox
             label="Include only at least once enrolled students"
@@ -546,24 +606,4 @@ ProtoC.defaultProps = {
   programme: '',
 }
 
-ProtoC.propTypes = {
-  isLoading: bool.isRequired,
-  data: shape({}).isRequired,
-  getProtoCDispatch: func.isRequired,
-  getProtoCProgrammeDispatch: func.isRequired,
-  protoCProgrammeLoading: bool.isRequired,
-  protoCProgrammeData: shape({}).isRequired,
-  programme: string,
-}
-
-const mapStateToProps = ({ coolDataScience }) => ({
-  data: coolDataScience.data.protoC,
-  protoCProgrammeData: coolDataScience.data.protoCProgramme,
-  isLoading: coolDataScience.pending.protoC,
-  protoCProgrammeLoading: coolDataScience.pending.protoCProgramme,
-})
-
-export default connect(mapStateToProps, {
-  getProtoCDispatch: getProtoC,
-  getProtoCProgrammeDispatch: getProtoCProgramme,
-})(ProtoC)
+export default ProtoC

services/frontend/src/components/CoolDataScience/Status.jsx

@@ -22,7 +22,7 @@ const settingDefinitions = _.map(
     {
       key: 'showByYear',
       type: 'checkbox',
-      defaultValue: false,
+      defaultValue: true,
     },
     {
       key: 'showYearlyValues',
@@ -59,6 +59,7 @@ const StatusContainer = ({
   max1,
   showYearlyValues,
   showRelativeValues,
+  showCountingFrom,
   showByYear,
   showStudentCounts,
   clickable,
@@ -84,11 +85,31 @@ const StatusContainer = ({
   let yearlyValues = null
 
   if (showYearlyValues) {
-    yearlyValues = _.orderBy(Object.entries(stats.yearly), ([y]) => y, ['desc']).map(([year, yearStats]) => ({
-      label: showByYear ? year : `${year}-${`${Number(year) + 1}`.slice(-2)}`,
-      accumulated: showStudentCounts && stats.type === 'course' ? yearStats.accStudents : yearStats.acc,
-      total: showStudentCounts && stats.type === 'course' ? yearStats.totalStudents : yearStats.total,
-    }))
+    yearlyValues = _.chain(stats.yearly)
+      .toPairs()
+      .map(([year, stats]) => ({
+        dateRange: {
+          start: showByYear ? moment([year]).startOf('year') : moment([year]).month(7).date(1),
+          end: showByYear
+            ? moment([year]).startOf('year')
+            : moment([parseInt(year, 10) + 1])
+                .month(6)
+                .date(31),
+        },
+        stats,
+      }))
+      .filter(({ dateRange: { start } }) => moment(showCountingFrom).isSameOrAfter(start))
+      .orderBy(({ dateRange: { start } }) => start, ['desc'])
+      .map(({ dateRange, stats }) => ({
+        label: _.chain([dateRange.start, dateRange.end])
+          .map(date => date.year())
+          .uniq()
+          .join('-')
+          .value(),
+        accumulated: showStudentCounts && stats.type === 'course' ? stats.accStudents : stats.acc,
+        total: showStudentCounts && stats.type === 'course' ? stats.totalStudents : stats.total,
+      }))
+      .value()
   }
 
   return (
@@ -108,7 +129,7 @@ const StatusContainer = ({
 }
 
 const StatusContent = ({ data, settings }) => {
-  const { showByYear, showRelativeValues, showYearlyValues, showStudentCounts } = settings
+  const { showByYear, showRelativeValues, showYearlyValues, showStudentCounts, showCountingFrom } = settings
 
   return (
     <DrillStack
@@ -124,6 +145,7 @@ const StatusContent = ({ data, settings }) => {
           showRelativeValues={showRelativeValues}
           showYearlyValues={showYearlyValues}
           showStudentCounts={showStudentCounts}
+          showCountingFrom={showCountingFrom}
           min1={-2 * medianDiff}
           max1={2 * medianDiff}
           showByYear={showByYear}

services/frontend/src/components/CoursePopulation/CoursePopulationGradeDist/index.jsx

@@ -10,7 +10,7 @@ const CoursePopulationCreditDist = ({ singleCourseStats, pending, selectedStuden
   const [courseGrades, setCourseGrades] = useState([])
 
   useEffect(() => {
-    if (samples && singleCourseStats.alternatives) {
+    if (samples && singleCourseStats?.unifyStats?.alternatives) {
       const filteredGradeArray = []
       const grades = {}
 

services/frontend/src/components/CoursePopulation/index.jsx

@@ -10,7 +10,7 @@ import { getCoursePopulation } from '../../redux/populations'
 import { getSingleCourseStats } from '../../redux/singleCourseStats'
 import { useGetStudentListCourseStatisticsQuery } from '../../redux/populationCourses'
 import { getFaculties } from '../../redux/faculties'
-import { getSemesters } from '../../redux/semesters'
+import { useGetSemestersQuery } from '../../redux/semesters'
 import { getElementDetails } from '../../redux/elementdetails'
 import PopulationStudents from '../PopulationStudents'
 import CoursePopulationGradeDist from './CoursePopulationGradeDist'
@@ -19,7 +19,7 @@ import CoursePopulationCreditGainTable from './CoursePopulationCreditGainTable'
 import CustomPopulationProgrammeDist from '../CustomPopulation/CustomPopulationProgrammeDist'
 import CustomPopulationCourses from '../CustomPopulation/CustomPopulationCourses'
 import ProgressBar from '../ProgressBar'
-import { getStudentToTargetCourseDateMap, getTextIn } from '../../common'
+import { getStudentToTargetCourseDateMap, getTextIn, getUnifyTextIn } from '../../common'
 import { useProgress, useTitle } from '../../common/hooks'
 import infotooltips from '../../common/InfoToolTips'
 import {
@@ -36,6 +36,12 @@ import FilterView from '../FilterView'
 import useLanguage from '../LanguagePicker/useLanguage'
 import { queryParamsFromUrl } from '../../common/query'
 
+const NO_PROGRAMME = {
+  code: '00000',
+  name: { en: 'No programme', fi: 'Ei ohjelmaa' },
+  startdate: '',
+}
+
 const CoursePopulation = ({
   getCoursePopulationDispatch,
   getSingleCourseStatsDispatch,
@@ -44,12 +50,10 @@ const CoursePopulation = ({
   pending,
   history,
   courseData,
-  getSemestersDispatch,
-  semesters,
   getFacultiesDispatch,
+  unifyCourses,
 }) => {
   const { language } = useLanguage()
-
   const [codes, setCodes] = useState([])
   const [headerYears, setYears] = useState('')
   const [dateFrom, setDateFrom] = useState(null)
@@ -75,9 +79,10 @@ const CoursePopulation = ({
     studentNumbers: studentData.students ? studentData.students.map(student => student.studentNumber) : [],
   })
 
+  const { data: semesters = {} } = useGetSemestersQuery()
+
   useEffect(() => {
     getElementDetails()
-    getSemestersDispatch()
   }, [])
 
   useEffect(() => {
@@ -100,9 +105,9 @@ const CoursePopulation = ({
 
   useEffect(() => {
     if (semesters.years && semesters.semesters) {
-      const { coursecodes, from, to, years, years2, separate } = queryParamsFromUrl(history.location)
+      const { coursecodes, from, to, years, years2, separate, unifyCourses } = queryParamsFromUrl(history.location)
       const parsedCourseCodes = JSON.parse(coursecodes)
-      getCoursePopulationDispatch({ coursecodes, from, to, onProgress, separate })
+      getCoursePopulationDispatch({ coursecodes, from, to, onProgress, separate, unifyCourses })
       getSingleCourseStatsDispatch({
         fromYear: from,
         toYear: to,
@@ -124,7 +129,8 @@ const CoursePopulation = ({
     }
   }, [semesters])
 
-  const header = courseData ? `${getTextIn(courseData.name, language)} ${headerYears}` : null
+  const avoin = getUnifyTextIn(unifyCourses)
+  const header = courseData ? `${getTextIn(courseData.name, language)} ${headerYears} ${avoin}` : null
 
   const subHeader = codes.join(', ')
 
@@ -301,7 +307,7 @@ const CoursePopulation = ({
       .max()
       .value()
 
-    return moment(date).isBetween(sre.startdate, sre.enddate)
+    return sre.code === NO_PROGRAMME.code || moment(date).isBetween(sre.startdate, sre.enddate, 'day', '[]')
   }
 
   return (
@@ -364,12 +370,12 @@ const CustomPopulationCoursesWrapper = props => {
   return <CustomPopulationCourses {...props} courses={courseStatistics} />
 }
 
-const mapStateToProps = ({ singleCourseStats, populations, semesters }) => {
+const mapStateToProps = ({ singleCourseStats, populations, courseSearch }) => {
   return {
     studentData: populations.data,
     pending: populations.pending,
-    courseData: singleCourseStats.stats || {},
-    semesters: semesters.data,
+    courseData: singleCourseStats.stats?.[courseSearch.openOrReqular],
+    unifyCourses: courseSearch.openOrReqular,
     elementDetails: populations?.data?.elementdetails?.data,
   }
 }
@@ -378,7 +384,6 @@ export default withRouter(
   connect(mapStateToProps, {
     getCoursePopulationDispatch: getCoursePopulation,
     getSingleCourseStatsDispatch: getSingleCourseStats,
-    getSemestersDispatch: getSemesters,
     getFacultiesDispatch: getFaculties,
     getElementDetails,
   })(CoursePopulation)

services/frontend/src/components/CourseStatistics/FacultyLevelStatistics/index.jsx

@@ -84,11 +84,13 @@ const CourseTable = ({ course, courseInstance, language, selectedYear }) => {
 
 const FacultyLevelStatistics = () => {
   const { language: activeLanguage } = useLanguage()
+  const openOrReqular = useSelector(state => state.courseSearch.openOrReqular)
   const { courseStats, language } = useSelector(({ courseStats }) => ({
     courseStats: courseStats.data,
     language: activeLanguage,
   }))
-  const yearcodes = uniq(flatten(Object.values(courseStats).map(c => Object.keys(c.facultyStats))))
+
+  const yearcodes = uniq(flatten(Object.values(courseStats).map(c => Object.keys(c[openOrReqular].facultyStats))))
     .sort()
     .reverse()
 
@@ -103,13 +105,13 @@ const FacultyLevelStatistics = () => {
   const yearsCourseStats = Object.values(courseStats)
     .map(course => ({
       course,
-      courseInstance: course.facultyStats[selectedYear],
+      courseInstance: course[openOrReqular].facultyStats[selectedYear],
     }))
     .sort((a, b) => (a.courseInstance === undefined) - (b.courseInstance === undefined))
 
   const courseTables = yearsCourseStats.map(({ course, courseInstance }) => (
     <CourseTable
-      course={course}
+      course={course[openOrReqular]}
       courseInstance={courseInstance}
       language={language}
       key={course.coursecode}

services/frontend/src/components/CourseStatistics/ResultTabs/HelpButton.jsx

@@ -0,0 +1,31 @@
+import React from 'react'
+import ReactMarkdown from 'react-markdown'
+
+import { Popup, Button, Icon } from 'semantic-ui-react'
+
+import infotooltips from '../../../common/InfoToolTips'
+
+export default ({ tab, viewMode }) => {
+  let content = infotooltips.CourseStatistics[tab]
+
+  if (viewMode) {
+    content = content[viewMode]
+  }
+
+  content = content.replace(/\n +/g, '\n')
+
+  return (
+    <Popup
+      on="click"
+      wide="very"
+      trigger={
+        <Button icon labelPosition="left">
+          <Icon name="question circle outline" />
+          Help
+        </Button>
+      }
+    >
+      <ReactMarkdown>{content}</ReactMarkdown>
+    </Popup>
+  )
+}