Show search by class to iam users

Make visible after a more limited access and obfuscation are working

.github/dependabot.yml

@@ -1,15 +1,47 @@
 version: 2
 updates:
-- package-ecosystem: npm
-  directory: "/"
-  schedule:
-    interval: weekly
-    time: "03:00"
-  open-pull-requests-limit: 10
-  target-branch: "trunk"
-- package-ecosystem: "github-actions"
-  directory: "/"
-  schedule:
-    interval: weekly
-    time: "03:00"
-  target-branch: "trunk"
+  - package-ecosystem: npm
+    directory: '/'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'infra'
+  - package-ecosystem: npm
+    directory: '/updater/sis-updater-scheduler'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'updater'
+  - package-ecosystem: npm
+    directory: '/updater/sis-updater-worker'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'updater'
+  - package-ecosystem: npm
+    directory: '/services/backend'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'back'
+  - package-ecosystem: npm
+    directory: '/services/frontend'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'front'
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: weekly
+      time: '03:00'

.github/workflows/test-and-release-updater.yaml

@@ -19,6 +19,9 @@ jobs:
       - name: 'Build the image'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=production
           context: updater/sis-updater-worker/
           target: production
           push: false
@@ -40,6 +43,9 @@ jobs:
       - name: 'Build the image'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=production
           context: updater/sis-updater-scheduler/
           target: production
           push: false
@@ -81,6 +87,11 @@ jobs:
         env:
           SCHEDULE_IMMEDIATE: hourly
           EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run daily jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: daily
+          EXIT_AFTER_IMMEDIATES: yes
       - name: 'Run weekly jobs'
         run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
         env:
@@ -128,6 +139,19 @@ jobs:
           push: true
           tags: toska/sis-updater-worker:latest
           target: production
+      - name: 'Create Sentry release'
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: updater
+        with:
+          environment: production
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/static/js'
+          version: ${{ github.sha }}
       - name: 'Send notification to slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:

.github/workflows/test-oodikone.yaml

@@ -505,6 +505,50 @@ jobs:
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
+  test_faculty_statistics:
+    name: 'Test faculty statistics'
+    needs:
+      - build_backend
+      - build_frontend
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built images'
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Run services, pull rest of the needed images'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: 'Run cypress'
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: cypress/integration/Faculty_statistics.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: 'Upload video if tests fail'
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
   # === Other tests ===
 
   test_service_exit_statuses:
@@ -560,6 +604,7 @@ jobs:
       - test_student_statistics
       - test_studyprogramme_overview
       - test_teachers_and_users
+      - test_faculty_statistics
       - test_service_exit_statuses
     steps:
       - uses: actions/checkout@v2
@@ -626,6 +671,7 @@ jobs:
       - test_student_statistics
       - test_studyprogramme_overview
       - test_teachers_and_users
+      - test_faculty_statistics
       - test_service_exit_statuses
     if: failure()
     steps:

README.md

@@ -8,7 +8,7 @@ To run Oodikone locally, you will need the following:
 
 1. Applications:
    - [Docker](https://docs.docker.com/install/) (version 20.10+),
-   - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
+   - [Docker Compose](https://docs.docker.com/compose/install/)(version 1.29+) and
    - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
 2. Access to Toska Docker Hub,
 3. For real data setup, access to Toskas production servers.
@@ -48,6 +48,10 @@ What different CLI options do:
 
 ## ⌨️ Development
 
+### Architecture
+
+![Oodikone architecture](Oodikone.png)
+
 ### Basics
 
 The development environment is entirely configured in the `docker-compose.yml` file located in this repository. The file defines the services for oodikone's two main components: **oodikone** and **updater**.

cypress/integration/Course_statistics.js

@@ -82,8 +82,8 @@ describe('Course Statistics tests', () => {
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by a course code']").type(courseCode)
       cy.contains('td', courseCode).click()
-      cy.contains('Separate statistics for Spring and Fall semesters').click()
       cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.get('[data-cy=separateToggle]', { force: true }).click({ force: true })
       cy.contains('Syksy 2019')
       cy.contains('Syksy 2017')
       cy.get('tbody').contains('19')
@@ -96,7 +96,7 @@ describe('Course Statistics tests', () => {
       cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
 
       cy.contains('Tietokantojen perusteet')
-      cy.contains('Avoin yo: Tietokantojen perusteet')
+      cy.contains('Tietokantojen perusteet')
       cy.contains('A581328')
       cy.contains('AYTKT10004')
       cy.contains('TKT10004, 581328')
@@ -105,46 +105,39 @@ describe('Course Statistics tests', () => {
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
 
-      cy.contains('TKT10004, 581328 Tietokantojen perusteet')
+      cy.contains('TKT10004, 581328, AYTKT10004, A581328 Tietokantojen perusteet')
       cy.get('.right').click()
       cy.contains('No results')
+    })
 
+    it('Provider organization toggle works', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
       cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
-      cy.contains('td', /^AYTKT10004/).click()
 
+      cy.contains('td', /^TKT10004/).click()
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
-      cy.contains('AYTKT10004 Avoin yo: Tietokantojen perusteet')
+
+      cy.contains('TKT10004, 581328, AYTKT10004, A581328 Tietokantojen perusteet')
+      cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.contains('TKT10004, 581328 Tietokantojen perusteet')
     })
 
     it('Searching course by name displays right courses, 10 credit courses', () => {
       cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
-
       cy.contains('Tietorakenteet ja algoritmit')
-      cy.contains('Avoin yo: Tietorakenteet ja algoritmit')
       cy.contains('AYTKT20001')
       cy.contains('TKT20001, 58131')
       cy.contains('td', /^TKT20001/).click()
-
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
 
-      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
       cy.get('.right').click()
       cy.contains('No results')
-
-      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
-      cy.contains('td', /^AYTKT20001/).click()
-
-      cy.contains('Fetch statistics').should('be.enabled').click()
-      cy.contains('Search for courses').should('not.exist')
-      cy.contains('AYTKT20001 Avoin yo: Tietorakenteet ja algoritmit')
-
-      cy.get('.right').click()
-      cy.contains('No results')
-      cy.get("input[name='unifyOpenUniCourses']").parent().click()
       cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
       cy.contains('td', 'TKT20001, 58131, AYTKT20001').click()
 
@@ -159,12 +152,11 @@ describe('Course Statistics tests', () => {
       cy.get("input[placeholder='Search by a course code']").type('TKT20003')
       cy.contains('tr', 'TKT20003').click()
       cy.contains('Fetch statistics').should('be.enabled').click()
-
       cy.contains('Käyttöjärjestelmät')
       cy.contains('TKT20003')
 
-      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
-      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020 (open and normal')
       cy.contains('TKT20003')
 
       cy.contains('Students (127)').click()
@@ -174,12 +166,12 @@ describe('Course Statistics tests', () => {
 
     it('Population of course shows grades for each student', () => {
       cy.get("input[placeholder='Search by a course code']").type('TKT20001')
-      cy.contains(/^TKT20001$/).click()
+      cy.contains(/^TKT20001, AYTKT20001$/).click()
       cy.contains('Fetch statistics').should('be.enabled').click()
-      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
-      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
-      cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019')
-      cy.contains('Students (197)').click()
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
+      cy.get('tbody >:nth-child(3) > :nth-child(2) .level').click()
+      cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019 (open and normal)')
+      cy.contains('Students (204)').click()
       cy.contains('td', '010262566').siblings().contains('4')
       cy.contains('td', '010674989').siblings().contains('1')
     })
@@ -189,10 +181,11 @@ describe('Course Statistics tests', () => {
       cy.contains(/^TKT20003$/).click()
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
-      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
       cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
       cy.contains('Language distribution').click()
-      cy.contains('td', 'finnish').siblings().contains('107')
+      cy.contains('td', 'finnish').siblings().contains('122')
       cy.contains('td', 'english').siblings().contains('5')
     })
 
@@ -200,7 +193,6 @@ describe('Course Statistics tests', () => {
       beforeEach(() => {
         cy.url().should('include', '/coursestatistics')
         cy.contains('Search for courses')
-        cy.get("input[name='unifyOpenUniCourses']").parent().click()
         cy.get("input[placeholder='Search by a course code']").type('TKT10002')
         cy.contains('td', 'TKT10002, AYTKT10002').click()
         cy.contains('Fetch statistics').should('be.enabled').click()
@@ -212,18 +204,28 @@ describe('Course Statistics tests', () => {
       const yearRange = { from: '2000-2001', to: '2020-2021' }
       const attemptsTableContents = [
         // [time, passed, failed, passrate]
-        ['Total', 511, 486, 25],
+        ['Total', null, 511, 486, 25],
+      ]
+
+      const attemptsTableContentsOpen = [
+        // [time, passed, failed, passrate]
+        ['Total', null, 175, 175, 0],
+      ]
+
+      const attemptsTableContentsReqular = [
+        // [time, passed, failed, passrate]
+        ['Total', null, 312, 298, 14],
       ]
 
       const gradesTableContents = [
         // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-        ['Total', 511, 25, 11, 17, 21, 59, 365, 13],
-        ['2020-2021', 1, 0, 0, 0, 0, 0, 1, 0],
-        ['2019-2020', 54, 4, 3, 1, 3, 4, 39, 0],
-        ['2018-2019', 154, 7, 2, 6, 5, 15, 119, 0],
-        ['2017-2018', 186, 9, 3, 5, 6, 16, 139, 8],
-        ['2016-2017', 71, 2, 1, 2, 5, 13, 48, 0],
-        ['2015-2016', 21, 1, 1, 1, 0, 7, 8, 3],
+        ['Total', null, 511, 25, 11, 17, 21, 59, 365, 13],
+        ['2020-2021', null, 1, 0, 0, 0, 0, 0, 1, 0],
+        ['2019-2020', null, 54, 4, 3, 1, 3, 4, 39, 0],
+        ['2018-2019', null, 154, 7, 2, 6, 5, 15, 119, 0],
+        ['2017-2018', null, 186, 9, 3, 5, 6, 16, 139, 8],
+        ['2016-2017', null, 71, 2, 1, 2, 5, 13, 48, 0],
+        ['2015-2016', null, 21, 1, 1, 1, 0, 7, 8, 3],
       ]
 
       it('shows stats', () => {
@@ -250,6 +252,7 @@ describe('Course Statistics tests', () => {
               .eq(trIndex)
               .within(() => {
                 values.forEach((value, tdIndex) => {
+                  if (value === null) return
                   cy.get('td').eq(tdIndex).contains(value)
                 })
               })
@@ -263,6 +266,7 @@ describe('Course Statistics tests', () => {
               .eq(trIndex)
               .within(() => {
                 values.forEach((value, tdIndex) => {
+                  if (value === null) return
                   cy.get('td').eq(tdIndex).contains(value)
                 })
               })
@@ -271,7 +275,45 @@ describe('Course Statistics tests', () => {
         })
       })
 
-      it('after changing time range shows same stats', () => {
+      it('Toggling course provider changes stats correctly', () => {
+        cy.get('[data-cy=unify_radio_reqular]').click()
+        cy.get("div[name='toYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', '2020-2021')
+          cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', yearRange.from)
+          cy.get("div[role='option']").should('have.length', 21)
+        })
+        cy.contains('#CourseStatPanes a.item', 'Table').click()
+        cy.contains('#CourseStatPanes a.item', 'Attempts').click()
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          attemptsTableContentsReqular.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  if (value === null) return
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
+              })
+          })
+        })
+        cy.get('[data-cy=unify_radio_open]').click()
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          attemptsTableContentsOpen.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  if (value === null) return
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
+              })
+          })
+        })
+        cy.get('[data-cy=unify_radio_unify]').click()
+      })
+
+      it('After changing time range shows same stats', () => {
         const newYearRange = { from: '2016-2017', to: '2019-2020' }
         cy.get("div[name='fromYear']")
           .click()
@@ -300,9 +342,25 @@ describe('Course Statistics tests', () => {
         cy.contains('Show population').should('be.enabled')
       })
     })
+
+    it('If no data available, unify toggle is disabled', () => {
+      cy.get("input[placeholder='Search by a course code']").type('AYCSM14111')
+      cy.contains('AYCSM14111').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('AYCSM14111 Avoin yo: Full Stack -websovelluskehitys: React Native')
+      cy.get('[data-cy=unify_radio_reqular]').find('input').should('be.disabled')
+      cy.get('[data-cy=unify_radio_unify]').find('input').should('not.be.disabled')
+    })
+
+    it('Has right to see all the sudents, because course provider is TKT', () => {
+      cy.visit('coursestatistics?courseCodes=%5B%22TKT10004%22%5D&cs_tab=0&separate=false')
+      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
+      cy.contains('Students (154)').click()
+      cy.contains('015224224')
+    })
   })
 
-  it.only('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
+  it.skip('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
     cy.init('/coursestatistics', 'onlycoursestatistics')
     cy.get('[data-cy=navbar-courseStatistics]').click()
     cy.get('[data-cy=course-code-input]').type('TKT20003')
@@ -316,13 +374,13 @@ describe('Course Statistics tests', () => {
 
     const attemptsTableContents = [
       // [time, passed, failed, passrate]
-      ['Total*', 288, 213, 75, '73.96 %'],
-      // ['Total', 295, 216, 79, '73.22 %'],
+      ['Total', 288, 213, 75, '73.96 %'], //Somehow cypress in CI can't find the number even if it is present in the artifact video
       ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
-      //['2020-2021', 5, 2, 3, '40.00 %'],
-      ['2019-2020', 164, 121, 43, '73.78 %'],
-      ['2018-2019', 85, 60, 25, '70.59 %'],
-      ['2017-2018', 39, 32, 7, '82.05 %'],
+      // ['Total', null, 295, 216, 79, '73.22 %'],
+      // ['2020-2021', null, 5, 2, 3, '40.00 %'],
+      ['2019-2020', null, 164, 121, 43, '73.78 %'],
+      ['2018-2019', null, 85, 60, 25, '70.59 %'],
+      ['2017-2018', null, 39, 32, 7, '82.05 %'],
     ]
 
     cy.contains('#CourseStatPanes a.item', 'Attempts').click()
@@ -332,6 +390,7 @@ describe('Course Statistics tests', () => {
           .eq(trIndex)
           .within(() => {
             values.forEach((value, tdIndex) => {
+              if (value === null) return
               cy.get('td').eq(tdIndex).contains(value)
             })
           })

cypress/integration/Faculty_statistics.js

@@ -0,0 +1,26 @@
+/// <reference types="Cypress" />
+
+// Change admin to regular user when feature is ready for general use
+
+describe('Faculty overview', () => {
+  describe('Faculty can be selected', () => {
+    it('Faculties are listed and one can be chosen', () => {
+      cy.init('/faculties', 'admin')
+      cy.get('[data-cy=select-faculty]').contains('td', 'H10')
+      cy.contains('td', 'H99').should('not.exist')
+      cy.contains('td', 'H90').click()
+      cy.contains('.header', 'Eläinlääketieteellinen tiedekunta')
+    })
+  })
+
+  describe('Faculty basic information', () => {
+    beforeEach(() => {
+      cy.init('/faculties', 'admin')
+      cy.contains('td', 'H90').click()
+    })
+    it('Credits produced by faculty are shown', () => {
+      cy.get(['data-cy=Section-CreditsProducedByTheFaculty'])
+      cy.get(['data-cy=Graph-CreditsProducedByTheFaculty'])
+    })
+  })
+})

cypress/integration/Filters.js

@@ -2,6 +2,8 @@
 
 // const baseUrl = Cypress.config().baseUrl
 
+const MOCKED_DATE = new Date(2022, 6, 12).getTime()
+
 const checkFilteringResult = studentCount => {
   cy.contains(`Students (${studentCount})`)
 }
@@ -17,6 +19,28 @@ const clearSingleDropdownSelection = dataCyAttribute => {
   cy.cs(dataCyAttribute).find('i.clear').click({ force: true })
 }
 
+const testRangeFilter = (parentEl, min, max, expected) => {
+  cy.cs(parentEl)
+    .cs('range-selector-min')
+    .find('input')
+    .invoke('val')
+    .then(initialMin => {
+      cy.cs(parentEl)
+        .cs('range-selector-max')
+        .find('input')
+        .invoke('val')
+        .then(initialMax => {
+          cy.cs(parentEl).cs('range-selector-min').find('input').type(`{selectall}{backspace}${min}`).blur()
+          cy.cs(parentEl).cs('range-selector-max').find('input').type(`{selectall}{backspace}${max}`).blur()
+
+          checkFilteringResult(expected)
+
+          cy.cs(parentEl).cs('range-selector-min').find('input').type(`{selectall}{backspace}${initialMin}`).blur()
+          cy.cs(parentEl).cs('range-selector-max').find('input').type(`{selectall}{backspace}${initialMax}`).blur()
+        })
+    })
+}
+
 // Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
 // to every it-function. Reason behind using test function wrapper is that Cypresses internal beforeEach / afterEach
 // functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
@@ -47,7 +71,7 @@ const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
 describe('Population Statistics', () => {
   const pathToCSBach2018 =
     '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2018'
-  const defaultAmountOfStudents = 148
+  const defaultAmountOfStudents = 159
   const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
   before(() => {
@@ -78,7 +102,10 @@ describe('Population Statistics', () => {
     })
   })
 
-  it('Study track filter works', () => {
+  // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
+  // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
+  // So fix this
+  it.skip('Study track filter works', () => {
     runTestStepWithPreAndPostParts('StudyTrack', () => {
       const card = cy.cs('StudyTrack-filter-card')
       const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
@@ -115,37 +142,30 @@ describe('Population Statistics', () => {
     runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
       cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
       cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(0)
-      checkFilteringResult(137)
+      checkFilteringResult(148)
       clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
     })
   })
 
   it('Credit filter works', () => {
     runTestStepWithPreAndPostParts('CreditsEarned', () => {
-      cy.cs('credit-filter-min').type('50')
-      cy.cs('credit-filter-max').type('150')
-      checkFilteringResult(116)
-      cy.cs('credit-filter-min').clear()
-      cy.cs('credit-filter-max').clear()
+      testRangeFilter('CreditsEarned-filter-card', 50, 150, 118)
     })
   })
 
   it('Age filter works', () => {
+    cy.clock(MOCKED_DATE, ['Date'])
     runTestStepWithPreAndPostParts('Age', () => {
-      cy.cs('ageFilter-min').type('20')
-      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(40)
-      cy.cs('ageFilter-min').find('input').clear()
-      cy.cs('ageFilter-max').find('input').clear()
+      testRangeFilter('Age-filter-card', 20, 40, 47)
     })
   })
 
   it('Gender filter works', () => {
     runTestStepWithPreAndPostParts('Gender', () => {
       cy.cs('genderFilter-dropdown').selectFromDropdown(0)
-      checkFilteringResult(36)
+      checkFilteringResult(42)
       cy.cs('genderFilter-dropdown').selectFromDropdown(1)
-      checkFilteringResult(112)
+      checkFilteringResult(117)
       cy.cs('genderFilter-dropdown').selectFromDropdown(2)
       checkFilteringResult(0)
       cy.cs('genderFilter-dropdown').selectFromDropdown(3)
@@ -160,7 +180,7 @@ describe('Population Statistics', () => {
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
       cy.cs('startYearAtUni-dropdown').selectFromDropdown(7)
-      checkFilteringResult(141)
+      checkFilteringResult(152)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
@@ -196,15 +216,12 @@ describe('Population Statistics', () => {
   })
 
   it('Filter combinations work', () => {
+    cy.clock(MOCKED_DATE, ['Date'])
     runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
       runTestStepWithPreAndPostParts('Age', () => {
         const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
         getCard().cs('option-graduated-true').click()
-        cy.cs('ageFilter-min').type('20')
-        cy.cs('ageFilter-max').type('30')
-        checkFilteringResult(11)
-        cy.cs('ageFilter-min').find('input').clear()
-        cy.cs('ageFilter-max').find('input').clear()
+        testRangeFilter('Age-filter-card', 20, 30, 1)
         getCard().cs('option-all').click()
       })
     })
@@ -213,7 +230,7 @@ describe('Population Statistics', () => {
 
 describe('Course Statistics', () => {
   const pathToDSAndAlgoSpring2019 =
-    '/coursepopulation?coursecodes=%5B%22TKT20001%22%2C%2258131%22%5D&from=138&separate=true&to=138&years=Kev%C3%A4t%202019'
+    '/coursepopulation?coursecodes=%5B%22TKT20001%22%2C%2258131%22%5D&from=138&separate=true&to=138&years=Kev%C3%A4t%202019&unifyCourses=unifyStats'
   const defaultAmountOfStudents = 118
   const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
@@ -247,7 +264,10 @@ describe('Course Statistics', () => {
     })
   })
 
-  it('Study track filter works', () => {
+  // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
+  // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
+  // So fix this
+  it.skip('Study track filter works', () => {
     runTestStepWithPreAndPostParts('StudyTrack', () => {
       const card = cy.cs('StudyTrack-filter-card')
       const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
@@ -257,12 +277,9 @@ describe('Course Statistics', () => {
   })
 
   it('Age filter works', () => {
+    cy.clock(MOCKED_DATE, ['Date'])
     runTestStepWithPreAndPostParts('Age', () => {
-      cy.cs('ageFilter-min').type('20')
-      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(31)
-      cy.cs('ageFilter-min').find('input').clear()
-      cy.cs('ageFilter-max').find('input').clear()
+      testRangeFilter('Age-filter-card', 20, 40, 33)
     })
   })
 
@@ -292,14 +309,11 @@ describe('Course Statistics', () => {
   })
 
   it('Filter combinations work', () => {
+    cy.clock(MOCKED_DATE, ['Date'])
     runTestStepWithPreAndPostParts('Grade', () => {
       runTestStepWithPreAndPostParts('Age', () => {
         cy.cs('gradeFilter-3').click()
-        cy.cs('ageFilter-min').type('20')
-        cy.cs('ageFilter-max').type('30')
-        checkFilteringResult(1)
-        cy.cs('ageFilter-min').find('input').clear()
-        cy.cs('ageFilter-max').find('input').clear()
+        testRangeFilter('Age-filter-card', 20, 30, 1)
         cy.cs('gradeFilter-3').click()
       })
     })
@@ -322,12 +336,9 @@ describe('Custom Population Statistics', () => {
   })
 
   it('Age filter works', () => {
+    cy.clock(MOCKED_DATE, ['Date'])
     runTestStepWithPreAndPostParts('Age', () => {
-      cy.cs('ageFilter-min').type('20')
-      cy.cs('ageFilter-max').type('30')
-      checkFilteringResult(1)
-      cy.cs('ageFilter-min').find('input').clear()
-      cy.cs('ageFilter-max').find('input').clear()
+      testRangeFilter('Age-filter-card', 20, 30, 1)
     })
   })
 

cypress/integration/Population_statistics.js

@@ -3,15 +3,15 @@
 const moment = require('moment')
 const _ = require('lodash')
 
-const setPopStatsUntil = (until, includeSettings = []) => {
-  cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
-  includeSettings.forEach(setting => {
-    cy.contains('Advanced settings').parent().siblings().contains(setting).click()
-  })
-  cy.get('.adv-stats-until > .form-control').click().clear().type(until)
-  cy.contains('Fetch population with new settings').click()
-  cy.contains('Advanced settings')
-}
+// const setPopStatsUntil = (until, includeSettings = []) => {
+//   cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
+//   includeSettings.forEach(setting => {
+//     cy.contains('Advanced settings').parent().siblings().contains(setting).click()
+//   })
+//   cy.get('.adv-stats-until > .form-control').click().clear().type(until)
+//   cy.contains('Fetch population with new settings').click()
+//   cy.contains('Advanced settings')
+// }
 
 describe('Population Statistics tests', () => {
   const pathToCSBach2017 =
@@ -65,13 +65,14 @@ describe('Population Statistics tests', () => {
     it('Population statistics is usable on general level', () => {
       cy.visit(pathToCSBach2017)
       cy.cs('filtered-students')
-      setPopStatsUntil('toukokuu 2020')
+      // This advanced settings does not work correctly
+      // see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3552
+      // setPopStatsUntil('toukokuu 2020')
 
       cy.get('.card').within(() => {
         cy.contains('Tietojenkäsittelytieteen kandiohjelma')
-        cy.contains('Sample size: 149 students')
+        cy.contains('Sample size: 170 students')
         cy.contains('Excludes exchange students')
-        cy.contains("Excludes students who haven't enrolled present nor absent")
         cy.contains('Excludes students with non-degree study right')
         cy.contains('Excludes students who have transferred out of this programme')
       })
@@ -80,17 +81,17 @@ describe('Population Statistics tests', () => {
 
       cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
       // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(15000) // a bit hacky way, wait until ui is ready
-      cy.get('[data-cy=expand-TKT1]').click()
-      cy.get('[data-cy=coursestats-link-TKT10002]').click()
+      cy.wait(25000) // a bit hacky way, wait until ui is ready
+      cy.get('[data-cy=toggle-group-module-TKT1]').click()
+      cy.contains('td', 'TKT10002').siblings().find('i.level.up').click()
       cy.wait('@coursePage')
       cy.url().should('include', '/coursestatistics')
-      cy.contains('TKT10002, 581325 Ohjelmoinnin perusteet')
+      cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
     })
 
     it("Empty 'tags' tab has a link to the page where tags can be created", () => {
       cy.visit(pathToCSBach2017)
-      cy.contains('Students (149)').click()
+      cy.contains('Students (170)').click()
       cy.get('[data-cy=student-table-tabs]').contains('Tags').click()
       cy.contains('No tags defined. You can define them here.').find('a').click()
       cy.contains('Tags').click()
@@ -119,12 +120,11 @@ describe('Population Statistics tests', () => {
 
       cy.contains('Fetch population').click()
 
-      cy.contains('Credit accumulation (for 149 students)')
+      cy.contains('Credit accumulation (for 170 students)')
 
-      // spring + fall and include cancelled
+      // spring + fall
       cy.get('[data-cy=advanced-toggle]').click()
       cy.cs('toggle-spring').click()
-      cy.cs('toggle-cancelled').click()
       cy.contains('Fetch population').click()
 
       cy.contains('Credit accumulation (for 170 students)')
@@ -133,6 +133,7 @@ describe('Population Statistics tests', () => {
     it('Credit Statistics, Credits Gained tab works', () => {
       cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
       cy.contains('Credit statistics').click()
+      cy.contains('Credits Gained').click()
       cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the population')
 
       const months = Math.ceil(moment.duration(moment().diff(moment('2017-08-1'))).asMonths())
@@ -140,7 +141,7 @@ describe('Population Statistics tests', () => {
       const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
       const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))
 
-      cy.get('.credits-gained-table').should('contain', '(n=149)')
+      cy.get('.credits-gained-table').should('contain', '(n=170)')
 
       for (const [start, end] of ranges) {
         let value = 'credits'
@@ -159,7 +160,7 @@ describe('Population Statistics tests', () => {
       cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
         const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)
 
-        expect(sum).to.equal(149)
+        expect(sum).to.equal(170)
       })
 
       cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('not.exist')
@@ -170,6 +171,7 @@ describe('Population Statistics tests', () => {
       cy.cs('filtered-students')
       cy.contains('Credit statistics')
       cy.get("[data-cy='credit-statistics']").click()
+      cy.contains('Credits Gained').click()
       cy.get('.credits-gained-divider').click()
       cy.get("[data-cy='credits-gained-table-Avoin väylä']").should('exist')
       cy.get("[data-cy='credits-gained-table-Yhteispisteet']").should('exist')
@@ -182,10 +184,10 @@ describe('Population Statistics tests', () => {
       cy.contains('Credit statistics').click({ force: true })
       cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
 
-      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
-      cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
-      cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
-      cy.get("[data-cy='credit-stats-min']").should('contain', '11')
+      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 170 Students')
+      cy.get("[data-cy='credit-stats-mean']").should('contain', '114.75')
+      cy.get("[data-cy='credit-stats-stdev']").should('contain', '65.53')
+      cy.get("[data-cy='credit-stats-min']").should('contain', '5')
       cy.get("[data-cy='credit-stats-max']").should('contain', '314')
     })
   })
@@ -198,7 +200,7 @@ describe('Population Statistics tests', () => {
       const existing = '010113437'
       const nonExisting = '66666666'
       cy.visit(pathToCSBach2017)
-      cy.contains('Students (149)').click()
+      cy.contains('Students (170)').click()
       cy.contains(existing)
       cy.contains(nonExisting).should('not.exist')
       cy.contains('button', 'Check studentnumbers').click()

cypress/integration/Student_statistics.js

@@ -11,6 +11,7 @@ describe('Student Statistics tests', () => {
     firstnames: 'Matti Jaakko',
     lastname: 'Nieminen',
     studentnumber: '010654019',
+    sis_person_id: 'hy-hlo-84830887',
   }
 
   it('Student statistics search form is usable', () => {
@@ -37,9 +38,9 @@ describe('Student Statistics tests', () => {
   it('Can get student specific page by clicking student', () => {
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
     cy.contains('Started: 01.08.2017')
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 01.08.2025')
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 31.07.2025')
     cy.contains(student.lastname).should('not.exist')
     cy.contains(student.firstnames).should('not.exist')
 
@@ -54,7 +55,7 @@ describe('Student Statistics tests', () => {
 
   it('Can get back to search menu', () => {
     cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
     cy.get('.remove').click()
     cy.contains('Student number').should('not.exist')
     cy.contains('Credits').should('not.exist')
@@ -62,7 +63,7 @@ describe('Student Statistics tests', () => {
 
   it('Can jump to course', () => {
     cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
     cy.contains('Introduction to Machine Learning (DATA11002)')
       .siblings()
       .within(() => {
@@ -72,6 +73,14 @@ describe('Student Statistics tests', () => {
     cy.contains('Introduction to Machine Learning')
   })
 
+  it('Has correct Sisu link', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains('td a', student.studentnumber).click()
+    cy.get('[data-cy=sisulink] > a')
+      .should('have.attr', 'href')
+      .and('include', 'https://sis-helsinki.funidata.fi/tutor/role/staff/student/hy-hlo-84830887/basic/basic-info')
+  })
+
   it('Searching with bad inputs doesnt yield results', () => {
     cy.get('.prompt').type('SWAG LITTINEN')
     cy.contains('Student number').should('not.exist')
@@ -82,7 +91,7 @@ describe('Student Statistics tests', () => {
 
   it('Can jump to population page', () => {
     cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
     cy.get('i.level.up.alternate.icon').eq(0).click()
     cy.contains('Population statistics')
     cy.contains('Tietojenkäsittelytieteen kandiohjelma')

cypress/integration/Studyprogramme_overview.js

@@ -1,83 +1,83 @@
 /// <reference types="Cypress" />
 
 const deleteTag = name => {
-  cy.contains(name).siblings().contains('Delete').click()
+  cy.contains('td', name).siblings().contains('Delete').click()
   cy.contains('Are you sure you want to delete tag')
   cy.contains('Confirm').click()
-  cy.contains(name).should('not.exist')
+  cy.contains('td', name).should('not.exist')
 }
 
-const getEmptyYears = () => {
+const getEmptyYears = isAcademicYear => {
   const thisYear = new Date().getFullYear()
   const years = []
   for (let year = thisYear; year > 2021; year--) {
-    years.push(year)
+    if (isAcademicYear) {
+      years.push(`${year} - ${year + 1}`)
+    } else {
+      years.push(year)
+    }
   }
   return years
 }
 
 describe('Studyprogramme overview', () => {
-  describe('when opening programme page with basic user', () => {
+  /* Basic information overview -tests*/
+  describe('Basic information -view works for basic user', () => {
     beforeEach(() => {
-      cy.init('/study-programme/KH50_005')
+      cy.init('/study-programme')
+      cy.viewport(1536, 960)
     })
 
     // If the backend breaks for one of the sections, the section header is not rendered and this will fail
     it('Basic information -tab loads', () => {
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('[data-cy=Section-StudentsOfTheStudyprogramme]')
       cy.get('[data-cy=Section-CreditsProducedByTheStudyprogramme]')
       cy.get('[data-cy=Section-GraduatedAndThesisWritersOfTheProgramme]')
-      cy.get('[data-cy=Section-ProgrammesAfterGraduation]')
-      cy.get('[data-cy=Section-AverageGraduationTimes]')
-    })
-
-    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
-    it('Studytracks and student populations -tab loads', () => {
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Studytracks and student populations').click()
-      cy.get('[data-cy=Section-StudytrackOverview]')
-      cy.get('[data-cy=Section-StudytrackProgress]')
+      cy.get('[data-cy=Section-ProgrammesBeforeOrAfter]')
       cy.get('[data-cy=Section-AverageGraduationTimes]')
     })
 
     it('Basic information contains correct students', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       const years = getEmptyYears()
       const tableContents = [
-        // [Year, All, Started, Graduated, Cancelled, Transferred Away, Transferred to]
-        ...years.map(year => [year, 0, 0, 0, 0, 0, 0]),
-        [2021, 0, 0, 0, 0, 1, 0],
-        [2020, 12, 10, 36, 16, 3, 2],
-        [2019, 87, 86, 17, 14, 0, 1],
-        [2018, 161, 162, 1, 3, 0, 0],
-        [2017, 171, 173, 0, 0, 0, 0],
+        // [Year, Started, Graduated, Transferred Away, Transferred to]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        [2021, 0, 0, 1, 0],
+        [2020, 12, 35, 3, 2],
+        [2019, 87, 17, 0, 1],
+        [2018, 161, 1, 0, 0],
+        [2017, 171, 0, 0, 0],
       ]
 
       cy.checkTableStats(tableContents, 'StudentsOfTheStudyprogramme')
     })
 
     it('Basic information contains correct credits', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       const years = getEmptyYears()
       const tableContents = [
-        // [Year, Total, Major credits, Non-major credits, Transferred credits]
-        ...years.map(year => [year, 0, 0, 0, 0]),
-        [2021, 0, 0, 0, 0],
-        [2020, 4928, 548, 4213, 167],
-        [2019, 7737, 1644, 5744, 349],
-        [2018, 6730, 1820, 4638, 272],
-        [2017, 2900, 788, 1724, 388],
+        // [Year, Total, Major credits, Non-major credits, Non-degree credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0, 0, 0]),
+        [2021, 0, 0, 0, 0, 0],
+        [2020, 5860, 5591, 102, 0, 167],
+        [2019, 8696, 7737, 587, 23, 349],
+        [2018, 7818, 6584, 857, 105, 272],
+        [2017, 3085, 2492, 158, 47, 388],
       ]
 
       cy.checkTableStats(tableContents, 'CreditsProducedByTheStudyprogramme')
     })
 
     it('Basic information contains correct thesis writers and graduates', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       const years = getEmptyYears()
       const tableContents = [
         // [Year, Graduated, Wrote thesis]
         ...years.map(year => [year, 0, 0]),
         [2021, 0, 0],
-        [2020, 36, 35],
+        [2020, 35, 35],
         [2019, 17, 25],
         [2018, 1, 5],
         [2017, 0, 0],
@@ -86,9 +86,361 @@ describe('Studyprogramme overview', () => {
       cy.checkTableStats(tableContents, 'GraduatedAndThesisWritersOfTheProgramme')
     })
 
+    it.skip('Special studyrights can be excluded and basic data changes accordingly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen maisteriohjelma').click()
+      cy.get('[data-cy=StudentToggle]').click()
+      const years = getEmptyYears()
+      const studentTableContents = [
+        // [Year, Started, Graduated]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 22, 0],
+        [2019, 12, 0],
+        [2018, 0, 0],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
+
+      const creditTableContents = [
+        // [Year, Total, Major students credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0]),
+        [2021, 0, 0, 0],
+        [2020, 117.5, 0, 7.5],
+        [2019, 67, 0, 0],
+        [2018, 0, 0, 0],
+        [2017, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+
+      const graduatedTableContents = [
+        // [Year, Graduated, Wrote thesis]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 0, 0],
+        [2019, 0, 0],
+        [2018, 0, 0],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(graduatedTableContents, 'GraduatedAndThesisWritersOfTheProgramme')
+      cy.get('[data-cy=StudentToggle]').click()
+    })
+
+    it('Year can be changed to academic year, and data changes accordingly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen maisteriohjelma').click()
+      cy.get('[data-cy=YearToggle]').click()
+      const isAcademicYear = true
+      const years = getEmptyYears(isAcademicYear)
+      const studentTableContents = [
+        // [Year, Started, Graduated, Transferred away, Transferred to]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0],
+        ['2020 - 2021', 5, 0, 0, 0],
+        ['2019 - 2020', 27, 0, 0, 0],
+        ['2018 - 2019', 4, 0, 0, 0],
+        ['2017 - 2018', 0, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
+
+      const creditTableContents = [
+        // [Year, Total, Major students credits, Non-major students credits, Non-degree credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0],
+        ['2020 - 2021', 0, 0, 0, 0],
+        ['2019 - 2020', 338.5, 177, 154, 0, 7.5],
+        ['2018 - 2019', 173, 0, 173, 0, 0],
+        ['2017 - 2018', 42, 0, 42, 0, 0],
+      ]
+
+      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+      cy.get('[data-cy=YearToggle]').click()
+    })
+
+    it('Basic information graphs render', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+
+      cy.get('[data-cy=Graph-StudentsOfTheStudyprogramme')
+        .should('contain', 'Started')
+        .should('contain', 'Graduated')
+        .should('contain', 'Transferred away')
+
+      cy.get('[data-cy=Graph-CreditsProducedByTheStudyprogramme')
+        .should('contain', 'Major students credits')
+        .should('contain', 'Non-major students credits')
+        .should('contain', 'Transferred credits')
+        .should('contain', 857)
+        .should('contain', 587)
+
+      cy.get('[data-cy=Graph-GraduatedAndThesisWritersOfTheProgramme')
+        .should('contain', 'Graduated students')
+        .should('contain', 'Wrote thesis')
+        .should('contain', 1)
+        .should('contain', 5)
+        .should('contain', 17)
+        .should('contain', 25)
+        .should('contain', 35)
+
+      cy.get('[data-cy=Graph-2019-AverageGraduationTimes]')
+        .should('contain', '2019')
+        .should('contain', 'n = 17')
+        .should('contain', '23 kk')
+
+      cy.get('[data-cy=Graph-ProgrammesBeforeOrAfter')
+        .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
+        .should('contain', 'Datatieteen maisteriohjelma')
+        .should('contain', 'Matematiikan ja tilastotieteen maisteriohjelma')
+        .should('contain', 1)
+        .should('contain', 4)
+        .should('contain', 13)
+    })
+  })
+
+  /* Studytrack overview -tests*/
+  describe('Studytrack overview works for basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme')
+    })
+
+    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
+    it('Studytracks and student populations -tab loads', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      cy.get('[data-cy=Section-StudytrackOverview]')
+      cy.get('[data-cy=Section-StudytrackProgress]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
+    })
+
+    it('Students of the studyprogramme are shown correctly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      const tableContents = [
+        // [Year, All, Started studying, Currently enrolled, Absent, Cancelled, Graduated, Men, Women, Finnish]
+        [
+          '2020 - 2021',
+          12,
+          '100.0 %',
+          10,
+          '83.3 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          10,
+          '83.3 %',
+          2,
+          '16.7 %',
+          12,
+          '100.0 %',
+        ],
+        [
+          '2019 - 2020',
+          86,
+          '100.0 %',
+          86,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          1,
+          '1.2 %',
+          65,
+          '75.6 %',
+          21,
+          '24.4 %',
+          86,
+          '100.0 %',
+        ],
+        [
+          '2018 - 2019',
+          162,
+          '100.0 %',
+          161,
+          '99.4 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          11,
+          '6.8 %',
+          118,
+          '72.8 %',
+          44,
+          '27.2 %',
+          161,
+          '99.4 %',
+        ],
+        [
+          '2017 - 2018',
+          171,
+          '100.0 %',
+          171,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          3,
+          '1.8 %',
+          42,
+          '24.6 %',
+          135,
+          '78.9 %',
+          36,
+          '21.1 %',
+          166,
+          '97.1 %',
+        ],
+        [
+          'Total',
+          431,
+          '100.0 %',
+          431,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          3,
+          '0.7 %',
+          54,
+          '12.5 %',
+          328,
+          '76.1 %',
+          103,
+          '23.9 %',
+          425,
+          '98.6 %',
+        ],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudytrackOverview')
+    })
+
+    it('Student progress data is shown correctly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      const tableContents = [
+        ['2020 - 2021', 12, 5, 2, 2, 0, 2, 0, 1],
+        ['2019 - 2020', 86, 13, 15, 32, 11, 2, 0, 13],
+        ['2018 - 2019', 162, 7, 16, 35, 33, 27, 18, 26],
+        ['2017 - 2018', 171, 10, 22, 18, 22, 24, 27, 48],
+        ['Total', 431, 35, 55, 87, 66, 55, 45, 88],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudytrackProgress')
+    })
+
+    it('Studytrack overview graphs render', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+
+      cy.get('[data-cy=Graph-StudytrackProgress]')
+        .should('contain', 'Less than 30 credits')
+        .should('contain', '30-59 credits')
+        .should('contain', 'More than 180 credits')
+        .should('contain', '5.8%') // The percentage for less than 30 credits in 2017-2018, to check that the graph renders
+        .should('contain', '4.3%') // The percentage for less than 30 credits in 2018-2019
+        .should('contain', '15.1%') // The percentage for less than 30 credits in 2019-2020
+
+      cy.get('[data-cy=Graph-2018-AverageGraduationTimes]')
+        .should('contain', '2018 - 2019')
+        .should('contain', 'n = 11 / 162')
+        .should('contain', '22 kk')
+    })
+  })
+
+  /* Programme courses -tests*/
+  describe('Programme courses works for basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme')
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Programme courses').click()
+    })
+
+    it('content loads', () => {
+      cy.get('[data-cy=CoursesYearFilter]')
+      cy.get('[data-cy=CourseTabs]')
+    })
+
+    it('time range selection works', () => {
+      cy.get('[data-cy=fromYear]').click().contains('2018').click()
+
+      cy.get('[data-cy=toYear').click().contains('2019').click()
+
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('684')
+      })
+    })
+
+    it('calendar year -> academic year toggle works', () => {
+      cy.get('[data-cy=fromYear]').click().contains('2018').click()
+
+      cy.get('[data-cy=toYear').click().contains('2019').click()
+
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('684')
+      })
+
+      cy.get('[data-cy=calendarAcademicYearToggle]').first().click()
+
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('772')
+      })
+    })
+
+    it('contains correct courses in alphabetical order', () => {
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('Aineopintojen harjoitustyö: Tietokantasovellus')
+        cy.get('tr').eq(-1).contains('Äidinkielinen viestintä')
+      })
+    })
+
+    it('different sorting options work', () => {
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        // Test sorting by different columns
+        cy.get('th').eq(0).click()
+        cy.get('tr').eq(1).contains('Tietojenkäsittelytieteen kisälliopetus: Java-ohjelmointi')
+        cy.get('th').eq(1).click()
+        cy.get('tr').eq(1).contains('Äidinkielinen viestintä')
+        cy.get('th').eq(2).click()
+        cy.get('tr').eq(1).contains('Tietorakenteet ja algoritmit')
+      })
+    })
+
+    it('show credits -> show students toggle works', () => {
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('1068')
+      })
+
+      cy.get('[data-cy=creditsStudentsToggle]').first().click()
+
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('267')
+      })
+    })
+  })
+
+  /* Tag-tests*/
+  describe('Basic information -view works for basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme')
+    })
+
     it('can create and delete tags for population', () => {
       const name = `tag-${new Date().getTime()}`
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('.attached').contains('Tags').click()
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
@@ -100,7 +452,7 @@ describe('Studyprogramme overview', () => {
 
     it('can create personal tags', () => {
       const name = `tag-${new Date().getTime()}`
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('.attached').contains('Tags').click()
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
@@ -116,7 +468,7 @@ describe('Studyprogramme overview', () => {
 
       const student = '010113437'
 
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('.attached').contains('Tags').click()
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
@@ -124,7 +476,7 @@ describe('Studyprogramme overview', () => {
       cy.contains(name)
 
       cy.contains('Add tags to students').click()
-      cy.get('.form > .field > .dropdown').click().get('.ui > .search').type(name).click()
+      cy.get('.form > .field > .dropdown').click().get('.ui > input.search').type(name).click()
 
       cy.get('.form > .field > .dropdown > .visible').contains(name).click()
 
@@ -133,7 +485,7 @@ describe('Studyprogramme overview', () => {
 
       cy.contains('Student statistics').click()
       cy.get('.prompt').type(student)
-      cy.contains(student).click()
+      cy.contains('a', student).click()
       cy.contains(name)
 
       cy.go('back')
@@ -143,7 +495,7 @@ describe('Studyprogramme overview', () => {
 
       cy.contains('Student statistics').click()
       cy.get('.prompt').type(student)
-      cy.contains(student).click()
+      cy.contains('a', student).click()
       cy.contains(name).should('not.exist')
     })
   })

cypress/integration/Teachers.js

@@ -16,7 +16,7 @@ describe('Teachers page tests', () => {
     cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
     cy.cs('course-providers').children('.icon').click()
     cy.get('.form > .fluid').click()
-    cy.contains('Name')
+    cy.contains('Teacher')
     cy.contains(teacher1)
     cy.contains(teacher2)
   })
@@ -32,21 +32,13 @@ describe('Teachers page tests', () => {
     cy.url().should('include', '/teachers')
     cy.get('.borderless > :nth-child(3)').click()
     cy.get('.prompt').type(teacher1.split()[0])
-    cy.contains(teacher1).click()
+    cy.contains('td', teacher1).click()
     cy.contains('Aineopintojen harjoitustyö: Tietokantasovellus')
   })
 
   it('Check leaderboad works', () => {
     cy.get('.borderless > :nth-child(2)').click({ force: true })
     cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.contains('2019-2020').click({ force: true })
-    cy.contains('Recalculate this year').click({ force: true })
-    cy.wait(5000)
-    cy.reload()
-    cy.contains('2019-2020').click({ force: true })
-    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.contains('Passed')
-    cy.contains('Ahokas Jarmo Ilkka')
+    cy.contains('2020-2021').click({ force: true })
   })
 })

cypress/integration/Users.js

@@ -39,9 +39,7 @@ describe('Users tests', () => {
 
     it("mocking normal user shows only the mocked user's programmes", () => {
       cy.contains('mocking').should('not.exist')
-      cy.contains('tr', 'basic').within(() => {
-        cy.contains('.button', 'Edit').click()
-      })
+      cy.cs('user-edit-button-basic').click()
       cy.get('i.spy').click()
       cy.contains('mocking as basic')
       cy.wait(1000)

cypress/support/commands.js

@@ -106,7 +106,7 @@ Cypress.Commands.add('selectStudyProgramme', name => {
 })
 
 Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {
-  cy.get(`[data-cy=${tableName}]>tbody`).within(() => {
+  cy.get(`[data-cy=Table-${tableName}]>tbody`).within(() => {
     correctStats.forEach((values, trIndex) => {
       cy.get('tr')
         .eq(trIndex)

cypress/support/index.js

@@ -14,3 +14,13 @@
 // ***********************************************************
 
 import './commands'
+
+// This error is benign and seems to be only present in Cypress
+const resizeObserverLoopErrRe = /^[^(ResizeObserver loop limit exceeded)]/
+
+Cypress.on('uncaught:exception', err => {
+  /* returning false here prevents Cypress from failing the test */
+  if (resizeObserverLoopErrRe.test(err.message)) {
+    return false
+  }
+})

package.json

@@ -28,24 +28,24 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@babel/eslint-parser": "^7.15.0",
-    "@babel/plugin-syntax-jsx": "^7.14.5",
-    "concurrently": "^6.2.1",
-    "cypress": "^8.3.1",
+    "@babel/eslint-parser": "^7.17.0",
+    "@babel/plugin-syntax-jsx": "^7.16.7",
+    "concurrently": "^6.5.1",
+    "cypress": "^8.7.0",
     "eslint": "^7.32.0",
     "eslint-config-airbnb": "^18.2.1",
-    "eslint-config-prettier": "^8.3.0",
+    "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-babel": "^5.3.1",
-    "eslint-plugin-cypress": "^2.11.3",
-    "eslint-plugin-import": "^2.24.2",
-    "eslint-plugin-jsx-a11y": "^6.4.1",
+    "eslint-plugin-cypress": "^2.12.1",
+    "eslint-plugin-import": "^2.25.4",
+    "eslint-plugin-jsx-a11y": "^6.5.1",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
     "eslint-plugin-prettier": "^3.4.1",
-    "eslint-plugin-react": "^7.24.0",
-    "eslint-plugin-react-hooks": "^4.2.0",
-    "husky": "^7.0.2",
-    "lint-staged": "^11.1.2",
-    "prettier": "^2.3.2",
+    "eslint-plugin-react": "^7.29.3",
+    "eslint-plugin-react-hooks": "^4.3.0",
+    "husky": "^7.0.4",
+    "lint-staged": "^11.2.6",
+    "prettier": "^2.5.1",
     "stylelint": "^13.13.1",
     "stylelint-config-prettier": "^8.0.2",
     "stylelint-config-standard": "^22.0.0",

services/backend/config/IAMConfig.js

@@ -0,0 +1,435 @@
+const { facultyMap } = require('./data')
+
+const joryMap = {
+  'hy-ttdk-tuk-jory': '100-K001',
+  'hy-ttdk-tum-jory': '100-M001',
+  'hy-oiktdk-on-jory': '200-K001',
+  'hy-oiktdk-otm-jory': '200-M001',
+  'hy-oiktdk-ibl-jory': '200-M002',
+  'hy-oiktdk-ggl-jory': '200-M003',
+  'hy-ltdk-psyk-jory': ['300-K001', '300-M004'],
+  'hy-ltdk-logo-jory': ['300-K002', '300-M005'],
+  'hy-ltdk-ll-jory': '300-M001',
+  'hy-ltdk-tmed-jory': '300-M002',
+  'hy-ltdk-hll-jory': '300-M003',
+  'hy-humtdk-filk-jory': '400-K001',
+  'hy-humtdk-ttk-jory': '400-K002',
+  'hy-humtdk-kik-jory': '400-K003',
+  'hy-humtdk-kok-jory': '400-K004',
+  'hy-humtdk-kuka-jory': '400-K005',
+  'hy-humtdk-hisk-jory': '400-K006',
+  'hy-humtdk-ttm-jory': '400-M001',
+  'hy-humtdk-kim-jory': '400-M002',
+  'hy-humtdk-eng-jory': '400-M003',
+  'hy-humtdk-rus-jory': '400-M004',
+  'hy-humtdk-lingdig-jory': '400-M005',
+  'hy-humtdk-tra-jory': '400-M006',
+  'hy-humtdk-suku-jory': '400-M007',
+  'hy-humtdk-nor-jory': '400-M008',
+  'hy-humtdk-kir-jory': '400-M009',
+  'hy-humtdk-kuma-jory': '400-M010',
+  'hy-humtdk-ice-jory': '400-M011',
+  'hy-humtdk-alku-jory': '400-M012',
+  'hy-humtdk-spt-jory': '400-M014',
+  'hy-humtdk-hism-jory': '400-M015',
+  'hy-mltdk-mat-jory': '500-K001',
+  'hy-mltdk-fys-jory': '500-K002',
+  'hy-mltdk-kemk-jory': '500-K003',
+  'hy-mltdk-mfkk-jory': '500-K004',
+  'hy-mltdk-tkt-jory': '500-K005',
+  'hy-mltdk-geok-jory': '500-K006',
+  'hy-mltdk-maa-jory': '500-K007',
+  'hy-mltdk-bsc-jory': '500-K008',
+  'hy-mltdk-mast-jory': '500-M001',
+  'hy-mltdk-lsi-jory': '500-M002',
+  'hy-mltdk-tcm-jory': '500-M003',
+  'hy-mltdk-paras-jory': '500-M004',
+  'hy-mltdk-matres-jory': '500-M005',
+  'hy-mltdk-atm-jory': '500-M006',
+  'hy-mltdk-kem-jory': '500-M007',
+  'hy-mltdk-mfkm-jory': '500-M008',
+  'hy-mltdk-csm-jory': '500-M009',
+  'hy-mltdk-data-jory': '500-M010',
+  'hy-mltdk-geom-jory': '500-M011',
+  'hy-mltdk-geog-jory': '500-M012',
+  'hy-mltdk-usp-jory': '500-M013',
+  'hy-ftdk-farm-jory': '550-K001',
+  'hy-ftdk-prov-jory': '550-M001',
+  'hy-ftdk-mpharm-jory': '550-M002',
+  'hy-bytdk-bio-jory': '570-K001',
+  'hy-bytdk-mole-jory': '570-K002',
+  'hy-bytdk-env-jory': '570-K003',
+  'hy-bytdk-eeb-jory': '570-M001',
+  'hy-bytdk-ips-jory': '570-M002',
+  'hy-bytdk-gmb-jory': '570-M003',
+  'hy-bytdk-neuro-jory': '570-M004',
+  'hy-bytdk-ecgs-jory': '570-M005',
+  'hy-ktdk-eduk-jory': '600-K001',
+  'hy-ktdk-edum-jory': '600-M001',
+  'hy-ktdk-ce-jory': '600-M002',
+  'hy-valttdk-pvk-jory': '700-K001',
+  'hy-valttdk-yk-jory': '700-K002',
+  'hy-valttdk-sosk-jory': '700-K003',
+  'hy-valttdk-ecok-jory': '700-K004',
+  'hy-valttdk-film-jory': '700-M001',
+  'hy-valttdk-pvm-jory': '700-M002',
+  'hy-valttdk-gpc-jory': '700-M003',
+  'hy-valttdk-ym-jory': '700-M004',
+  'hy-valttdk-cos-jory': '700-M005',
+  'hy-valttdk-ens-jory': '700-M006',
+  'hy-valttdk-msv-jory': '700-M007',
+  'hy-valttdk-sosm-jory': '700-M008',
+  'hy-valttdk-econ-jory': '700-M009',
+  'hy-valttdk-sote-jory': '700-M011',
+  'hy-sskh-ksv-jory': '740-K001',
+  'hy-mmtdk-maatk-jory': '800-K001',
+  'hy-mmtdk-metsak-jory': '800-K002',
+  'hy-mmtdk-etk-jory': '800-K003',
+  'hy-mmtdk-yet-jory': '800-K004',
+  'hy-mmtdk-agri-jory': '800-M001',
+  'hy-mmtdk-agere-jory': '800-M002',
+  'hy-mmtdk-for-jory': '800-M003',
+  'hy-mmtdk-food-jory': '800-M004',
+  'hy-mmtdk-hnfb-jory': '800-M005',
+  'hy-mmtdk-ekm-jory': '800-M006',
+  'hy-mmtdk-mmb-jory': '800-M007',
+  'hy-eltdk-elk-jory': '900-K001',
+  'hy-eltdk-ell-jory': '900-M001',
+  'hy-dp-dptheol-jory': 'T920101',
+  'hy-dp-dplaw-jory': 'T920102',
+  'hy-dp-philartsoc-jory': 'T920103',
+  'hy-dp-dphistcult-jory': 'T920104',
+  'hy-dp-dplang-jory': 'T920105',
+  'hy-dp-sky-jory': 'T920106',
+  'hy-dp-dpsocs-jory': 'T920107',
+  'hy-dp-pyam-jory': 'T920108',
+  'hy-dp-dpe-jory': 'T920109',
+  'hy-dp-seduce-jory': 'T920110',
+  'hy-dp-clic-jory': 'T920111',
+  'hy-dp-dpbm-jory': 'T921101',
+  'hy-dp-klto-jory': 'T921102',
+  'hy-dp-docpop-jory': 'T921103',
+  'hy-dp-findos-jory': 'T921104',
+  'hy-dp-dpdr-jory': 'T921105',
+  'hy-dp-ils-jory': 'T921106',
+  'hy-dp-bandm-jory': 'T921107',
+  'hy-dp-cvm-jory': 'T921108',
+  'hy-dp-dphub-jory': 'T921109',
+  'hy-dp-luova-jory': 'T922101',
+  'hy-dp-dpps-jory': 'T922102',
+  'hy-dp-denvi-jory': 'T922103',
+  'hy-dp-agforee-jory': 'T922104',
+  'hy-dp-mbdp-jory': 'T922105',
+  'hy-dp-foodhealth-jory': 'T922106',
+  'hy-dp-papu-jory': 'T923101',
+  'hy-dp-geodoc-jory': 'T923102',
+  'hy-dp-atm-dp-jory': 'T923103',
+  'hy-dp-chems-jory': 'T923104',
+  'hy-dp-domast-jory': 'T923105',
+  'hy-dp-matrena-jory': 'T923106',
+  'hy-dp-docs-jory': 'T923107',
+}
+
+const kojoMap = {
+  'hy-ttdk-tuk-jory': 'hy-ttdk-kandi-kojot',
+  'hy-ttdk-tum-jory': 'hy-ttdk-maisteri-kojot',
+  'hy-oiktdk-on-jory': 'hy-oiktdk-kandi-kojot',
+  'hy-oiktdk-otm-jory': 'hy-oiktdk-maisteri-kojot',
+  'hy-oiktdk-ibl-jory': 'hy-oiktdk-maisteri-kojot',
+  'hy-oiktdk-ggl-jory': 'hy-oiktdk-maisteri-kojot',
+  'hy-ltdk-psyk-jory': 'hy-ltdk-maisteri-kojot',
+  'hy-ltdk-logo-jory': 'hy-ltdk-maisteri-kojot',
+  'hy-ltdk-ll-jory': 'hy-ltdk-maisteri-kojot',
+  'hy-ltdk-tmed-jory': 'hy-ltdk-maisteri-kojot',
+  'hy-ltdk-hll-jory': 'hy-ltdk-maisteri-kojot',
+  'hy-humtdk-filk-jory': 'hy-humtdk-kandi-kojot',
+  'hy-humtdk-ttk-jory': 'hy-humtdk-kandi-kojot',
+  'hy-humtdk-kik-jory': 'hy-humtdk-kandi-kojot',
+  'hy-humtdk-kok-jory': 'hy-humtdk-kandi-kojot',
+  'hy-humtdk-kuka-jory': 'hy-humtdk-kandi-kojot',
+  'hy-humtdk-hisk-jory': 'hy-humtdk-kandi-kojot',
+  'hy-humtdk-ttm-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-kim-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-eng-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-rus-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-lingdig-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-tra-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-suku-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-nor-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-kir-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-kuma-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-ice-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-alku-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-spt-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-hism-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-mltdk-mat-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-fys-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-kemk-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-mfkk-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-tkt-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-geok-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-maa-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-bsc-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-mast-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-lsi-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-tcm-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-paras-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-matres-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-atm-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-kem-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-mfkm-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-csm-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-data-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-geom-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-geog-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-usp-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-ftdk-farm-jory': 'hy-ftdk-kandi-kojot',
+  'hy-ftdk-prov-jory': 'hy-ftdk-maisteri-kojot',
+  'hy-ftdk-mpharm-jory': 'hy-ftdk-maisteri-kojot',
+  'hy-bytdk-bio-jory': 'hy-bytdk-kandi-kojot',
+  'hy-bytdk-mole-jory': 'hy-bytdk-kandi-kojot',
+  'hy-bytdk-env-jory': 'hy-bytdk-kandi-kojot',
+  'hy-bytdk-eeb-jory': 'hy-bytdk-maisteri-kojot',
+  'hy-bytdk-ips-jory': 'hy-bytdk-maisteri-kojot',
+  'hy-bytdk-gmb-jory': 'hy-bytdk-maisteri-kojot',
+  'hy-bytdk-neuro-jory': 'hy-bytdk-maisteri-kojot',
+  'hy-bytdk-ecgs-jory': 'hy-bytdk-maisteri-kojot',
+  'hy-ktdk-eduk-jory': 'hy-ktdk-kandi-kojot',
+  'hy-ktdk-edum-jory': 'hy-ktdk-maisteri-kojot',
+  'hy-ktdk-ce-jory': 'hy-ktdk-maisteri-kojot',
+  'hy-valttdk-pvk-jory': 'hy-valttdk-kandi-kojot',
+  'hy-valttdk-yk-jory': 'hy-valttdk-kandi-kojot',
+  'hy-valttdk-sosk-jory': 'hy-valttdk-kandi-kojot',
+  'hy-valttdk-ecok-jory': 'hy-valttdk-kandi-kojot',
+  'hy-valttdk-film-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-pvm-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-gpc-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-ym-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-cos-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-ens-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-msv-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-sosm-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-econ-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-sote-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-sskh-ksv-jory': 'hy-sskh-kandi-kojot',
+  'hy-mmtdk-maatk-jory': 'hy-mmtdk-kandi-kojot',
+  'hy-mmtdk-metsak-jory': 'hy-mmtdk-kandi-kojot',
+  'hy-mmtdk-etk-jory': 'hy-mmtdk-kandi-kojot',
+  'hy-mmtdk-yet-jory': 'hy-mmtdk-kandi-kojot',
+  'hy-mmtdk-agri-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-mmtdk-agere-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-mmtdk-for-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-mmtdk-food-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-mmtdk-hnfb-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-mmtdk-ekm-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-mmtdk-mmb-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-eltdk-elk-jory': 'hy-eltdk-kandi-kojot',
+  'hy-eltdk-ell-jory': 'hy-eltdk-maisteri-kojot',
+  'hy-dp-dptheol-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dplaw-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-philartsoc-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dphistcult-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dplang-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-sky-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dpsocs-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-pyam-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dpe-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-seduce-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-clic-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dpbm-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-klto-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-docpop-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-findos-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dpdr-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-ils-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-bandm-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-cvm-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dphub-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-luova-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dpps-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-denvi-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-agforee-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-mbdp-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-foodhealth-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-papu-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-geodoc-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-atm-dp-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-chems-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-domast-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-matrena-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-docs-jory': 'hy-tohtoriohjelma-johtajat',
+}
+
+/**
+ * Maps kosu IAM to the name of TDKs, defined in data.js facultyMap
+ */
+const kosuFacultyMap = {
+  'hy-ypa-opa-kosu-kumpula': 'matemaattis-luonnontieteellinen',
+  'hy-ypa-opa-kosu-meilahti': ['lääketieteellinen'],
+  'hy-ypa-opa-kosu-kruununhaka': [
+    'oikeustieteellinen',
+    'valtiotieteellinen',
+    'kasvatustieteellinen',
+    'humanistinen',
+    'teologinen',
+    'svenska',
+  ],
+  'hy-ypa-opa-kosu-metsatalo': [
+    'oikeustieteellinen',
+    'valtiotieteellinen',
+    'kasvatustieteellinen',
+    'humanistinen',
+    'teologinen',
+    'svenska',
+  ],
+  'hy-ypa-opa-kosu-porthania': [
+    'oikeustieteellinen',
+    'valtiotieteellinen',
+    'kasvatustieteellinen',
+    'humanistinen',
+    'teologinen',
+    'svenska',
+  ],
+  'hy-ypa-opa-kosu-siltavuori': [
+    'oikeustieteellinen',
+    'valtiotieteellinen',
+    'kasvatustieteellinen',
+    'humanistinen',
+    'teologinen',
+    'svenska',
+  ],
+  'hy-ypa-opa-kosu-viikki': [
+    'bio- ja ympäristötieteellinen',
+    'farmasia',
+    'maatalous -metsätieteellinen',
+    'eläinlääketieteellinen',
+  ],
+}
+
+const doctoralIams = ['hy-tohtorikoulutus-johtoryhma', 'hy-tine']
+
+const doctoralSchoolMap = {
+  'hy-tutkijakoulut-hymy-jory': [
+    'T920101',
+    'T920102',
+    'T920103',
+    'T920104',
+    'T920105',
+    'T920106',
+    'T920107',
+    'T920108',
+    'T920109',
+    'T920110',
+    'T920111',
+  ],
+  'hy-tutkijakoulut-dshealth-jory': [
+    'T921101',
+    'T921102',
+    'T921103',
+    'T921104',
+    'T921105',
+    'T921106',
+    'T921107',
+    'T921108',
+    'T921109',
+  ],
+  'hy-tutkijakoulut-yeb-jory': ['T922101', 'T922102', 'T922103', 'T922104', 'T922105', 'T922106'],
+  'hy-tutkijakoulut-donasci-jory': ['T923101', 'T923102', 'T923103', 'T923104', 'T923105', 'T923106', 'T923107'],
+}
+
+const opetusVaradekaani = 'hy-varadekaanit-opetus'
+
+const dekaaniFacultyMap = {
+  'hy-ttdk-dekanaatti': 'teologinen',
+  'hy-oiktdk-dekanaatti': 'oikeustieteellinen',
+  'hy-ltdk-dekanaatti': 'lääketieteellinen',
+  'hy-humtdk-dekanaatti': 'humanistinen',
+  'hy-mltdk-dekanaatti': 'matemaattis-luonnontieteellinen',
+  'hy-ftdk-dekanaatti': 'farmasia',
+  'hy-bytdk-dekanaatti': 'bio- ja ympäristötieteellinen',
+  'hy-ktdk-dekanaatti': 'kasvatustieteellinen',
+  'hy-valttdk-dekanaatti': 'valtiotieteellinen',
+  'hy-sskh-rehtoraatti': 'svenska',
+  'hy-mmtdk-dekanaatti': 'maatalous -metsätieteellinen',
+  'hy-eltdk-dekanaatti': 'eläinlääketieteellinen',
+}
+
+const oodikoneIams = [
+  'grp-oodikone-staging-users',
+  'grp-oodikone-basic-staging-users',
+  'grp-oodikone-users',
+  'grp-oodikone-basic-users',
+]
+
+const universityWideGroups = ['hy-ypa-opa-opintoasiainpaallikot', 'hy-rehtoraatti']
+
+const superAdminGroups = ['grp-toska']
+
+const adminGroups = ['grp-ospa']
+
+const employeeGroups = ['hy-employees']
+
+const isSuperAdminIam = iam => superAdminGroups.includes(iam)
+
+const isAdminIam = iam => adminGroups.includes(iam)
+
+const isUniversityWideIam = iam => universityWideGroups.includes(iam)
+
+const isDoctoralIam = iam => doctoralIams.includes(iam)
+
+const isEmployeeIam = iam => employeeGroups.includes(iam)
+
+const iamToDoctoralSchool = iam => doctoralSchoolMap[iam]
+
+const getStudyLeaderGroup = iam => kojoMap[iam]
+
+const iamToOrganisationCode = iam => {
+  const organisationCodes = joryMap[iam]
+  if (Array.isArray(organisationCodes)) {
+    return organisationCodes
+  }
+  return [organisationCodes]
+}
+
+const kosuIamToFaculties = iam => {
+  const faculties = kosuFacultyMap[iam]
+  if (!faculties?.length > 0) return []
+
+  return faculties.map(f => facultyMap[f])
+}
+
+const dekaaniIamToFaculty = iam => {
+  const faculty = dekaaniFacultyMap[iam]
+  if (!faculty) return null
+  return facultyMap[faculty]
+}
+
+const relevantIAMs = []
+  .concat(Object.keys(joryMap))
+  .concat(Object.keys(kojoMap))
+  .concat(Object.values(kojoMap))
+  .concat(Object.keys(kosuFacultyMap))
+  .concat(Object.keys(dekaaniFacultyMap))
+  .concat(opetusVaradekaani)
+  .concat(doctoralIams)
+  .concat(Object.keys(doctoralSchoolMap))
+  .concat(universityWideGroups)
+  .concat(superAdminGroups)
+  .concat(adminGroups)
+  .concat(employeeGroups)
+  .concat(oodikoneIams)
+
+module.exports = {
+  isSuperAdminIam,
+  isAdminIam,
+  isUniversityWideIam,
+  isDoctoralIam,
+  isEmployeeIam,
+  iamToDoctoralSchool,
+  iamToOrganisationCode,
+  kosuIamToFaculties,
+  dekaaniIamToFaculty,
+  opetusVaradekaani,
+  getStudyLeaderGroup,
+  relevantIAMs,
+}

services/backend/config/common.js

@@ -0,0 +1,19 @@
+const mapToDegreeCode = organisationCode => {
+  if (!organisationCode) return ''
+  if (organisationCode.length < 7) return ''
+  const doctoral = organisationCode[0] === 'T'
+  if (doctoral) {
+    return organisationCode
+  }
+
+  const [start, end] = organisationCode.split('-')
+  if (end && end.length < 3) return ''
+  if (start.length < 2) return ''
+  const masters = end[0] === 'M'
+  const code = `${masters ? 'M' : 'K'}H${start.substr(0, 2)}_${end.substr(-3)}`
+  return code
+}
+
+module.exports = {
+  mapToDegreeCode,
+}

services/backend/package-lock.json

@@ -3,10 +3,15 @@
   "requires": true,
   "lockfileVersion": 1,
   "dependencies": {
+    "@colors/colors": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz",
+      "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ=="
+    },
     "@dabh/diagnostics": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz",
-      "integrity": "sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==",
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz",
+      "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==",
       "requires": {
         "colorspace": "1.1.x",
         "enabled": "2.0.x",
@@ -76,48 +81,48 @@
       }
     },
     "@sentry/tracing": {
-      "version": "6.11.0",
-      "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.11.0.tgz",
-      "integrity": "sha512-9VA1/SY++WeoMQI4K6n/sYgIdRtCu9NLWqmGqu/5kbOtESYFgAt1DqSyqGCr00ZjQiC2s7tkDkTNZb38K6KytQ==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.18.2.tgz",
+      "integrity": "sha512-hg6NLqrqJ5sUPTyWEQ2RqdnhQVnyLtx8II0IyWxQLDWD8UCe3Mu6G7mroDtakPWcP+lWz6OnKfMEfuhMcxR8fw==",
       "requires": {
-        "@sentry/hub": "6.11.0",
-        "@sentry/minimal": "6.11.0",
-        "@sentry/types": "6.11.0",
-        "@sentry/utils": "6.11.0",
+        "@sentry/hub": "6.18.2",
+        "@sentry/minimal": "6.18.2",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
         "@sentry/hub": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.11.0.tgz",
-          "integrity": "sha512-pT9hf+ZJfVFpoZopoC+yJmFNclr4NPqPcl2cgguqCHb69DklD1NxgBNWK8D6X05qjnNFDF991U6t1mxP9HrGuw==",
+          "version": "6.18.2",
+          "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.18.2.tgz",
+          "integrity": "sha512-d0AugekMkbnN12b4EXMjseJxtLPc9S20DGobCPUb4oAQT6S2oDQEj1jwP6PQ5vtgyy+GMYWxBMgqAQ4pjVYISQ==",
           "requires": {
-            "@sentry/types": "6.11.0",
-            "@sentry/utils": "6.11.0",
+            "@sentry/types": "6.18.2",
+            "@sentry/utils": "6.18.2",
             "tslib": "^1.9.3"
           }
         },
         "@sentry/minimal": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.11.0.tgz",
-          "integrity": "sha512-XkZ7qrdlGp4IM/gjGxf1Q575yIbl5RvPbg+WFeekpo16Ufvzx37Mr8c2xsZaWosISVyE6eyFpooORjUlzy8EDw==",
+          "version": "6.18.2",
+          "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.18.2.tgz",
+          "integrity": "sha512-n7KYuo34W2LxE+3dnZ47of7XHuORINCnXq66XH72eoj67tf0XeWbIhEJrYGmoLRyRfoCYYrBLWiDl/uTjLzrzQ==",
           "requires": {
-            "@sentry/hub": "6.11.0",
-            "@sentry/types": "6.11.0",
+            "@sentry/hub": "6.18.2",
+            "@sentry/types": "6.18.2",
             "tslib": "^1.9.3"
           }
         },
         "@sentry/types": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.11.0.tgz",
-          "integrity": "sha512-gm5H9eZhL6bsIy/h3T+/Fzzz2vINhHhqd92CjHle3w7uXdTdFV98i2pDpErBGNTSNzbntqOMifYEB5ENtZAvcg=="
+          "version": "6.18.2",
+          "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.18.2.tgz",
+          "integrity": "sha512-WzpJf/Q5aORTzrSwer/As1NlO90dBAQpaHV2ikDDKqOyMWEgjKb5/4gh59p9gH8JMMnLetP1AvQel0fOj5UnUw=="
         },
         "@sentry/utils": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.11.0.tgz",
-          "integrity": "sha512-IOvyFHcnbRQxa++jO+ZUzRvFHEJ1cZjrBIQaNVc0IYF0twUOB5PTP6joTcix38ldaLeapaPZ9LGfudbvYvxkdg==",
+          "version": "6.18.2",
+          "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.18.2.tgz",
+          "integrity": "sha512-EC619jesknyu4xpwud5WC/5odYLz6JUy7OSFy5405PpdGeh/m8XUvuJAx4zDx0Iz/Mlk0S1Md+ZcQwqkv39dkw==",
           "requires": {
-            "@sentry/types": "6.11.0",
+            "@sentry/types": "6.18.2",
             "tslib": "^1.9.3"
           }
         }
@@ -137,10 +142,28 @@
         "tslib": "^1.9.3"
       }
     },
+    "@types/debug": {
+      "version": "4.1.7",
+      "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz",
+      "integrity": "sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==",
+      "requires": {
+        "@types/ms": "*"
+      }
+    },
+    "@types/ms": {
+      "version": "0.7.31",
+      "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz",
+      "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA=="
+    },
     "@types/node": {
-      "version": "14.14.39",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.39.tgz",
-      "integrity": "sha512-Qipn7rfTxGEDqZiezH+wxqWYR8vcXq5LRpZrETD19Gs4o8LbklbmqotSUsMU+s5G3PJwMRDfNEYoxrcBwIxOuw=="
+      "version": "17.0.21",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz",
+      "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ=="
+    },
+    "@types/validator": {
+      "version": "13.7.1",
+      "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.7.1.tgz",
+      "integrity": "sha512-I6OUIZ5cYRk5lp14xSOAiXjWrfVoMZVjDuevBYgQDYzZIjsf2CAISpEcXOkFAtpAHbmWIDLcZObejqny/9xq5Q=="
     },
     "abbrev": {
       "version": "1.1.1",
@@ -149,12 +172,12 @@
       "dev": true
     },
     "accepts": {
-      "version": "1.3.7",
-      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz",
-      "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==",
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
+      "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==",
       "requires": {
-        "mime-types": "~2.1.24",
-        "negotiator": "0.6.2"
+        "mime-types": "~2.1.34",
+        "negotiator": "0.6.3"
       }
     },
     "agent-base": {
@@ -165,11 +188,6 @@
         "debug": "4"
       }
     },
-    "any-promise": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
-      "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8="
-    },
     "arr-diff": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
@@ -206,9 +224,9 @@
       "dev": true
     },
     "async": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/async/-/async-3.2.1.tgz",
-      "integrity": "sha512-XdD5lRO/87udXCMC9meWdYiR+Nq6ZjUfXidViUZGu2F1MO4T3XwZ1et0hb2++BgLfhyJwy44BGB/yx80ABx8hg=="
+      "version": "3.2.3",
+      "resolved": "https://registry.npmjs.org/async/-/async-3.2.3.tgz",
+      "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g=="
     },
     "async-each": {
       "version": "1.0.3",
@@ -217,9 +235,9 @@
       "dev": true
     },
     "async-lock": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.3.0.tgz",
-      "integrity": "sha512-8A7SkiisnEgME2zEedtDYPxUPzdv3x//E7n5IFktPAtMYSEAV7eNJF0rMwrVyUFj6d/8rgajLantbjcNRQYXIg=="
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.3.1.tgz",
+      "integrity": "sha512-zK7xap9UnttfbE23JmcrNIyueAn6jWshihJqA33U/hEnKprF/lVGBDsBv/bqLm2YMMl1DnpHhUY044eA0t1TUw=="
     },
     "atob": {
       "version": "2.1.2",
@@ -228,9 +246,9 @@
       "dev": true
     },
     "axios": {
-      "version": "0.21.2",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.2.tgz",
-      "integrity": "sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg==",
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
       "requires": {
         "follow-redirects": "^1.14.0"
       }
@@ -291,20 +309,20 @@
       "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
     },
     "body-parser": {
-      "version": "1.19.0",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
-      "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
+      "version": "1.19.2",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz",
+      "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==",
       "requires": {
-        "bytes": "3.1.0",
+        "bytes": "3.1.2",
         "content-type": "~1.0.4",
         "debug": "2.6.9",
         "depd": "~1.1.2",
-        "http-errors": "1.7.2",
+        "http-errors": "1.8.1",
         "iconv-lite": "0.4.24",
         "on-finished": "~2.3.0",
-        "qs": "6.7.0",
-        "raw-body": "2.4.0",
-        "type-is": "~1.6.17"
+        "qs": "6.9.7",
+        "raw-body": "2.4.3",
+        "type-is": "~1.6.18"
       },
       "dependencies": {
         "debug": {
@@ -338,9 +356,9 @@
       "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw=="
     },
     "bytes": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
-      "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg=="
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
+      "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="
     },
     "cache-base": {
       "version": "1.0.1",
@@ -514,12 +532,12 @@
       }
     },
     "color": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz",
-      "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==",
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz",
+      "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==",
       "requires": {
-        "color-convert": "^1.9.1",
-        "color-string": "^1.5.2"
+        "color-convert": "^1.9.3",
+        "color-string": "^1.6.0"
       }
     },
     "color-convert": {
@@ -536,25 +554,20 @@
       "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
     },
     "color-string": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.6.0.tgz",
-      "integrity": "sha512-c/hGS+kRWJutUBEngKKmk4iH3sD59MBkoxVapS/0wgpCz2u7XsNloxknyvBhzwEs1IbV36D9PwqLPJ2DTu3vMA==",
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.0.tgz",
+      "integrity": "sha512-9Mrz2AQLefkH1UvASKj6v6hj/7eWgjnT/cVsR8CumieLoT+g900exWeNogqtweI8dxloXN9BDQTYro1oWu/5CQ==",
       "requires": {
         "color-name": "^1.0.0",
         "simple-swizzle": "^0.2.2"
       }
     },
-    "colors": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz",
-      "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA=="
-    },
     "colorspace": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz",
-      "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==",
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz",
+      "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==",
       "requires": {
-        "color": "3.0.x",
+        "color": "^3.1.3",
         "text-hex": "1.0.x"
       }
     },
@@ -571,11 +584,18 @@
       "dev": true
     },
     "content-disposition": {
-      "version": "0.5.3",
-      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
-      "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==",
+      "version": "0.5.4",
+      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
+      "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==",
       "requires": {
-        "safe-buffer": "5.1.2"
+        "safe-buffer": "5.2.1"
+      },
+      "dependencies": {
+        "safe-buffer": {
+          "version": "5.2.1",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+          "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
+        }
       }
     },
     "content-type": {
@@ -602,7 +622,8 @@
     "core-util-is": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
-      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
+      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
+      "dev": true
     },
     "cors": {
       "version": "2.8.5",
@@ -838,16 +859,16 @@
       }
     },
     "express": {
-      "version": "4.17.1",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz",
-      "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==",
+      "version": "4.17.3",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz",
+      "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==",
       "requires": {
-        "accepts": "~1.3.7",
+        "accepts": "~1.3.8",
         "array-flatten": "1.1.1",
-        "body-parser": "1.19.0",
-        "content-disposition": "0.5.3",
+        "body-parser": "1.19.2",
+        "content-disposition": "0.5.4",
         "content-type": "~1.0.4",
-        "cookie": "0.4.0",
+        "cookie": "0.4.2",
         "cookie-signature": "1.0.6",
         "debug": "2.6.9",
         "depd": "~1.1.2",
@@ -861,13 +882,13 @@
         "on-finished": "~2.3.0",
         "parseurl": "~1.3.3",
         "path-to-regexp": "0.1.7",
-        "proxy-addr": "~2.0.5",
-        "qs": "6.7.0",
+        "proxy-addr": "~2.0.7",
+        "qs": "6.9.7",
         "range-parser": "~1.2.1",
-        "safe-buffer": "5.1.2",
-        "send": "0.17.1",
-        "serve-static": "1.14.1",
-        "setprototypeof": "1.1.1",
+        "safe-buffer": "5.2.1",
+        "send": "0.17.2",
+        "serve-static": "1.14.2",
+        "setprototypeof": "1.2.0",
         "statuses": "~1.5.0",
         "type-is": "~1.6.18",
         "utils-merge": "1.0.1",
@@ -875,9 +896,9 @@
       },
       "dependencies": {
         "cookie": {
-          "version": "0.4.0",
-          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz",
-          "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg=="
+          "version": "0.4.2",
+          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz",
+          "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA=="
         },
         "debug": {
           "version": "2.6.9",
@@ -891,6 +912,11 @@
           "version": "2.0.0",
           "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
           "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+        },
+        "safe-buffer": {
+          "version": "5.2.1",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+          "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
         }
       }
     },
@@ -951,11 +977,6 @@
         }
       }
     },
-    "fast-safe-stringify": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz",
-      "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA=="
-    },
     "fecha": {
       "version": "4.2.1",
       "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.1.tgz",
@@ -1003,9 +1024,9 @@
       "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="
     },
     "follow-redirects": {
-      "version": "1.14.3",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.3.tgz",
-      "integrity": "sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw=="
+      "version": "1.14.9",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
+      "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w=="
     },
     "for-in": {
       "version": "1.0.2",
@@ -1014,9 +1035,9 @@
       "dev": true
     },
     "forwarded": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
-      "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
+      "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="
     },
     "fragment-cache": {
       "version": "0.2.1",
@@ -1106,22 +1127,15 @@
       }
     },
     "http-errors": {
-      "version": "1.7.2",
-      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
-      "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz",
+      "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==",
       "requires": {
         "depd": "~1.1.2",
-        "inherits": "2.0.3",
-        "setprototypeof": "1.1.1",
+        "inherits": "2.0.4",
+        "setprototypeof": "1.2.0",
         "statuses": ">= 1.5.0 < 2",
-        "toidentifier": "1.0.0"
-      },
-      "dependencies": {
-        "inherits": {
-          "version": "2.0.3",
-          "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
-          "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
-        }
+        "toidentifier": "1.0.1"
       }
     },
     "https-proxy-agent": {
@@ -1160,9 +1174,9 @@
       "dev": true
     },
     "inflection": {
-      "version": "1.13.1",
-      "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.1.tgz",
-      "integrity": "sha512-dldYtl2WlN0QDkIDtg8+xFwOS2Tbmp12t1cHa5/YClU6ZQjTFm7B66UcVbh9NQB+HvT5BAd2t5+yKsBkw5pcqA=="
+      "version": "1.13.2",
+      "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.2.tgz",
+      "integrity": "sha512-cmZlljCRTBFouT8UzMzrGcVEvkv6D/wBdcdKG7J1QH5cXjtU75Dm+P27v9EKu/Y43UYyCJd1WC4zLebRrC8NBw=="
     },
     "inherits": {
       "version": "2.0.4",
@@ -1303,7 +1317,8 @@
     "isarray": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
+      "dev": true
     },
     "isexe": {
       "version": "2.0.0",
@@ -1334,14 +1349,14 @@
       "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
     },
     "logform": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz",
-      "integrity": "sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg==",
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/logform/-/logform-2.4.0.tgz",
+      "integrity": "sha512-CPSJw4ftjf517EhXZGGvTHHkYobo7ZCc0kvwUoOYcjfR2UVrI66RHj8MCrfAdEitdmFqbu2BYdYs8FHHZSb6iw==",
       "requires": {
-        "colors": "^1.2.1",
-        "fast-safe-stringify": "^2.0.4",
+        "@colors/colors": "1.5.0",
         "fecha": "^4.2.0",
         "ms": "^2.1.1",
+        "safe-stable-stringify": "^2.3.1",
         "triple-beam": "^1.3.0"
       }
     },
@@ -1400,16 +1415,16 @@
       "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
     },
     "mime-db": {
-      "version": "1.47.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz",
-      "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw=="
+      "version": "1.51.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz",
+      "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g=="
     },
     "mime-types": {
-      "version": "2.1.30",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz",
-      "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==",
+      "version": "2.1.34",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz",
+      "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==",
       "requires": {
-        "mime-db": "1.47.0"
+        "mime-db": "1.51.0"
       }
     },
     "minimatch": {
@@ -1443,9 +1458,9 @@
       "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ=="
     },
     "moment-timezone": {
-      "version": "0.5.33",
-      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz",
-      "integrity": "sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==",
+      "version": "0.5.34",
+      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.34.tgz",
+      "integrity": "sha512-3zAEHh2hKUs3EXLESx/wsgw6IQdusOT8Bxm3D9UrHPQR7zlMmzwybC8zHEM1tQ4LJwP7fcxrWr8tuBg05fFCbg==",
       "requires": {
         "moment": ">= 2.9.0"
       }
@@ -1514,9 +1529,9 @@
       }
     },
     "negotiator": {
-      "version": "0.6.2",
-      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
-      "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
+      "version": "0.6.3",
+      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
+      "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="
     },
     "nodemon": {
       "version": "1.19.4",
@@ -2205,14 +2220,14 @@
       "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
     },
     "pg": {
-      "version": "8.7.1",
-      "resolved": "https://registry.npmjs.org/pg/-/pg-8.7.1.tgz",
-      "integrity": "sha512-7bdYcv7V6U3KAtWjpQJJBww0UEsWuh4yQ/EjNf2HeO/NnvKjpvhEIe/A/TleP6wtmSKnUnghs5A9jUoK6iDdkA==",
+      "version": "8.7.3",
+      "resolved": "https://registry.npmjs.org/pg/-/pg-8.7.3.tgz",
+      "integrity": "sha512-HPmH4GH4H3AOprDJOazoIcpI49XFsHCe8xlrjHkWiapdbHK+HLtbm/GQzXYAZwmPju/kzKhjaSfMACG+8cgJcw==",
       "requires": {
         "buffer-writer": "2.0.0",
         "packet-reader": "1.0.0",
         "pg-connection-string": "^2.5.0",
-        "pg-pool": "^3.4.1",
+        "pg-pool": "^3.5.1",
         "pg-protocol": "^1.5.0",
         "pg-types": "^2.1.0",
         "pgpass": "1.x"
@@ -2229,9 +2244,9 @@
       "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="
     },
     "pg-pool": {
-      "version": "3.4.1",
-      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.4.1.tgz",
-      "integrity": "sha512-TVHxR/gf3MeJRvchgNHxsYsTCHQ+4wm3VIHSS19z8NC0+gioEhq1okDY1sm/TYbfoP6JLFx01s0ShvZ3puP/iQ=="
+      "version": "3.5.1",
+      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.1.tgz",
+      "integrity": "sha512-6iCR0wVrro6OOHFsyavV+i6KYL4lVNyYAB9RD18w66xSzN+d8b66HiwuP30Gp1SH5O9T82fckkzsRjlrhD0ioQ=="
     },
     "pg-protocol": {
       "version": "1.5.0",
@@ -2251,11 +2266,11 @@
       }
     },
     "pgpass": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.4.tgz",
-      "integrity": "sha512-YmuA56alyBq7M59vxVBfPJrGSozru8QAdoNlWuW3cz8l+UX3cWge0vTvjKhsSHSJpo3Bom8/Mm6hf0TR5GY0+w==",
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+      "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
       "requires": {
-        "split2": "^3.1.1"
+        "split2": "^4.1.0"
       }
     },
     "posix-character-classes": {
@@ -2290,14 +2305,15 @@
     "process-nextick-args": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
-      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
+      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
+      "dev": true
     },
     "proxy-addr": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz",
-      "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==",
+      "version": "2.0.7",
+      "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
+      "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==",
       "requires": {
-        "forwarded": "~0.1.2",
+        "forwarded": "0.2.0",
         "ipaddr.js": "1.9.1"
       }
     },
@@ -2314,9 +2330,9 @@
       "dev": true
     },
     "qs": {
-      "version": "6.7.0",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
-      "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ=="
+      "version": "6.9.7",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz",
+      "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw=="
     },
     "range-parser": {
       "version": "1.2.1",
@@ -2324,12 +2340,12 @@
       "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
     },
     "raw-body": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
-      "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
+      "version": "2.4.3",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz",
+      "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==",
       "requires": {
-        "bytes": "3.1.0",
-        "http-errors": "1.7.2",
+        "bytes": "3.1.2",
+        "http-errors": "1.8.1",
         "iconv-lite": "0.4.24",
         "unpipe": "1.0.0"
       }
@@ -2350,6 +2366,7 @@
       "version": "2.3.7",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
       "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
+      "dev": true,
       "requires": {
         "core-util-is": "~1.0.0",
         "inherits": "~2.0.3",
@@ -2430,12 +2447,9 @@
       "dev": true
     },
     "retry-as-promised": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-3.2.0.tgz",
-      "integrity": "sha512-CybGs60B7oYU/qSQ6kuaFmRd9sTZ6oXSc0toqePvV74Ac6/IFZSI1ReFQmtCN+uvW1Mtqdwpvt/LGOiCBAY2Mg==",
-      "requires": {
-        "any-promise": "^1.3.0"
-      }
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-5.0.0.tgz",
+      "integrity": "sha512-6S+5LvtTl2ggBumk04hBo/4Uf6fRJUwIgunGZ7CYEBCeufGFW1Pu6ucUf/UskHeWOIsUcLOGLFXPig5tR5V1nA=="
     },
     "safe-buffer": {
       "version": "5.1.2",
@@ -2451,11 +2465,39 @@
         "ret": "~0.1.10"
       }
     },
+    "safe-stable-stringify": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.3.1.tgz",
+      "integrity": "sha512-kYBSfT+troD9cDA85VDnHZ1rpHC50O0g1e6WlGHVCz/g+JS+9WKLj+XwFYyR8UbrZN8ll9HUpDAAddY58MGisg=="
+    },
     "safer-buffer": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
+    "semver": {
+      "version": "7.3.5",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
+      "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
+      "requires": {
+        "lru-cache": "^6.0.0"
+      },
+      "dependencies": {
+        "lru-cache": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+          "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+          "requires": {
+            "yallist": "^4.0.0"
+          }
+        },
+        "yallist": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
+        }
+      }
+    },
     "semver-diff": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-2.1.0.tgz",
@@ -2474,9 +2516,9 @@
       }
     },
     "send": {
-      "version": "0.17.1",
-      "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz",
-      "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==",
+      "version": "0.17.2",
+      "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz",
+      "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==",
       "requires": {
         "debug": "2.6.9",
         "depd": "~1.1.2",
@@ -2485,9 +2527,9 @@
         "escape-html": "~1.0.3",
         "etag": "~1.8.1",
         "fresh": "0.5.2",
-        "http-errors": "~1.7.2",
+        "http-errors": "1.8.1",
         "mime": "1.6.0",
-        "ms": "2.1.1",
+        "ms": "2.1.3",
         "on-finished": "~2.3.0",
         "range-parser": "~1.2.1",
         "statuses": "~1.5.0"
@@ -2507,76 +2549,66 @@
               "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
             }
           }
-        },
-        "ms": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
-          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
         }
       }
     },
     "sequelize": {
-      "version": "6.6.5",
-      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.6.5.tgz",
-      "integrity": "sha512-QyRrJrDRiwuiILqTMHUA1yWOPIL12KlfmgZ3hnzQwbMvp2vJ6fzu9bYJQB+qPMosck4mBUggY4Cjoc6Et8FBIQ==",
+      "version": "6.17.0",
+      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.17.0.tgz",
+      "integrity": "sha512-AZus+0YZDq91Zg0hzDaO5atTzHgJruI23V8nBlAhkLuI81Z53nSRdAe/4R1A6vGOZ/RfCLP9idF4tfQnoAsM5A==",
       "requires": {
-        "debug": "^4.1.1",
-        "dottie": "^2.0.0",
-        "inflection": "1.13.1",
-        "lodash": "^4.17.20",
-        "moment": "^2.26.0",
-        "moment-timezone": "^0.5.31",
-        "retry-as-promised": "^3.2.0",
-        "semver": "^7.3.2",
-        "sequelize-pool": "^6.0.0",
+        "@types/debug": "^4.1.7",
+        "@types/validator": "^13.7.1",
+        "debug": "^4.3.3",
+        "dottie": "^2.0.2",
+        "inflection": "^1.13.2",
+        "lodash": "^4.17.21",
+        "moment": "^2.29.1",
+        "moment-timezone": "^0.5.34",
+        "pg-connection-string": "^2.5.0",
+        "retry-as-promised": "^5.0.0",
+        "semver": "^7.3.5",
+        "sequelize-pool": "^7.1.0",
         "toposort-class": "^1.0.1",
-        "uuid": "^8.1.0",
-        "validator": "^13.6.0",
+        "uuid": "^8.3.2",
+        "validator": "^13.7.0",
         "wkx": "^0.5.0"
       },
       "dependencies": {
-        "lru-cache": {
-          "version": "6.0.0",
-          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
-          "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+        "debug": {
+          "version": "4.3.3",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz",
+          "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==",
           "requires": {
-            "yallist": "^4.0.0"
+            "ms": "2.1.2"
           }
         },
-        "semver": {
-          "version": "7.3.5",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
-          "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
-          "requires": {
-            "lru-cache": "^6.0.0"
-          }
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
         },
         "uuid": {
           "version": "8.3.2",
           "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
           "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
-        },
-        "yallist": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
         }
       }
     },
     "sequelize-pool": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-6.1.0.tgz",
-      "integrity": "sha512-4YwEw3ZgK/tY/so+GfnSgXkdwIJJ1I32uZJztIEgZeAO6HMgj64OzySbWLgxj+tXhZCJnzRfkY9gINw8Ft8ZMg=="
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-7.1.0.tgz",
+      "integrity": "sha512-G9c0qlIWQSK29pR/5U2JF5dDQeqqHRragoyahj/Nx4KOOQ3CPPfzxnfqFPCSB7x5UgjOgnZ61nSxz+fjDpRlJg=="
     },
     "serve-static": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",
-      "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==",
+      "version": "1.14.2",
+      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz",
+      "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==",
       "requires": {
         "encodeurl": "~1.0.2",
         "escape-html": "~1.0.3",
         "parseurl": "~1.3.3",
-        "send": "0.17.1"
+        "send": "0.17.2"
       }
     },
     "set-value": {
@@ -2618,9 +2650,9 @@
       }
     },
     "setprototypeof": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
-      "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw=="
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
+      "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
     },
     "shebang-command": {
       "version": "1.2.0",
@@ -2842,24 +2874,9 @@
       }
     },
     "split2": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz",
-      "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==",
-      "requires": {
-        "readable-stream": "^3.0.0"
-      },
-      "dependencies": {
-        "readable-stream": {
-          "version": "3.6.0",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
-          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
-          "requires": {
-            "inherits": "^2.0.3",
-            "string_decoder": "^1.1.1",
-            "util-deprecate": "^1.0.1"
-          }
-        }
-      }
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz",
+      "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ=="
     },
     "stack-trace": {
       "version": "0.0.10",
@@ -3072,9 +3089,9 @@
       }
     },
     "toidentifier": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
-      "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw=="
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
+      "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="
     },
     "toposort-class": {
       "version": "1.0.1",
@@ -3323,19 +3340,20 @@
       }
     },
     "winston": {
-      "version": "3.3.3",
-      "resolved": "https://registry.npmjs.org/winston/-/winston-3.3.3.tgz",
-      "integrity": "sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw==",
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/winston/-/winston-3.6.0.tgz",
+      "integrity": "sha512-9j8T75p+bcN6D00sF/zjFVmPp+t8KMPB1MzbbzYjeN9VWxdsYnTB40TkbNUEXAmILEfChMvAMgidlX64OG3p6w==",
       "requires": {
         "@dabh/diagnostics": "^2.0.2",
-        "async": "^3.1.0",
+        "async": "^3.2.3",
         "is-stream": "^2.0.0",
-        "logform": "^2.2.0",
+        "logform": "^2.4.0",
         "one-time": "^1.0.0",
         "readable-stream": "^3.4.0",
+        "safe-stable-stringify": "^2.3.1",
         "stack-trace": "0.0.x",
         "triple-beam": "^1.3.0",
-        "winston-transport": "^4.4.0"
+        "winston-transport": "^4.5.0"
       },
       "dependencies": {
         "is-stream": {
@@ -3356,12 +3374,25 @@
       }
     },
     "winston-transport": {
-      "version": "4.4.0",
-      "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz",
-      "integrity": "sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw==",
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.5.0.tgz",
+      "integrity": "sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q==",
       "requires": {
-        "readable-stream": "^2.3.7",
-        "triple-beam": "^1.2.0"
+        "logform": "^2.3.2",
+        "readable-stream": "^3.6.0",
+        "triple-beam": "^1.3.0"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "3.6.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
+          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
+          "requires": {
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
+          }
+        }
       }
     },
     "wkx": {

services/backend/package.json

@@ -8,27 +8,27 @@
   },
   "dependencies": {
     "@sentry/node": "^5.30.0",
-    "@sentry/tracing": "^6.11.0",
-    "async": "^3.2.1",
-    "async-lock": "^1.3.0",
-    "axios": "^0.21.2",
+    "@sentry/tracing": "^6.18.2",
+    "async": "^3.2.3",
+    "async-lock": "^1.3.1",
+    "axios": "^0.21.4",
     "bluebird": "^3.7.2",
     "cors": "^2.8.5",
     "cron": "^1.8.2",
-    "express": "^4.17.1",
+    "express": "^4.17.3",
     "express-async-errors": "^3.1.1",
     "lodash": "^4.17.21",
     "lru-cache": "^5.1.1",
     "moment": "^2.29.1",
-    "moment-timezone": "^0.5.33",
+    "moment-timezone": "^0.5.34",
     "morgan": "^1.10.0",
-    "pg": "^8.7.1",
+    "pg": "^8.7.3",
     "redis": "^3.1.2",
-    "sequelize": "^6.6.5",
+    "sequelize": "^6.17.0",
     "umzug": "^2.3.0",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "uuid": "^3.4.0",
-    "winston": "^3.3.3"
+    "winston": "^3.6.0"
   },
   "devDependencies": {
     "nodemon": "^1.19.4"

services/backend/src/database/migrations_kone/20220307_00_remove_thesis_courses.js

@@ -0,0 +1,6 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('thesis_courses')
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_user/20220712_00_add_iam_groups.js

@@ -0,0 +1,14 @@
+const { ARRAY, STRING } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.addColumn('users', 'iam_groups', {
+      type: ARRAY(STRING),
+      allowNull: false,
+      defaultValue: [],
+    })
+  },
+  down: async queryInterface => {
+    await queryInterface.removeColumn('users', 'iam_groups')
+  },
+}

services/backend/src/events.js

@@ -1,13 +1,6 @@
 const { CronJob } = require('cron')
 const moment = require('moment')
-const {
-  refreshProtoC,
-  refreshStatus,
-  refreshStatusGraduated,
-  refreshUber,
-  refreshProtoCProgramme,
-  getStartYears,
-} = require('./services/trends')
+const { refreshStatus, refreshStatusGraduated, refreshUber, getStartYears } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
 const { updateBasicView, updateStudytrackView } = require('./services/studyprogrammeUpdates')
@@ -25,7 +18,9 @@ const refreshStudyrightAssociations = async () => {
 
 const refreshNewOverviews = async () => {
   logger.info('Refreshing studyprogramme and studytrack overview statistics for all programmes')
-  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH') || code.includes('MH'))
+  const codes = (await getAllProgrammes())
+    .map(p => p.code)
+    .filter(code => code.includes('KH') || code.includes('MH') || /^(T)[0-9]{6}$/.test(code))
   let ready = 0
   for (const code of codes) {
     try {
@@ -46,19 +41,6 @@ const refreshTeacherLeaderboard = async () => {
   await findAndSaveTeachers(currentSemestersYearCode - 1, currentSemestersYearCode)
 }
 
-const refreshProtoCtoRedis = async () => {
-  logger.info('Refreshing CDS ProtoC')
-  const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false' }
-  const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false' }
-  const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true' }
-  const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true' }
-  await refreshProtoC(defaultQuery)
-  await refreshProtoC(onlyOld)
-  await refreshProtoC(onlyEnr)
-  await refreshProtoC(bothToggles)
-  logger.info('Refreshing CDS ProtoC doned')
-}
-
 const refreshStatusToRedis = async () => {
   const unixMillis = moment().valueOf()
   const date = new Date(Number(unixMillis))
@@ -95,30 +77,6 @@ const refreshUberToRedis = async () => {
   logger.info(`Refreshing CDS Uber data doned`)
 }
 
-const refreshProtoCProgrammeToRedis = async () => {
-  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH'))
-  for (const code of codes) {
-    logger.info(`Refreshing ProtoCProgramme code ${code}`)
-    const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
-    const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
-    const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
-    const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true', code }
-    const queries = [defaultQuery, onlyOld, onlyEnr, bothToggles]
-    for (const query of queries) {
-      try {
-        await refreshProtoCProgramme(query)
-      } catch (e) {
-        logger.error({
-          message: `Error when refreshing ProtocProgramme code ${code}, query ${Object.keys({ query })[0]}`,
-          meta: e,
-        }) // object.keys is hax to print query variable name
-      }
-    }
-    logger.info(`Refreshing ProtoCProgramme code ${code} doned`)
-  }
-  logger.info(`Refreshing protocprogramme doned`)
-}
-
 const refreshStatistics = async () => {
   const statfuncs = [refreshStudyrightAssociations, refreshTeacherLeaderboard]
   logger.info('Refreshing statistics')
@@ -129,7 +87,7 @@ const refreshStatistics = async () => {
 }
 
 const refreshTrends = async () => {
-  const trendfuncs = [refreshProtoCtoRedis, refreshStatusToRedis, refreshUberToRedis, refreshProtoCProgrammeToRedis]
+  const trendfuncs = [refreshStatusToRedis, refreshUberToRedis]
   logger.info('Refreshing trends')
   for (const func of trendfuncs) {
     await func()

services/backend/src/middleware/currentUserMiddleware.js

@@ -3,6 +3,7 @@ const { ApplicationError } = require('../util/customErrors')
 const { getUser, getMockedUser } = require('../services/userService')
 const { requiredGroup } = require('../conf-backend')
 const _ = require('lodash')
+const { relevantIAMs } = require('../../config/IAMConfig')
 
 const parseIamGroups = iamGroups => iamGroups?.split(';').filter(Boolean) ?? []
 
@@ -24,7 +25,7 @@ const currentUserMiddleware = async (req, _res, next) => {
     throw new ApplicationError('Not enough data in request headers', 403, { logoutUrl })
   }
 
-  const iamGroups = parseIamGroups(hygroupcn)
+  const iamGroups = parseIamGroups(hygroupcn).filter(iam => relevantIAMs.includes(iam))
 
   if (!hasRequiredIamGroup(iamGroups)) {
     throw new ApplicationError('User does not have required iam group', 403, { logoutUrl })

services/backend/src/models/course.js

@@ -50,6 +50,10 @@ Course.init(
     course_unit_type: {
       type: STRING,
     },
+    mainCourseCode: {
+      field: 'main_course_code',
+      type: STRING,
+    },
   },
   {
     underscored: false,

services/backend/src/models/enrollment.js

@@ -0,0 +1,69 @@
+const { Model, DATE, INTEGER, STRING, BOOLEAN } = require('sequelize')
+const { dbConnections } = require('../database/connection')
+
+class Enrollment extends Model {}
+
+Enrollment.init(
+  {
+    id: {
+      primaryKey: true,
+      type: STRING,
+    },
+    studentnumber: {
+      type: STRING,
+      references: {
+        model: 'student',
+        key: 'studentnumber',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+    },
+    course_code: {
+      type: STRING,
+    },
+    state: {
+      type: STRING,
+    },
+    enrollment_date_time: {
+      type: DATE,
+    },
+    course_id: {
+      type: STRING,
+      references: {
+        model: 'course',
+        key: 'id',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+    },
+    semester_composite: {
+      type: STRING,
+      references: {
+        model: 'semesters',
+        key: 'composite',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+    },
+    semestercode: {
+      type: INTEGER,
+    },
+    createdAt: {
+      type: DATE,
+    },
+    updatedAt: {
+      type: DATE,
+    },
+    is_open: {
+      type: BOOLEAN,
+    },
+  },
+  {
+    underscored: false,
+    sequelize: dbConnections.sequelize,
+    modelName: 'enrollment',
+    tableName: 'enrollment',
+  }
+)
+
+module.exports = Enrollment

services/backend/src/models/index.js

@@ -10,6 +10,7 @@ const CreditType = require('./creditType')
 const Credit = require('./credit')
 const CreditTeacher = require('./creditTeacher')
 const ElementDetail = require('./elementDetail')
+const Studyplan = require('./studyplan')
 const StudyrightExtent = require('./studyrightExtent')
 const Studyright = require('./studyright')
 const StudyrightElement = require('./studyrightElement')
@@ -17,6 +18,7 @@ const Transfer = require('./transfer')
 const ProgrammeModule = require('./programmeModule')
 const ProgrammeModuleChild = require('./programmeModuleChild')
 const ExcludedCourse = require('./excludedCourse')
+const Enrollment = require('./enrollment')
 
 CourseType.hasMany(Course, { foreignKey: 'coursetypecode', sourceKey: 'coursetypecode' })
 Course.belongsTo(CourseType, { foreignKey: 'coursetypecode', targetKey: 'coursetypecode' })
@@ -62,6 +64,9 @@ Teacher.belongsToMany(Credit, { through: CreditTeacher, foreignKey: 'teacher_id'
 Organization.hasMany(Studyright, { foreignKey: 'facultyCode', sourceKey: 'code' })
 Studyright.belongsTo(Organization, { foreignKey: 'facultyCode', targetKey: 'code' })
 
+Studyplan.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
+Student.hasMany(Studyplan, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
+
 Studyright.belongsTo(Student, { foreignKey: 'studentStudentnumber', targetKey: 'studentnumber' })
 Student.hasMany(Studyright, { foreignKey: 'studentStudentnumber', sourceKey: 'studentnumber' })
 
@@ -88,6 +93,12 @@ Studyright.hasMany(Transfer, { foreignKey: 'studyrightid', sourceKey: 'studyrigh
 Transfer.belongsTo(ElementDetail, { as: 'source', foreignKey: 'sourcecode' })
 Transfer.belongsTo(ElementDetail, { as: 'target', foreignKey: 'targetcode' })
 
+Enrollment.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
+Student.hasMany(Enrollment, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
+Enrollment.belongsTo(Course, { foreignKey: 'course_id' })
+Course.hasMany(Enrollment, { foreignKey: 'course_id' })
+Enrollment.belongsTo(Semester, { foreignKey: { name: 'semester_composite', allowNull: false } })
+
 ProgrammeModule.belongsToMany(ProgrammeModule, {
   as: 'parents',
   through: ProgrammeModuleChild,
@@ -114,6 +125,7 @@ module.exports = {
   Credit,
   CreditTeacher,
   ElementDetail,
+  Studyplan,
   StudyrightExtent,
   Studyright,
   StudyrightElement,
@@ -121,4 +133,5 @@ module.exports = {
   ProgrammeModule,
   ProgrammeModuleChild,
   ExcludedCourse,
+  Enrollment,
 }

services/backend/src/models/models_kone.js

@@ -104,20 +104,6 @@ const MandatoryCourseLabels = sequelizeKone.define('mandatory_course_labels', {
   },
 })
 
-const ThesisCourse = sequelizeKone.define('thesis_courses', {
-  programmeCode: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  courseCode: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  thesisType: {
-    type: Sequelize.ENUM([ThesisTypeEnums.BACHELOR, ThesisTypeEnums.MASTER]),
-  },
-})
-
 const CustomPopulationSearch = sequelizeKone.define('custom_population_searches', {
   id: {
     primaryKey: true,
@@ -173,7 +159,6 @@ module.exports = {
   MandatoryCourse,
   TagStudent,
   Tag,
-  ThesisCourse,
   ThesisTypeEnums,
   MandatoryCourseLabels,
   CustomPopulationSearch,

services/backend/src/models/models_user.js

@@ -76,6 +76,11 @@ const User = sequelizeUser.define(
     email: { type: Sequelize.STRING },
     language: { type: Sequelize.STRING },
     sisu_person_id: { type: Sequelize.STRING },
+    iam_groups: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+      allowNull: false,
+      defaultValue: [],
+    },
     last_login: { type: Sequelize.DATE },
   },
   {

services/backend/src/models/studyplan.js

@@ -0,0 +1,48 @@
+const { Model, DATE, ARRAY, STRING, INTEGER } = require('sequelize')
+const { dbConnections } = require('../database/connection')
+
+class Studyplan extends Model {}
+
+Studyplan.init(
+  {
+    id: {
+      primaryKey: true,
+      type: STRING,
+      allowNull: false,
+    },
+    studentnumber: {
+      type: STRING,
+      references: {
+        model: 'student',
+        key: 'studentnumber',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+      allowNull: false,
+    },
+    programme_code: {
+      type: STRING,
+      allowNull: false,
+    },
+    included_courses: {
+      type: ARRAY(STRING),
+    },
+    completed_credits: {
+      type: INTEGER,
+    },
+    createdAt: {
+      type: DATE,
+    },
+    updatedAt: {
+      type: DATE,
+    },
+  },
+  {
+    underscored: false,
+    sequelize: dbConnections.sequelize,
+    modelName: 'studyplan',
+    tableName: 'studyplan',
+  }
+)
+
+module.exports = Studyplan

services/backend/src/models/studyright.js

@@ -9,9 +9,6 @@ Studyright.init(
       primaryKey: true,
       type: STRING,
     },
-    canceldate: {
-      type: DATE,
-    },
     startdate: {
       type: DATE,
     },
@@ -27,6 +24,9 @@ Studyright.init(
     graduated: {
       type: INTEGER,
     },
+    active: {
+      type: INTEGER,
+    },
     // irtisanomisperuste
     studentStudentnumber: {
       type: STRING,

services/backend/src/rapodiff.js

@@ -1,41 +1,69 @@
 /* eslint no-console: 0 */
 
-// script needs rapo certs, so you need to run this in production pannu
-//
-// possible workflows:
-// - go to oodikone pannu, then "docker exec -it backend sh", then run this script with "npm
-// run rapodiff". Modify with vim / nano until you're happy, then copy changes to local version and commit.
-// - or bind mount a file to backend container in pannu, overriding this file. Modify that file
-// inside or outside container, however you want. Keep file in pannu, or put it to pannu's git repo (oodikone-server-setup).
+/*
+  depends on nodeproxy running in importer.cs.helsinki.fi, and env RAPO_NODEPROXY (see importer for a proper value)
+  needs also acces to importer db api, uses env IMPORTER_API
+
+  known differences in APIs
+    - rapo does not show students that have graduated (it however takes some days for rapo to notice a graduation)
+    - rapo does not handle the starting year of transfers properly (shows in old staring population)
+    - rapo show some students incorrectly passive, see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3521
+*/
 
 const axios = require('axios').default
-const fs = require('fs')
-const https = require('https')
+
 const _ = require('lodash')
 
 const { optimizedStatisticsOf } = require('./services/populations')
+const { Student, Transfer, StudyrightElement, Studyright } = require('./models')
 
-const { RAPO_CERT_PATH, RAPO_KEY_PATH, RAPO_API_KEY } = process.env
+const token = process.env.RAPO_NODEPROXY
 
-const agent = new https.Agent({
-  cert: fs.readFileSync(RAPO_CERT_PATH, 'utf8'),
-  key: fs.readFileSync(RAPO_KEY_PATH, 'utf8'),
-})
+const getFromRapo = async (urlStart, urlEnd) => {
+  const { data: rapoStuffLasna } = await axios.get(urlStart)
+  const { data: rapoStuffLasnaMinus } = await axios.get(urlEnd)
 
-const api = axios.create({
-  baseURL: 'https://gw.api.helsinki.fi/secure/rapo/report/',
-  headers: { 'X-Api-Key': RAPO_API_KEY },
-  httpsAgent: agent,
-})
+  const rapoStudents = _.difference(
+    rapoStuffLasna.map(s => s.opiskelijanumero),
+    rapoStuffLasnaMinus.map(s => s.opiskelijanumero)
+  )
 
-const main = async () => {
-  const facultyCode = 'H50'
-  const programme = 'KH50_005'
-  const studyrightStartDate = '2020-08-01'
-  const presence = 'LÄSNÄOLEVA'
+  return rapoStudents
+}
 
-  const months = 12 // some amount of months, dunno
-  const year = 2020
+const known_problems_in_oodikone = {
+  KH30_001: ['013896191'],
+  KH40_003: ['013200457'],
+  KH55_001: ['013195834', '015119708', '015138336'],
+}
+
+const known_problems_in_rapo = {
+  KH55_001: ['015160142', '015135575', '015119708', '015138336'],
+}
+
+const getFromOodikone = async (programme, year) => {
+  const isNotAmongKnownProblems = student => {
+    const hasProblems = known_problems_in_oodikone[programme] ? known_problems_in_oodikone[programme] : []
+    return !hasProblems.includes(student.studentNumber)
+  }
+
+  const theStudyrights = student => {
+    const theOne = s => s.studyright_elements.find(e => e.code === programme)
+    // in some rare cases student may have more than one studyright for same programme
+    return student.studyrights.filter(theOne)
+  }
+
+  const notGraduated = student => {
+    const graduation = theStudyrights(student).filter(r => r.graduated === 1)
+    return graduation.length === 0
+  }
+
+  const notTransferred = student => {
+    const transfer = student.transfers.find(t => t.targetcode === programme)
+    return transfer === undefined
+  }
+
+  const months = 12
 
   const oodikoneQuery = {
     semesters: ['FALL', 'SPRING'],
@@ -44,21 +72,292 @@ const main = async () => {
     year,
   }
 
-  const oodikoneStuff = await optimizedStatisticsOf(oodikoneQuery)
-  const oodikoneStudents = oodikoneStuff.students.map(s => s.studentNumber)
+  const oodikoneStuff = (await optimizedStatisticsOf(oodikoneQuery)).students
 
-  const rapoUrl = encodeURI(
-    `studyrights?faculty=${facultyCode}&presence=${presence}&education=${programme}&studyrightStartDate=${studyrightStartDate}`
+  return oodikoneStuff
+    .filter(notGraduated)
+    .filter(notTransferred)
+    .filter(isNotAmongKnownProblems)
+    .map(s => s.studentNumber)
+}
+
+const isAcuallyActive = async student => {
+  const currentSemester = 144
+  const tokenImporterApi = process.env.IMPORTER_API
+
+  const url = `https://importer.cs.helsinki.fi/api/importer/students/${student}/rapo_semester_enrollments?token=${tokenImporterApi}`
+  const { data } = await axios.get(url)
+  const terms = data.reduce((all, s) => all.concat(s.terms), []).filter(t => t.semester_code === currentSemester)
+  const active = terms.filter(t => t.type !== 'NEGLECTED')
+  const negl = terms.filter(t => t.type === 'NEGLECTED')
+  return active.length > 0 && negl.length > 0
+}
+
+const hasTransferred = async (student, programme) => {
+  const data = await Student.findByPk(student, {
+    include: [
+      {
+        model: Transfer,
+      },
+    ],
+  })
+
+  const transferTo = data.transfers.find(t => t.targetcode === programme)
+  return transferTo !== undefined
+}
+
+const graduatedVeryRecently = async (student, programme) => {
+  const data = await Student.findByPk(student, {
+    include: [
+      {
+        model: Studyright,
+        include: [
+          {
+            model: StudyrightElement,
+            required: true,
+            where: {
+              code: programme,
+            },
+          },
+        ],
+        where: {
+          graduated: 1,
+        },
+      },
+    ],
+  })
+
+  if (!data || data.studyrights.length === 0) {
+    return false
+  }
+
+  const graduated = new Date(data.studyrights[0].enddate)
+  const weekAgo = new Date(new Date().getTime() - 7 * 24 * 60 * 60 * 1000)
+  return graduated > weekAgo
+}
+
+const programme_diff_year = async (programme, facultyCode, year, brief = false) => {
+  const oodikoneStudents = await getFromOodikone(programme, year)
+
+  const studyrightStartDate = `${year}-08-01`
+  const studyrightEndDate = `${year + 1}-08-01`
+
+  const rapoUrlStart = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+    )
+  const rapoUrlEnd = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+    )
+
+  const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
+  const rapoStudentsPoissa = await getFromRapo(rapoUrlStart('Poissa'), rapoUrlEnd('Poissa'))
+  const rapoStudentPoissaLaki = await getFromRapo(
+    rapoUrlStart('Poissa ei kuluta opintoaikaa'),
+    rapoUrlEnd('Poissa ei kuluta opintoaikaa')
   )
 
-  const { data: rapoStuff } = await api.get(rapoUrl)
-  const rapoStudents = rapoStuff.map(s => s.opiskelijanumero)
+  const isNotAmongKnownProblems = student => {
+    const hasProblems = known_problems_in_rapo[programme] ? known_problems_in_rapo[programme] : []
+    return !hasProblems.includes(student.studentNumber)
+  }
+
+  const rapoStudents = rapoStudentsLasna
+    .concat(rapoStudentsPoissa)
+    .concat(rapoStudentPoissaLaki)
+    .filter(isNotAmongKnownProblems)
 
   const onlyInOodikone = _.difference(oodikoneStudents, rapoStudents)
   const onlyInRapo = _.difference(rapoStudents, oodikoneStudents)
 
-  console.log('Only in Oodikone:', onlyInOodikone)
-  console.log('Only in Rapo:', onlyInRapo)
+  const wrongMissingInRapo = []
+  for (let student of onlyInOodikone) {
+    const status = await isAcuallyActive(student)
+    if (status) {
+      wrongMissingInRapo.push(student)
+    }
+  }
+
+  // these are in wrong RAPO-population
+  const transferred = []
+  for (let student of onlyInRapo) {
+    const isTransfer = await hasTransferred(student, programme)
+    if (isTransfer) {
+      transferred.push(student)
+    } else {
+      const isRecentGraduation = await graduatedVeryRecently(student, programme)
+      if (isRecentGraduation) {
+        transferred.push(student)
+      }
+    }
+  }
+
+  const checkOodikoneOnly = _.difference(onlyInOodikone, wrongMissingInRapo)
+  const checkRapoOnly = _.difference(onlyInRapo, transferred)
+
+  if (!brief) {
+    console.log('vuosi', year)
+    console.log('')
+    console.log('oodikone', oodikoneStudents.length, ' transferred', transferred.length)
+    console.log('rapo', rapoStudents.length, ' activeness wrong rapo', wrongMissingInRapo.length)
+    console.log('only rapo', checkRapoOnly.length)
+    console.log('only oodikone', checkOodikoneOnly.length)
+
+    console.log('')
+
+    if (checkRapoOnly.length > 0) {
+      console.log('Only in Rapo:')
+      console.log(checkRapoOnly.join('\n'))
+    }
+
+    if (checkOodikoneOnly.length > 0) {
+      console.log('Only in Oodikone:')
+      console.log(checkOodikoneOnly.join('\n'))
+    }
+  }
+
+  if (checkRapoOnly.length + checkOodikoneOnly.length) {
+    console.log('******************************', year)
+    if (checkRapoOnly.length > 0) {
+      console.log('Only in Rapo:')
+      console.log(checkRapoOnly.join('\n'))
+    }
+
+    if (checkOodikoneOnly.length > 0) {
+      console.log('Only in Oodikone:')
+      console.log(checkOodikoneOnly.join('\n'))
+    }
+  }
+
+  if (!brief) {
+    console.log('')
+  }
+}
+
+const programme_diff = async (programme, faculty, brief) => {
+  console.log()
+  console.log(programme)
+  console.log('--------', '\n')
+
+  await programme_diff_year(programme, faculty, 2017, brief)
+  await programme_diff_year(programme, faculty, 2018, brief)
+  await programme_diff_year(programme, faculty, 2019, brief)
+  await programme_diff_year(programme, faculty, 2020, brief)
+  await programme_diff_year(programme, faculty, 2021, brief)
+}
+
+const faculty_diff = async (faculty, programmes, brief) => {
+  console.log('\n')
+  console.log(faculty)
+  console.log('==')
+  for (let programme of programmes) {
+    await programme_diff(programme, faculty, brief)
+  }
+}
+
+// eslint-disable-next-line no-unused-vars
+const msc_programme_diff_year = async (programme, facultyCode, year, brief = false) => {
+  const oodikoneStudents = await getFromOodikone(programme, year)
+
+  const studyrightStartDate = `${year}-08-01`
+  const studyrightEndDate = `${year + 1}-08-01`
+
+  const rapoUrlStart = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+    )
+  const rapoUrlEnd = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+    )
+
+  const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
+  const rapoStudentsPoissa = await getFromRapo(rapoUrlStart('Poissa'), rapoUrlEnd('Poissa'))
+  const rapoStudentPoissaLaki = await getFromRapo(
+    rapoUrlStart('Poissa ei kuluta opintoaikaa'),
+    rapoUrlEnd('Poissa ei kuluta opintoaikaa')
+  )
+
+  const rapoStudents = rapoStudentsLasna.concat(rapoStudentsPoissa).concat(rapoStudentPoissaLaki)
+
+  const onlyInOodikone = _.difference(oodikoneStudents, rapoStudents)
+  const onlyInRapo = _.difference(rapoStudents, oodikoneStudents)
+
+  const wrongInRapo = []
+  for (let student of onlyInOodikone) {
+    const status = await isAcuallyActive(student)
+    if (status) {
+      //wrongInRapo.push(student)
+    }
+  }
+
+  // in wrong RAPO population
+  const transferred = []
+  for (let student of onlyInRapo) {
+    const status = await hasTransferred(student, programme)
+    if (status) {
+      transferred.push(student)
+    }
+  }
+
+  const checkOodikoneOnly = _.difference(onlyInOodikone, wrongInRapo)
+  const checkRapoOnly = _.difference(onlyInRapo, transferred)
+  const both = _.intersection(oodikoneStudents, rapoStudents)
+
+  console.log('vuosi', year)
+
+  if (!brief) {
+    console.log('')
+    console.log('oodikone', oodikoneStudents.length, ' transferred', transferred.length)
+    console.log('rapo', rapoStudents.length, ' activeness wrong rapo', wrongInRapo.length)
+    console.log('both', both.length)
+    console.log('only rapo', checkRapoOnly.length)
+    console.log('only oodikone', checkOodikoneOnly.length)
+
+    console.log('')
+
+    if (checkRapoOnly.length > 0) {
+      console.log('Only in Rapo:')
+      console.log(checkRapoOnly.join('\n'))
+    }
+
+    if (checkOodikoneOnly.length > 0) {
+      console.log('Only in Oodikone:')
+      console.log(checkOodikoneOnly.join('\n'))
+    }
+  }
+
+  if (checkRapoOnly.length + checkOodikoneOnly.length) {
+    console.log('******************************************************')
+  }
+
+  if (!brief) {
+    console.log('')
+  }
+}
+
+const bsc_priogrammes_of_faculties = {
+  H10: ['KH10_001'],
+  H20: ['KH20_001'],
+  H30: ['KH30_001', 'KH30_002'],
+  H40: ['KH40_001', 'KH40_002', 'KH40_003', 'KH40_004', 'KH40_005', 'KH40_006'],
+  H50: ['KH50_001', 'KH50_002', 'KH50_003', 'KH50_004', 'KH50_005', 'KH50_006', 'KH50_007', 'KH50_008'],
+  H55: ['KH55_001'],
+  H60: ['KH60_001'],
+  H70: ['KH70_001', 'KH70_002', 'KH70_003', 'KH70_004'],
+  H74: ['KH74_001'],
+  H80: ['KH80_001', 'KH80_002', 'KH80_003', 'KH80_004'],
+  H90: ['KH90_001'],
+}
+
+const main = async () => {
+  for (let faculty of Object.keys(bsc_priogrammes_of_faculties)) {
+    await faculty_diff(faculty, bsc_priogrammes_of_faculties[faculty], true)
+  }
+
+  //await programme_diff_year('KH55_001', 'H55', 2021, true)
+  //await programme_diff('KH60_001', 'H60', true)
 }
 
 main().then(() => process.exit(0))

services/backend/src/routes.js

@@ -57,7 +57,7 @@ module.exports = (app, url) => {
   app.use(url, tags)
   app.use(url, programmeModules)
   app.use(url, studyProgramme)
-  app.use(`${url}/faculties`, faculties)
+  app.use(url, faculties)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
   app.use(`${url}/users`, auth.roles(['admin']), users)

services/backend/src/routes/courses.js

@@ -28,10 +28,9 @@ router.get('/v3/courseyearlystats', async (req, res) => {
     return res.status(403).json({ error: 'No programmes so no access to course stats' })
   }
 
-  const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
+  const { codes, separate: sep } = req.query
 
   const separate = !sep ? false : JSON.parse(sep)
-  const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
   if (!codes) {
     res.status(422).send('Missing required query parameters')
   } else {
@@ -39,7 +38,7 @@ router.get('/v3/courseyearlystats', async (req, res) => {
     // and users with rights to any specific study programmes
     const anonymize = allowedRoles !== 'admin' && rights.length < 1
     const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
-    const results = await Course.courseYearlyStats(codes, separate, unifyOpenUniCourses, anonymizationSalt)
+    const results = await Course.courseYearlyStats(codes, separate, anonymizationSalt)
     res.json(results)
   }
 })

services/backend/src/routes/elementdetails.js

@@ -1,23 +1,6 @@
 const router = require('express').Router()
 const MandatoryCourses = require('../services/mandatoryCourses')
-const {
-  productivityStatsForStudytrack,
-  throughputStatsForStudytrack,
-  optionData,
-} = require('../services/studyprogramme')
-const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
-const {
-  getProductivity,
-  setProductivity,
-  getThroughput,
-  setThroughput,
-  patchProductivity,
-  patchThroughput,
-} = require('../services/analyticsService')
 const { getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
-const logger = require('../util/logger')
-
-const programmeStatsSince = new Date('2017-07-31')
 
 router.get('/elementdetails/all', async (req, res) => {
   const elementdetails = await getAllElementDetails()
@@ -29,22 +12,6 @@ router.get('/studyprogrammes', async (req, res) => {
   res.json(studyrights)
 })
 
-router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
-  const code = req.params.id
-
-  let level
-  if (code.includes('MH')) {
-    level = 'MSC'
-  } else if (code.includes('KH')) {
-    level = 'BSC'
-  } else {
-    return res.json([])
-  }
-
-  const data = await optionData(new Date('2017-07-31'), new Date(), code, level)
-  res.json(data)
-})
-
 router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
   if (req.params.id) {
     const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
@@ -54,138 +21,4 @@ router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
   }
 })
 
-router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    let data = null
-    try {
-      data = await getProductivity(code)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} productivity`)
-    }
-    if (!data) {
-      try {
-        const stats = await productivityStatsForStudytrack(code, programmeStatsSince)
-        data = await setProductivity(stats)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      }
-    }
-    return res.json(data)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-  res.status(200).end()
-
-  try {
-    if (code.includes('MH') || code.includes('KH')) {
-      const data = await productivityStatsForStudytrack(code, new Date('2017-07-31'))
-      await setProductivity(data)
-    } else {
-      const data = await productivityStatsForStudytrack(code, new Date('2000-07-31'))
-      await setProductivity(data)
-    }
-  } catch (e) {
-    try {
-      await patchProductivity({
-        [code]: { status: 'RECALCULATION ERRORED' },
-      })
-    } catch (e) {
-      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      return
-    }
-
-    logger.error(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-  }
-  logger.info(`Produdtiviy stats recalculation for studyprogramme ${code} done`)
-})
-
-router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    const thesisPromise = findProgrammeTheses(req.params.id)
-
-    let data = null
-    try {
-      data = await getThroughput(code)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} throughput`)
-    }
-    if (!data) {
-      try {
-        const result = await throughputStatsForStudytrack(req.params.id, programmeStatsSince.getFullYear())
-        data = await setThroughput(result)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      }
-    }
-    return res.json({ ...data, thesis: await thesisPromise })
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-  res.status(200).end()
-
-  try {
-    if (code.includes('MH') || code.includes('KH')) {
-      const data = await throughputStatsForStudytrack(code, 2017)
-      await setThroughput(data)
-    } else {
-      const data = await throughputStatsForStudytrack(code, 2000)
-      await setThroughput(data)
-    }
-  } catch (e) {
-    try {
-      await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-    } catch (e) {
-      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      return
-    }
-    logger.error(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
-  }
-  logger.info(`Throughput stats recalculation for studyprogramme ${code} done`)
-})
-
-router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {
-  const { id } = req.params
-  if (id) {
-    const thesis = await findProgrammeTheses(id)
-    res.json(thesis)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.post('/v2/studyprogrammes/:id/thesis', async (req, res) => {
-  const { id } = req.params
-  const { course, thesisType } = req.body
-  if (id && course && thesisType) {
-    const thesis = await createThesisCourse(id, course, thesisType)
-    res.status(201).json(thesis)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
-  const { id, course } = req.params
-  if (id && course) {
-    const deleted = await deleteThesisCourse(id, course)
-    res.status(204).json(deleted)
-  } else {
-    res.status(422).end()
-  }
-})
-
 module.exports = router

services/backend/src/routes/faculties.js

@@ -1,9 +1,101 @@
 const router = require('express').Router()
-const { faculties } = require('../services/organisations')
+const { faculties, degreeProgrammeCodesOfFaculty } = require('../services/organisations')
+const { combineFacultyBasics } = require('../services/facultyBasics')
+const { combineFacultyCredits } = require('../services/facultyCredits')
 
-router.get('/', async (req, res) => {
+// Faculty uses a lot of tools designed for Study programme.
+// Some of them have been copied here and slightly edited for faculty purpose.
+
+router.get('/faculties', async (req, res) => {
   const facultyList = await faculties()
   res.json(facultyList)
 })
 
+router.get('/faculties/:id/basicstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+
+  if (!code) return res.status(422).end()
+
+  let allBasics = {
+    id: code,
+    years: [],
+    tableStats: [],
+    graphStats: [],
+    titles: [
+      '',
+      'Started studying',
+      'All graduated',
+      'Graduated bachelors',
+      'Graduated masters',
+      'Graduated doctors',
+      'Other graduations',
+      'Transferred inside',
+      'Transferred away',
+      'Transferred to',
+    ],
+    status: 'DONE',
+    lastUpdated: '',
+  }
+
+  const programmes = await degreeProgrammeCodesOfFaculty(code)
+  if (programmes) {
+    await combineFacultyBasics(allBasics, code, programmes, yearType)
+  }
+
+  return res.json(allBasics)
+})
+
+router.get('/faculties/:id/creditstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+  const specialGroups = 'SPECIAL_INCLUDED' // req.query?.special_groups
+
+  if (!code) return res.status(422).end()
+
+  let counts = {}
+  let years = []
+  let allCredits = {
+    id: code,
+    years: [],
+    tableStats: [],
+    graphStats: [],
+    titles: [
+      '',
+      'Total',
+      'Major students credits',
+      'Non-major faculty students credits',
+      'Non-major other faculty students credits',
+      'Non-degree student credits',
+    ],
+    status: 'DONE',
+    lastUpdated: '',
+  }
+
+  const programmes = await degreeProgrammeCodesOfFaculty(code)
+
+  if (programmes) {
+    await combineFacultyCredits(allCredits, programmes, yearType, specialGroups, counts, years)
+  }
+  let majors = []
+  let facultyNonMajor = []
+  let otherNonMajor = []
+  let nonDegree = []
+
+  years.forEach(year => {
+    majors.push(counts[year][1])
+    facultyNonMajor.push(counts[year][2])
+    otherNonMajor.push(counts[year][3])
+    nonDegree.push(counts[year][4])
+  })
+
+  allCredits.graphStats = [
+    { name: 'Major students credits', data: majors },
+    { name: 'Non-major faculty students credits', data: facultyNonMajor },
+    { name: 'Non-major other faculty students credits', data: otherNonMajor },
+    { name: 'Non-degree credits', data: nonDegree },
+  ]
+  return res.json(allCredits)
+})
+
 module.exports = router

services/backend/src/routes/population.js

@@ -8,6 +8,8 @@ const StudyrightService = require('../services/studyrights')
 const TagService = require('../services/tags')
 const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
+const { mapToProviders } = require('../util/utils')
+
 const { ApplicationError } = require('../util/customErrors')
 
 const filterPersonalTags = (population, userId) => {
@@ -141,7 +143,7 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
 router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
   const {
     user: { isAdmin, studentsUserCanAccess },
-    body: { studentnumberlist: studentnumbersInReq },
+    body: { studentnumberlist: studentnumbersInReq, studentCountLimit, ...query },
   } = req
 
   if (!studentnumbersInReq) {
@@ -152,10 +154,11 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
 
   const result = await Population.bottlenecksOf(
     {
-      year: 1900,
-      studyRights: [],
-      semesters: ['FALL', 'SPRING'],
-      months: 10000,
+      year: query?.year ?? 1900,
+      studyRights: query?.studyRights ?? [],
+      semesters: query?.semesters ?? ['FALL', 'SPRING'],
+      months: query?.months ?? 10000,
+      studentCountLimit,
     },
     studentnumberlist
   )
@@ -177,13 +180,13 @@ router.get('/v3/populationstatistics', async (req, res) => {
     return
   }
   let studyRights = null
-  try {
-    studyRights = JSON.parse(studyRightsJSON)
-    const {
-      user: { rights, isAdmin },
-    } = req
+  studyRights = JSON.parse(studyRightsJSON)
+  const {
+    user: { rights, iamRights, isAdmin },
+  } = req
 
-    if (!isAdmin && !rights.includes(studyRights.programme)) {
+  try {
+    if (!isAdmin && !rights.includes(studyRights.programme) && !iamRights.includes(studyRights.programme)) {
       res.status(403).json([])
       return
     }
@@ -229,6 +232,27 @@ router.get('/v3/populationstatistics', async (req, res) => {
       return
     }
 
+    // Obfuscate if user has only iam rights
+    if (!isAdmin && !rights.includes(studyRights.programme)) {
+      const randomHash = crypto.randomBytes(12).toString('hex')
+      result.students = result.students.map(student => {
+        const studentNumberHash = crypto.createHash('md5').update(`${student.studentNumber}${randomHash}`).digest('hex')
+        const obfuscatedBirthDate = new Date(new Date(student.birthdate).setMonth(0, 0)) // only birth year for age distribution
+        return {
+          ...student,
+          firstnames: '',
+          lastname: '',
+          started: null,
+          studentNumber: studentNumberHash,
+          name: '',
+          email: '',
+          tags: [],
+          birthdate: obfuscatedBirthDate,
+          obfuscated: true,
+        }
+      })
+    }
+
     res.json(filterPersonalTags(result, user.id))
   }
 })
@@ -274,9 +298,9 @@ router.get('/v3/populationstatisticsbytag', async (req, res) => {
 })
 
 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
-  const { coursecodes, from, to, separate: sep } = req.query
+  const { coursecodes, from, to, separate: sep, unifyCourses } = req.query
   const {
-    user: { id, isAdmin, studentsUserCanAccess: allStudentsUserCanAccess },
+    user: { id, isAdmin, studentsUserCanAccess: allStudentsUserCanAccess, rights },
   } = req
   const separate = sep ? JSON.parse(sep) : false
 
@@ -293,7 +317,13 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
   }
 
   const semesters = ['FALL', 'SPRING']
-  const studentnumbers = await Student.findByCourseAndSemesters(JSON.parse(coursecodes), from, to, separate)
+  const studentnumbers = await Student.findByCourseAndSemesters(
+    JSON.parse(coursecodes),
+    from,
+    to,
+    separate,
+    unifyCourses
+  )
   const result = await Population.optimizedStatisticsOf(
     {
       // Useless, because studentnumbers are already filtered above by from & to.
@@ -305,10 +335,15 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
     },
     studentnumbers
   )
+  let courseproviders = []
+  if (!isAdmin) {
+    courseproviders = await CourseService.getCourseProvidersForCourses(JSON.parse(coursecodes))
+  }
+  const rightsMappedToProviders = mapToProviders(rights)
+  const found = courseproviders.some(r => rightsMappedToProviders.includes(r))
 
-  const studentsUserCanAccess = isAdmin
-    ? new Set(studentnumbers)
-    : new Set(_.intersection(studentnumbers, allStudentsUserCanAccess))
+  const studentsUserCanAccess =
+    isAdmin || found ? new Set(studentnumbers) : new Set(_.intersection(studentnumbers, allStudentsUserCanAccess))
 
   const randomHash = crypto.randomBytes(12).toString('hex')
   const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
@@ -363,13 +398,13 @@ router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
 
 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
   const {
-    user: { rights, roles },
+    user: { rights, iamRights, roles },
   } = req
   if (roles?.includes('admin')) {
     const studyrights = await StudyrightService.getAssociations()
     res.json(studyrights)
   } else {
-    const studyrights = await StudyrightService.getFilteredAssociations(rights)
+    const studyrights = await StudyrightService.getFilteredAssociations(rights.concat(iamRights))
     res.json(studyrights)
   }
 })
@@ -381,8 +416,23 @@ router.get('/v3/populationstatistics/studyprogrammes/unfiltered', async (req, re
 
 router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
   const { courseCodes } = req.query
-  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(courseCodes))
-  return res.json(maxYearsToCreatePopulationFrom)
+  const maxYearsToCreatePopulationFromOpen = await CourseService.maxYearsToCreatePopulationFrom(
+    JSON.parse(courseCodes),
+    'openStats'
+  )
+  const maxYearsToCreatePopulationFromUni = await CourseService.maxYearsToCreatePopulationFrom(
+    JSON.parse(courseCodes),
+    'reqularStats'
+  )
+  const maxYearsToCreatePopulationFromBoth = await CourseService.maxYearsToCreatePopulationFrom(
+    JSON.parse(courseCodes),
+    'unifyStats'
+  )
+  return res.json({
+    openCourses: maxYearsToCreatePopulationFromOpen,
+    uniCourses: maxYearsToCreatePopulationFromUni,
+    unifyCourses: maxYearsToCreatePopulationFromBoth,
+  })
 })
 
 module.exports = router

services/backend/src/routes/studyProgramme.js

@@ -3,6 +3,7 @@ const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics
 const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
 const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
 const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
+const { getStudyprogrammeCoursesForStudytrack } = require('../services/studyprogrammeCourses')
 const {
   getBasicStats,
   setBasicStats,
@@ -21,29 +22,20 @@ router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
   const yearType = req.query?.year_type
   const specialGroups = req.query?.special_groups
 
-  if (code) {
-    let data = null
-    try {
-      data = await getBasicStats(code, yearType, specialGroups)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} basic stats: ${e}`)
-    }
-    if (!data) {
-      try {
-        const result = await getBasicStatsForStudytrack({
-          studyprogramme: req.params.id,
-          yearType,
-          specialGroups,
-        })
-        data = await setBasicStats(result, yearType, specialGroups)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} basic stats: ${e}`)
-      }
-    }
-    return res.json(data)
-  } else {
-    res.status(422).end()
-  }
+  if (!code) return res.status(422).end()
+
+  const data = await getBasicStats(code, yearType, specialGroups)
+  if (data) return res.json(data)
+
+  const updated = await getBasicStatsForStudytrack({
+    studyprogramme: req.params.id,
+    settings: {
+      isAcademicYear: yearType === 'ACADEMIC_YEAR',
+      includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
+    },
+  })
+  if (updated) await setBasicStats(updated, yearType, specialGroups)
+  return res.json(updated)
 })
 
 router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
@@ -51,29 +43,19 @@ router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
   const yearType = req.query?.year_type
   const specialGroups = req.query?.special_groups
 
-  if (code) {
-    let data = null
-    try {
-      data = await getCreditStats(code, yearType, specialGroups)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} credit stats`)
-    }
-    if (!data) {
-      try {
-        const result = await getCreditStatsForStudytrack({
-          studyprogramme: req.params.id,
-          yearType,
-          specialGroups,
-        })
-        data = await setCreditStats(result, yearType, specialGroups)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} credit stats`)
-      }
-    }
-    return res.json(data)
-  } else {
-    res.status(422).end()
-  }
+  if (!code) return res.status(422).end()
+
+  const data = await getCreditStats(code, yearType, specialGroups)
+  if (data) return res.json(data)
+  const updatedStats = await getCreditStatsForStudytrack({
+    studyprogramme: req.params.id,
+    settings: {
+      isAcademicYear: yearType === 'ACADEMIC_YEAR',
+      includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
+    },
+  })
+  if (updatedStats) await setCreditStats(updatedStats, yearType, specialGroups)
+  return res.json(updatedStats)
 })
 
 router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
@@ -81,28 +63,32 @@ router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
   const yearType = req.query?.year_type
   const specialGroups = req.query?.special_groups
 
-  if (code) {
-    let data = null
-    try {
-      data = await getGraduationStats(code, yearType, specialGroups)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} graduation stats`)
-    }
-    if (!data) {
-      try {
-        const result = await getGraduationStatsForStudytrack({
-          studyprogramme: req.params.id,
-          yearType,
-          specialGroups,
-        })
-        data = await setGraduationStats(result, yearType, specialGroups)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} graduation stats`)
-      }
-    }
+  if (!code) return res.status(422).end()
+
+  const data = await getGraduationStats(code, yearType, specialGroups)
+  if (data) return res.json(data)
+  const updatedStats = await getGraduationStatsForStudytrack({
+    studyprogramme: req.params.id,
+    settings: {
+      isAcademicYear: yearType === 'ACADEMIC_YEAR',
+      includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
+    },
+  })
+  if (updatedStats) await setGraduationStats(updatedStats, yearType, specialGroups)
+  return res.json(updatedStats)
+})
+
+router.get('/v2/studyprogrammes/:id/coursestats', async (req, res) => {
+  const code = req.params.id
+  const showByYear = req.query.academicyear
+  const date = new Date()
+  date.setHours(23, 59, 59, 999)
+
+  try {
+    const data = await getStudyprogrammeCoursesForStudytrack(date.getTime(), code, showByYear)
     return res.json(data)
-  } else {
-    res.status(422).end()
+  } catch (e) {
+    logger.error(`Failed to get code ${code} programme courses stats: ${e}`)
   }
 })
 
@@ -111,25 +97,20 @@ router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
   const graduated = req.query?.graduated
   const specialGroups = req.query?.special_groups
 
-  if (code) {
-    let data = null
-    try {
-      data = await getStudytrackStats(code, graduated, specialGroups)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} studytrack stats: ${e}`)
-    }
-    if (!data) {
-      try {
-        const result = await getStudytrackStatsForStudyprogramme({ studyprogramme: code, specialGroups, graduated })
-        data = await setStudytrackStats(result, graduated, specialGroups)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
-      }
-    }
-    return res.json(data)
-  } else {
-    res.status(422).end()
-  }
+  if (!code) return res.status(422).end()
+
+  const data = await getStudytrackStats(code, graduated, specialGroups)
+  if (data) return res.json(data)
+
+  const updated = await getStudytrackStatsForStudyprogramme({
+    studyprogramme: code,
+    settings: {
+      graduated: graduated === 'GRADUATED_INCLUDED',
+      specialGroups: specialGroups === 'SPECIAL_INCLUDED',
+    },
+  })
+  if (updated) await setStudytrackStats(updated, graduated, specialGroups)
+  return res.json(updated)
 })
 
 router.get('/v2/studyprogrammes/:id/update_basicview', async (req, res) => {

services/backend/src/services/analyticsService.js

@@ -1,8 +1,8 @@
 const { redisClient } = require('../services/redis')
 const moment = require('moment')
 
-const createRedisKeyForProductivity = id => `PRODUCTIVITY_${id}`
-const createRedisKeyForThroughput = id => `THROUGHPUT_${id}`
+// Only new bachelor, masters and doctoral programmes get their data updated in redis every night, use redis for them
+const isUpdatedNewProgramme = code => code.includes('KH') || code.includes('MH') || /^(T)[0-9]{6}$/.test(code)
 const createRedisKeyForBasicStats = (id, yearType, specialGroups) => `BASIC_STATS_${id}_${yearType}_${specialGroups}`
 const createRedisKeyForCreditStats = (id, yearType, specialGroups) => `CREDIT_STATS_${id}_${yearType}_${specialGroups}`
 const createRedisKeyForGraduationStats = (id, yearType, specialGroups) =>
@@ -10,90 +10,8 @@ const createRedisKeyForGraduationStats = (id, yearType, specialGroups) =>
 const createRedisKeyForStudytrackStats = (id, graduated, specialGroups) =>
   `STUDYTRACK_STATS_${id}_${graduated}_${specialGroups}`
 
-const getProductivity = async id => {
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return {
-    [id]: JSON.parse(dataFromRedis),
-  }
-}
-
-const setProductivity = async data => {
-  const { id } = data
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: dataToRedis,
-  }
-}
-
-const patchProductivity = async data => {
-  const { id } = data
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataFromRedis = JSON.parse(await redisClient.getAsync(redisKey))
-  const patchedData = {
-    ...dataFromRedis,
-    ...data,
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(patchedData))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: patchedData,
-  }
-}
-
-const getThroughput = async id => {
-  const redisKey = createRedisKeyForThroughput(id)
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return {
-    [id]: JSON.parse(dataFromRedis),
-  }
-}
-
-const setThroughput = async data => {
-  const { id, data: dataToSet } = data
-  const redisKey = createRedisKeyForThroughput(id)
-  const dataToRedis = {
-    data: dataToSet.years,
-    totals: dataToSet.totals,
-    stTotals: dataToSet.stTotals,
-    status: 'DONE',
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: dataToRedis,
-  }
-}
-
-const patchThroughput = async data => {
-  const [id, dataToPatch] = Object.entries(data)[0]
-  const redisKey = createRedisKeyForThroughput(id)
-  const { status } = dataToPatch
-  const dataFromRedis = JSON.parse(await redisClient.getAsync(redisKey))
-  const patchedData = {
-    ...dataFromRedis,
-    status,
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(patchedData))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: patchedData,
-  }
-}
-
 const getBasicStats = async (id, yearType, specialGroups) => {
+  if (!isUpdatedNewProgramme(id)) return null
   const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
@@ -108,12 +26,14 @@ const setBasicStats = async (data, yearType, specialGroups) => {
     status: 'DONE',
     lastUpdated: moment().format(),
   }
+  if (!isUpdatedNewProgramme(id)) return dataToRedis
   const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
   if (setOperationStatus !== 'OK') return null
   return dataToRedis
 }
 
 const getCreditStats = async (id, yearType, specialGroups) => {
+  if (!isUpdatedNewProgramme(id)) return null
   const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
@@ -128,12 +48,14 @@ const setCreditStats = async (data, yearType, specialGroups) => {
     status: 'DONE',
     lastUpdated: moment().format(),
   }
+  if (!isUpdatedNewProgramme(id)) return dataToRedis
   const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
   if (setOperationStatus !== 'OK') return null
   return dataToRedis
 }
 
 const getGraduationStats = async (id, yearType, specialGroups) => {
+  if (!isUpdatedNewProgramme(id)) return null
   const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
@@ -148,12 +70,14 @@ const setGraduationStats = async (data, yearType, specialGroups) => {
     status: 'DONE',
     lastUpdated: moment().format(),
   }
+  if (!isUpdatedNewProgramme(id)) return dataToRedis
   const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
   if (setOperationStatus !== 'OK') return null
   return dataToRedis
 }
 
 const getStudytrackStats = async (id, graduated, specialGroups) => {
+  if (!isUpdatedNewProgramme(id)) return null
   const redisKey = createRedisKeyForStudytrackStats(id, graduated, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
@@ -168,18 +92,13 @@ const setStudytrackStats = async (data, graduated, specialGroups) => {
     status: 'DONE',
     lastUpdated: moment().format(),
   }
+  if (!isUpdatedNewProgramme(id)) return dataToRedis
   const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
   if (setOperationStatus !== 'OK') return null
   return dataToRedis
 }
 
 module.exports = {
-  getProductivity,
-  setProductivity,
-  patchProductivity,
-  getThroughput,
-  setThroughput,
-  patchThroughput,
   getBasicStats,
   setBasicStats,
   getCreditStats,

services/backend/src/services/course_stats_counter.js

@@ -40,18 +40,22 @@ class CourseStatsCounter {
       triedOfPopulation: undefined,
       passingSemesters: this.initializePassingSemesters(),
     }
+    this.enrollments = {
+      enrolledNoGrade: new Set(),
+      semesters: this.initializePassingSemesters({}),
+    }
     this.grades = {}
   }
 
-  initializePassingSemesters() {
+  initializePassingSemesters(initialValue = 0) {
     const passingSemesters = {
-      BEFORE: 0,
-      LATER: 0,
+      BEFORE: initialValue,
+      LATER: initialValue,
     }
 
     for (let i = 0; i < 7; i++) {
-      passingSemesters[fall[i]] = 0
-      passingSemesters[spring[i]] = 0
+      passingSemesters[fall[i]] = initialValue
+      passingSemesters[spring[i]] = initialValue
     }
 
     return passingSemesters
@@ -91,6 +95,13 @@ class CourseStatsCounter {
     }
   }
 
+  markEnrollment(studentnumber, state, semester) {
+    if (!this.enrollments[state]) this.enrollments[state] = new Set()
+    this.enrollments[state].add(studentnumber)
+    if (!this.enrollments.semesters[semester][state]) this.enrollments.semesters[semester][state] = new Set()
+    this.enrollments.semesters[semester][state].add(studentnumber)
+  }
+
   failedBefore(studentnumber) {
     return this.students.failed[studentnumber] !== undefined
   }
@@ -173,6 +184,10 @@ class CourseStatsCounter {
   getFinalStats() {
     const stats = this.stats
     const students = this.students
+    const allStudents = Object.keys(students.all)
+    const totalEnrolledNoGrade = [...(this.enrollments.ENROLLED || [])].filter(
+      student => !allStudents.includes(student)
+    ).length
 
     stats.students = lengthOf(students.all)
     stats.passed = lengthOf(students.passed)
@@ -185,6 +200,9 @@ class CourseStatsCounter {
     stats.triedOfPopulation = percentageOf(stats.students, this.studentsInTotal)
     stats.perStudent = stats.attempts / (stats.passed + stats.failed)
     stats.passingSemestersCumulative = this.getPassingSemestersCumulative()
+    stats.totalStudents = stats.students + totalEnrolledNoGrade
+    stats.totalEnrolledNoGrade = totalEnrolledNoGrade
+    stats.percentageWithEnrollments = percentageOf(stats.passed, stats.totalStudents)
 
     return {
       stats,

services/backend/src/services/course_yearly_stats_counter.js

@@ -8,12 +8,23 @@ class CourseYearlyStatsCounter {
   }
 
   initProgramme(code, name) {
-    this.programmes[code] = { name, students: {}, passed: {}, credits: {} }
+    this.programmes[code] = {
+      name,
+      students: {},
+      passed: {},
+      credits: {},
+    }
   }
 
   initFacultyYear(code) {
     const year = `${1949 + Number(code)}-${1950 + Number(code)}`
-    this.facultyStats[code] = { year, allStudents: [], allPassed: [], faculties: {}, allCredits: 0 }
+    this.facultyStats[code] = {
+      year,
+      allStudents: [],
+      allPassed: [],
+      faculties: {},
+      allCredits: 0,
+    }
   }
 
   initFaculty(yearcode, faculty_code, organization) {
@@ -43,8 +54,11 @@ class CourseYearlyStatsCounter {
           passedEventually: [],
           neverPassed: [],
         },
+        grades: {},
         studentnumbers: [],
       },
+      enrollments: [],
+      allEnrollments: [],
       yearcode,
     }
   }
@@ -97,11 +111,41 @@ class CourseYearlyStatsCounter {
     })
   }
 
+  markStudyProgrammesEnrollment(studentnumber, programmes, yearcode, state, enrollment_date_time) {
+    programmes.forEach(({ code, name, faculty_code, organization }) => {
+      this.markStudyProgrammeEnrollment(
+        code,
+        name,
+        studentnumber,
+        yearcode,
+        state,
+        enrollment_date_time,
+        faculty_code,
+        organization
+      )
+    })
+  }
+
   markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode) {
     if (!this.groups[groupcode]) {
       this.initGroup(groupcode, groupname, coursecode, yearcode)
     }
     this.markCreditToAttempts(studentnumber, passed, grade, groupcode)
+    this.markBestEffortGrade(studentnumber, passed, grade, groupcode)
+  }
+
+  markEnrollmentToGroup(studentnumber, state, enrollment_date_time, groupcode, groupname, coursecode, yearcode) {
+    if (!this.groups[groupcode]) this.initGroup(groupcode, groupname, coursecode, yearcode)
+
+    const enrollment = { studentnumber, state, enrollment_date_time }
+    this.groups[groupcode].allEnrollments.push(enrollment)
+    const oldEnrollment = this.groups[groupcode].enrollments.find(e => e.studentnumber === studentnumber)
+    if (!oldEnrollment) return this.groups[groupcode].enrollments.push({ studentnumber, state, enrollment_date_time })
+    if (oldEnrollment.state === 'ENROLLED' || oldEnrollment.state === 'CONFIRMED') return
+    if (state !== 'ENROLLED' || state !== 'CONFIRMED') return
+    this.groups[groupcode].enrollments = this.groups[groupcode].enrollments
+      .filter(e => e.studentnumber !== studentnumber)
+      .concat([enrollment])
   }
 
   markCreditToAttempts(studentnumber, passed, grade, groupcode) {
@@ -118,6 +162,19 @@ class CourseYearlyStatsCounter {
     }
   }
 
+  markBestEffortGrade(studentnumber, passed, grade, groupcode) {
+    const current = this.groups[groupcode].students.grades[studentnumber]
+
+    if (!current) {
+      this.groups[groupcode].students.grades[studentnumber] = { grade, passed }
+      return
+    }
+
+    if (current.passed && !passed) return
+    if (current.passed && Number(current.grade) >= Number(grade)) return
+    this.groups[groupcode].students.grades[studentnumber] = { grade, passed }
+  }
+
   markCreditToStudentCategories(studentnumber, passed, attainment_date, groupcode) {
     if (!this.students.has(studentnumber)) {
       if (passed) {
@@ -183,8 +240,17 @@ class CourseYearlyStatsCounter {
     }
 
     const groupStatistics = Object.values(this.groups).map(({ ...rest }) => {
+      const { students } = rest
+      const grades = {}
+      Object.keys(students.grades).forEach(student => {
+        const { grade, passed } = students.grades[student]
+        const parsed = passed ? grade : '0'
+        if (!grades[parsed]) grades[parsed] = []
+        grades[parsed].push(student)
+      })
       const normalStats = {
         ...rest,
+        students: { ...students, grades },
       }
 
       if (anonymizationSalt && normalStats.students.studentnumbers.length < 6) {

services/backend/src/services/courses.js

@@ -8,12 +8,14 @@ const {
   Studyright,
   Semester,
   Organization,
+  Enrollment,
 } = require('../models')
 
 const { parseCredit } = require('./parseCredits')
 const Op = Sequelize.Op
 const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
 const { sortMainCode, getSortRank } = require('../util/utils')
+const { parseEnrollment } = require('./parseEnrollments')
 
 const byNameOrCode = (searchTerm, language) =>
   Course.findAll({
@@ -59,10 +61,12 @@ const findOneByCode = code => {
   })
 }
 
-const creditsForCourses = async (codes, anonymizationSalt, unifyOpenUniCourses) => {
+const creditsForCourses = async (codes, anonymizationSalt, unification) => {
   let is_open = false
 
-  if (unifyOpenUniCourses) {
+  if (unification === 'open') is_open = true
+
+  if (unification === 'unify') {
     is_open = {
       [Op.in]: [false, true],
     }
@@ -104,6 +108,11 @@ const creditsForCourses = async (codes, anonymizationSalt, unifyOpenUniCourses)
       {
         model: Semester,
         attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+        where: {
+          startdate: {
+            [Op.lte]: new Date(),
+          },
+        },
       },
     ],
     where: {
@@ -122,6 +131,75 @@ const creditsForCourses = async (codes, anonymizationSalt, unifyOpenUniCourses)
   return parsedCredits
 }
 
+const enrollmentsForCourses = async (codes, anonymizationSalt, unification) => {
+  let is_open = false
+
+  if (unification === 'open') is_open = true
+
+  if (unification === 'unify') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
+  const enrollments = await Enrollment.findAll({
+    include: [
+      {
+        model: Student,
+        attributes: ['studentnumber'],
+        include: {
+          model: StudyrightElement,
+          attributes: ['code', 'startdate'],
+          include: [
+            {
+              model: ElementDetail,
+              attributes: ['name', 'type'],
+              where: {
+                type: {
+                  [Op.eq]: 20,
+                },
+              },
+            },
+            {
+              model: Studyright,
+              attributes: ['prioritycode', 'faculty_code'],
+              where: {
+                prioritycode: {
+                  [Op.eq]: 1,
+                },
+              },
+              include: {
+                model: Organization,
+              },
+            },
+          ],
+        },
+      },
+      {
+        model: Semester,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+        where: {
+          startdate: {
+            [Op.lte]: new Date(),
+          },
+        },
+      },
+    ],
+    where: {
+      course_code: {
+        [Op.in]: codes,
+      },
+      studentnumber: {
+        [Op.ne]: null,
+      },
+      enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
+      state: ['ENROLLED', 'CONFIRMED'],
+      [Op.or]: [{ is_open }, { is_open: null }],
+    },
+  })
+  return enrollments.map(enrollment => parseEnrollment(enrollment, anonymizationSalt))
+}
+
 const bySearchTerm = async (term, language) => {
   const formatCourse = course => ({
     name: course.name[language],
@@ -162,23 +240,20 @@ const allCodeAlternatives = async code => {
   return sortMainCode([...course.substitutions, code])
 }
 
-const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anonymizationSalt) => {
+const yearlyStatsOfNew = async (coursecode, separate, unification, anonymizationSalt) => {
   const courseForSubs = await Course.findOne({
     where: { code: coursecode },
   })
 
   let codes = courseForSubs.substitutions ? sortMainCode([...courseForSubs.substitutions, coursecode]) : [coursecode]
 
-  if (isOpenUniCourseCode(coursecode) && !unifyOpenUniCourses) {
-    codes = [coursecode]
+  if (unification === 'reqular') {
+    codes = codes.filter(course => !isOpenUniCourseCode(course))
   }
 
-  if (!isOpenUniCourseCode(coursecode) && !unifyOpenUniCourses) {
-    codes = codes.filter(code => !code.match(/^[A][0-9]|^AY/))
-  }
-
-  const [credits, course] = await Promise.all([
-    creditsForCourses(codes, anonymizationSalt, unifyOpenUniCourses),
+  const [credits, enrollments, course] = await Promise.all([
+    creditsForCourses(codes, anonymizationSalt, unification),
+    enrollmentsForCourses(codes, anonymizationSalt, unification),
     Course.findOne({
       where: {
         code: coursecode,
@@ -233,7 +308,27 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
     counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode)
     counter.markCreditToStudentCategories(studentnumber, passed, attainment_date, groupcode)
   }
+
+  enrollments.forEach(enrollment => {
+    const { studentnumber, semestercode, semestername, yearcode, yearname, coursecode, state, enrollment_date_time } =
+      enrollment
+
+    const groupcode = separate ? semestercode : yearcode
+    const groupname = separate ? semestername : yearname
+
+    counter.markEnrollmentToGroup(
+      studentnumber,
+      state,
+      enrollment_date_time,
+      groupcode,
+      groupname,
+      coursecode,
+      yearcode
+    )
+  })
+
   const statistics = counter.getFinalStatistics(anonymizationSalt)
+
   return {
     ...statistics,
     coursecode,
@@ -242,7 +337,17 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
   }
 }
 
-const maxYearsToCreatePopulationFrom = async coursecodes => {
+const maxYearsToCreatePopulationFrom = async (coursecodes, unifyCourses) => {
+  let is_open = false
+
+  if (unifyCourses === 'openStats') is_open = true
+
+  if (unifyCourses === 'unifyStats') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
   const maxAttainmentDate = new Date(
     Math.max(
       ...(
@@ -257,6 +362,7 @@ const maxYearsToCreatePopulationFrom = async coursecodes => {
       ).map(c => new Date(c.max_attainment_date).getTime())
     )
   )
+
   const attainmentThreshold = new Date(maxAttainmentDate.getFullYear(), 0, 1)
   attainmentThreshold.setFullYear(attainmentThreshold.getFullYear() - 6)
 
@@ -268,6 +374,7 @@ const maxYearsToCreatePopulationFrom = async coursecodes => {
       attainment_date: {
         [Op.gt]: attainmentThreshold,
       },
+      is_open,
     },
     order: [['attainment_date', 'ASC']],
   })
@@ -291,11 +398,18 @@ const maxYearsToCreatePopulationFrom = async coursecodes => {
   return maxYearsToCreatePopulationFrom
 }
 
-const courseYearlyStats = async (coursecodes, separate, unifyOpenUniCourses, anonymizationSalt) => {
-  const stats = await Promise.all(
-    coursecodes.map(code => yearlyStatsOfNew(code, separate, unifyOpenUniCourses, anonymizationSalt))
+const courseYearlyStats = async (coursecodes, separate, anonymizationSalt) => {
+  const statsReqular = await Promise.all(
+    coursecodes.map(async code => {
+      const unifyStats = await yearlyStatsOfNew(code, separate, 'unify', anonymizationSalt)
+      const reqularStats = await yearlyStatsOfNew(code, separate, 'reqular', anonymizationSalt)
+      const openStats = await yearlyStatsOfNew(code, separate, 'open', anonymizationSalt)
+
+      return { unifyStats, reqularStats, openStats }
+    })
   )
-  return stats
+
+  return statsReqular
 }
 
 const nameLikeTerm = name => {
@@ -381,6 +495,23 @@ const byNameAndOrCodeLike = async (name, code) => {
   return { courses }
 }
 
+const getCourseProvidersForCourses = async codes => {
+  return (
+    await Organization.findAll({
+      attributes: ['code'],
+      include: {
+        model: Course,
+        where: {
+          code: {
+            [Op.in]: codes,
+          },
+        },
+      },
+      raw: true,
+    })
+  ).map(({ code }) => code)
+}
+
 module.exports = {
   byCode,
   byName,
@@ -393,4 +524,5 @@ module.exports = {
   unifyOpenUniversity,
   allCodeAlternatives,
   findOneByCode,
+  getCourseProvidersForCourses,
 }

services/backend/src/services/faculty.js

@@ -0,0 +1,90 @@
+const Sequelize = require('sequelize')
+const { Op } = Sequelize
+const { Studyright, Student, Transfer } = require('../models')
+const { formatStudyright, formatTransfer } = require('./studyprogrammeHelpers')
+
+const startedStudyrights = async (faculty, since) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        faculty_code: faculty,
+        studystartdate: {
+          [Op.gte]: since,
+        },
+        student_studentnumber: { [Op.not]: null },
+      },
+    })
+  ).map(formatStudyright)
+
+const graduatedStudyrights = async (faculty, since) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        faculty_code: faculty,
+        enddate: {
+          [Op.gte]: since,
+        },
+        graduated: 1,
+        student_studentnumber: { [Op.not]: null },
+      },
+    })
+  ).map(formatStudyright)
+
+const transferredInsideFaculty = async (programmes, since) =>
+  (
+    await Transfer.findAll({
+      where: {
+        transferdate: {
+          [Op.gte]: since,
+        },
+        sourcecode: programmes,
+        targetcode: programmes,
+      },
+    })
+  ).map(formatTransfer)
+
+const transferredAway = async (programmes, since) =>
+  (
+    await Transfer.findAll({
+      where: {
+        transferdate: {
+          [Op.gte]: since,
+        },
+        sourcecode: programmes,
+        targetcode: {
+          [Op.notIn]: programmes,
+        },
+      },
+    })
+  ).map(formatTransfer)
+
+const transferredTo = async (programmes, since) =>
+  (
+    await Transfer.findAll({
+      where: {
+        transferdate: {
+          [Op.gte]: since,
+        },
+        sourcecode: {
+          [Op.notIn]: programmes,
+        },
+        targetcode: programmes,
+      },
+    })
+  ).map(formatTransfer)
+
+module.exports = { startedStudyrights, graduatedStudyrights, transferredInsideFaculty, transferredAway, transferredTo }

services/backend/src/services/facultyBasics.js

@@ -0,0 +1,175 @@
+const { indexOf } = require('lodash')
+const {
+  startedStudyrights,
+  graduatedStudyrights,
+  transferredInsideFaculty,
+  transferredAway,
+  transferredTo,
+} = require('./faculty')
+const { getStatsBasis, getYearsArray, defineYear } = require('./studyprogrammeHelpers')
+
+const filterDuplicateStudyrights = studyrights => {
+  // bachelor+master students have two studyrights (separated by two last digits in studyrightid)
+  // choose only the earlier started one, so we don't count start of masters as starting in faculty
+  let rightsToCount = {}
+
+  studyrights.forEach(right => {
+    const id = right.studyrightid.slice(0, -2)
+    const start = new Date(right.studystartdate)
+    if (id in rightsToCount) {
+      if (new Date(rightsToCount[id].studystartdate) > start) {
+        rightsToCount[id] = right
+      }
+    } else {
+      rightsToCount[id] = right
+    }
+  })
+  return Object.values(rightsToCount)
+}
+
+const getFacultyStarters = async (
+  faculty,
+  since,
+  isAcademicYear,
+  yearsArray,
+  graphStats,
+  tableStats,
+  counts,
+  allBasics
+) => {
+  const startedGraphStats = [...graphStats]
+  const startedTableStats = { ...tableStats }
+  const studyrights = await startedStudyrights(faculty, since)
+  const filteredStudyrights = filterDuplicateStudyrights(studyrights)
+
+  filteredStudyrights.forEach(({ studystartdate }) => {
+    const startYear = defineYear(studystartdate, isAcademicYear)
+    startedGraphStats[indexOf(yearsArray, startYear)] += 1
+    startedTableStats[startYear] += 1
+  })
+  allBasics.graphStats.push({ name: 'Started studying', data: startedGraphStats })
+
+  Object.keys(startedTableStats).forEach(year => {
+    counts[year] = [startedTableStats[year]]
+  })
+}
+
+const getFacultyGraduates = async (
+  faculty,
+  since,
+  isAcademicYear,
+  yearsArray,
+  graphStats,
+  tableStats,
+  allBasics,
+  counts
+) => {
+  const graduatedGraphStats = [[...graphStats], [...graphStats], [...graphStats], [...graphStats], [...graphStats]]
+  const graduatedTableStats = {}
+  Object.keys(tableStats).forEach(year => (graduatedTableStats[year] = [0, 0, 0, 0, 0]))
+
+  const graduatedRights = await graduatedStudyrights(faculty, since)
+
+  graduatedRights.forEach(({ enddate, extentcode }) => {
+    const endYear = defineYear(enddate, isAcademicYear)
+    graduatedGraphStats[0][indexOf(yearsArray, endYear)] += 1
+    graduatedTableStats[endYear][0] += 1
+
+    if (extentcode === 1) {
+      graduatedGraphStats[1][indexOf(yearsArray, endYear)] += 1
+      graduatedTableStats[endYear][1] += 1
+    } else if (extentcode === 2) {
+      graduatedGraphStats[2][indexOf(yearsArray, endYear)] += 1
+      graduatedTableStats[endYear][2] += 1
+    } else if (extentcode === 4) {
+      graduatedGraphStats[3][indexOf(yearsArray, endYear)] += 1
+      graduatedTableStats[endYear][3] += 1
+    } else {
+      graduatedGraphStats[4][indexOf(yearsArray, endYear)] += 1
+      graduatedTableStats[endYear][4] += 1
+    }
+  })
+
+  allBasics.graphStats.push({ name: 'All graduated', data: graduatedGraphStats[0] })
+  allBasics.graphStats.push({ name: 'Graduated bachelors', data: graduatedGraphStats[1] })
+  allBasics.graphStats.push({ name: 'Graduated masters', data: graduatedGraphStats[2] })
+  allBasics.graphStats.push({ name: 'Graduated doctors', data: graduatedGraphStats[3] })
+  allBasics.graphStats.push({ name: 'Other graduations', data: graduatedGraphStats[4] })
+
+  Object.keys(graduatedTableStats).forEach(year => {
+    counts[year] = counts[year].concat(graduatedTableStats[year])
+  })
+}
+
+const getFacultyTransfers = async (
+  programmes,
+  since,
+  isAcademicYear,
+  yearsArray,
+  graphStats,
+  tableStats,
+  allBasics,
+  counts
+) => {
+  const insideTransfers = await transferredInsideFaculty(programmes, since)
+  const awayTransfers = await transferredAway(programmes, since)
+  const toTransfers = await transferredTo(programmes, since)
+
+  const transferGraphStats = [[...graphStats], [...graphStats], [...graphStats]]
+  const transferTableStats = {}
+  Object.keys(tableStats).forEach(year => (transferTableStats[year] = [0, 0, 0]))
+
+  insideTransfers.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    transferGraphStats[0][indexOf(yearsArray, transferYear)] += 1
+    transferTableStats[transferYear][0] += 1
+  })
+
+  awayTransfers.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    transferGraphStats[1][indexOf(yearsArray, transferYear)] += 1
+    transferTableStats[transferYear][1] += 1
+  })
+
+  toTransfers.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    transferGraphStats[2][indexOf(yearsArray, transferYear)] += 1
+    transferTableStats[transferYear][2] += 1
+  })
+
+  allBasics.graphStats.push({ name: 'Transferred inside', data: transferGraphStats[0] })
+  allBasics.graphStats.push({ name: 'Transferred away', data: transferGraphStats[1] })
+  allBasics.graphStats.push({ name: 'Transferred to', data: transferGraphStats[2] })
+
+  Object.keys(transferTableStats).forEach(year => {
+    counts[year] = counts[year].concat(transferTableStats[year])
+  })
+}
+
+const combineFacultyBasics = async (allBasics, faculty, programmes, yearType) => {
+  let counts = {}
+  let years = []
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const since = isAcademicYear ? new Date('2017-08-01') : new Date('2017-01-01')
+  const yearsArray = getYearsArray(since.getFullYear(), isAcademicYear)
+  const { graphStats, tableStats } = getStatsBasis(yearsArray)
+  Object.keys(tableStats).forEach(year => years.push(year))
+
+  // Started studying in faculty
+  await getFacultyStarters(faculty, since, isAcademicYear, yearsArray, graphStats, tableStats, counts, allBasics)
+
+  // Graduated in faculty
+  await getFacultyGraduates(faculty, since, isAcademicYear, yearsArray, graphStats, tableStats, allBasics, counts)
+
+  // Transfers
+  await getFacultyTransfers(programmes, since, isAcademicYear, yearsArray, graphStats, tableStats, allBasics, counts)
+
+  // combine tableStats from all categories
+  allBasics.years = years
+  years.forEach(year => {
+    allBasics.tableStats.push([year, ...counts[year]])
+  })
+  allBasics.tableStats.reverse()
+}
+
+module.exports = { combineFacultyBasics }

services/backend/src/services/facultyCredits.js

@@ -0,0 +1,230 @@
+const { indexOf } = require('lodash')
+const { mapToProviders } = require('../util/utils')
+const {
+  getStatsBasis,
+  defineYear,
+  isMajorStudentCredit,
+  isNonMajorCredit,
+  getYearsArray,
+  getStartDate,
+  isSpecialGroupCredit,
+  tableTitles,
+} = require('./studyprogrammeHelpers')
+const {
+  getStudyRights,
+  getCreditsForStudyProgramme,
+  allTransfers,
+  getCourseCodesForStudyProgramme,
+} = require('./studyprogramme')
+
+const isFacultyNonMajorCredit = (studyrights, attainment_date, facultyProgrammes) => {
+  let right = ''
+  studyrights.forEach(studyright => {
+    if (studyright) {
+      if (!studyright.graduated && new Date(attainment_date) >= new Date(studyright.studystartdate)) {
+        if (!right) right = studyright.code
+        else if (right && !facultyProgrammes.includes(right) && facultyProgrammes.includes(studyright.code))
+          right = studyright.code
+      } else {
+        if (
+          new Date(attainment_date) >= new Date(studyright.studystartdate) &&
+          new Date(attainment_date) <= new Date(studyright.enddate)
+        ) {
+          if (!right) right = studyright.code
+          else if (right && !facultyProgrammes.includes(right) && facultyProgrammes.includes(studyright.code))
+            right = studyright.code
+        }
+      }
+    }
+  })
+  return facultyProgrammes.includes(right)
+}
+
+// Fetches all the credits for the studyprogramme and divides them into major-students, non-major faculty students,
+// non-major other faculty students adn non-degree student credits
+// Division is done on the basis that whether the student had a primary studyright to the programme on the attainment_date
+const getFacultyRegularCreditStats = async ({
+  studyprogramme,
+  since,
+  years,
+  isAcademicYear,
+  includeAllSpecials,
+  facultyProgrammes,
+}) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const courses = await getCourseCodesForStudyProgramme(providercode)
+  const credits = await getCreditsForStudyProgramme(courses, since)
+  const students = [...new Set(credits.map(({ student_studentnumber }) => student_studentnumber))]
+
+  let studyrights = await getStudyRights(students, since)
+  const transfers = (await allTransfers(studyprogramme, since)).map(t => t.studyrightid)
+  if (!includeAllSpecials) {
+    studyrights = studyrights.filter(s => !transfers.includes(s.studyrightid))
+  }
+
+  let majors = getStatsBasis(years)
+  let allNonMajors = getStatsBasis(years)
+  let facultyNonMajors = getStatsBasis(years)
+  let otherNonMajors = getStatsBasis(years)
+  let nonDegree = getStatsBasis(years)
+
+  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
+    const studentStudyrights = studyrights.filter(studyright => studyright.studentnumber === student_studentnumber)
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+
+    if (!includeAllSpecials && isSpecialGroupCredit(studentStudyrights, attainment_date, transfers)) {
+      return
+    }
+
+    if (isMajorStudentCredit(studentStudyrights, attainment_date, studyprogramme)) {
+      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      majors.tableStats[attainmentYear] += credits || 0
+    } else if (isNonMajorCredit(studentStudyrights, attainment_date)) {
+      // for comparison
+      allNonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      allNonMajors.tableStats[attainmentYear] += credits || 0
+
+      if (isFacultyNonMajorCredit(studentStudyrights, attainment_date, facultyProgrammes)) {
+        facultyNonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+        facultyNonMajors.tableStats[attainmentYear] += credits || 0
+      } else {
+        otherNonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+        otherNonMajors.tableStats[attainmentYear] += credits || 0
+      }
+    } else {
+      nonDegree.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      nonDegree.tableStats[attainmentYear] += credits || 0
+    }
+  })
+
+  if (
+    majors.graphStats.every(year => year === 0) &&
+    facultyNonMajors.graphStats.every(year => year === 0) &&
+    otherNonMajors.graphStats.every(year => year === 0) &&
+    nonDegree.graphStats.every(year => year === 0)
+  ) {
+    return {
+      majors: { graphStats: [], tableStats: {} },
+      facultyNonMajors: { graphStats: [], tableStats: {} },
+      otherNonMajors: { graphStats: [], tableStats: {} },
+      nonDegree: { graphStats: [], tablestats: {} },
+    }
+  }
+
+  return { majors, facultyNonMajors, otherNonMajors, nonDegree }
+}
+
+// Fetches all credits for the studytrack and combines them into statistics
+// These are for the faculty view which differs from study programme overview
+// (no tranfers, non majors split to in- and outside-faculty)
+const getFacultyCreditStatsForStudytrack = async ({ studyprogramme, facultyProgrammes, settings }) => {
+  const { isAcademicYear, includeAllSpecials } = settings
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials, facultyProgrammes }
+  const { majors, facultyNonMajors, otherNonMajors, nonDegree } = await getFacultyRegularCreditStats(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const titles = tableTitles['credits'][includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED']
+  const dataFound = [majors, facultyNonMajors, otherNonMajors].some(d => d.graphStats.length) // , transferred
+
+  if (!dataFound) return null
+
+  const tableStats = reversedYears.map(year =>
+    includeAllSpecials
+      ? [
+          year,
+          majors.tableStats[year] +
+            facultyNonMajors.tableStats[year] +
+            otherNonMajors.tableStats[year] +
+            nonDegree.tableStats[year],
+          majors.tableStats[year],
+          facultyNonMajors.tableStats[year],
+          otherNonMajors.tableStats[year],
+          nonDegree.tableStats[year],
+        ]
+      : [
+          year,
+          majors.tableStats[year] +
+            facultyNonMajors.tableStats[year] +
+            otherNonMajors.tableStats[year] +
+            nonDegree.tableStats[year],
+          majors.tableStats[year],
+        ]
+  )
+
+  const graphStats = [
+    {
+      name: 'Major students credits',
+      data: majors.graphStats,
+    },
+    {
+      name: 'Non-major faculty students credits',
+      data: facultyNonMajors.graphStats,
+    },
+    {
+      name: 'Non-major other faculty students credits 2',
+      data: otherNonMajors.graphStats,
+    },
+    {
+      name: 'Non-degree credits',
+      data: nonDegree.graphStats,
+    },
+  ]
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    graphStats,
+    titles,
+  }
+}
+
+const getProgrammeCredits = async (code, yearType, specialGroups, facultyProgrammes) => {
+  // redis for faculty? Create faculty versions for out-commented functions
+  // const data = await getCreditStats(code, yearType, specialGroups)
+  // if (data) return data
+  const updatedStats = await getFacultyCreditStatsForStudytrack({
+    studyprogramme: code,
+    facultyProgrammes,
+    settings: {
+      isAcademicYear: yearType === 'ACADEMIC_YEAR',
+      includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
+    },
+  })
+
+  // if (updatedStats) await setCreditStats(updatedStats, yearType, specialGroups)
+  return updatedStats
+}
+
+const combineFacultyCredits = async (allCredits, programmes, yearType, specialGroups, counts, years) => {
+  for (const prog of programmes) {
+    const data = await getProgrammeCredits(prog, yearType, specialGroups, programmes)
+    if (data) {
+      if (!allCredits.lastUpdated || new Date(data.lastUpdated) > new Date(allCredits.lastUpdated))
+        allCredits.lastUpdated = data.lastUpdated
+
+      data.tableStats.forEach(row => {
+        if (!(row[0] in counts)) {
+          counts[row[0]] = row.slice(1)
+          years.push(row[0])
+        } else {
+          counts[row[0]] = row.slice(1).map((value, i) => {
+            return counts[row[0]][i] + value
+          })
+        }
+      })
+    }
+  }
+  // save table stats and graph stats
+  years.forEach(year => {
+    allCredits.tableStats.push([year, ...counts[year]])
+  })
+
+  years.sort()
+  allCredits.years = years
+}
+
+module.exports = { combineFacultyCredits, getFacultyCreditStatsForStudytrack }

services/backend/src/services/mailservice.js

@@ -3,7 +3,7 @@ const { isProduction, pateToken } = require('../conf-backend')
 const { ApplicationError } = require('../util/customErrors')
 
 const pateClient = axios.create({
-  baseURL: 'https://pate.toska.cs.helsinki.fi',
+  baseURL: 'https://importer.cs.helsinki.fi/api/pate/',
   params: {
     token: pateToken,
   },
@@ -22,7 +22,7 @@ const baseSettings = {
 const sendEmail = async (options = {}) => {
   if (!isProduction) throw new ApplicationError('Email sending is disabled in development mode.')
   if (!pateToken) throw new ApplicationError('Email sending failed because pate token is missing.')
-  await pateClient.post('/', options)
+  return await pateClient.post('/', options)
 }
 
 const sendFeedbackToToska = async ({ feedbackContent, user }) => {

services/backend/src/services/newStudyprogramme.js

@@ -1,402 +0,0 @@
-const sequelize = require('sequelize')
-const { Op } = sequelize
-const {
-  Credit,
-  Course,
-  Organization,
-  Studyright,
-  StudyrightElement,
-  ElementDetail,
-  Transfer,
-  Student,
-  SemesterEnrollment,
-  Semester,
-} = require('../models')
-const { formatStudyright, formatStudent, formatTransfer } = require('./studyprogrammeHelpers')
-const { getCurrentSemester } = require('./semesters')
-
-const whereStudents = studentnumbers => {
-  return studentnumbers ? studentnumbers : { [Op.not]: null }
-}
-
-const sinceDate = since => {
-  return since ? { [Op.gte]: since } : { [Op.not]: null }
-}
-
-const studytrackStudents = async studentnumbers =>
-  (
-    await Student.findAll({
-      where: {
-        studentnumber: {
-          [Op.in]: studentnumbers,
-        },
-      },
-    })
-  ).map(formatStudent)
-
-const enrolledStudents = async (studytrack, since, studentnumbers) => {
-  const currentSemester = await getCurrentSemester()
-
-  const students = await Student.findAll({
-    attributes: ['studentnumber'],
-    include: [
-      {
-        model: Studyright,
-        include: [
-          {
-            model: StudyrightElement,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        ],
-        attributes: ['studyrightid'],
-      },
-      {
-        model: SemesterEnrollment,
-        attributes: ['semestercode'],
-        include: [
-          {
-            model: Semester,
-            required: true,
-            where: {
-              semestercode: currentSemester.semestercode,
-            },
-          },
-        ],
-        where: {
-          enrollmenttype: 1,
-        },
-      },
-    ],
-    where: {
-      studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-    },
-  })
-
-  return students.filter(s => s.semester_enrollments?.length)
-}
-
-const absentStudents = async (studytrack, studentnumbers) => {
-  const currentSemester = await getCurrentSemester()
-  const students = await Student.findAll({
-    attributes: ['studentnumber'],
-    include: [
-      {
-        model: Studyright,
-        include: [
-          {
-            model: StudyrightElement,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        ],
-        attributes: ['studyrightid'],
-      },
-      {
-        model: SemesterEnrollment,
-        attributes: ['semestercode'],
-        where: {
-          enrollmenttype: 2,
-          semestercode: currentSemester.semestercode,
-        },
-      },
-    ],
-    where: {
-      studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-    },
-  })
-
-  return students
-}
-
-const allStudyrights = async (studytrack, since, studentnumbers) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        student_studentnumber: whereStudents(studentnumbers),
-      },
-    })
-  ).map(formatStudyright)
-
-const startedStudyrights = async (studytrack, since, studentnumbers) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        studystartdate: {
-          [Op.gte]: since,
-        },
-        student_studentnumber: whereStudents(studentnumbers),
-      },
-    })
-  ).map(formatStudyright)
-
-const graduatedStudyRights = async (studytrack, since, studentnumbers) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        graduated: 1,
-        enddate: sinceDate(since),
-        student_studentnumber: whereStudents(studentnumbers),
-      },
-    })
-  ).map(formatStudyright)
-
-const followingStudyrights = async (since, studentnumbers) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          include: [
-            {
-              model: ElementDetail,
-              attributes: ['name'],
-            },
-          ],
-          attributes: ['code'],
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        studystartdate: {
-          [Op.gte]: since,
-        },
-        extentcode: 2,
-        student_studentnumber: whereStudents(studentnumbers),
-      },
-    })
-  ).map(formatStudyright)
-
-const transfersAway = async (studytrack, since) =>
-  (
-    await Transfer.findAll({
-      where: {
-        transferdate: {
-          [Op.gte]: since,
-        },
-        sourcecode: studytrack,
-      },
-      distinct: true,
-      col: 'studentnumber',
-    })
-  ).map(formatTransfer)
-
-const transfersTo = async (studytrack, since) =>
-  (
-    await Transfer.findAll({
-      where: {
-        transferdate: {
-          [Op.gte]: since,
-        },
-        targetcode: studytrack,
-      },
-      distinct: true,
-      col: 'studentnumber',
-    })
-  ).map(formatTransfer)
-
-const allTransfers = async (studytrack, since) => {
-  const transferredTo = await transfersTo(studytrack, since)
-  const transferredAway = await transfersAway(studytrack, since)
-  return [...transferredTo, ...transferredAway]
-}
-
-const getProgrammesStudyrights = async studyprogramme =>
-  (
-    await Studyright.findAll({
-      attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode'],
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          where: {
-            code: {
-              [Op.in]: [studyprogramme],
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-    })
-  ).map(formatStudyright)
-
-const getCreditsForStudyProgramme = async (provider, since) =>
-  await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
-      where: {
-        is_study_module: false,
-      },
-      include: {
-        model: Organization,
-        required: true,
-        where: {
-          code: provider,
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.notIn]: [10, 9, 7],
-      },
-      isStudyModule: {
-        [Op.not]: true,
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-    },
-  })
-
-const getTransferredCredits = async (provider, since) =>
-  await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
-      include: {
-        model: Organization,
-        required: true,
-        where: {
-          code: provider,
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.eq]: [9],
-      },
-      isStudyModule: {
-        [Op.not]: true,
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-    },
-  })
-
-const getThesisCredits = async (provider, since, thesisType, studentnumbers) =>
-  await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
-      where: {
-        course_unit_type: {
-          [Op.in]: thesisType,
-        },
-      },
-      include: {
-        model: Organization,
-        required: true,
-        where: {
-          code: provider,
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.notIn]: [10, 9, 7],
-      },
-      isStudyModule: {
-        [Op.not]: true,
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-      student_studentnumber: whereStudents(studentnumbers),
-    },
-  })
-
-module.exports = {
-  studytrackStudents,
-  enrolledStudents,
-  absentStudents,
-  allStudyrights,
-  startedStudyrights,
-  graduatedStudyRights,
-  followingStudyrights,
-  transfersAway,
-  transfersTo,
-  allTransfers,
-  getProgrammesStudyrights,
-  getCreditsForStudyProgramme,
-  getTransferredCredits,
-  getThesisCredits,
-}

services/backend/src/services/parseCredits.js

@@ -65,4 +65,4 @@ const parseCredit = (credit, anonymizationSalt) => {
   }
 }
 
-module.exports = { parseCredit }
+module.exports = { parseCredit, formatStudyrightElement }

services/backend/src/services/parseEnrollments.js

@@ -0,0 +1,48 @@
+const crypto = require('crypto')
+const { formatStudyrightElement } = require('./parseCredits')
+
+const parseEnrollment = (enrollment, anonymizationSalt) => {
+  const { student, semester, state, enrollment_date_time, course_code } = enrollment
+  const { studentnumber, studyright_elements } = student
+  const { yearcode, yearname, semestercode, name: semestername } = semester
+
+  if (anonymizationSalt) {
+    const anonymizedStudentNumber = crypto
+      .createHash('sha256')
+      .update(`${studentnumber}${anonymizationSalt}`)
+      .digest('hex')
+
+    const anonymizedStudent = {
+      studentnumber: anonymizedStudentNumber,
+    }
+
+    return {
+      obfuscated: true,
+      student: anonymizedStudent,
+      yearcode,
+      yearname,
+      semestercode,
+      semestername,
+      coursecode: course_code,
+      state,
+      enrollment_date_time,
+      studentnumber: anonymizedStudentNumber,
+      programmes: studyright_elements.map(formatStudyrightElement),
+    }
+  }
+
+  return {
+    student,
+    yearcode,
+    yearname,
+    semestercode,
+    semestername,
+    coursecode: course_code,
+    state,
+    enrollment_date_time,
+    studentnumber,
+    programmes: studyright_elements.map(formatStudyrightElement),
+  }
+}
+
+module.exports = { parseEnrollment }

services/backend/src/services/populations.js

@@ -1,7 +1,7 @@
-const { Op } = require('sequelize')
+const Sequelize = require('sequelize')
+const { Op } = Sequelize
 const moment = require('moment')
-const { sortBy } = require('lodash')
-const { sortMainCode } = require('../util/utils')
+const { sortBy, keyBy } = require('lodash')
 
 const {
   Student,
@@ -10,11 +10,12 @@ const {
   Studyright,
   StudyrightExtent,
   ElementDetail,
-  CourseType,
+  Studyplan,
   SemesterEnrollment,
   Semester,
   Transfer,
   StudyrightElement,
+  Enrollment,
 } = require('../models')
 const {
   dbConnections: { sequelize },
@@ -47,6 +48,7 @@ const formatStudentForPopulationStatistics = (
     abbreviatedname,
     email,
     studyrights,
+    studyplans,
     semester_enrollments,
     transfers,
     updatedAt,
@@ -58,7 +60,9 @@ const formatStudentForPopulationStatistics = (
     tags,
     option,
     birthdate,
+    sis_person_id,
   },
+  enrollments,
   credits,
   startDate,
   endDate,
@@ -93,6 +97,7 @@ const formatStudentForPopulationStatistics = (
     studentNumber: studentnumber,
     credits: creditcount || 0,
     courses: credits[studentnumber] ? credits[studentnumber].map(toCourse) : [],
+    enrollments: enrollments[studentnumber],
     name: abbreviatedname,
     transfers: transfers || [],
     gender_code,
@@ -105,6 +110,8 @@ const formatStudentForPopulationStatistics = (
     starting: moment(started).isBetween(startDateMoment, endDateMoment, null, '[]'),
     option,
     birthdate,
+    studyplans,
+    sis_person_id,
   }
 }
 
@@ -140,10 +147,24 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
   if (tag) studentnumbers = studentnumbersWithTag
 
   const attainmentDateFrom = tag ? moment(startDate).year(tag.year) : startDate
+  const studyPlans = await Studyplan.findAll({
+    where: { studentnumber: studentnumbers },
+    attributes: ['included_courses'],
+    raw: true,
+  })
+  const studyPlanCourses = Array.from(new Set([...studyPlans.map(plan => plan.included_courses)].flat()))
+
   const creditsOfStudentOther = {
-    attainment_date: {
-      [Op.between]: [attainmentDateFrom, endDate],
-    },
+    [Op.or]: [
+      {
+        attainment_date: {
+          [Op.between]: [attainmentDateFrom, endDate],
+        },
+      },
+      {
+        course_code: studyPlanCourses,
+      },
+    ],
     student_studentnumber: {
       [Op.in]: studentnumbers,
     },
@@ -162,7 +183,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
       },
       {
         course_code: {
-          [Op.in]: ['375063', '339101'],
+          [Op.in]: ['375063', '339101'].concat(studyPlanCourses),
         },
       },
     ],
@@ -173,8 +194,9 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
     : creditsOfStudentOther
 
   if (studentnumbers.length === 0)
-    return { students: [], credits: [], extents: [], semesters: [], elementdetails: [], courses: [] }
-  const [courses, students, credits, extents, semesters, elementdetails] = await Promise.all([
+    return { students: [], enrollments: [], credits: [], extents: [], semesters: [], elementdetails: [], courses: [] }
+
+  const [courses, enrollments, students, credits, extents, semesters, elementdetails] = await Promise.all([
     Course.findAll({
       attributes: [sequelize.literal('DISTINCT ON("code") code'), 'name', 'coursetypecode'],
       include: [
@@ -186,6 +208,19 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
       ],
       raw: true,
     }),
+    Enrollment.findAll({
+      attributes: ['course_code', 'state', 'enrollment_date_time', 'studentnumber'],
+      where: {
+        enrollment_date_time: {
+          [Op.between]: [attainmentDateFrom, endDate],
+        },
+        studentnumber: {
+          [Op.in]: studentnumbers,
+        },
+        state: ['ENROLLED', 'CONFIRMED'],
+      },
+      raw: true,
+    }),
     Student.findAll({
       attributes: [
         'firstnames',
@@ -199,6 +234,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
         'updatedAt',
         'gender_code',
         'birthdate',
+        'sis_person_id',
         /* 'gender_fi',
         'gender_sv',
         'gender_en' */
@@ -217,7 +253,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
             'enddate',
             'extentcode',
             'graduated',
-            'canceldate',
+            'active',
             'prioritycode',
             'faculty_code',
             'studystartdate',
@@ -250,6 +286,11 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
             },
           },
         },
+        {
+          model: Studyplan,
+          attributes: ['included_courses', 'programme_code', 'completed_credits'],
+          separate: true,
+        },
       ],
       where: {
         studentnumber: {
@@ -330,7 +371,7 @@ EXISTS (SELECT 1 FROM studyright_elements WHERE studentnumber IN (:studentnumber
     student.tags = studentNumberToTags[student.studentnumber] || []
   })
 
-  return { students, credits, extents, semesters, elementdetails, courses }
+  return { students, enrollments, credits, extents, semesters, elementdetails, courses }
 }
 
 const count = (column, count, distinct = false) => {
@@ -361,7 +402,6 @@ const studentnumbersWithAllStudyrightElements = async (
   startDate,
   endDate,
   exchangeStudents,
-  cancelledStudents,
   nondegreeStudents,
   transferredOutStudents,
   tag,
@@ -385,9 +425,6 @@ const studentnumbersWithAllStudyrightElements = async (
   if (!nondegreeStudents) {
     filteredExtents.push(33, 99, 14, 13)
   }
-  if (!cancelledStudents) {
-    studyrightWhere.canceldate = null
-  }
 
   let studentWhere = {}
   if (tag) {
@@ -405,7 +442,7 @@ const studentnumbersWithAllStudyrightElements = async (
   }
 
   const students = await Studyright.findAll({
-    attributes: ['student_studentnumber', 'graduated'],
+    attributes: ['student_studentnumber', 'graduated', 'enddate'],
     include: {
       model: StudyrightElement,
       attributes: [],
@@ -446,7 +483,7 @@ const studentnumbersWithAllStudyrightElements = async (
     raw: true,
   })
 
-  const studentnumbers = [...new Set(students.map(s => s.student_studentnumber))]
+  let studentnumbers = [...new Set(students.map(s => s.student_studentnumber))]
 
   // bit hacky solution, but this is used to filter out studentnumbers who have since changed studytracks
   const rights = await Studyright.findAll({
@@ -493,13 +530,17 @@ const studentnumbersWithAllStudyrightElements = async (
     return acc
   }, {})
 
+  // Take the newest studytrack primarily by latest starting date in the track, secondarily by the latest enddate
   const filteredStudentnumbers = studentnumbers.filter(studentnumber => {
-    const newestStudytrack = sortBy(formattedStudytracks[studentnumber], ['enddate', 'startdate']).reverse()[0]
+    const newestStudytrack = sortBy(formattedStudytracks[studentnumber], ['startdate', 'enddate']).reverse()[0]
     if (!newestStudytrack) return false
     return studyRights.includes(newestStudytrack.code)
   })
 
-  let studentnumberlist = filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers
+  // Use the filtered list, if the search includes studytracks
+  // Then the studyrights length is > 1, which means that there is [studyright, studytrack].
+  // When searching only for studyprogramme, there is [studyright]
+  let studentnumberlist = studyRights.length > 1 ? filteredStudentnumbers : studentnumbers
 
   // fetch students that have transferred out of the programme and filter out these studentnumbers
   if (!transferredOutStudents) {
@@ -592,13 +633,11 @@ const parseQueryParams = query => {
     ? `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd[semesters.find(s => s === 'SPRING')]}`
     : `${year}-${semesterEnd[semesters.find(s => s === 'FALL')]}`
   const exchangeStudents = studentStatuses && studentStatuses.includes('EXCHANGE')
-  const cancelledStudents = studentStatuses && studentStatuses.includes('CANCELLED')
   const nondegreeStudents = studentStatuses && studentStatuses.includes('NONDEGREE')
   const transferredStudents = studentStatuses && studentStatuses.includes('TRANSFERRED')
 
   return {
     exchangeStudents,
-    cancelledStudents,
     nondegreeStudents,
     transferredStudents,
     // if someone passes something falsy like null as the studyright, remove it so it doesn't break
@@ -703,7 +742,7 @@ const getOptionsForStudents = async (students, code, level) => {
 }
 
 const formatStudentsForApi = async (
-  { students, credits, extents, semesters, elementdetails, courses },
+  { students, enrollments, credits, extents, semesters, elementdetails, courses },
   startDate,
   endDate,
   { studyRights },
@@ -724,6 +763,11 @@ const formatStudentsForApi = async (
     acc[e.student_studentnumber].push(e)
     return acc
   }, {})
+  enrollments = enrollments.reduce((acc, e) => {
+    acc[e.studentnumber] = acc[e.studentnumber] || []
+    acc[e.studentnumber].push(e)
+    return acc
+  }, {})
   const result = students.reduce(
     (stats, student) => {
       student.transfers.forEach(transfer => {
@@ -737,7 +781,15 @@ const formatStudentsForApi = async (
       if (student.studentnumber in optionData) student.option = optionData[student.studentnumber]
       else student.option = null
       stats.students.push(
-        formatStudentForPopulationStatistics(student, credits, startDate, endDate, startDateMoment, endDateMoment)
+        formatStudentForPopulationStatistics(
+          student,
+          enrollments,
+          credits,
+          startDate,
+          endDate,
+          startDateMoment,
+          endDateMoment
+        )
       )
       return stats
     },
@@ -800,22 +852,13 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
   if (
     formattedQueryParams.studentStatuses &&
     !formattedQueryParams.studentStatuses.every(
-      status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
+      status => status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
     )
   ) {
-    return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE or TRANSFERRED' }
+    return { error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' }
   }
-  const {
-    studyRights,
-    startDate,
-    months,
-    endDate,
-    exchangeStudents,
-    cancelledStudents,
-    nondegreeStudents,
-    transferredStudents,
-    tag,
-  } = parseQueryParams(formattedQueryParams)
+  const { studyRights, startDate, months, endDate, exchangeStudents, nondegreeStudents, transferredStudents, tag } =
+    parseQueryParams(formattedQueryParams)
 
   // db startdate is formatted to utc so need to change it when querying
   const formattedStartDate = new Date(moment.tz(startDate, 'Europe/Helsinki').format()).toUTCString()
@@ -827,7 +870,6 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
         formattedStartDate,
         endDate,
         exchangeStudents,
-        cancelledStudents,
         nondegreeStudents,
         transferredStudents
       )
@@ -841,48 +883,89 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
     optionData = await getOptionsForStudents(studentnumbers, code, 'BSC')
   }
 
-  const { students, credits, extents, semesters, elementdetails, courses } = await getStudentsIncludeCoursesBetween(
-    studentnumbers,
-    startDate,
-    dateMonthsFromNow(startDate, months),
-    studyRights,
-    tag
-  )
+  const { students, enrollments, credits, extents, semesters, elementdetails, courses } =
+    await getStudentsIncludeCoursesBetween(
+      studentnumbers,
+      startDate,
+      dateMonthsFromNow(startDate, months),
+      studyRights,
+      tag
+    )
 
   const formattedStudents = await formatStudentsForApi(
-    { students, credits, extents, semesters, elementdetails, courses },
+    { students, enrollments, credits, extents, semesters, elementdetails, courses },
     startDate,
     endDate,
     formattedQueryParams,
     optionData
   )
+
   return formattedStudents
 }
 
-const findCourses = async (studentnumbers, beforeDate) => {
-  const res = await Course.findAll({
-    attributes: ['code', 'name', 'coursetypecode', 'substitutions'],
-    include: [
-      {
-        required: true,
-        model: Credit,
-        attributes: ['grade', 'student_studentnumber', 'credittypecode', 'attainment_date', 'course_code'],
-        where: {
-          student_studentnumber: {
-            [Op.in]: studentnumbers,
-          },
-          attainment_date: {
-            [Op.lt]: beforeDate,
-          },
-        },
+const getSubstitutions = async codes => {
+  const courses = await Course.findAll({ where: { code: codes }, attributes: ['code', 'substitutions'], raw: true })
+  return [...new Set(courses.map(({ code, substitutions }) => [code, ...substitutions]).flat())]
+}
+
+const findCourses = async (studentnumbers, beforeDate, courses = [], studentCountLimit = 0) => {
+  const courseCodes = courses.length > 0 ? await getSubstitutions(courses) : ['DUMMY']
+
+  const res = await sequelize.query(
+    `
+      SELECT DISTINCT ON (course.id)
+        course.code,
+        course.name,
+        course.coursetypecode,
+        course.substitutions,
+        course.main_course_code,
+        course_types.name AS course_type,
+        credit.data AS credits,
+        enrollment.data AS enrollments
+      FROM course
+      INNER JOIN (
+        SELECT
+          course_code,
+          ARRAY_AGG(JSON_BUILD_OBJECT(
+            'grade', grade,
+            'student_studentnumber', student_studentnumber,
+            'attainment_date', attainment_date,
+            'credittypecode', credittypecode,
+            'course_code', course_code
+          )) AS data
+        FROM credit
+        WHERE student_studentnumber IN (:studentnumbers) AND attainment_date < :beforeDate
+        GROUP BY credit.course_code
+        HAVING COUNT(DISTINCT credit.student_studentnumber) >= :studentCountLimit
+      ) credit ON credit.course_code = course.code
+      INNER JOIN course_types ON course_types.coursetypecode = course.coursetypecode
+      LEFT JOIN (
+        SELECT
+          course_id,
+          ARRAY_AGG(JSON_BUILD_OBJECT(
+            'studentnumber', studentnumber,
+            'state', state,
+            'enrollment_date_time', enrollment_date_time
+          )) AS data
+        FROM enrollment
+        WHERE enrollment.studentnumber IN (:studentnumbers) AND enrollment.enrollment_date_time < :beforeDate
+        GROUP BY enrollment.course_id
+      ) enrollment ON enrollment.course_id = course.id 
+      WHERE :skipCourseCodeFilter OR course.code IN (:courseCodes)
+      -- GROUP BY 1, 2, 3, 4, 5, 6
+    `,
+    {
+      replacements: {
+        studentnumbers: studentnumbers.length > 0 ? studentnumbers : ['DUMMY'],
+        beforeDate,
+        studentCountLimit,
+        courseCodes,
+        skipCourseCodeFilter: courses.length === 0,
       },
-      {
-        attributes: ['coursetypecode', 'name'],
-        model: CourseType,
-        required: true,
-      },
-    ],
-  })
+      type: sequelize.QueryTypes.SELECT,
+    }
+  )
+
   return res
 }
 
@@ -898,37 +981,9 @@ const parseCreditInfo = credit => ({
   date: credit.attainment_date,
 })
 
-const findMainCourse = async code => {
-  const course = await Course.findOne({
-    where: { code: code },
-  })
-
-  let result = []
-
-  if (course.substitutions) {
-    const sortedCourses = sortMainCode([...course.substitutions, code])
-    result = await Course.findOne({
-      attributes: ['code', 'name'],
-      where: { code: sortedCourses[0] },
-    })
-  } else {
-    return course
-  }
-
-  return result
-}
-
 const bottlenecksOf = async (query, studentnumberlist) => {
   const isValidRequest = async (query, params) => {
-    const {
-      studyRights,
-      startDate,
-      endDate,
-      exchangeStudents,
-      cancelledStudents,
-      nondegreeStudents,
-      transferredStudents,
-    } = params
+    const { studyRights, startDate, endDate, exchangeStudents, nondegreeStudents, transferredStudents } = params
 
     if (!query.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
       return { error: 'Semester should be either SPRING OR FALL' }
@@ -936,10 +991,10 @@ const bottlenecksOf = async (query, studentnumberlist) => {
     if (
       query.studentStatuses &&
       !query.studentStatuses.every(
-        status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
+        status => status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
       )
     ) {
-      return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE or TRANSFERRED' }
+      return { error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' }
     }
     if (query.selectedStudents) {
       const allStudents = await studentnumbersWithAllStudyrightElements(
@@ -947,7 +1002,6 @@ const bottlenecksOf = async (query, studentnumberlist) => {
         startDate,
         endDate,
         exchangeStudents,
-        cancelledStudents,
         nondegreeStudents,
         transferredStudents,
         query.tag
@@ -961,19 +1015,10 @@ const bottlenecksOf = async (query, studentnumberlist) => {
     return null
   }
 
-  const getStudentsAndCourses = async (selectedStudents, studentnumberlist) => {
+  const getStudentsAndCourses = async (selectedStudents, studentnumberlist, courseCodes) => {
     if (!studentnumberlist) {
-      const {
-        months,
-        studyRights,
-        startDate,
-        endDate,
-        exchangeStudents,
-        cancelledStudents,
-        nondegreeStudents,
-        transferredStudents,
-        tag,
-      } = params
+      const { months, studyRights, startDate, endDate, exchangeStudents, nondegreeStudents, transferredStudents, tag } =
+        params
       const studentnumbers =
         selectedStudents ||
         (await studentnumbersWithAllStudyrightElements(
@@ -981,7 +1026,6 @@ const bottlenecksOf = async (query, studentnumberlist) => {
           startDate,
           endDate,
           exchangeStudents,
-          cancelledStudents,
           nondegreeStudents,
           transferredStudents,
           tag
@@ -992,20 +1036,28 @@ const bottlenecksOf = async (query, studentnumberlist) => {
         return numbers
       }, {})
 
-      const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months))
+      const courses = await findCourses(
+        studentnumbers,
+        dateMonthsFromNow(startDate, months),
+        courseCodes,
+        query.studentCountLimit
+      )
       return [allstudents, courses]
     } else {
+      const { months, startDate } = params
+      const beforeDate = months && startDate ? dateMonthsFromNow(startDate, months) : new Date()
+
       const allstudents = studentnumberlist.reduce((numbers, num) => {
         numbers[num] = true
         return numbers
       }, {})
-      const courses = await findCourses(studentnumberlist, new Date())
+      const courses = await findCourses(studentnumberlist, beforeDate, courseCodes, query.studentCountLimit)
       return [allstudents, courses]
     }
   }
   const params = parseQueryParams(query)
   const [[allstudents, courses], error] = await Promise.all([
-    getStudentsAndCourses(query.selectedStudents, studentnumberlist),
+    getStudentsAndCourses(query.selectedStudents, studentnumberlist, query.courses),
     isValidRequest(query, params),
   ])
   if (error) return error
@@ -1019,17 +1071,19 @@ const bottlenecksOf = async (query, studentnumberlist) => {
 
   const startYear = parseInt(query.year, 10)
   const allstudentslength = Object.keys(allstudents).length
+  const coursesByCode = keyBy(courses, 'code')
 
   for (const course of courses) {
     let { course_type } = course
     let maincourse = course
-    const sortedMaincourse = await findMainCourse(course.code)
-    if (sortedMaincourse) maincourse = sortedMaincourse
 
-    /*  if (isOpenUniCourseCode) {
-      const nonOpen = unifyOpenUniversity(course.code)
-      const response = await findOneByCode(nonOpen)
- */
+    if (course.main_course_code && course.main_course_code !== course.code) {
+      let newmain = coursesByCode[course.main_course_code]
+
+      if (newmain) {
+        maincourse = newmain
+      }
+    }
 
     if (!stats[maincourse.code]) {
       stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudentslength)
@@ -1046,6 +1100,14 @@ const bottlenecksOf = async (query, studentnumberlist) => {
       const semester = getPassingSemester(startYear, date)
       coursestats.markCredit(studentnumber, grade, passingGrade, failingGrade, improvedGrade, semester)
     })
+
+    if (course.enrollments) {
+      course.enrollments.forEach(({ studentnumber, state, enrollment_date_time }) => {
+        const semester = getPassingSemester(startYear, enrollment_date_time)
+        coursestats.markEnrollment(studentnumber, state, semester)
+      })
+    }
+
     stats[maincourse.code] = coursestats
   }
 

services/backend/src/services/semesters.js

@@ -1,3 +1,4 @@
+const moment = require('moment')
 const sequelize = require('sequelize')
 const { Semester } = require('../models')
 const Op = sequelize.Op
@@ -28,8 +29,8 @@ const getSemestersAndYears = async () => {
         acc.years[yearcode] = {
           yearcode,
           yearname,
-          startdate: Math.min(acc.years[yearcode].startdate, startdate),
-          enddate: Math.max(acc.years[yearcode].enddate, enddate),
+          startdate: moment.min(moment(acc.years[yearcode].startdate), moment(startdate)),
+          enddate: moment.max(moment(acc.years[yearcode].enddate), moment(enddate)),
         }
       }
       return acc

services/backend/src/services/statMerger.js

@@ -26,7 +26,7 @@ const populationCourseStatsMerger = multiyearstats => {
         stats.coursestatistics[index].stats.retryPassed += c.stats.retryPassed
 
         stats.coursestatistics[index].stats.perStudent =
-          (stats.coursestatistics[index].stats.attempts / stats.coursestatistics[index].stats.students) * 100
+          stats.coursestatistics[index].stats.attempts / stats.coursestatistics[index].stats.students
 
         stats.coursestatistics[index].stats.percentage =
           (stats.coursestatistics[index].stats.passed / stats.coursestatistics[index].stats.students) * 100

services/backend/src/services/students.js

@@ -12,6 +12,7 @@ const {
   ElementDetail,
   SemesterEnrollment,
   Semester,
+  Studyplan,
 } = require('../models')
 const { TagStudent, Tag } = require('../models/models_kone')
 const Op = Sequelize.Op
@@ -57,6 +58,10 @@ const byId = async id => {
         {
           model: SemesterEnrollment,
         },
+        {
+          model: Studyplan,
+          attributes: ['included_courses', 'programme_code', 'completed_credits'],
+        },
       ],
     }),
     TagStudent.findAll({
@@ -99,33 +104,79 @@ const byId = async id => {
   return student
 }
 
-const findByCourseAndSemesters = async (coursecodes, from, to, separate) =>
-  (
+const getUnifyStatus = unifyCourses => {
+  switch (unifyCourses) {
+    case 'unifyStats':
+      return [true, false]
+
+    case 'openStats':
+      return [true]
+
+    case 'reqularStats':
+      return [false]
+
+    default:
+      return [true, false]
+  }
+}
+
+const findByCourseAndSemesters = async (coursecodes, from, to, separate, unifyCourses = 'unifyStats') => {
+  const { startdate } = await Semester.findOne({
+    where: {
+      [separate ? 'semestercode' : 'yearcode']: from,
+    },
+    order: ['semestercode'],
+    limit: 1,
+    raw: true,
+  })
+  const { enddate } = await Semester.findOne({
+    where: {
+      [separate ? 'semestercode' : 'yearcode']: to,
+    },
+    order: [['semestercode', 'DESC']],
+    limit: 1,
+    raw: true,
+  })
+  return (
     await sequelize.query(
       `
   SELECT
     studentnumber
-  FROM student
-  INNER JOIN credit ON
-    student.studentnumber=credit.student_studentnumber
+  FROM
+    student s
   WHERE
-    course_code IN (:coursecodes) AND
-    attainment_date
-  BETWEEN
-    (select startdate FROM semesters where ${
-      separate ? 'semestercode' : 'yearcode'
-    }=:minYearCode ORDER BY semestercode LIMIT 1) AND
-    (select enddate FROM semesters where ${
-      separate ? 'semestercode' : 'yearcode'
-    }=:maxYearCode ORDER BY semestercode DESC LIMIT 1);
+    EXISTS (
+      SELECT
+        1
+      FROM
+        credit c
+      WHERE
+        c.student_studentnumber = s.studentnumber
+        AND c.course_code IN (:coursecodes)
+        AND c.is_open IN (:isOpen)
+        AND c.attainment_date BETWEEN '${startdate.toISOString()}' AND '${enddate.toISOString()}'
+    )
+    OR EXISTS (
+      SELECT
+        1
+      FROM
+        enrollment e
+      WHERE
+        e.studentnumber = s.studentnumber
+        AND e.course_code IN (:coursecodes)
+        AND e.enrollment_date_time BETWEEN '${startdate.toISOString()}' AND '${enddate.toISOString()}'
+        AND e.enrollment_date_time >= '2021-05-31'
+        AND e.state IN ('ENROLLED', 'CONFIRMED')
+    );
 `,
       {
-        replacements: { coursecodes, minYearCode: from, maxYearCode: to },
+        replacements: { coursecodes, minYearCode: from, maxYearCode: to, isOpen: getUnifyStatus(unifyCourses) },
         type: sequelize.QueryTypes.SELECT,
         raw: true,
       }
     )
   ).map(st => st.studentnumber)
+}
 
 const findByTag = async tag => {
   return (
@@ -140,7 +191,7 @@ const findByTag = async tag => {
   ).map(st => st.studentnumber)
 }
 
-const formatStudent = ({
+const formatStudent = async ({
   firstnames,
   lastname,
   studentnumber,
@@ -153,11 +204,13 @@ const formatStudent = ({
   studyrights,
   semester_enrollments,
   transfers,
+  studyplans,
   updatedAt,
   createdAt,
   tags,
+  sis_person_id,
 }) => {
-  const toCourse = ({ id, grade, credits, credittypecode, attainment_date, course, isStudyModule }) => {
+  const toCourse = ({ id, grade, credits, credittypecode, is_open, attainment_date, course, isStudyModule }) => {
     try {
       course = course.get()
     } catch (e) {
@@ -183,18 +236,21 @@ const formatStudent = ({
       credits,
       credittypecode,
       isStudyModuleCredit: isStudyModule,
+      isOpenCourse: is_open,
     }
   }
 
   studyrights = studyrights || []
   semester_enrollments = semester_enrollments || []
+  studyplans = studyplans || []
   const semesterenrollments = semester_enrollments.map(
-    ({ semestercode, enrollmenttype, name, yearname, startYear }) => ({
+    ({ semestercode, enrollmenttype, name, yearname, startYear, statutory_absence: statutoryAbsence }) => ({
       yearname,
       name,
       startYear,
       semestercode,
       enrollmenttype,
+      statutoryAbsence,
     })
   )
 
@@ -211,6 +267,13 @@ const formatStudent = ({
     .map(toCourse)
     .filter(c => c.course.name !== 'missing')
 
+  const hopsCourses = studyplans.map(sp => sp.included_courses).flat()
+
+  const allCourses = await sequelize.query(
+    'SELECT DISTINCT ON(course_code) course_code, credits FROM credit WHERE course_code IN (:hopsCourses)',
+    { replacements: { hopsCourses: hopsCourses.concat('id-to-fill-array') } }
+  )
+
   return {
     firstnames,
     lastname,
@@ -225,14 +288,17 @@ const formatStudent = ({
     email,
     semesterenrollments,
     updatedAt: updatedAt || createdAt,
+    studyplans,
+    allCourses: allCourses[0],
     tags,
+    sis_person_id: sis_person_id,
   }
 }
 
 const withId = async id => {
   try {
     const result = await byId(id)
-    return formatStudent(result)
+    return await formatStudent(result)
   } catch (e) {
     logger.error({ message: 'error when fetching single student', meta: e })
     return {
@@ -285,7 +351,7 @@ const bySearchTerm = async searchterm => {
       [Op.or]: [nameLike(terms), studentnumberLike(terms)],
     },
   })
-  return matches.map(formatStudent)
+  return await Promise.all(matches.map(async s => await formatStudent(s)))
 }
 
 const bySearchTermAndStudentNumbers = async (searchterm, studentnumbers) => {
@@ -300,7 +366,7 @@ const bySearchTermAndStudentNumbers = async (searchterm, studentnumbers) => {
       },
     },
   })
-  return matches.map(formatStudent)
+  return await Promise.all(matches.map(async s => await formatStudent(s)))
 }
 
 const filterStudentnumbersByAccessrights = async (studentnumbers, codes) =>

services/backend/src/services/studyprogrammeBasics.js

@@ -6,37 +6,12 @@ const {
   getYearsArray,
   getStartDate,
   tableTitles,
+  alltimeStartDate,
+  alltimeEndDate,
 } = require('./studyprogrammeHelpers')
-const {
-  allStudyrights,
-  graduatedStudyRights,
-  startedStudyrights,
-  transfersAway,
-  transfersTo,
-} = require('./newStudyprogramme')
+const { graduatedStudyRights, startedStudyrights, transfersAway, transfersTo } = require('./studyprogramme')
 const { getYearStartAndEndDates } = require('../util/semester')
 
-const getAllStats = async ({ studyprogramme, years, isAcademicYear, includeAllSpecials }) => {
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  await Promise.all(
-    years.map(async year => {
-      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
-      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
-        codes: [studyprogramme],
-        startDate,
-        endDate,
-        includeAllSpecials,
-      })
-      const all = await allStudyrights(studyprogramme, null, studentnumbersOfTheYear)
-
-      graphStats[indexOf(years, year)] = all.length
-      tableStats[year] += all.length
-    })
-  )
-  return { graphStats, tableStats }
-}
-
 const getStartedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
   const { graphStats, tableStats } = getStatsBasis(years)
 
@@ -64,36 +39,23 @@ const getStartedStats = async ({ studyprogramme, since, years, isAcademicYear, i
 const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
   const { graphStats, tableStats } = getStatsBasis(years)
 
-  await Promise.all(
-    years.map(async year => {
-      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
-      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
-        codes: [studyprogramme],
-        startDate,
-        endDate,
-        includeAllSpecials,
-      })
+  const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+    codes: [studyprogramme],
+    startDate: alltimeStartDate,
+    endDate: alltimeEndDate,
+    includeAllSpecials,
+  })
 
-      const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
-      studyrights.forEach(({ enddate }) => {
-        const graduationYear = defineYear(enddate, isAcademicYear)
-        graphStats[indexOf(years, graduationYear)] += 1
-        tableStats[graduationYear] += 1
-      })
-    })
-  )
-  return { graphStats, tableStats }
-}
-
-const getCancelledStats = async ({ studyprogramme, years, isAcademicYear }) => {
-  const studyrights = await allStudyrights(studyprogramme)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ canceldate }) => {
-    if (canceldate !== null && canceldate !== undefined) {
-      const cancelYear = defineYear(canceldate, isAcademicYear)
-      graphStats[indexOf(years, cancelYear)] += 1
-      tableStats[cancelYear] += 1
+  const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
+  studyrights.forEach(({ enddate, studyrightElements }) => {
+    // Check that the study right element ending to graduation belong to study programme
+    const elements = studyrightElements.filter(
+      sre => new Date(sre.enddate).toDateString() === new Date(enddate).toDateString()
+    )
+    if (elements.length) {
+      const graduationYear = defineYear(enddate, isAcademicYear)
+      graphStats[indexOf(years, graduationYear)] += 1
+      tableStats[graduationYear] += 1
     }
   })
 
@@ -126,34 +88,29 @@ const getTransferredToStats = async ({ studyprogramme, since, years, isAcademicY
   return { graphStats, tableStats }
 }
 
-const getBasicStatsForStudytrack = async ({ studyprogramme, yearType, specialGroups }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+const getBasicStatsForStudytrack = async ({ studyprogramme, settings }) => {
+  const { includeAllSpecials, isAcademicYear } = settings
   const since = getStartDate(studyprogramme, isAcademicYear)
   const years = getYearsArray(since.getFullYear(), isAcademicYear)
 
   const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
-  const all = await getAllStats(queryParameters)
   const started = await getStartedStats(queryParameters)
   const graduated = await getGraduatedStats(queryParameters)
-  const cancelled = await getCancelledStats(queryParameters)
   const transferredAway = await getTransferredAwayStats(queryParameters)
   const transferredTo = await getTransferredToStats(queryParameters)
 
   const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const titles = tableTitles['basics'][specialGroups]
+  const titles = tableTitles['basics'][includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED']
   const tableStats = reversedYears.map(year =>
     includeAllSpecials
       ? [
           year,
-          all.tableStats[year],
           started.tableStats[year],
           graduated.tableStats[year],
-          cancelled.tableStats[year],
           transferredAway.tableStats[year],
           transferredTo.tableStats[year],
         ]
-      : [year, all.tableStats[year], started.tableStats[year], graduated.tableStats[year]]
+      : [year, started.tableStats[year], graduated.tableStats[year]]
   )
 
   return {
@@ -169,10 +126,6 @@ const getBasicStatsForStudytrack = async ({ studyprogramme, yearType, specialGro
             name: 'Graduated',
             data: graduated.graphStats,
           },
-          {
-            name: 'Cancelled',
-            data: cancelled.graphStats,
-          },
           {
             name: 'Transferred away',
             data: transferredAway.graphStats,
@@ -198,5 +151,6 @@ const getBasicStatsForStudytrack = async ({ studyprogramme, yearType, specialGro
 }
 
 module.exports = {
+  getGraduatedStats,
   getBasicStatsForStudytrack,
 }

services/backend/src/services/studyprogrammeCourses.js

@@ -0,0 +1,209 @@
+const _ = require('lodash')
+
+const {
+  getStudentsForProgrammeCourses,
+  getCurrentStudyYearStartDate,
+  getOwnStudentsForProgrammeCourses,
+  getStudentsWithoutStudyrightForProgrammeCourses,
+  getOtherStudentsForProgrammeCourses,
+  getAllProgrammeCourses,
+} = require('./studyprogramme')
+const { mapToProviders } = require('../util/utils')
+
+const getCurrentYearStartDate = () => {
+  return new Date(new Date().getFullYear(), 0, 1)
+}
+
+const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+
+const getAllStudyprogrammeCourses = async studyprogramme => {
+  const providerCode = mapToProviders([studyprogramme])[0]
+  const normalCourses = await getAllProgrammeCourses(providerCode)
+  return normalCourses.reduce((acc, curr) => {
+    acc.push(curr.code)
+    if (curr.substitutions && curr.substitutions.includes('AY' + curr.code)) {
+      acc.push('AY' + curr.code)
+    }
+    return acc
+  }, [])
+}
+
+const makeYearlyPromises = (years, academicYear, type, programmeCourses, studyprogramme) => {
+  return years.map(
+    year =>
+      new Promise(async res => {
+        const from = academicYear == 'true' ? new Date(year, 7, 1, 0, 0, 0) : new Date(year, 0, 1, 0, 0, 0)
+        const to = academicYear == 'true' ? new Date(year + 1, 6, 31, 23, 59, 59) : new Date(year, 11, 31, 23, 59, 59)
+
+        let result = null
+
+        switch (type) {
+          case 'allStudents':
+            result = await getStudentsForProgrammeCourses(from, to, programmeCourses)
+            break
+          case 'ownStudents':
+            result = await getOwnStudentsForProgrammeCourses(from, to, programmeCourses, studyprogramme)
+            break
+          case 'withoutStudyright':
+            result = await getStudentsWithoutStudyrightForProgrammeCourses(from, to, programmeCourses)
+            break
+          case 'otherStudents':
+            result = await getOtherStudentsForProgrammeCourses(from, to, programmeCourses, studyprogramme)
+            break
+          default:
+            result = await getStudentsForProgrammeCourses(from, to, programmeCourses)
+        }
+
+        res(
+          result.map(c => {
+            c['year'] = year
+            return c
+          })
+        )
+      })
+  )
+}
+
+const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme, academicYear) => {
+  const startDate = academicYear == 'true' ? await getCurrentStudyYearStartDate(unixMillis) : getCurrentYearStartDate()
+  const startYear = startDate.getFullYear()
+  const yearRange = _.range(2017, startYear + 1)
+  const programmeCourses = await getAllStudyprogrammeCourses(studyprogramme)
+
+  const yearlyStudentByCoursePromises = makeYearlyPromises(yearRange, academicYear, 'allStudents', programmeCourses)
+  const yearlyProgrammeStudentsPromises = makeYearlyPromises(
+    yearRange,
+    academicYear,
+    'ownStudents',
+    programmeCourses,
+    studyprogramme
+  )
+  const yearlyStudentsWithoutStudyrightPromises = makeYearlyPromises(
+    yearRange,
+    academicYear,
+    'withoutStudyright',
+    programmeCourses
+  )
+  const yearlyOtherProgrammeStudentsPromises = makeYearlyPromises(
+    yearRange,
+    academicYear,
+    'otherStudents',
+    programmeCourses,
+    studyprogramme
+  )
+
+  const [
+    yearlyStudentByCourse,
+    yearlyProgrammeStudents,
+    yearlyStudentsWithoutStudyright,
+    yearlyOtherProgrammeStudents,
+  ] = await Promise.all([
+    Promise.all(yearlyStudentByCoursePromises),
+    Promise.all(yearlyProgrammeStudentsPromises),
+    Promise.all(yearlyStudentsWithoutStudyrightPromises),
+    Promise.all(yearlyOtherProgrammeStudentsPromises),
+  ])
+
+  let maxYear = 0
+  const allCourses = [
+    ...yearlyStudentByCourse.flat(),
+    ...yearlyProgrammeStudents.flat(),
+    ...yearlyStudentsWithoutStudyright.flat(),
+    ...yearlyOtherProgrammeStudents.flat(),
+  ].reduce((acc, curr) => {
+    if (curr.year > maxYear) maxYear = curr.year
+    if (!acc[curr.code]) {
+      acc[curr.code] = {
+        code: curr.code,
+        name: curr.name,
+        years: {},
+      }
+    }
+    if (!acc[curr.code]['years'][curr.year]) {
+      acc[curr.code]['years'][curr.year] = {
+        totalAllStudents: 0,
+        totalAllCredits: 0,
+        totalProgrammeStudents: 0,
+        totalProgrammeCredits: 0,
+        totalOtherProgrammeStudents: 0,
+        totalOtherProgrammeCredits: 0,
+        totalWithoutStudyrightStudents: 0,
+        totalWithoutStudyrightCredits: 0,
+      }
+    }
+    switch (curr.type) {
+      case 'total':
+        acc[curr.code]['years'][curr.year]['totalAllStudents'] += curr.totalAllStudents
+        acc[curr.code]['years'][curr.year]['totalAllCredits'] += curr.totalAllcredits
+        break
+      case 'ownProgramme':
+        acc[curr.code]['years'][curr.year]['totalProgrammeStudents'] += curr.totalProgrammeStudents
+        acc[curr.code]['years'][curr.year]['totalProgrammeCredits'] += curr.totalProgrammeCredits
+        break
+
+      case 'otherProgramme':
+        acc[curr.code]['years'][curr.year]['totalOtherProgrammeStudents'] += curr.totalOtherProgrammeStudents
+        acc[curr.code]['years'][curr.year]['totalOtherProgrammeCredits'] += curr.totalOtherProgrammeCredits
+        break
+
+      case 'noStudyright':
+        acc[curr.code]['years'][curr.year]['totalWithoutStudyrightStudents'] += curr.totalWithoutStudyrightStudents
+        acc[curr.code]['years'][curr.year]['totalWithoutStudyrightCredits'] += curr.totalWithoutStudyrightCredits
+        break
+    }
+
+    return acc
+  }, {})
+  const ayCourses = Object.keys(allCourses).filter(c => c.startsWith('AY'))
+  const properties = [
+    'totalAllStudents',
+    'totalAllCredits',
+    'totalProgrammeStudents',
+    'totalProgrammeCredits',
+    'totalOtherProgrammeStudents',
+    'totalOtherProgrammeCredits',
+    'totalWithoutStudyrightStudents',
+    'totalWithoutStudyrightCredits',
+  ]
+  ayCourses.forEach(ayCourse => {
+    const normCode = isOpenUniCourseCode(ayCourse)[1]
+
+    if (allCourses[normCode]) {
+      const mergedCourse = {}
+      mergedCourse['code'] = allCourses[normCode].code
+      mergedCourse['name'] = allCourses[normCode].name
+      mergedCourse['years'] = {}
+
+      yearRange
+        .filter(year => year <= maxYear)
+        .forEach(year => {
+          if (!allCourses[normCode]['years'][year]) {
+            mergedCourse['years'][year] = {
+              totalAllStudents: 0,
+              totalAllCredits: 0,
+              totalProgrammeStudents: 0,
+              totalProgrammeCredits: 0,
+              totalOtherProgrammeStudents: 0,
+              totalOtherProgrammeCredits: 0,
+              totalWithoutStudyrightStudents: 0,
+              totalWithoutStudyrightCredits: 0,
+            }
+          } else {
+            mergedCourse['years'][year] = { ...allCourses[normCode]['years'][year] }
+          }
+
+          if (allCourses[ayCourse]['years'][year]) {
+            properties.forEach(prop => {
+              mergedCourse['years'][year][prop] =
+                mergedCourse['years'][year][prop] + allCourses[ayCourse]['years'][year][prop]
+            })
+          }
+        })
+      allCourses[normCode] = mergedCourse
+      delete allCourses[ayCourse]
+    }
+  })
+  return Object.values(allCourses)
+}
+
+module.exports = { getStudyprogrammeCoursesForStudytrack, getAllStudyprogrammeCourses }

services/backend/src/services/studyprogrammeCredits.js

@@ -5,53 +5,73 @@ const {
   getStatsBasis,
   defineYear,
   isMajorStudentCredit,
+  isNonMajorCredit,
   getYearsArray,
   getStartDate,
   isSpecialGroupCredit,
   tableTitles,
 } = require('./studyprogrammeHelpers')
 const {
-  getProgrammesStudyrights,
+  getStudyRights,
   getCreditsForStudyProgramme,
   getTransferredCredits,
   allTransfers,
-} = require('./newStudyprogramme')
+  getCourseCodesForStudyProgramme,
+} = require('./studyprogramme')
 
 // Fetches all the credits for the studyprogramme and divides them into major-students and non-major students credits
 // Division is done on the basis that whether the student had a primary studyright to the programme on the attainment_date
 // If special groups are excluded, the transfer students are filtered away from the major-students as well
 const getRegularCreditStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
   const providercode = mapToProviders([studyprogramme])[0]
-  let studyrights = await getProgrammesStudyrights(studyprogramme, since)
+  const courses = await getCourseCodesForStudyProgramme(providercode)
+  const credits = await getCreditsForStudyProgramme(courses, since)
+  const students = [...new Set(credits.map(({ student_studentnumber }) => student_studentnumber))]
+
+  //let studyrights = await getProgrammesStudyrights(studyprogramme, since)
+  let studyrights = await getStudyRights(students, since)
   const transfers = (await allTransfers(studyprogramme, since)).map(t => t.studyrightid)
-
   if (!includeAllSpecials) {
-    studyrights.filter(s => !transfers.includes(s.studyrightid))
+    studyrights = studyrights.filter(s => !transfers.includes(s.studyrightid))
   }
 
-  const credits = await getCreditsForStudyProgramme(providercode, since)
-
   let majors = getStatsBasis(years)
   let nonMajors = getStatsBasis(years)
+  let nonDegree = getStatsBasis(years)
 
   credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
-    const studyright = studyrights.find(studyright => studyright.studentnumber == student_studentnumber)
+    const studentStudyrights = studyrights.filter(studyright => studyright.studentnumber === student_studentnumber)
     const attainmentYear = defineYear(attainment_date, isAcademicYear)
 
-    if (!includeAllSpecials && isSpecialGroupCredit(studyright, attainment_date, transfers)) {
+    if (!includeAllSpecials && isSpecialGroupCredit(studentStudyrights, attainment_date, transfers)) {
       return
     }
 
-    if (isMajorStudentCredit(studyright, attainment_date)) {
+    if (isMajorStudentCredit(studentStudyrights, attainment_date, studyprogramme)) {
       majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
       majors.tableStats[attainmentYear] += credits || 0
-    } else {
+    } else if (isNonMajorCredit(studentStudyrights, attainment_date, studyprogramme)) {
       nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
       nonMajors.tableStats[attainmentYear] += credits || 0
+    } else {
+      nonDegree.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      nonDegree.tableStats[attainmentYear] += credits || 0
     }
   })
 
-  return { majors, nonMajors }
+  if (
+    majors.graphStats.every(year => year === 0) &&
+    nonMajors.graphStats.every(year => year === 0) &&
+    nonDegree.graphStats.every(year => year === 0)
+  ) {
+    return {
+      majors: { graphStats: [], tableStats: {} },
+      nonMajors: { graphStats: [], tableStats: {} },
+      nonDegree: { graphStats: [], tablestats: {} },
+    }
+  }
+
+  return { majors, nonMajors, nonDegree }
 }
 
 // Fetches all the credits with the type "transferred / hyväksiluettu" and divides them by year
@@ -66,57 +86,78 @@ const getTransferredCreditStats = async ({ studyprogramme, since, years, isAcade
     tableStats[attainmentYear] += credits || 0
   })
 
+  if (graphStats.every(year => year === 0)) {
+    return { graphStats: [], tableStats: {} }
+  }
+
   return { graphStats, tableStats }
 }
 
-const getCreditStatsForStudytrack = async ({ studyprogramme, yearType, specialGroups }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+// Fetches all credits for the studytrack and combines them into the statistics for the table
+// and graph in the studyprogramme overview
+const getCreditStatsForStudytrack = async ({ studyprogramme, settings }) => {
+  const { isAcademicYear, includeAllSpecials } = settings
   const since = getStartDate(studyprogramme, isAcademicYear)
   const years = getYearsArray(since.getFullYear(), isAcademicYear)
 
   const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
-  const { majors, nonMajors } = await getRegularCreditStats(queryParameters)
+  const { majors, nonMajors, nonDegree } = await getRegularCreditStats(queryParameters)
   const transferred = await getTransferredCreditStats(queryParameters)
 
   const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const titles = tableTitles['credits'][specialGroups]
+  const titles = tableTitles['credits'][includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED']
+  const dataFound = [majors, nonMajors, transferred].some(d => d.graphStats.length)
+
+  if (!dataFound) return null
 
   const tableStats = reversedYears.map(year =>
     includeAllSpecials
       ? [
           year,
-          majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
+          majors.tableStats[year] +
+            nonMajors.tableStats[year] +
+            transferred.tableStats[year] +
+            nonDegree.tableStats[year],
           majors.tableStats[year],
           nonMajors.tableStats[year],
+          nonDegree.tableStats[year],
           transferred.tableStats[year],
         ]
       : [
           year,
-          majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
+          majors.tableStats[year] +
+            nonMajors.tableStats[year] +
+            transferred.tableStats[year] +
+            nonDegree.tableStats[year],
           majors.tableStats[year],
           transferred.tableStats[year],
         ]
   )
 
+  const graphStats = [
+    {
+      name: 'Major students credits',
+      data: majors.graphStats,
+    },
+    {
+      name: 'Non-major students credits',
+      data: nonMajors.graphStats,
+    },
+    {
+      name: 'Non-degree credits',
+      data: nonDegree.graphStats,
+    },
+    {
+      name: 'Transferred credits',
+      data: transferred.graphStats,
+    },
+  ]
+
   return {
     id: studyprogramme,
     years,
     tableStats,
-    graphStats: [
-      {
-        name: 'Major students credits',
-        data: majors.graphStats,
-      },
-      {
-        name: 'Non-major students credits',
-        data: nonMajors.graphStats,
-      },
-      {
-        name: 'Transferred credits',
-        data: transferred.graphStats,
-      },
-    ],
+    graphStats,
     titles,
   }
 }

services/backend/src/services/studyprogrammeGraduations.js

@@ -7,43 +7,47 @@ const {
   getStatsBasis,
   defineYear,
   getYearsObject,
-  getAcademicYearsObject,
   getYearsArray,
   getMedian,
   getMean,
   getStartDate,
   getThesisType,
+  alltimeStartDate,
+  alltimeEndDate,
 } = require('./studyprogrammeHelpers')
 const {
   graduatedStudyRights,
-  transfersAway,
-  transfersTo,
   getThesisCredits,
   followingStudyrights,
-} = require('./newStudyprogramme')
-const { getYearStartAndEndDates } = require('../util/semester')
+  previousStudyrights,
+  allStudyrights,
+  allTransfers,
+} = require('./studyprogramme')
+const { getAllProgrammes } = require('./studyrights')
 
 const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
   const { graphStats, tableStats } = getStatsBasis(years)
 
-  await Promise.all(
-    years.map(async year => {
-      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
-      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
-        codes: [studyprogramme],
-        startDate,
-        endDate,
-        includeAllSpecials,
-      })
+  const studentnumbers = await getCorrectStudentnumbers({
+    codes: [studyprogramme],
+    startDate: alltimeStartDate,
+    endDate: alltimeEndDate,
+    includeAllSpecials,
+  })
+
+  const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbers)
+  studyrights.forEach(({ enddate, studyrightElements }) => {
+    // Check that the study right element ending to graduation belong to study programme
+    const elements = studyrightElements.filter(
+      sre => new Date(sre.enddate).toDateString() === new Date(enddate).toDateString()
+    )
+    if (elements.length) {
+      const graduationYear = defineYear(enddate, isAcademicYear)
+      graphStats[indexOf(years, graduationYear)] += 1
+      tableStats[graduationYear] += 1
+    }
+  })
 
-      const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
-      studyrights.forEach(({ enddate }) => {
-        const graduationYear = defineYear(enddate, isAcademicYear)
-        graphStats[indexOf(years, graduationYear)] += 1
-        tableStats[graduationYear] += 1
-      })
-    })
-  )
   return { graphStats, tableStats }
 }
 
@@ -52,60 +56,48 @@ const getThesisStats = async ({ studyprogramme, since, years, isAcademicYear, in
   const thesisType = getThesisType(studyprogramme)
   const { graphStats, tableStats } = getStatsBasis(years)
 
-  await Promise.all(
-    years.map(async year => {
-      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
-      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
-        codes: [studyprogramme],
-        startDate,
-        endDate,
-        includeAllSpecials,
-      })
+  const studentnumbers = await getCorrectStudentnumbers({
+    codes: [studyprogramme],
+    startDate: alltimeStartDate,
+    endDate: alltimeEndDate,
+    includeAllSpecials,
+  })
 
-      const credits = await getThesisCredits(providercode, since, thesisType, studentnumbersOfTheYear)
+  const credits = await getThesisCredits(providercode, since, thesisType, studentnumbers)
 
-      credits?.forEach(({ attainment_date }) => {
-        const attainmentYear = defineYear(attainment_date, isAcademicYear)
-        graphStats[indexOf(years, attainmentYear)] += 1
-        tableStats[attainmentYear] += 1
-      })
-    })
-  )
+  credits?.forEach(({ attainment_date }) => {
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+    graphStats[indexOf(years, attainmentYear)] += 1
+    tableStats[attainmentYear] += 1
+  })
 
   return { graphStats, tableStats }
 }
 
 const getGraduationTimeStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
-  let graduationAmounts = getYearsObject(years)
-  let graduationTimes = getYearsObject(years, true)
-  let totalAmounts = isAcademicYear ? getAcademicYearsObject(years) : getYearsObject(years)
+  let graduationAmounts = getYearsObject({ years })
+  let graduationTimes = getYearsObject({ years, emptyArrays: true })
 
-  await Promise.all(
-    years.map(async year => {
-      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
-      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
-        codes: [studyprogramme],
-        startDate,
-        endDate,
-        includeAllSpecials,
-      })
+  const studentnumbers = await getCorrectStudentnumbers({
+    codes: [studyprogramme],
+    startDate: alltimeStartDate,
+    endDate: alltimeEndDate,
+    includeAllSpecials,
+  })
 
-      const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
-      totalAmounts[year] = studentnumbersOfTheYear.length
+  const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbers)
 
-      studyrights.forEach(({ enddate, studystartdate }) => {
-        const graduationYear = defineYear(enddate, isAcademicYear)
-        const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
-        graduationAmounts[graduationYear] += 1
-        graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
-      })
-    })
-  )
+  studyrights.forEach(({ enddate, studystartdate }) => {
+    const graduationYear = defineYear(enddate, isAcademicYear)
+    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+    graduationAmounts[graduationYear] += 1
+    graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
+  })
   // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
   const comparisonValue = studyprogramme.includes('KH') ? 72 : 48
 
-  const medians = getYearsObject(years, true)
-  const means = getYearsObject(years, true)
+  const medians = getYearsObject({ years, emptyArrays: true })
+  const means = getYearsObject({ years, emptyArrays: true })
 
   // HighCharts graph require the data to have this format (ie. actual value, "empty value")
   years.forEach(year => {
@@ -120,59 +112,124 @@ const getGraduationTimeStats = async ({ studyprogramme, since, years, isAcademic
       ['', comparisonValue - mean],
     ]
   })
-  return { medians, means, graduationAmounts, totalAmounts }
+  return { medians, means, graduationAmounts }
+}
+
+// Creates (studyrightid, transfer)-map out of programmes transfers
+const getTransferStudyrightMap = async (studyprogramme, since) => {
+  const transfers = await allTransfers(studyprogramme, since)
+  const studyrightTransferMap = new Map()
+  transfers.forEach(t => studyrightTransferMap.set(t.studyrightid, t))
+  return studyrightTransferMap
+}
+
+const formatStats = (stats, years) => {
+  const tableStats = Object.values(stats)
+    .filter(p => years.map(year => p[year]).find(started => started !== 0)) // Filter out programmes with no-one started between the selected years
+    .map(p => [p.code, p.name['fi'], ...years.map(year => p[year])])
+
+  const graphStats = Object.values(stats)
+    .filter(p => years.map(year => p[year]).find(started => started !== 0)) // Filter out programmes with no-one started between the selected years
+    .map(p => ({ name: p.name['fi'], code: p.code, data: years.map(year => p[year]) }))
+
+  return { tableStats, graphStats }
+}
+
+const getProgrammesBeforeStarting = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const all = await allStudyrights(studyprogramme)
+  const allStudents = all.map(s => s.studentnumber)
+
+  // Get studyrights for bachelor studyprogrammes. This excludes studytracks.
+  const programmes = await getAllProgrammes()
+  let studyrightsBeforeMasters = await previousStudyrights(programmes, allStudents)
+  const studyprogrammeCodes = uniqBy(studyrightsBeforeMasters, 'code').map(s => ({ code: s.code, name: s.name }))
+  const stats = {}
+  studyprogrammeCodes.forEach(c => (stats[c.code] = { ...c, ...getYearsObject({ years }) }))
+
+  // Map transfers to the Master's programme
+  const transfers = await getTransferStudyrightMap(studyprogramme, since)
+
+  studyrightsBeforeMasters.forEach(({ studentnumber, code, givendate }) => {
+    const masterStudyright = all.find(s => s.studentnumber === studentnumber)
+    const transfer = transfers.get(masterStudyright.studyrightid)
+
+    // If special studyrights are excluded, transfers to the master's programme should be excluded
+    if (!includeAllSpecials && transfers.get(masterStudyright.studyrightid)) {
+      return
+    }
+
+    // If all studyrights are included, and transfer exists, define the start year in the programme, based on when the student transferred to it
+    // Otherwise, the year is defined based on when the student started in the chosen master's programme
+    const programmeStartDate = includeAllSpecials && transfer ? transfer.transferdate : masterStudyright?.studystartdate
+    const startYear = defineYear(programmeStartDate, isAcademicYear)
+
+    // The master studyright should have been given on the same date as the bachelor's
+    if (years.includes(startYear) && code && masterStudyright.givendate.getTime() === givendate.getTime()) {
+      stats[code][startYear] += 1
+    }
+  })
+
+  const graphAndTableStats = formatStats(stats, years)
+  return graphAndTableStats
 }
 
 const getProgrammesAfterGraduation = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
   const graduated = await graduatedStudyRights(studyprogramme, since)
   const graduatedStudents = graduated.map(s => s.studentnumber)
-  let studyrightsAfterThisProgramme = await followingStudyrights(since, graduatedStudents)
+
+  // Get studyrights for masters studyprogramme. This excludes studytracks.
+  const programmes = await getAllProgrammes()
+  let studyrightsAfterThisProgramme = await followingStudyrights(since, programmes, graduatedStudents)
   const studyprogrammeCodes = uniqBy(studyrightsAfterThisProgramme, 'code').map(s => ({ code: s.code, name: s.name }))
-
   const stats = {}
-  studyprogrammeCodes.forEach(c => (stats[c.code] = { ...c, ...getYearsObject(years) }))
+  studyprogrammeCodes.forEach(c => (stats[c.code] = { ...c, ...getYearsObject({ years }) }))
 
-  if (!includeAllSpecials) {
-    const transferredAway = await transfersAway(studyprogramme, since)
-    const transferredTo = await transfersTo(studyprogramme, since)
-    const transfers = [...transferredAway, ...transferredTo].map(s => s.studyrightid)
-    studyrightsAfterThisProgramme.filter(s => !transfers.includes(s.studyrightid) && !s.canceldate)
-  }
+  const transfers = await getTransferStudyrightMap(studyprogramme, since)
 
-  studyrightsAfterThisProgramme.forEach(({ studystartdate, code }) => {
+  studyrightsAfterThisProgramme.forEach(({ studentnumber, studystartdate, code, givendate }) => {
+    const bachelorStudyright = graduated.find(s => s.studentnumber === studentnumber)
+
+    // If special studyrights are excluded, transfers to the bachelor programme should be excluded
+    if (!includeAllSpecials && transfers.get(bachelorStudyright.studyrightid)) {
+      return
+    }
+
+    // If all studyrights are included, and transfer exists, define the start year in the programme, based on when the student transferred to it
+    // Otherwise, the year is defined based on when the student started in the chosen master's programme
     const startYear = defineYear(studystartdate, isAcademicYear)
-    if (years.includes(startYear) && code) {
+
+    // The master studyright should have been given on the same date as the bachelor's
+    if (years.includes(startYear) && code && bachelorStudyright.givendate.getTime() === givendate.getTime()) {
       stats[code][startYear] += 1
     }
   })
 
-  const tableStats = Object.values(stats)
-    .map(p => [p.code, p.name['fi'], ...years.map(year => p[year])])
-    .filter(p => p[0].includes('MH'))
-
-  const graphStats = Object.values(stats)
-    .map(p => ({ name: p.name['fi'], code: p.code, data: years.map(year => p[year]) }))
-    .filter(p => p.code.includes('MH'))
-
-  return { tableStats, graphStats }
+  const graphAndTableStats = formatStats(stats, years)
+  return graphAndTableStats
 }
 
-const getGraduationStatsForStudytrack = async ({ studyprogramme, yearType, specialGroups }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+const getProgrammesBeforeOrAfter = async (studyprogramme, queryParameters) => {
+  if (studyprogramme.includes('KH')) return await getProgrammesAfterGraduation(queryParameters)
+  if (studyprogramme.includes('MH')) return await getProgrammesBeforeStarting(queryParameters)
+  return null
+}
+
+const getGraduationStatsForStudytrack = async ({ studyprogramme, settings }) => {
+  const { isAcademicYear, includeAllSpecials } = settings
   const since = getStartDate(studyprogramme, isAcademicYear)
   const years = getYearsArray(since.getFullYear(), isAcademicYear)
 
   const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
   const thesis = await getThesisStats(queryParameters)
+
   const graduated = await getGraduatedStats(queryParameters)
   const graduationTimeStats = await getGraduationTimeStats(queryParameters)
-  const programmesAfterGraduation = await getProgrammesAfterGraduation(queryParameters)
+  const programmesBeforeOrAfter = await getProgrammesBeforeOrAfter(studyprogramme, queryParameters)
 
   const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
 
   const titles = ['', 'Graduated', 'Wrote thesis']
-  const programmesAfterTitles = ['Code', 'Programme', ...years]
+  const programmesBeforeOrAfterTitles = ['Code', 'Programme', ...years]
   const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
 
   return {
@@ -193,10 +250,9 @@ const getGraduationStatsForStudytrack = async ({ studyprogramme, yearType, speci
     graduationMedianTime: graduationTimeStats.medians,
     graduationMeanTime: graduationTimeStats.means,
     graduationAmounts: graduationTimeStats.graduationAmounts,
-    totalAmounts: graduationTimeStats.totalAmounts,
-    programmesAfterTableStats: programmesAfterGraduation.tableStats,
-    programmesAfterGraphStats: programmesAfterGraduation.graphStats,
-    programmesAfterTitles,
+    programmesBeforeOrAfterTableStats: programmesBeforeOrAfter?.tableStats,
+    programmesBeforeOrAfterGraphStats: programmesBeforeOrAfter?.graphStats,
+    programmesBeforeOrAfterTitles,
   }
 }
 

services/backend/src/services/studyprogrammeHelpers.js

@@ -6,7 +6,6 @@ const { studentnumbersWithAllStudyrightElements } = require('./populations')
 const getCorrectStudentnumbers = async ({ codes, startDate, endDate, includeAllSpecials, includeGraduated = true }) => {
   let studentnumbers = []
   const exchangeStudents = includeAllSpecials
-  const cancelledStudents = includeAllSpecials
   const nondegreeStudents = includeAllSpecials
   const transferredOutStudents = includeAllSpecials
   const transferredToStudents = !includeAllSpecials
@@ -17,7 +16,6 @@ const getCorrectStudentnumbers = async ({ codes, startDate, endDate, includeAllS
     startDate,
     endDate,
     exchangeStudents,
-    cancelledStudents,
     nondegreeStudents,
     transferredOutStudents,
     null,
@@ -28,13 +26,30 @@ const getCorrectStudentnumbers = async ({ codes, startDate, endDate, includeAllS
   return studentnumbers
 }
 
+const defineStartDate = (studyright_elements, studystartdate) => {
+  if (studyright_elements?.length && studyright_elements[0].startdate > studystartdate) {
+    return studyright_elements[0].startdate
+  }
+  return studystartdate
+}
+
+const resolveStudyRightCode = studyright_elements => {
+  if (!studyright_elements) return null
+  const studyRightElement = studyright_elements
+    .filter(sre => sre.element_detail.type === 20)
+    .sort((a, b) => new Date(b.startdate) - new Date(a.startdate))[0]
+  if (studyRightElement) return studyRightElement.code
+  return null
+}
+
 const formatStudyright = studyright => {
   const {
     studyrightid,
     studystartdate,
     enddate,
-    canceldate,
+    givendate,
     graduated,
+    active,
     prioritycode,
     extentcode,
     student,
@@ -43,14 +58,16 @@ const formatStudyright = studyright => {
 
   return {
     studyrightid,
-    studystartdate,
+    studystartdate: defineStartDate(studyright_elements, studystartdate),
     enddate,
-    canceldate,
+    givendate,
     graduated,
+    active,
     prioritycode,
     extentcode,
     studentnumber: student.studentnumber,
-    code: studyright_elements?.length ? studyright_elements[0].code : null,
+    code: resolveStudyRightCode(studyright_elements),
+    studyrightElements: studyright_elements,
     name:
       studyright_elements?.length && studyright_elements[0].element_detail && studyright_elements[0].element_detail.name
         ? studyright_elements[0].element_detail.name
@@ -69,17 +86,19 @@ const formatStudent = student => {
 }
 
 const formatTransfer = transfer => {
-  const { sourcecode, transfercode, transferdate, studyrightid } = transfer
+  const { sourcecode, targetcode, transferdate, studyrightid } = transfer
   return {
     sourcecode,
-    transfercode,
+    targetcode,
     transferdate,
     studyrightid,
   }
 }
 
-const getYearsArray = (since, isAcademicYear) => {
+const getYearsArray = (since, isAcademicYear, yearsCombined) => {
   const years = []
+  const allYears = 'Total'
+  if (yearsCombined) years.push(allYears)
   for (let i = since; i <= new Date().getFullYear(); i++) {
     const year = isAcademicYear ? `${i} - ${i + 1}` : i
     years.push(year)
@@ -87,7 +106,7 @@ const getYearsArray = (since, isAcademicYear) => {
   return years
 }
 
-const getYearsObject = (years, emptyArrays = false) => {
+const getYearsObject = ({ years, emptyArrays = false }) => {
   let yearsObject = {}
   for (const year of years) {
     yearsObject = { ...yearsObject, [year]: emptyArrays ? [] : 0 }
@@ -95,7 +114,7 @@ const getYearsObject = (years, emptyArrays = false) => {
   return yearsObject
 }
 
-const getAcademicYearsObject = (years, emptyArrays = false) => {
+const getAcademicYearsObject = ({ years, emptyArrays = false }) => {
   let yearsObject = {}
   for (const year of years) {
     yearsObject = { ...yearsObject, [`${year} - ${year + 1}`]: emptyArrays ? [] : 0 }
@@ -106,27 +125,40 @@ const getAcademicYearsObject = (years, emptyArrays = false) => {
 const getStatsBasis = years => {
   return {
     graphStats: new Array(years.length).fill(0),
-    tableStats: getYearsObject(years),
+    tableStats: getYearsObject({ years }),
   }
 }
 
-const isMajorStudentCredit = (studyright, attainment_date) =>
-  studyright &&
-  (studyright.prioritycode === 1 || studyright.prioritycode === 30) && // Is studyright state = MAIN or state = GRADUATED
-  studyright.studystartdate && // The student has started studying in the programme
-  studyright.studystartdate <= attainment_date && // Has the credit been attained after studying in the programme started
-  studyright.enddate >= attainment_date && // Has the credit been attained before the studyright ended
-  (!studyright.canceldate || studyright.canceldate >= attainment_date) // If the studyright was cancelled, was the credit attained before it was cancelled
+const isMajorStudentCredit = (studyrights, attainment_date, code) =>
+  studyrights.some(studyright => {
+    if (!studyright) return false
+    if (studyright.code !== code) return false
+    if (!studyright.graduated) return new Date(attainment_date) >= new Date(studyright.studystartdate)
+    return (
+      new Date(attainment_date) >= new Date(studyright.studystartdate) &&
+      new Date(attainment_date) <= new Date(studyright.enddate)
+    )
+  })
 
-const isSpecialGroupCredit = (studyright, attainment_date, transfers) => {
-  if (!studyright) return true // If there is no studyright matching the credit, is not a major student credit
-  if (studyright.canceldate) return true // Cancelled studyrights are in the special groups
-  if (studyright.studystartdate > attainment_date) return true // Credits before the studyright started are not major student credits
-  if (studyright.enddate && attainment_date > studyright.enddate) return true // Credits after studyright are not major student credits
-  if ([7, 9, 16, 34, 33, 99, 14, 13].includes(studyright.extentcode)) return true // Excludes non-degree studyrights and exchange students
-  if (transfers.includes(studyright.studyrightid)) return true // Excludes both transfers in and out of the programme
-  return false
-}
+const isNonMajorCredit = (studyrights, attainment_date) =>
+  studyrights.some(studyright => {
+    if (!studyright) return false
+    if (!studyright.graduated) return new Date(attainment_date) >= new Date(studyright.studystartdate)
+    return (
+      new Date(attainment_date) >= new Date(studyright.studystartdate) &&
+      new Date(attainment_date) <= new Date(studyright.enddate)
+    )
+  })
+
+const isSpecialGroupCredit = (studyrights, attainment_date, transfers) =>
+  studyrights.some(studyright => {
+    if (!studyright) return true // If there is no studyright matching the credit, is not a major student credit
+    if (studyright.studystartdate > attainment_date) return true // Credits before the studyright started are not major student credits
+    if (studyright.enddate && attainment_date > studyright.enddate) return true // Credits after studyright are not major student credits
+    if ([7, 9, 16, 34, 33, 99, 14, 13].includes(studyright.extentcode)) return true // Excludes non-degree studyrights and exchange students
+    if (transfers.includes(studyright.studyrightid)) return true // Excludes both transfers in and out of the programme
+    return false
+  })
 
 const getMedian = values => {
   if (values.length === 0) return 0
@@ -156,6 +188,9 @@ const getStartDate = (studyprogramme, isAcademicYear) => {
   return new Date('2017-01-01')
 }
 
+const alltimeStartDate = new Date('1900-01-01')
+const alltimeEndDate = new Date()
+
 // There are 9 course_unit_types
 // 1. urn:code:course-unit-type:regular
 // 2. urn:code:course-unit-type:bachelors-thesis
@@ -175,7 +210,7 @@ const getThesisType = studyprogramme => {
     return ['urn:code:course-unit-type:bachelors-thesis']
   if (/^(T)[0-9]{6}$/.test(studyprogramme))
     return ['urn:code:course-unit-type:doctors-thesis', 'urn:code:course-unit-type:licentiate-thesis']
-  return 'thesis-type-not-defined'
+  return ['urn:code:course-unit-type:doctors-thesis', 'urn:code:course-unit-type:licentiate-thesis']
 }
 
 const getPercentage = (value, total) => {
@@ -208,8 +243,12 @@ const getBachelorCreditGraphStats = years => ({
     name: '120-149 credits',
     data: getEmptyArray(years.length),
   },
-  mte150: {
-    name: 'More than 150 credits',
+  lte180: {
+    name: '150-179 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte180: {
+    name: 'More than 180 credits',
     data: getEmptyArray(years.length),
   },
 })
@@ -235,35 +274,85 @@ const getMasterCreditGraphStats = years => ({
     name: '260-279 credits',
     data: getEmptyArray(years.length),
   },
-  mte280: {
-    name: 'More than 280 credits',
+  lte300: {
+    name: '280-299 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte300: {
+    name: 'More than 300 credits',
     data: getEmptyArray(years.length),
   },
 })
 
-const getCreditGraphStats = (studyprogramme, years) =>
-  studyprogramme.includes('KH') ? getBachelorCreditGraphStats(years) : getMasterCreditGraphStats(years)
+const getDoctoralCreditGraphStats = years => ({
+  lte50: {
+    name: 'Less than 50 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte100: {
+    name: '50-99 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte150: {
+    name: '100-149 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte200: {
+    name: '150-199 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte250: {
+    name: '200-249 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte300: {
+    name: '250-299 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte300: {
+    name: 'More than 300 credits',
+    data: getEmptyArray(years.length),
+  },
+})
 
-const bachelorCreditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'mte150']
-const masterCreditThresholds = ['lte200', 'lte220', 'lte240', 'lte260', 'lte280', 'mte280']
-const bachelorCreditAmounts = [30, 60, 90, 120, 150, 150]
-const masterCreditAmounts = [200, 220, 240, 260, 280, 280]
+const getCreditGraphStats = (studyprogramme, years) => {
+  if (studyprogramme.includes('KH')) return getBachelorCreditGraphStats(years)
+  if (studyprogramme.includes('MH')) return getMasterCreditGraphStats(years)
+  return getDoctoralCreditGraphStats(years)
+}
+
+const bachelorCreditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'lte180', 'mte180']
+const masterCreditThresholds = ['lte200', 'lte220', 'lte240', 'lte260', 'lte280', 'lte300', 'mte300']
+const doctoralCreditThresholds = ['lte50', 'lte100', 'lte150', 'lte200', 'lte250', 'lte300', 'mte300']
+const bachelorCreditAmounts = [30, 60, 90, 120, 150, 180, 180]
+const masterCreditAmounts = [200, 220, 240, 260, 280, 300, 300]
+const doctoralCreditAmounts = [50, 100, 150, 200, 250, 300, 300]
 
 const getCreditThresholds = studyprogramme => {
   if (studyprogramme.includes('KH')) {
     return { creditThresholdKeys: bachelorCreditThresholds, creditThresholdAmounts: bachelorCreditAmounts }
   }
-  return { creditThresholdKeys: masterCreditThresholds, creditThresholdAmounts: masterCreditAmounts }
+  if (studyprogramme.includes('MH')) {
+    return { creditThresholdKeys: masterCreditThresholds, creditThresholdAmounts: masterCreditAmounts }
+  }
+  return { creditThresholdKeys: doctoralCreditThresholds, creditThresholdAmounts: doctoralCreditAmounts }
 }
 
 const tableTitles = {
   basics: {
-    SPECIAL_EXCLUDED: ['', 'All', 'Started studying', 'Graduated'],
-    SPECIAL_INCLUDED: ['', 'All', 'Started studying', 'Graduated', 'Cancelled', 'Transferred away', 'Transferred to'],
+    SPECIAL_EXCLUDED: ['', 'Started studying', 'Graduated'],
+    SPECIAL_INCLUDED: ['', 'Started studying', 'Graduated', 'Transferred away', 'Transferred to'],
   },
   credits: {
     SPECIAL_EXCLUDED: ['', 'Total', 'Major students credits', 'Transferred credits'],
-    SPECIAL_INCLUDED: ['', 'Total', 'Major students credits', 'Non major students credits', 'Transferred credits'],
+    SPECIAL_INCLUDED: [
+      '',
+      'Total',
+      'Major students credits',
+      'Non major students credits',
+      'Non degree students credits',
+      'Transferred credits',
+    ],
   },
   creditProgress: {
     bachelor: [
@@ -274,7 +363,8 @@ const tableTitles = {
       '60-89 credits',
       '90-119 credits',
       '120-149 credits',
-      '> 150 credits',
+      '150-179 credits',
+      '> 180 credits',
     ],
     master: [
       '',
@@ -284,7 +374,19 @@ const tableTitles = {
       '220-239 credits',
       '240-259 credits',
       '260-279 credits',
-      '> 280 credits',
+      '280-299 credits',
+      '> 300 credits',
+    ],
+    doctoral: [
+      '',
+      'All',
+      '< 50 credits',
+      '50-99 credits',
+      '100-149 credits',
+      '150-199 credits',
+      '200-249 credits',
+      '250-299 credits',
+      '> 300 credits',
     ],
   },
   studytracks: [
@@ -293,7 +395,7 @@ const tableTitles = {
     'Started studying',
     'Currently enrolled',
     'Absent',
-    'Cancelled',
+    'Inactive',
     'Graduated',
     'Men',
     'Women',
@@ -301,8 +403,11 @@ const tableTitles = {
   ],
 }
 
-const getCreditProgressTableTitles = studyprogramme =>
-  studyprogramme.includes('KH') ? tableTitles.bachelorCreditProgress : tableTitles.masterCreditProgress
+const getCreditProgressTableTitles = studyprogramme => {
+  if (studyprogramme.includes('KH')) return tableTitles.creditProgress.bachelor
+  if (studyprogramme.includes('MH')) return tableTitles.creditProgress.master
+  return tableTitles.creditProgress.doctoral
+}
 
 module.exports = {
   getCorrectStudentnumbers,
@@ -319,12 +424,16 @@ module.exports = {
   getMean,
   defineYear,
   getStartDate,
+  alltimeStartDate,
+  alltimeEndDate,
   getThesisType,
   getPercentage,
   getBachelorCreditGraphStats,
   getMasterCreditGraphStats,
+  getDoctoralCreditGraphStats,
   getCreditGraphStats,
   getCreditThresholds,
   tableTitles,
   getCreditProgressTableTitles,
+  isNonMajorCredit,
 }

services/backend/src/services/studyprogrammeUpdates.js

@@ -8,72 +8,103 @@ const {
   setGraduationStats,
   setStudytrackStats,
 } = require('../services/analyticsService')
+const logger = require('../util/logger')
 
 const updateBasicView = async code => {
-  const yearTypes = ['ACADEMIC_YEAR', 'CALENDAR_YEAR']
-  const special = ['SPECIAL_INCLUDED', 'SPECIAL_EXCLUDED']
+  const specialCalendar = {
+    yearType: 'CALENDAR_YEAR',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const specialAcademic = {
+    yearType: 'ACADEMIC_YEAR',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const specialExcludedCalendar = {
+    yearType: 'CALENDAR_YEAR',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
 
-  yearTypes.forEach(async yearType => {
-    special.forEach(async specialGroups => {
+  const specialExcludedAcademic = {
+    yearType: 'ACADEMIC_YEAR',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
+
+  const options = [specialCalendar, specialAcademic, specialExcludedCalendar, specialExcludedAcademic]
+
+  for (const option of options) {
+    try {
       const basicStats = await getBasicStatsForStudytrack({
         studyprogramme: code,
-        yearType,
-        specialGroups,
+        settings: {
+          isAcademicYear: option.yearType === 'ACADEMIC_YEAR',
+          includeAllSpecials: option.specialGroups === 'SPECIAL_INCLUDED',
+        },
       })
-      await setBasicStats(basicStats, yearType, specialGroups)
+      await setBasicStats(basicStats, option.yearType, option.specialGroups)
 
       const creditStats = await getCreditStatsForStudytrack({
         studyprogramme: code,
-        yearType,
-        specialGroups,
+        settings: {
+          isAcademicYear: option.yearType === 'ACADEMIC_YEAR',
+          includeAllSpecials: option.specialGroups === 'SPECIAL_INCLUDED',
+        },
       })
-      await setCreditStats(creditStats, yearType, specialGroups)
+      await setCreditStats(creditStats, option.yearType, option.specialGroups)
 
       const graduationStats = await getGraduationStatsForStudytrack({
         studyprogramme: code,
-        yearType,
-        specialGroups,
+        settings: {
+          isAcademicYear: option.yearType === 'ACADEMIC_YEAR',
+          includeAllSpecials: option.specialGroups === 'SPECIAL_INCLUDED',
+        },
       })
-      await setGraduationStats(graduationStats, yearType, specialGroups)
-    })
-  })
-
-  return 'OK'
+      await setGraduationStats(graduationStats, option.yearType, option.specialGroups)
+    } catch (e) {
+      logger.error(e)
+    }
+    return 'OK'
+  }
 }
 
 const updateStudytrackView = async code => {
-  const specialIncluded = 'SPECIAL_INCLUDED'
-  const specialExcluded = 'SPECIAL_EXCLUDED'
-  const graduatedIncluded = 'GRADUATED_INCLUDED'
-  const graduatedExcluded = 'GRADUATED_EXCLUDED'
+  const specialGraduated = {
+    graduated: 'GRADUATED_INCLUDED',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const specialExcludedGraduated = {
+    graduated: 'GRADUATED_INCLUDED',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
+  const specialGraduatedExcluded = {
+    graduated: 'GRADUATED_EXCLUDED',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const specialExcludedGraduatedExcluded = {
+    graduated: 'GRADUATED_EXCLUDED',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
 
-  const statsSpecialAndGraduated = await getStudytrackStatsForStudyprogramme({
-    studyprogramme: code,
-    specialGroups: specialIncluded,
-    graduated: graduatedIncluded,
-  })
-  await setStudytrackStats(statsSpecialAndGraduated, graduatedIncluded, specialIncluded)
+  const options = [
+    specialGraduated,
+    specialExcludedGraduated,
+    specialGraduatedExcluded,
+    specialExcludedGraduatedExcluded,
+  ]
 
-  const statsSpecialNotGraduated = await getStudytrackStatsForStudyprogramme({
-    studyprogramme: code,
-    specialGroups: specialIncluded,
-    graduated: graduatedExcluded,
-  })
-  await setStudytrackStats(statsSpecialNotGraduated, graduatedExcluded, specialIncluded)
-
-  const statsNotSpecialAndGraduated = await getStudytrackStatsForStudyprogramme({
-    studyprogramme: code,
-    specialGroups: specialExcluded,
-    graduated: graduatedIncluded,
-  })
-  await setStudytrackStats(statsNotSpecialAndGraduated, graduatedIncluded, specialExcluded)
-
-  const statsNotSpecialNotGraduated = await getStudytrackStatsForStudyprogramme({
-    studyprogramme: code,
-    specialGroups: specialExcluded,
-    graduated: graduatedExcluded,
-  })
-  await setStudytrackStats(statsNotSpecialNotGraduated, graduatedExcluded, specialExcluded)
+  for (const option of options) {
+    try {
+      const stats = await getStudytrackStatsForStudyprogramme({
+        studyprogramme: code,
+        settings: {
+          specialGroups: option.specialGroups === 'SPECIAL_INCLUDED',
+          graduated: option.graduated === 'GRADUATED_INCLUDED',
+        },
+      })
+      await setStudytrackStats(stats, option.graduated, option.specialGroups)
+    } catch (e) {
+      logger.error(e)
+    }
+  }
 
   return 'OK'
 }

services/backend/src/services/studytrackStats.js

@@ -9,20 +9,21 @@ const {
   getYearsArray,
   getPercentage,
   getYearsObject,
-  getAcademicYearsObject,
   getCorrectStudentnumbers,
   getCreditThresholds,
   getCreditGraphStats,
   tableTitles,
+  getCreditProgressTableTitles,
 } = require('./studyprogrammeHelpers')
 const {
   studytrackStudents,
   allStudyrights,
   startedStudyrights,
+  inactiveStudyrights,
   enrolledStudents,
   absentStudents,
   graduatedStudyRights,
-} = require('./newStudyprogramme')
+} = require('./studyprogramme')
 const { getAcademicYearDates } = require('../util/semester')
 
 const getStudentData = (students, thresholdKeys, thresholdAmounts) => {
@@ -38,7 +39,8 @@ const getStudentData = (students, thresholdKeys, thresholdAmounts) => {
     data[thresholdKeys[2]] += creditcount >= thresholdAmounts[1] && creditcount < thresholdAmounts[2] ? 1 : 0
     data[thresholdKeys[3]] += creditcount >= thresholdAmounts[2] && creditcount < thresholdAmounts[3] ? 1 : 0
     data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
-    data[thresholdKeys[5]] += creditcount >= thresholdAmounts[5] ? 1 : 0
+    data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
+    data[thresholdKeys[6]] += creditcount >= thresholdAmounts[6] ? 1 : 0
   })
   return data
 }
@@ -79,8 +81,8 @@ const getGraduationTimeStats = async ({
 // Goes through the programme and all its studytracks for the said year and adds the wanted stats to the data objects
 const getStudytrackDataForTheYear = async ({
   studyprogramme,
-  special,
-  includeGraduated,
+  since,
+  settings,
   studytracks,
   studytrackNames,
   year,
@@ -89,6 +91,9 @@ const getStudytrackDataForTheYear = async ({
   creditThresholdKeys,
   creditThresholdAmounts,
 }) => {
+  const { specialGroups: includeAllSpecials, graduated: includeGraduated } = settings
+  const { startDate, endDate } = getAcademicYearDates(year, since)
+
   const {
     mainStatsByYear,
     mainStatsByTrack,
@@ -100,7 +105,6 @@ const getStudytrackDataForTheYear = async ({
     totalAmounts,
     emptyTracks,
   } = data
-  const { startDate, endDate } = getAcademicYearDates(year)
 
   await Promise.all(
     studytracks.map(async track => {
@@ -110,19 +114,19 @@ const getStudytrackDataForTheYear = async ({
         codes,
         startDate,
         endDate,
-        includeAllSpecials: special,
+        includeAllSpecials,
         includeGraduated,
       })
 
       // Get all the studyrights and students for the calculations
-      const all = await allStudyrights(track, null, studentnumbers)
+      const all = await allStudyrights(track, studentnumbers)
       const students = await studytrackStudents(studentnumbers)
       const studentData = getStudentData(students, creditThresholdKeys, creditThresholdAmounts)
       const started = await startedStudyrights(track, startDate, studentnumbers)
       const enrolled = await enrolledStudents(track, startDate, studentnumbers)
       const absent = await absentStudents(track, studentnumbers)
-      const cancelled = all.filter(s => s.canceldate !== null && s.canceldate !== undefined)
-      const graduated = await graduatedStudyRights(track, null, studentnumbers)
+      const inactive = await inactiveStudyrights(track, studentnumbers)
+      const graduated = await graduatedStudyRights(track, startDate, studentnumbers)
 
       // If the track has no stats for that year, it should be removed from the table and dropdown options
       if (all.length === 0) {
@@ -134,17 +138,18 @@ const getStudytrackDataForTheYear = async ({
       creditThresholdKeys.forEach(
         threshold => (creditGraphStats[track][threshold].data[indexOf(years, year)] = studentData[threshold])
       )
+
       // Count stats for the credit progress table for the year per track
       creditTableStats[track] = [
         ...creditTableStats[track],
-        [`${year} - ${year + 1}`, all.length, ...creditThresholdKeys.map(threshold => studentData[threshold])],
+        [year, all.length, ...creditThresholdKeys.map(threshold => studentData[threshold])],
       ]
 
       // Count stats for the main studytrack table grouped by tracks
       mainStatsByTrack[track] = [
         ...mainStatsByTrack[track],
         [
-          `${year} - ${year + 1}`,
+          year,
           all.length,
           getPercentage(all.length, all.length),
           started.length,
@@ -153,8 +158,8 @@ const getStudytrackDataForTheYear = async ({
           getPercentage(enrolled.length, all.length),
           absent.length,
           getPercentage(absent.length, all.length),
-          cancelled.length,
-          getPercentage(cancelled.length, all.length),
+          inactive.length,
+          getPercentage(inactive.length, all.length),
           graduated.length,
           getPercentage(graduated.length, all.length),
           studentData.male,
@@ -170,9 +175,7 @@ const getStudytrackDataForTheYear = async ({
       mainStatsByYear[year] = [
         ...mainStatsByYear[year],
         [
-          studytrackNames[track]?.name['fi']
-            ? `${studytrackNames[track]?.name['fi']}, ${track}`
-            : `${year} - ${year + 1}`,
+          studytrackNames[track]?.name['fi'] ? `${studytrackNames[track]?.name['fi']}, ${track}` : year,
           all.length,
           getPercentage(all.length, all.length),
           started.length,
@@ -181,8 +184,8 @@ const getStudytrackDataForTheYear = async ({
           getPercentage(enrolled.length, all.length),
           absent.length,
           getPercentage(absent.length, all.length),
-          cancelled.length,
-          getPercentage(cancelled.length, all.length),
+          inactive.length,
+          getPercentage(inactive.length, all.length),
           graduated.length,
           getPercentage(graduated.length, all.length),
           studentData.male,
@@ -195,9 +198,9 @@ const getStudytrackDataForTheYear = async ({
       ]
 
       // Count stats for the graduation time charts grouped by year
-      totalAmounts[track][`${year} - ${year + 1}`] = await all.length
+      totalAmounts[track][year] = await all.length
       await getGraduationTimeStats({
-        year: `${year} - ${year + 1}`,
+        year,
         graduated,
         studyprogramme,
         track,
@@ -224,7 +227,7 @@ const getStudytrackOptions = (studyprogramme, studytrackNames, studytracks, empt
 
 // Creates empty objects for each statistic type, which are then updated with the studytrack data
 const getEmptyStatsObjects = (years, studytracks, studyprogramme) => {
-  const mainStatsByYear = getYearsObject(years, true)
+  const mainStatsByYear = getYearsObject({ years, emptyArrays: true })
   const mainStatsByTrack = {}
   const creditGraphStats = {}
   const creditTableStats = {}
@@ -238,10 +241,10 @@ const getEmptyStatsObjects = (years, studytracks, studyprogramme) => {
     mainStatsByTrack[track] = []
     creditGraphStats[track] = getCreditGraphStats(studyprogramme, years)
     creditTableStats[track] = []
-    graduationMedianTime[track] = getAcademicYearsObject(years, true)
-    graduationMeanTime[track] = getAcademicYearsObject(years, true)
-    graduationAmounts[track] = getAcademicYearsObject(years)
-    totalAmounts[track] = getAcademicYearsObject(years)
+    graduationMedianTime[track] = getYearsObject({ years, emptyArrays: true })
+    graduationMeanTime[track] = getYearsObject({ years, emptyArrays: true })
+    graduationAmounts[track] = getYearsObject({ years })
+    totalAmounts[track] = getYearsObject({ years })
   })
 
   return {
@@ -258,12 +261,11 @@ const getEmptyStatsObjects = (years, studytracks, studyprogramme) => {
 }
 
 // Combines all the data for the Populations and Studytracks -view
-const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, graduated, specialGroups }) => {
+const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, settings }) => {
   const isAcademicYear = true
-  const special = specialGroups === 'SPECIAL_INCLUDED'
-  const includeGraduated = graduated === 'GRADUATED_INCLUDED'
+  const includeYearsCombined = true
   const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear())
+  const years = getYearsArray(since.getFullYear(), isAcademicYear, includeYearsCombined)
 
   const associations = await getAssociations()
   const studytracks = associations.programmes[studyprogramme]
@@ -279,8 +281,8 @@ const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, graduated,
     years.map(async year => {
       return await getStudytrackDataForTheYear({
         studyprogramme,
-        special,
-        includeGraduated,
+        since,
+        settings,
         studytracks,
         studytrackNames,
         year,
@@ -296,19 +298,12 @@ const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, graduated,
 
   return {
     id: studyprogramme,
-    years: getYearsArray(since.getFullYear(), isAcademicYear),
-    mainStatsByTrack: data.mainStatsByTrack,
-    mainStatsByYear: data.mainStatsByYear,
-    creditTableStats: data.creditTableStats,
-    creditGraphStats: data.creditGraphStats,
-    graduationMedianTime: data.graduationMedianTime,
-    graduationMeanTime: data.graduationMeanTime,
-    graduationAmounts: data.graduationAmounts,
-    totalAmounts: data.totalAmounts,
+    years: getYearsArray(since.getFullYear(), isAcademicYear, includeYearsCombined),
+    ...data,
     studytrackOptions,
-    includeGraduated,
+    includeGraduated: settings.graduated,
     populationTitles: tableTitles['studytracks'],
-    creditTableTitles: tableTitles['creditProgress'][studyprogramme.includes('KH') ? 'bachelor' : 'master'],
+    creditTableTitles: getCreditProgressTableTitles(studyprogramme),
   }
 }
 

services/backend/src/services/thesis.js

@@ -1,29 +0,0 @@
-const { ThesisCourse } = require('../models/models_kone')
-
-const createThesisCourse = (programme, course, type) =>
-  ThesisCourse.create({
-    programmeCode: programme,
-    courseCode: course,
-    thesisType: type,
-  })
-
-const deleteThesisCourse = (programme, course) =>
-  ThesisCourse.destroy({
-    where: {
-      programmeCode: programme,
-      courseCode: course,
-    },
-  })
-
-const findProgrammeTheses = programme =>
-  ThesisCourse.findAll({
-    where: {
-      programmeCode: programme,
-    },
-  })
-
-module.exports = {
-  createThesisCourse,
-  deleteThesisCourse,
-  findProgrammeTheses,
-}

services/backend/src/services/topteachers.js

@@ -107,7 +107,9 @@ const findTopTeachers = async yearcode => {
     .filter(isRegularCourse)
     .map(credit => {
       const { credits, course, credittypecode } = credit
-      const teachers = credit.teachers.map(({ id, name }) => ({ id, name }))
+      const teachers = credit.teachers
+        .filter(({ id }) => !id.includes('hy-hlo-org')) // Remove faculties from the leaderboards
+        .map(({ id, name }) => ({ id, name }))
       const passed = Credit.passed(credit) || Credit.improved(credit)
       const failed = Credit.failed(credit)
       const transferred = credittypecode === 9

services/backend/src/services/trends/protoC.js

@@ -2,15 +2,28 @@ const _ = require('lodash')
 const { getRedisCDS, saveToRedis, getTargetStudentCounts } = require('./shared')
 const { getAssociations } = require('../studyrights')
 
-const REDIS_KEY_PROTOC = 'PROTOC_DATA_V2'
+const REDIS_KEY_PROTOC = 'PROTOC_DATA_V3.3'
 const REDIS_KEY_PROTOC_PROGRAMME = 'PROTOC_PROGRAMME_DATA_V2'
 
+const tryParseInt = (number, defaultValue) => {
+  try {
+    return parseInt(number, 10)
+  } catch (e) {
+    return defaultValue
+  }
+}
+
 const calculateProtoC = async query => {
   const associations = await getAssociations()
 
+  const startYear = tryParseInt(query.startYear, null)
+  const endYear = tryParseInt(query.endYear, null)
+
   const data = await getTargetStudentCounts({
     includeOldAttainments: query.include_old_attainments === 'true',
     excludeNonEnrolled: query.exclude_non_enrolled === 'true',
+    startYear,
+    endYear,
   })
 
   const programmeData = data.filter(d => d.programmeType !== 30)
@@ -97,10 +110,16 @@ const calculateProtoCProgramme = async query => {
   const codes = associations.programmes[query.code]
     ? [...associations.programmes[query.code].studytracks, query.code]
     : []
+
+  const startYear = tryParseInt(query.startYear, null)
+  const endYear = tryParseInt(query.endYear, null)
+
   const data = await getTargetStudentCounts({
-    codes: codes,
+    codes,
     includeOldAttainments: query.include_old_attainments === 'true',
     excludeNonEnrolled: query.exclude_non_enrolled === 'true',
+    startYear,
+    endYear,
   })
 
   const programmeData = data.find(d => d.programmeType === 20)
@@ -157,10 +176,10 @@ const calculateProtoCProgramme = async query => {
 }
 
 const getProtoC = async (query, doRefresh = false) => {
-  const { include_old_attainments, exclude_non_enrolled } = query
+  const { include_old_attainments, exclude_non_enrolled, startYear, endYear } = query
 
   // redis keys for different queries
-  const KEY = `${REDIS_KEY_PROTOC}_OLD_${include_old_attainments.toUpperCase()}_ENR_${exclude_non_enrolled.toUpperCase()}`
+  const KEY = `${REDIS_KEY_PROTOC}_OLD_${include_old_attainments.toUpperCase()}_ENR_${exclude_non_enrolled.toUpperCase()}_FROM_${startYear}_TO_${endYear}`
   const protoC = await getRedisCDS(KEY)
   if (!protoC || doRefresh) {
     const data = await calculateProtoC(query)
@@ -172,8 +191,8 @@ const getProtoC = async (query, doRefresh = false) => {
 
 // used for studytrack view
 const getProtoCProgramme = async (query, doRefresh = false) => {
-  const { include_old_attainments, exclude_non_enrolled, code } = query
-  const KEY = `${REDIS_KEY_PROTOC_PROGRAMME}_CODE_${code}_OLD_${include_old_attainments.toUpperCase()}_ENR_${exclude_non_enrolled.toUpperCase()}`
+  const { include_old_attainments, exclude_non_enrolled, code, startYear, endYear } = query
+  const KEY = `${REDIS_KEY_PROTOC_PROGRAMME}_CODE_${code}_OLD_${include_old_attainments.toUpperCase()}_ENR_${exclude_non_enrolled.toUpperCase()}_FROM_${startYear}_TO_${endYear}`
   const protoCProgramme = await getRedisCDS(KEY)
 
   if (!protoCProgramme || doRefresh) {

services/backend/src/services/trends/shared.js

@@ -16,7 +16,16 @@ const saveToRedis = async (data, REDIS_KEY, expire = false) => {
   }
 }
 
-const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNonEnrolled }) => {
+const getTargetStudentCounts = async ({
+  codes,
+  includeOldAttainments,
+  excludeNonEnrolled,
+  startYear = 2017,
+  endYear = 2020,
+}) => {
+  // only pick studyrights during which the student has enrolled at least once, whether it's
+  // a present or non-present enrollment
+
   return await sequelize.query(
     `
     SELECT
@@ -25,39 +34,42 @@ const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNon
         ss.programme_code "programmeCode",
         ss.programme_name "programmeName",
         ss.programme_type "programmeType",
-        COUNT(ss.studentnumber) "programmeTotalStudents",
-        SUM(
-             public.is_in_target(
-                 CURRENT_TIMESTAMP,
-                 ss.studystartdate,
-                 public.next_date_occurrence(ss.studystartdate), -- e.g if studystartdate = 2017-07-31 and it's now 2020-02-02, this returns 2020-07-31. However, if it's now 2020-11-01 (we've passed the date already), it returns 2021-07-31.
-                 ss.credits,
-                 CASE
-                     -- HAX: instead of trying to guess which category the student is in if
-                     --      the studyright is currently cancelled, just make the target HUGE
-                     --      so we can subtract these from the non-goal students
-                     WHEN ss.currently_cancelled = 1 THEN 999999999
-                     -- credit target: 60 credits per year since starting
-                     ELSE 60 * ceil(EXTRACT(EPOCH FROM (now() - ss.studystartdate) / 365) / 86400)
-                 END
-             )
+        0 AS "students3y",
+        COUNT(ss.studentnumber) FILTER (
+          WHERE
+            public.is_in_target(
+                CURRENT_TIMESTAMP,
+                ss.studystartdate,
+                public.next_date_occurrence(ss.studystartdate), -- e.g if studystartdate = 2017-07-31 and it's now 2020-02-02, this returns 2020-07-31. However, if it's now 2020-11-01 (we've passed the date already), it returns 2021-07-31.
+                ss.credits,
+                CASE
+                    -- HAX: instead of trying to guess which category the student is in if
+                    --      the studyright is currently cancelled, just make the target HUGE
+                    --      so we can subtract these from the non-goal students
+                    WHEN ss.currently_cancelled = 1 THEN 999999999
+                    -- credit target: 60 credits per year since starting
+                    ELSE 60 * ceil(EXTRACT(EPOCH FROM (now() - ss.studystartdate) / 365) / 86400)
+                END
+            ) = 1
         ) "students3y",
-        SUM(
-             public.is_in_target(
-                 CURRENT_TIMESTAMP,
-                 ss.studystartdate,
-                 public.next_date_occurrence(ss.studystartdate),
-                 ss.credits,
-                 CASE
-                     -- HAX: instead of trying to guess which category the student is in if
-                     --      the studyright is currently cancelled, just make the target HUGE
-                     --      so we can subtract these from the non-goal students
-                     WHEN ss.currently_cancelled = 1 THEN 999999999
-                     -- credit target: 45 credits per year since starting
-                     ELSE 45 * ceil(EXTRACT(EPOCH FROM (now() - ss.studystartdate) / 365) / 86400)
-                 END
-             )
+        COUNT(ss.studentnumber) FILTER (
+          WHERE
+            public.is_in_target(
+                CURRENT_TIMESTAMP,
+                ss.studystartdate,
+                public.next_date_occurrence(ss.studystartdate),
+                ss.credits,
+                CASE
+                    -- HAX: instead of trying to guess which category the student is in if
+                    --      the studyright is currently cancelled, just make the target HUGE
+                    --      so we can subtract these from the non-goal students
+                    WHEN ss.currently_cancelled = 1 THEN 999999999
+                    -- credit target: 45 credits per year since starting
+                    ELSE 45 * ceil(EXTRACT(EPOCH FROM (now() - ss.studystartdate) / 365) / 86400)
+                END
+            ) = 1
         ) "students4y",
+        COUNT(ss.studentnumber) "programmeTotalStudents",
         SUM(ss.currently_cancelled) "currentlyCancelled"
     FROM (
         SELECT
@@ -68,92 +80,16 @@ const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNon
             programme_type,
             studystartdate,
             studentnumber,
-            SUM(credits) credits,
+            SUM(credits) FILTER (WHERE NOT credits.is_old OR :include_old_attainments) credits,
             CASE
                 WHEN SUM(currently_cancelled) > 0 THEN 1
                 ELSE 0
             END currently_cancelled
-        FROM (
-            SELECT
-                org.code org_code,
-                org.name->>'fi' org_name,
-                element_details.code programme_code,
-                element_details.name->>'fi' programme_name,
-                element_details.type programme_type,
-                studyright.studystartdate studystartdate,
-                studyright.student_studentnumber studentnumber,
-                credit.credits credits,
-                CASE
-                    WHEN studyright.canceldate IS NOT NULL AND studyright.graduated != 1 THEN 1
-                    ELSE 0
-                END currently_cancelled
-            FROM
-                organization org
-                ${
-                  excludeNonEnrolled
-                    ? `
-                -- only pick studyrights during which the student has enrolled at least once, whether it's
-                -- a present or non-present enrollment
-                INNER JOIN (
-                  SELECT DISTINCT -- remove duplicate join rows from se with DISTINCT
-                      studyright.*
-                  FROM
-                      studyright
-                      INNER JOIN (
-                          SELECT
-                              studentnumber,
-                              startdate
-                          FROM
-                              semester_enrollments
-                              INNER JOIN semesters
-                                  ON semester_enrollments.semestercode = semesters.semestercode
-                      ) se
-                          ON se.studentnumber = studyright.student_studentnumber
-                  WHERE
-                      se.startdate IS NOT NULL
-                      AND se.startdate >= studyright.studystartdate
-                ) studyright
-                    ON org.code = studyright.faculty_code`
-                    : `
-                INNER JOIN studyright
-                      ON org.code = studyright.faculty_code`
-                }
-                
-                INNER JOIN (
-                    -- HACK: fix updater writing duplicates where enddate has changed
-                    SELECT DISTINCT ON (studyrightid, startdate, code, studentnumber)
-                        *
-                    FROM studyright_elements
-                ) s_elements
-                    ON studyright.studyrightid = s_elements.studyrightid
-                INNER JOIN element_details
-                    ON s_elements.code = element_details.code
-                LEFT JOIN transfers
-                    ON studyright.studyrightid = transfers.studyrightid
-                LEFT JOIN LATERAL (
-                    SELECT
-                        student_studentnumber,
-                        attainment_date,
-                        credits
-                    FROM credit
-                    WHERE credit.credittypecode IN (4, 9) -- Completed or Transferred
-                        AND (credit."isStudyModule" = false OR credit."isStudyModule" IS NULL)
-                        ${
-                          includeOldAttainments
-                            ? ''
-                            : "AND credit.attainment_date >= studyright.studystartdate -- only include credits attained during studyright's time"
-                        }
-                ) credit
-                    ON credit.student_studentnumber = studyright.student_studentnumber
-            WHERE
-                studyright.extentcode = 1 -- Bachelor's
-                AND org.code NOT IN ('01', 'H02955')
-                AND studyright.prioritycode IN (1, 30) -- Primary or Graduated
-                AND studyright.studystartdate IN ('2017-08-01 00:00:00+00', '2018-08-01 00:00:00+00', '2019-08-01 00:00:00+00')
-                AND element_details.type IN (20,30) -- programme
-                ${!!codes && codes.length > 0 ? 'AND element_details.code IN (:codes)' : ''}
-                AND transfers.studyrightid IS NULL -- Not transferred within faculty
-        ) s
+        FROM organization_yearly_credits credits
+        WHERE
+          (NOT :exclude_non_enrolled OR credits.enrollment_exists) AND
+          (:ignore_codes OR credits.programme_code IN (:codes)) AND
+          credits.startyear BETWEEN :start_year AND :end_year
         GROUP BY (1,2), (3,4) ,5 , 6, 7
     ) ss
     GROUP BY (1, 2), (3, 4), 5;
@@ -161,7 +97,12 @@ const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNon
     {
       type: sequelize.QueryTypes.SELECT,
       replacements: {
-        codes: codes,
+        codes: !codes || codes.length === 0 ? ['DUMMY'] : codes,
+        ignore_codes: !codes || codes.length === 0,
+        exclude_non_enrolled: excludeNonEnrolled,
+        start_year: startYear,
+        end_year: endYear,
+        include_old_attainments: includeOldAttainments,
       },
     }
   )

services/backend/src/services/trends/statusAttainments.js

@@ -200,11 +200,14 @@ const getTotalOpenUniCreditsOfCoursesBetween = async (a, b, alias = 'sum', alias
     INNER JOIN organization o ON o.id = cp.organizationcode
     AND cr.attainment_date BETWEEN :a AND :b
     AND (cr."isStudyModule" = false OR cr."isStudyModule" IS NULL)
-    AND cr.credittypecode IN (4, 9) AND cp.organizationcode = 'hy-org-48645785'
+    AND cr.credittypecode IN (4, 9) 
+    AND cr.is_open = true
     AND cr.student_studentnumber NOT IN
-      (SELECT student_studentnumber FROM studyright
-      WHERE studyright.student_studentnumber = cr.student_studentnumber
-      AND cr.attainment_date BETWEEN studyright.startdate AND studyright.enddate)
+        (
+        SELECT student_studentnumber FROM studyright_elements
+        WHERE studyright_elements.studentnumber = cr.student_studentnumber
+        AND cr.attainment_date BETWEEN studyright_elements.startdate AND studyright_elements.enddate
+      )
     GROUP BY co.id, o.code -- HAVING SUM(cr.credits) > 0
     `,
     {
@@ -284,6 +287,7 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
     from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
     to: new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS),
   })
+
   const yearlyAccCreditsPromises = makeYearlyCreditsPromises(
     startYear,
     yearRange,
@@ -342,7 +346,7 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
     if (!regexresult) return code
     return regexresult[1]
   }
-
+  // console.log('yearly open uni acc credits: ', yearlyOpenUniAccCredits)
   const fromAyToNormalCourses = [
     ...new Set(_.flatten(yearlyOpenUniAccCredits).map(c => unifyOpenUniversity(c.code))),
     ...new Set(_.flatten(yearlyOpenUniTotalCredits).map(c => unifyOpenUniversity(c.code))),

services/backend/src/services/trends/statusGraduated.js

@@ -1,4 +1,5 @@
 const _ = require('lodash')
+const Sentry = require('@sentry/node')
 const {
   dbConnections: { sequelize },
 } = require('../../database/connection')
@@ -93,10 +94,15 @@ const calculateStatusGraduated = async (unixMillis, showByYear) => {
       // init faculty
       if (!acc[curr.faculty_code]) {
         const org = orgs.find(o => o.code === curr.faculty_code)
-        acc[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        if (org) {
+          acc[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        } else {
+          Sentry.captureException(new Error(`Failed to find org for faculty code: ${curr.faculty_code}`))
+        }
       }
-      if (!acc[curr.faculty_code]['yearly'][data.year])
+      if (!acc[curr.faculty_code]['yearly'][data.year]) {
         acc[curr.faculty_code]['yearly'][data.year] = { acc: 0, total: 0 }
+      }
       // init programme
       if (!acc[curr.faculty_code]['drill'][curr.code]) {
         const element = elements.find(e => e.code === curr.code)
@@ -128,7 +134,11 @@ const calculateStatusGraduated = async (unixMillis, showByYear) => {
       // if no accumulated init faculty
       if (!mankeled[curr.faculty_code]) {
         const org = orgs.find(o => o.code === curr.faculty_code)
-        mankeled[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        if (org) {
+          mankeled[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        } else {
+          Sentry.captureException(new Error(`Failed to find org for faculty code: ${curr.faculty_code}`))
+        }
       }
       // if no accumulated init programme
       if (!mankeled[curr.faculty_code]['drill'][curr.code]) {

services/backend/src/services/userService.js

@@ -7,6 +7,7 @@ const { sendNotificationAboutNewUser } = require('../services/mailservice')
 const { checkStudyGuidanceGroupsAccess, getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')
 const _ = require('lodash')
 const { User, UserElementDetails, AccessGroup, UserFaculties, sequelizeUser } = require('../models/models_user')
+const { getOrganizationAccess } = require('../util/organizationAccess')
 
 const courseStatisticsGroup = 'grp-oodikone-basic-users'
 const hyOneGroup = 'hy-one'
@@ -190,8 +191,18 @@ const formatUser = async userFromDb => {
 
   const rights = _.uniqBy([...programmes, ...enrichProgrammesFromFaculties(faculties)])
 
+  const iamRights = Object.keys(await getOrganizationAccess(userFromDb))
+
   const {
-    dataValues: { id, username: userId, full_name: name, email, language, sisu_person_id: sisPersonId },
+    dataValues: {
+      id,
+      username: userId,
+      full_name: name,
+      email,
+      language,
+      sisu_person_id: sisPersonId,
+      iam_groups: iamGroups,
+    },
   } = userFromDb
 
   const studentsUserCanAccess = _.uniqBy(
@@ -205,9 +216,11 @@ const formatUser = async userFromDb => {
     name,
     language,
     sisPersonId, //
+    iamGroups,
     email,
     is_enabled: true, // this is probably not needed: is here mainly because old userservice created users for every logged in user, even if they hadn't correct iamgroups
     rights,
+    iamRights,
     roles: accessGroups,
     studentsUserCanAccess, // studentnumbers used in various parts of backend. For admin this is usually empty, since no programmes / faculties are set.
     isAdmin: accessGroups.includes('admin'),
@@ -240,6 +253,7 @@ const getUser = async ({ username, name, email, iamGroups, sisId }) => {
     email,
     language,
     sisu_person_id: sisId,
+    iam_groups: iamGroups,
     last_login: new Date(),
   })
 

services/backend/src/util/IAMRights.js

@@ -0,0 +1,239 @@
+const {
+  isSuperAdminIam,
+  isAdminIam,
+  isUniversityWideIam,
+  isDoctoralIam,
+  getStudyLeaderGroup,
+  iamToOrganisationCode,
+  isEmployeeIam,
+  iamToDoctoralSchool,
+  kosuIamToFaculties,
+  dekaaniIamToFaculty,
+  opetusVaradekaani,
+} = require('../../config/IAMConfig')
+const { data } = require('../../config/data')
+const { mapToDegreeCode } = require('../../config/common')
+
+/**
+ * Return given access to all programmes where predicate is true
+ * (all if no predicate defined)
+ * @param {object} accessLevel
+ * @param {(program: object) => boolean} where
+ * @returns {object} access to programmes
+ */
+const getAllProgrammeAccess = (accessLevel, where) => {
+  const access = {}
+  data.forEach(faculty => {
+    faculty.programmes.forEach(program => {
+      if (where?.(program) === false) return
+      access[program.key] = { ...accessLevel }
+    })
+  })
+  return access
+}
+
+/**
+ * NOT USED
+ * Grant super-admin rights if the user has correct iams (eg. grp-toska)
+ * @param {string[]} hyGroups
+ * @returns superAdmin special group
+ */
+const getSuperAdmin = hyGroups => {
+  const isToska = hyGroups.some(isSuperAdminIam)
+  if (isToska) {
+    return { specialGroup: { superAdmin: true } }
+  }
+  return {}
+}
+
+/**
+ * NOT USED
+ * Grant admin rights if the user has correct iams (eg. grp-ospa)
+ * @param {string[]} hyGroups
+ * @returns admin special group
+ */
+const getAdmin = hyGroups => {
+  const isOspa = hyGroups.some(isAdminIam)
+  if (isOspa) {
+    return { specialGroup: { admin: true } }
+  }
+  return {}
+}
+
+/**
+ * Grant reading rights to all programmes if user has uni wide IAM (eg. hy-rehtoraatti)
+ * @param {string[]} hyGroups
+ * @returns read access to ALL programmes
+ */
+const getUniversityReadingRights = hyGroups => {
+  const hasUniversityReadingRights = hyGroups.some(isUniversityWideIam)
+  if (!hasUniversityReadingRights) {
+    return {}
+  }
+
+  const access = getAllProgrammeAccess({ read: true })
+  const specialGroup = { allProgrammes: true }
+
+  return { access, specialGroup }
+}
+
+/**
+ * Grant reading rights to programmes of faculties if user is kosu or dekaanaatti of some faculties
+ * @param {string[]} hyGroups
+ */
+const getFacultyReadingRights = hyGroups => {
+  // faculty codes from kosu iam
+  const facultyCodes = hyGroups
+    .flatMap(kosuIamToFaculties)
+    // faculty codes from dekanaatti iam
+    .concat(hyGroups.map(dekaaniIamToFaculty))
+    .filter(Boolean)
+  const access = {}
+  facultyCodes.forEach(fc => {
+    const faculty = data.find(faculty => faculty.code === fc)
+    const programmeCodes = faculty.programmes.map(p => p.key)
+    programmeCodes.forEach(code => {
+      access[code] = { read: true }
+    })
+  })
+  return { access, specialGroup: {} }
+}
+
+/**
+ * Grant admin rights to faculty programmes if user is opetusvaradekaani of that faculty
+ * @param {string[]} hyGroups
+ */
+const getFacultyAdminRights = hyGroups => {
+  if (!hyGroups.includes(opetusVaradekaani)) return {}
+  const facultyCodes = hyGroups.map(dekaaniIamToFaculty).filter(Boolean)
+
+  const access = {}
+  facultyCodes.forEach(fc => {
+    const faculty = data.find(faculty => faculty.code === fc)
+    const programmeCodes = faculty.programmes.map(p => p.key)
+    programmeCodes.forEach(code => {
+      access[code] = { read: true, write: true, admin: true }
+    })
+  })
+  return { access, specialGroup: {} }
+}
+
+/**
+ * Grant reading rights to all doctoral programmes if the user belongs to doctoral IAM
+ * @param {string[]} hyGroups
+ * @returns read access to ALL doctoral programs
+ */
+const getDoctoralAccess = hyGroups => {
+  const hasDoctoralReadingRights = hyGroups.some(isDoctoralIam)
+  if (!hasDoctoralReadingRights) return {}
+  const access = getAllProgrammeAccess({ read: true }, program => program.level === 'doctoral')
+  const specialGroup = { doctoral: true }
+
+  return { access, specialGroup }
+}
+
+/**
+ * Grants reading rights to all doctoral programmes that belong to user's
+ * doctoral school IAMs
+ * @param {string[]} hyGroups
+ * @returns read access to doctoral programs
+ */
+const getDoctoralSchoolAccess = hyGroups => {
+  const doctoralProgrammeCodes = hyGroups.flatMap(iamToDoctoralSchool)
+  const access = {}
+  doctoralProgrammeCodes.forEach(code => {
+    if (!code) return
+    access[code] = { read: true }
+  })
+  return { access }
+}
+
+/**
+ * Grant admin access if the user belongs to studyprogramme's manager group and is a study program leader
+ * @param {string[]} hyGroups
+ */
+const getProgrammeAdminAccess = hyGroups => {
+  const orgCodes = hyGroups
+    .filter(iam => hyGroups.includes(getStudyLeaderGroup(iam)))
+    .map(iam => iamToOrganisationCode(iam))
+    .filter(Boolean)
+
+  const degreeCodes = orgCodes.flatMap(codes => codes.map(mapToDegreeCode))
+
+  if (!degreeCodes?.length > 0) {
+    return {}
+  }
+
+  const access = {}
+  degreeCodes.forEach(code => {
+    access[code] = { read: true, write: true, admin: true }
+  })
+  return { access }
+}
+
+/**
+ * Grant write and read access if the user belongs to employees group and studyprogramme's manager group
+ * @param {string[]} hyGroups
+ */
+const getProgrammeWriteAccess = hyGroups => {
+  if (!hyGroups.some(isEmployeeIam)) return {}
+  const orgCodes = hyGroups.map(iam => iamToOrganisationCode(iam)).filter(Boolean)
+  const degreeCodes = orgCodes.flatMap(codes => codes.map(mapToDegreeCode))
+  const access = {}
+  degreeCodes.forEach(code => {
+    if (!code) return
+    access[code] = { read: true, write: true }
+  })
+
+  return { access }
+}
+
+/**
+ * Grant read access if the user belongs to studyprogramme's manager group
+ * @param {string[]} hyGroups
+ */
+const getProgrammeReadAccess = hyGroups => {
+  const orgCodes = hyGroups.map(iam => iamToOrganisationCode(iam)).filter(Boolean)
+  const degreeCodes = orgCodes.flatMap(codes => codes.map(mapToDegreeCode))
+  const access = {}
+  degreeCodes.forEach(code => {
+    if (!code) return
+    access[code] = { read: true }
+  })
+
+  return { access }
+}
+
+/**
+ * Gets access rights and special groups,
+ * based on IAM-groups in IAM header string
+ * @param {string} hyGroupsHeader
+ */
+const getIAMRights = hyGroups => {
+  let access = {}
+  let specialGroup = {}
+
+  ;[
+    getUniversityReadingRights,
+    getFacultyReadingRights,
+    getDoctoralAccess,
+    getDoctoralSchoolAccess,
+    getProgrammeReadAccess,
+    getProgrammeWriteAccess,
+    getProgrammeAdminAccess,
+    getFacultyAdminRights,
+    getAdmin,
+    getSuperAdmin,
+  ]
+    .map(f => f(hyGroups))
+    .forEach(({ access: newAccess, specialGroup: newSpecialGroup }) => {
+      access = { ...access, ...newAccess }
+      specialGroup = { ...specialGroup, ...newSpecialGroup }
+    })
+
+  return { access, specialGroup }
+}
+
+module.exports = {
+  getIAMRights,
+}

services/backend/src/util/organizationAccess.js

@@ -0,0 +1,96 @@
+/* eslint-disable no-unused-vars */
+const _ = require('lodash')
+const Organization = require('../models/organization')
+const { getIAMRights } = require('./IAMRights')
+
+const isNumber = value => !Number.isNaN(parseInt(value, 10))
+
+const normalizeOrganizationCode = r => {
+  if (!r.includes('_')) {
+    return r
+  }
+
+  const [left, right] = r.split('_')
+  const prefix = [...left].filter(isNumber).join('')
+  const suffix = `${left[0]}${right}`
+  const providercode = `${prefix}0-${suffix}`
+  return providercode
+}
+
+const RELEVANT_ORGANIZATION_CODES = [
+  'H906', // Kielikeskus
+  'H930', // Avoin yliopisto
+]
+
+const ORGANIZATION_ACCESS_BY_IAM_GROUP = {
+  'grp-kielikeskus-esihenkilot': {
+    // Kielikeskus
+    H906: {
+      read: true,
+      write: true,
+      admin: true,
+    },
+  },
+  'grp-avoin-johto': {
+    // Avoin yliopisto
+    H930: {
+      read: true,
+    },
+  },
+}
+
+const getOrganizationAccessFromIamGroups = user => {
+  const access = {}
+
+  const iamAccess = (user.iamGroups ?? []).reduce(
+    (access, group) => _.merge(access, ORGANIZATION_ACCESS_BY_IAM_GROUP[group] ?? {}),
+    {}
+  )
+
+  Object.keys(iamAccess).forEach(code => {
+    access[code] = iamAccess[code]
+  })
+
+  return access
+}
+
+const organizationIsRelevant = organization => {
+  const { code } = organization
+
+  return code.includes('-') || RELEVANT_ORGANIZATION_CODES.includes(code)
+}
+
+const getAccessToAll = async accessLevel => {
+  const access = {}
+  const allOrganizations = await Organization.findAll({ attributes: ['code'] })
+  allOrganizations.filter(organizationIsRelevant).forEach(({ code }) => {
+    access[code] = accessLevel
+  })
+  return access
+}
+
+const getLomakeAccess = async user => {
+  if (!user.iamGroups) user.iamGroups = user.iam_groups
+
+  const access = {}
+  const { access: iamAccess } = getIAMRights(user.iamGroups)
+
+  if (!_.isObject(iamAccess)) return access
+  Object.keys(iamAccess).forEach(code => {
+    access[code] = iamAccess[code]
+  })
+  return access
+}
+
+const getOrganizationAccess = async user => {
+  const access = {
+    ...(await getLomakeAccess(user)),
+    ...getOrganizationAccessFromIamGroups(user),
+  }
+
+  return access
+}
+
+module.exports = {
+  getOrganizationAccess,
+}

services/backend/src/util/semester.js

@@ -29,10 +29,19 @@ const getPassingSemester = (startYear, date) => {
   return yearCount < 6 ? `${yearCount}-${semester}` : 'LATER'
 }
 
-const getAcademicYearDates = year => {
+const getAcademicYearDates = (year, since) => {
+  if (year === 'Total') {
+    return {
+      startDate: `${since.getFullYear()}-${semesterStart['FALL']}`,
+      endDate: `${moment(new Date()).add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`,
+    }
+  }
+  const startYear = typeof year === 'number' ? year : Number(year.slice(0, 4))
+  const endYear =
+    typeof year === 'number' ? moment(year, 'YYYY').add(1, 'years').format('YYYY') : Number(year.slice(7, 11))
   return {
-    startDate: `${year}-${semesterStart['FALL']}`,
-    endDate: `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`,
+    startDate: `${startYear}-${semesterStart['FALL']}`,
+    endDate: `${endYear}-${semesterEnd['SPRING']}`,
   }
 }
 

services/backend/src/util/utils.js

@@ -1,5 +1,5 @@
-const mapToProviders = elementDetails =>
-  elementDetails.map(r => {
+const mapToProviders = elementDetails => {
+  return elementDetails.map(r => {
     const isNumber = str => !Number.isNaN(Number(str))
     if (r.includes('_')) {
       const [left, right] = r.split('_')
@@ -8,8 +8,13 @@ const mapToProviders = elementDetails =>
       const providercode = `${prefix}0-${suffix}`
       return providercode
     }
+    if (/^(T)[0-9]{6}$/.test(r)) {
+      const numbers = r.substring(1)
+      return `7${numbers}`
+    }
     return r
   })
+}
 
 // sort substitutions so that main code is first
 const newLetterBasedCode = /^[A-Za-z]/ // new letter based codes come first

services/frontend/package-lock.json

@@ -1908,26 +1908,19 @@
       }
     },
     "@popperjs/core": {
-      "version": "2.9.2",
-      "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.9.2.tgz",
-      "integrity": "sha512-VZMYa7+fXHdwIq1TDhSXoVmSPEGM/aa+6Aiq3nVVJ9bXr24zScr+NlKFKC3iPljA7ho/GAZr+d2jOf5GIRC30Q=="
+      "version": "2.11.2",
+      "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.2.tgz",
+      "integrity": "sha512-92FRmppjjqz29VMJ2dn+xdyXZBrMlE42AV6Kq6BwjWV7CNUW1hs2FtxSNLQE+gJhaZ6AAmYuO9y8dshhcBl7vA=="
     },
     "@reduxjs/toolkit": {
-      "version": "1.6.2",
-      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.6.2.tgz",
-      "integrity": "sha512-HbfI/hOVrAcMGAYsMWxw3UJyIoAS9JTdwddsjlr5w3S50tXhWb+EMyhIw+IAvCVCLETkzdjgH91RjDSYZekVBA==",
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.8.0.tgz",
+      "integrity": "sha512-cdfHWfcvLyhBUDicoFwG1u32JqvwKDxLxDd7zSmSoFw/RhYLOygIRtmaMjPRUUHmVmmAGAvquLLsKKU/677kSQ==",
       "requires": {
-        "immer": "^9.0.6",
-        "redux": "^4.1.0",
-        "redux-thunk": "^2.3.0",
-        "reselect": "^4.0.0"
-      },
-      "dependencies": {
-        "immer": {
-          "version": "9.0.6",
-          "resolved": "https://registry.npmjs.org/immer/-/immer-9.0.6.tgz",
-          "integrity": "sha512-G95ivKpy+EvVAnAab4fVa4YGYn24J1SpEktnJX7JJ45Bd7xqME/SCplFzYFmTbrkwZbQ4xJK1xMTUYBkN6pWsQ=="
-        }
+        "immer": "^9.0.7",
+        "redux": "^4.1.2",
+        "redux-thunk": "^2.4.1",
+        "reselect": "^4.1.5"
       }
     },
     "@rollup/plugin-node-resolve": {
@@ -1978,13 +1971,13 @@
       }
     },
     "@sentry/browser": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-6.9.0.tgz",
-      "integrity": "sha512-4JnEPcwoNs6JqeEd4wscBq+hxpotEJ0DJ4eOIsaNZIMyqEHXBHTXCk/gfrSsiZFrkHM4PgvUHOxaC0HcZ92oBA==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-6.18.2.tgz",
+      "integrity": "sha512-EsqKSNboi2gOiMuEwQranLucxrARi00y2vgUnaPXcqTKTlVlHDetoWHvq8/r29idA1JHGka5tDrwrmWccWIkrg==",
       "requires": {
-        "@sentry/core": "6.9.0",
-        "@sentry/types": "6.9.0",
-        "@sentry/utils": "6.9.0",
+        "@sentry/core": "6.18.2",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
@@ -1996,14 +1989,14 @@
       }
     },
     "@sentry/core": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/core/-/core-6.9.0.tgz",
-      "integrity": "sha512-oFX2qQcMLujCeIuCQGlhpTUIOXiU5n6V2lqDnvMXUV8gKpplBPalwdlR9bgbSi+VO8u7LjHR1IKM0RAPWgNHWw==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/core/-/core-6.18.2.tgz",
+      "integrity": "sha512-r5ad/gq5S/JHc9sd5CUhZQT9ojQ+f+thk/AoGeGawX/8HURZYAgIqD565d6FK0VsZEDkdRMl58z1Qon20h3y1g==",
       "requires": {
-        "@sentry/hub": "6.9.0",
-        "@sentry/minimal": "6.9.0",
-        "@sentry/types": "6.9.0",
-        "@sentry/utils": "6.9.0",
+        "@sentry/hub": "6.18.2",
+        "@sentry/minimal": "6.18.2",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
@@ -2015,12 +2008,12 @@
       }
     },
     "@sentry/hub": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.9.0.tgz",
-      "integrity": "sha512-5mors7ojbo7G85ZmoVPQBgFBMONAJwyZfV0LNLy14GenoaVNuxTPyvAQiJb1FYq+x6YZ3CvqGX6r74KRKQU87w==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.18.2.tgz",
+      "integrity": "sha512-d0AugekMkbnN12b4EXMjseJxtLPc9S20DGobCPUb4oAQT6S2oDQEj1jwP6PQ5vtgyy+GMYWxBMgqAQ4pjVYISQ==",
       "requires": {
-        "@sentry/types": "6.9.0",
-        "@sentry/utils": "6.9.0",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
@@ -2032,12 +2025,12 @@
       }
     },
     "@sentry/minimal": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.9.0.tgz",
-      "integrity": "sha512-GBZ6wG2Rc1wInYEl2BZTZc/t57O1Da876ifLsSPpEQAEnGWbqZWb8RLjZskH09ZIL/K4XCIDDi5ySzN8kFUWJw==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.18.2.tgz",
+      "integrity": "sha512-n7KYuo34W2LxE+3dnZ47of7XHuORINCnXq66XH72eoj67tf0XeWbIhEJrYGmoLRyRfoCYYrBLWiDl/uTjLzrzQ==",
       "requires": {
-        "@sentry/hub": "6.9.0",
-        "@sentry/types": "6.9.0",
+        "@sentry/hub": "6.18.2",
+        "@sentry/types": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
@@ -2049,51 +2042,17 @@
       }
     },
     "@sentry/tracing": {
-      "version": "6.11.0",
-      "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.11.0.tgz",
-      "integrity": "sha512-9VA1/SY++WeoMQI4K6n/sYgIdRtCu9NLWqmGqu/5kbOtESYFgAt1DqSyqGCr00ZjQiC2s7tkDkTNZb38K6KytQ==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.18.2.tgz",
+      "integrity": "sha512-hg6NLqrqJ5sUPTyWEQ2RqdnhQVnyLtx8II0IyWxQLDWD8UCe3Mu6G7mroDtakPWcP+lWz6OnKfMEfuhMcxR8fw==",
       "requires": {
-        "@sentry/hub": "6.11.0",
-        "@sentry/minimal": "6.11.0",
-        "@sentry/types": "6.11.0",
-        "@sentry/utils": "6.11.0",
+        "@sentry/hub": "6.18.2",
+        "@sentry/minimal": "6.18.2",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
-        "@sentry/hub": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.11.0.tgz",
-          "integrity": "sha512-pT9hf+ZJfVFpoZopoC+yJmFNclr4NPqPcl2cgguqCHb69DklD1NxgBNWK8D6X05qjnNFDF991U6t1mxP9HrGuw==",
-          "requires": {
-            "@sentry/types": "6.11.0",
-            "@sentry/utils": "6.11.0",
-            "tslib": "^1.9.3"
-          }
-        },
-        "@sentry/minimal": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.11.0.tgz",
-          "integrity": "sha512-XkZ7qrdlGp4IM/gjGxf1Q575yIbl5RvPbg+WFeekpo16Ufvzx37Mr8c2xsZaWosISVyE6eyFpooORjUlzy8EDw==",
-          "requires": {
-            "@sentry/hub": "6.11.0",
-            "@sentry/types": "6.11.0",
-            "tslib": "^1.9.3"
-          }
-        },
-        "@sentry/types": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.11.0.tgz",
-          "integrity": "sha512-gm5H9eZhL6bsIy/h3T+/Fzzz2vINhHhqd92CjHle3w7uXdTdFV98i2pDpErBGNTSNzbntqOMifYEB5ENtZAvcg=="
-        },
-        "@sentry/utils": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.11.0.tgz",
-          "integrity": "sha512-IOvyFHcnbRQxa++jO+ZUzRvFHEJ1cZjrBIQaNVc0IYF0twUOB5PTP6joTcix38ldaLeapaPZ9LGfudbvYvxkdg==",
-          "requires": {
-            "@sentry/types": "6.11.0",
-            "tslib": "^1.9.3"
-          }
-        },
         "tslib": {
           "version": "1.14.1",
           "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
@@ -2102,16 +2061,16 @@
       }
     },
     "@sentry/types": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.9.0.tgz",
-      "integrity": "sha512-v52HJqLoLapEnqS2NdVtUXPvT+aezQgNXQkp8hiQ3RUdTm5cffwBVG7wlbpE6OsOOIZxd6p1zKylFkwCypiIIA=="
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.18.2.tgz",
+      "integrity": "sha512-WzpJf/Q5aORTzrSwer/As1NlO90dBAQpaHV2ikDDKqOyMWEgjKb5/4gh59p9gH8JMMnLetP1AvQel0fOj5UnUw=="
     },
     "@sentry/utils": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.9.0.tgz",
-      "integrity": "sha512-PimDr6KAi4cCp5hQZ8Az2/pDcdfhTu7WAU30Dd9MZwknpHSTmD4G6QvkdrB5er6kMMnNQOC7rMo6w/Do3m6X3w==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.18.2.tgz",
+      "integrity": "sha512-EC619jesknyu4xpwud5WC/5odYLz6JUy7OSFy5405PpdGeh/m8XUvuJAx4zDx0Iz/Mlk0S1Md+ZcQwqkv39dkw==",
       "requires": {
-        "@sentry/types": "6.9.0",
+        "@sentry/types": "6.18.2",
         "tslib": "^1.9.3"
       },
       "dependencies": {
@@ -2330,9 +2289,9 @@
       }
     },
     "@types/hast": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.2.tgz",
-      "integrity": "sha512-Op5W7jYgZI7AWKY5wQ0/QNMzQM7dGQPyW1rXKNiymVCy5iTfdPuGu4HhYNOM2sIv8gUfIuIdcYlXmAepwaowow==",
+      "version": "2.3.4",
+      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.4.tgz",
+      "integrity": "sha512-wLEm0QvaoawEDoTRwzTXp4b4jpwiJDvR5KMnFnVodm3scufTlBOWRD6N1OBf9TZMhjlNsSfcO5V+7AF4+Vy+9g==",
       "requires": {
         "@types/unist": "*"
       }
@@ -2378,9 +2337,9 @@
       "integrity": "sha512-YSBPTLTVm2e2OoQIDYx8HaeWJ5tTToLH67kXR7zYNGupXMEHa2++G8k+DczX2cFVgalypqtyZIcU19AFcmOpmg=="
     },
     "@types/mdast": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.4.tgz",
-      "integrity": "sha512-gIdhbLDFlspL53xzol2hVzrXAbzt71erJHoOwQZWssjaiouOotf03lNtMmFm9VfFkvnLWccSVjUAZGQ5Kqw+jA==",
+      "version": "3.0.10",
+      "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.10.tgz",
+      "integrity": "sha512-W864tg/Osz1+9f4lrGTZpCSO5/z4608eUp19tbozkq2HJK6i3z1kT0H9tlADXuYIb1YYOBByU4Jsqkk75q48qA==",
       "requires": {
         "@types/unist": "*"
       }
@@ -2485,9 +2444,9 @@
       }
     },
     "@types/unist": {
-      "version": "2.0.5",
-      "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.5.tgz",
-      "integrity": "sha512-wnra4Vw9dopnuybR6HBywJ/URYpYrKLoepBTEtgfJup8Ahoi2zJECPP2cwiXp7btTvOT2CULv87aQRA4eZSP6g=="
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.6.tgz",
+      "integrity": "sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ=="
     },
     "@types/webpack": {
       "version": "4.41.30",
@@ -3160,9 +3119,9 @@
       "integrity": "sha512-99FZt8qS/xukgxU/8daV8WT7wAakqBzt6lF3XCweO6pwcf50/NgxxBj6ZC7/ejR+F4PWeFpkb9lAMH3y2quBXA=="
     },
     "axios": {
-      "version": "0.21.2",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.2.tgz",
-      "integrity": "sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg==",
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
       "requires": {
         "follow-redirects": "^1.14.0"
       }
@@ -4087,13 +4046,35 @@
       "integrity": "sha512-/4YgnZS8y1UXXmC02xD5rRrBEu6T5ub+mQHLNRj0fzTRbgdBYhsNo2V5EqwgqrExjxsjtF/OpAKAMkKsxbD5XQ=="
     },
     "cfb": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.0.tgz",
-      "integrity": "sha512-sXMvHsKCICVR3Naq+J556K+ExBo9n50iKl6LGarlnvuA2035uMlGA/qVrc0wQtow5P1vJEw9UyrKLCbtIKz+TQ==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.1.tgz",
+      "integrity": "sha512-wT2ScPAFGSVy7CY+aauMezZBnNrfnaLSrxHUHdea+Td/86vrk6ZquggV+ssBR88zNs0OnBkL2+lf9q0K+zVGzQ==",
       "requires": {
-        "adler-32": "~1.2.0",
+        "adler-32": "~1.3.0",
         "crc-32": "~1.2.0",
-        "printj": "~1.1.2"
+        "printj": "~1.3.0"
+      },
+      "dependencies": {
+        "adler-32": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.0.tgz",
+          "integrity": "sha512-f5nltvjl+PRUh6YNfUstRaXwJxtfnKEWhAWWlmKvh+Y3J2+98a0KKVYDEhz6NdKGqswLhjNGznxfSsZGOvOd9g==",
+          "requires": {
+            "printj": "~1.2.2"
+          },
+          "dependencies": {
+            "printj": {
+              "version": "1.2.3",
+              "resolved": "https://registry.npmjs.org/printj/-/printj-1.2.3.tgz",
+              "integrity": "sha512-sanczS6xOJOg7IKDvi4sGOUOe7c1tsEzjwlLFH/zgwx/uyImVM9/rgBkc8AfiQa/Vg54nRd8mkm9yI7WV/O+WA=="
+            }
+          }
+        },
+        "printj": {
+          "version": "1.3.1",
+          "resolved": "https://registry.npmjs.org/printj/-/printj-1.3.1.tgz",
+          "integrity": "sha512-GA3TdL8szPK4AQ2YnOe/b+Y1jUFwmmGMMK/qbY7VcE3Z7FU8JstbKiKRzO6CIiAKPhTO8m01NoQ0V5f3jc4OGg=="
+        }
       }
     },
     "chalk": {
@@ -4152,6 +4133,11 @@
       "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
       "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="
     },
+    "chroma-js": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/chroma-js/-/chroma-js-2.4.2.tgz",
+      "integrity": "sha512-U9eDw6+wt7V8z5NncY2jJfZa+hUH8XEj8FQHgFJTrUFnJfXYf4Ml4adI2vXZOjqRDpFWtYVWypDfZwnJ+HIR4A=="
+    },
     "chrome-trace-event": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz",
@@ -4299,20 +4285,9 @@
       }
     },
     "codepage": {
-      "version": "1.14.0",
-      "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.14.0.tgz",
-      "integrity": "sha1-jL4lSBMjVZ19MHVxsP/5HnodL5k=",
-      "requires": {
-        "commander": "~2.14.1",
-        "exit-on-epipe": "~1.0.1"
-      },
-      "dependencies": {
-        "commander": {
-          "version": "2.14.1",
-          "resolved": "https://registry.npmjs.org/commander/-/commander-2.14.1.tgz",
-          "integrity": "sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw=="
-        }
-      }
+      "version": "1.15.0",
+      "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz",
+      "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA=="
     },
     "collect-v8-coverage": {
       "version": "1.0.1",
@@ -4617,12 +4592,19 @@
       }
     },
     "crc-32": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.0.tgz",
-      "integrity": "sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.1.tgz",
+      "integrity": "sha512-Dn/xm/1vFFgs3nfrpEVScHoIslO9NZRITWGz/1E/St6u4xw99vfZzVkW0OSnzx2h9egej9xwMCEut6sqwokM/w==",
       "requires": {
         "exit-on-epipe": "~1.0.1",
-        "printj": "~1.1.0"
+        "printj": "~1.3.1"
+      },
+      "dependencies": {
+        "printj": {
+          "version": "1.3.1",
+          "resolved": "https://registry.npmjs.org/printj/-/printj-1.3.1.tgz",
+          "integrity": "sha512-GA3TdL8szPK4AQ2YnOe/b+Y1jUFwmmGMMK/qbY7VcE3Z7FU8JstbKiKRzO6CIiAKPhTO8m01NoQ0V5f3jc4OGg=="
+        }
       }
     },
     "create-ecdh": {
@@ -6657,11 +6639,6 @@
         "bser": "2.1.1"
       }
     },
-    "fflate": {
-      "version": "0.3.11",
-      "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.3.11.tgz",
-      "integrity": "sha512-Rr5QlUeGN1mbOHlaqcSYMKVpPbgLy0AWT/W0EHxA6NGI12yO1jpoui2zBBvU2G824ltM6Ut8BFgfHSBGfkmS0A=="
-    },
     "figgy-pudding": {
       "version": "3.5.2",
       "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz",
@@ -6837,9 +6814,9 @@
       }
     },
     "follow-redirects": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz",
-      "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg=="
+      "version": "1.14.8",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz",
+      "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA=="
     },
     "for-in": {
       "version": "1.0.2",
@@ -7355,9 +7332,9 @@
       "integrity": "sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ=="
     },
     "highcharts": {
-      "version": "9.1.2",
-      "resolved": "https://registry.npmjs.org/highcharts/-/highcharts-9.1.2.tgz",
-      "integrity": "sha512-RwCnJnxAUCGG4R/WDG+QP/4VJNtOpkQTkXcuf58DjDvm9ZILplSMdZvrbMRC9Y9ecuJQCjjZsiifvyS5E2IBvQ=="
+      "version": "9.3.3",
+      "resolved": "https://registry.npmjs.org/highcharts/-/highcharts-9.3.3.tgz",
+      "integrity": "sha512-QeOvm6cifeZYYdTLm4IxZsXcOE9c4xqfs0z0OJJ0z7hhA9WG0rmcVAyuIp5HBl/znjA/ayYHmpYjBYD/9PG4Fg=="
     },
     "highcharts-custom-events": {
       "version": "3.0.10",
@@ -7741,9 +7718,9 @@
       "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg=="
     },
     "immer": {
-      "version": "9.0.7",
-      "resolved": "https://registry.npmjs.org/immer/-/immer-9.0.7.tgz",
-      "integrity": "sha512-KGllzpbamZDvOIxnmJ0jI840g7Oikx58lBPWV0hUh7dtAyZpFqqrBZdKka5GlTwMTZ1Tjc/bKKW4VSFAt6BqMA=="
+      "version": "9.0.12",
+      "resolved": "https://registry.npmjs.org/immer/-/immer-9.0.12.tgz",
+      "integrity": "sha512-lk7UNmSbAukB5B6dh9fnh5D0bJTOFKxVg2cyJWTYrWRfhLrLMBquONcUs3aFq507hNoIZEDDh8lb8UtOizSMhA=="
     },
     "import-cwd": {
       "version": "2.1.0",
@@ -10412,9 +10389,9 @@
       "optional": true
     },
     "nanoid": {
-      "version": "3.1.23",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.23.tgz",
-      "integrity": "sha512-FiB0kzdP0FFVGDKlRLEQ1BgDzU87dy5NnzjeW9YZNt+/c3+q82EQDUwniSAUxp/F0gFNI1ZhKU1FqYsMuqZVnw=="
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.2.0.tgz",
+      "integrity": "sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA=="
     },
     "nanomatch": {
       "version": "1.2.13",
@@ -12351,13 +12328,13 @@
       }
     },
     "prop-types": {
-      "version": "15.7.2",
-      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz",
-      "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==",
+      "version": "15.8.1",
+      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
+      "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
       "requires": {
         "loose-envify": "^1.4.0",
         "object-assign": "^4.1.1",
-        "react-is": "^16.8.1"
+        "react-is": "^16.13.1"
       },
       "dependencies": {
         "react-is": {
@@ -12455,9 +12432,9 @@
       "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc="
     },
     "qs": {
-      "version": "6.10.1",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.1.tgz",
-      "integrity": "sha512-M528Hph6wsSVOBiYUnGf+K/7w0hNshs/duGsNXPUCLH5XAqjEtiPGwNONLV0tBH8NoGb0mvD5JubnUTrujKDTg==",
+      "version": "6.10.3",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz",
+      "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==",
       "requires": {
         "side-channel": "^1.0.4"
       }
@@ -12565,9 +12542,9 @@
       }
     },
     "react-datetime": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/react-datetime/-/react-datetime-3.0.4.tgz",
-      "integrity": "sha512-v6MVwCve+DRaLN2f22LTO5TlrPpkUXumPkp1zfrbhaFtSYGl2grZ2JtwJfLxRj/T4ACyePAV4srCR6cMSiQ/Iw==",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/react-datetime/-/react-datetime-3.1.1.tgz",
+      "integrity": "sha512-gHCTjAniCcMb6jdXpz+MpVe/uCeaHNDOofg+l41nLlJI3uBLBMV40CQbGB2TCTUpCzGT1mCs4vQzKGMjXO/WWQ==",
       "requires": {
         "prop-types": "^15.5.7"
       }
@@ -12734,15 +12711,15 @@
       "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w=="
     },
     "react-markdown": {
-      "version": "6.0.2",
-      "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-6.0.2.tgz",
-      "integrity": "sha512-Et2AjXAsbmPP1nLQQRqmVgcqzfwcz8uQJ8VAdADs8Nk/aaUA0YeU9RDLuCtD+GwajCnm/+Iiu2KPmXzmD/M3vA==",
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-6.0.3.tgz",
+      "integrity": "sha512-kQbpWiMoBHnj9myLlmZG9T1JdoT/OEyHK7hqM6CqFT14MAkgWiWBUYijLyBmxbntaN6dCDicPcUhWhci1QYodg==",
       "requires": {
         "@types/hast": "^2.0.0",
         "@types/unist": "^2.0.3",
         "comma-separated-tokens": "^1.0.0",
         "prop-types": "^15.7.2",
-        "property-information": "^5.0.0",
+        "property-information": "^5.3.0",
         "react-is": "^17.0.0",
         "remark-parse": "^9.0.0",
         "remark-rehype": "^8.0.0",
@@ -12772,6 +12749,11 @@
         }
       }
     },
+    "react-range": {
+      "version": "1.8.12",
+      "resolved": "https://registry.npmjs.org/react-range/-/react-range-1.8.12.tgz",
+      "integrity": "sha512-vY2eCNDvKlXtEpl8xx5AQVIBeE4ZZ46dLVL+82f4FcS9XrPqz4+XagPHmq3IPqYuVQ2H4b8UArXzvVAsd/B03Q=="
+    },
     "react-redux": {
       "version": "7.2.6",
       "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-7.2.6.tgz",
@@ -12800,12 +12782,17 @@
       "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.8.3.tgz",
       "integrity": "sha512-X8jZHc7nCMjaCqoU+V2I0cOhNW+QMBwSUkeXnTi8IPe6zaRWfn60ZzvFDZqWPfmSJfjub7dDW1SP0jaHWLu/hg=="
     },
+    "react-reverse-portal": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/react-reverse-portal/-/react-reverse-portal-2.1.0.tgz",
+      "integrity": "sha512-ck1igNI4tUFozJ89X7qqFtWy1wSL7WEd/5iZbjBjyf+6vjZ3leg4JJxdVPbFt9+zwMLcQSPsIkw9xGu9E+VqCQ=="
+    },
     "react-router": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/react-router/-/react-router-5.2.0.tgz",
-      "integrity": "sha512-smz1DUuFHRKdcJC0jobGo8cVbhO3x50tCL4icacOlcwDOEQPq4TMqwx3sY1TP+DvtTgz4nm3thuo7A+BK2U0Dw==",
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/react-router/-/react-router-5.2.1.tgz",
+      "integrity": "sha512-lIboRiOtDLFdg1VTemMwud9vRVuOCZmUIT/7lUoZiSpPODiiH1UQlfXy+vPLC/7IWdFYnhRwAyNqA/+I7wnvKQ==",
       "requires": {
-        "@babel/runtime": "^7.1.2",
+        "@babel/runtime": "^7.12.13",
         "history": "^4.9.0",
         "hoist-non-react-statics": "^3.1.0",
         "loose-envify": "^1.3.1",
@@ -12817,6 +12804,14 @@
         "tiny-warning": "^1.0.0"
       },
       "dependencies": {
+        "@babel/runtime": {
+          "version": "7.17.2",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.2.tgz",
+          "integrity": "sha512-hzeyJyMA1YGdJTuWU0e/j4wKXrU4OMFvY2MSlaI9B7VQb0r5cxTE3EAIS2Q7Tn2RIcDkRvTA/v2JsAEhxe99uw==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        },
         "isarray": {
           "version": "0.0.1",
           "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
@@ -12838,17 +12833,27 @@
       }
     },
     "react-router-dom": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-5.2.0.tgz",
-      "integrity": "sha512-gxAmfylo2QUjcwxI63RhQ5G85Qqt4voZpUXSEqCwykV0baaOTQDR1f0PmY8AELqIyVc0NEZUj0Gov5lNGcXgsA==",
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-5.3.0.tgz",
+      "integrity": "sha512-ObVBLjUZsphUUMVycibxgMdh5jJ1e3o+KpAZBVeHcNQZ4W+uUGGWsokurzlF4YOldQYRQL4y6yFRWM4m3svmuQ==",
       "requires": {
-        "@babel/runtime": "^7.1.2",
+        "@babel/runtime": "^7.12.13",
         "history": "^4.9.0",
         "loose-envify": "^1.3.1",
         "prop-types": "^15.6.2",
-        "react-router": "5.2.0",
+        "react-router": "5.2.1",
         "tiny-invariant": "^1.0.2",
         "tiny-warning": "^1.0.0"
+      },
+      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.17.2",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.2.tgz",
+          "integrity": "sha512-hzeyJyMA1YGdJTuWU0e/j4wKXrU4OMFvY2MSlaI9B7VQb0r5cxTE3EAIS2Q7Tn2RIcDkRvTA/v2JsAEhxe99uw==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        }
       }
     },
     "react-scripts": {
@@ -13086,9 +13091,9 @@
       }
     },
     "redux-thunk": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.4.0.tgz",
-      "integrity": "sha512-/y6ZKQNU/0u8Bm7ROLq9Pt/7lU93cT0IucYMrubo89ENjxPa7i8pqLKu6V4X7/TvYovQ6x01unTeyeZ9lgXiTA=="
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.4.1.tgz",
+      "integrity": "sha512-OOYGNY5Jy2TWvTL1KgAlVy6dcx3siPJ1wTq741EPyUKfn6W6nChdICjZwCd0p8AZBs5kWpZlbkXW2nE/zjUa+Q=="
     },
     "regenerate": {
       "version": "1.4.2",
@@ -13261,9 +13266,9 @@
       "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
     },
     "reselect": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/reselect/-/reselect-4.1.2.tgz",
-      "integrity": "sha512-wg60ebcPOtxcptIUfrr7Jt3h4BR86cCW3R7y4qt65lnNb4yz4QgrXcbSioVsIOYguyz42+XTHIyJ5TEruzkFgQ=="
+      "version": "4.1.5",
+      "resolved": "https://registry.npmjs.org/reselect/-/reselect-4.1.5.tgz",
+      "integrity": "sha512-uVdlz8J7OO+ASpBYoz1Zypgx0KasCY20H+N8JD13oUMtPvSHQuscrHop4KbXrbsBcdB9Ds7lVK7eRkBIfO43vQ=="
     },
     "resolve": {
       "version": "1.18.1",
@@ -13837,9 +13842,9 @@
       }
     },
     "semantic-ui-react": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/semantic-ui-react/-/semantic-ui-react-2.0.3.tgz",
-      "integrity": "sha512-a0hGN6XXw64sRSKwWqMCKSI/AGLohxNeWuErS39eswvBbUnLjBij8ZoEdiqDiz/PuWpwYIRjgmQVrut+7h3b2g==",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/semantic-ui-react/-/semantic-ui-react-2.1.2.tgz",
+      "integrity": "sha512-NG7qMUi2Ab2gPWzOnGyBO9Ao2tjf9MZjjMMPbmg8uEUcEFm+z1QPqCKhW+eYMiZekH8L9azw9M+hzoTIB1lZgQ==",
       "requires": {
         "@babel/runtime": "^7.10.5",
         "@fluentui/react-component-event-listener": "~0.51.6",
@@ -13848,8 +13853,8 @@
         "@semantic-ui-react/event-stack": "^3.1.2",
         "clsx": "^1.1.1",
         "keyboard-key": "^1.1.0",
-        "lodash": "^4.17.19",
-        "lodash-es": "^4.17.15",
+        "lodash": "^4.17.21",
+        "lodash-es": "^4.17.21",
         "prop-types": "^15.7.2",
         "react-is": "^16.8.6 || ^17.0.0",
         "react-popper": "^2.2.4",
@@ -15343,9 +15348,9 @@
       }
     },
     "unfuck-spa-shibboleth-session": {
-      "version": "3.1.3",
-      "resolved": "https://registry.npmjs.org/unfuck-spa-shibboleth-session/-/unfuck-spa-shibboleth-session-3.1.3.tgz",
-      "integrity": "sha512-ZchOl94XvUz0eTLfZtx1iuqoKcNFN7WePB9Qz+6vPAkHO+F+H1zBatJo0lLgEdZGm1wqCl7gArrRuilcnsUp/A==",
+      "version": "3.1.4",
+      "resolved": "https://registry.npmjs.org/unfuck-spa-shibboleth-session/-/unfuck-spa-shibboleth-session-3.1.4.tgz",
+      "integrity": "sha512-eh20O3yTDRlRYmV8DQhELvnwFS78I8mdMUjetdvvIpO/tGSO5mzDooCrF9ssOUxGtCA6VtlcoVBs3FjFSm1Wng==",
       "requires": {
         "axios": "^0.21.1"
       }
@@ -15375,9 +15380,9 @@
       "integrity": "sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg=="
     },
     "unified": {
-      "version": "9.2.1",
-      "resolved": "https://registry.npmjs.org/unified/-/unified-9.2.1.tgz",
-      "integrity": "sha512-juWjuI8Z4xFg8pJbnEZ41b5xjGUWGHqXALmBZ3FC3WX0PIx1CZBIIJ6mXbYMcf6Yw4Fi0rFUTA1cdz/BglbOhA==",
+      "version": "9.2.2",
+      "resolved": "https://registry.npmjs.org/unified/-/unified-9.2.2.tgz",
+      "integrity": "sha512-Sg7j110mtefBD+qunSLO1lqOEKdrwBFBrR6Qd8f4uwkhWNlbkaqwHse6e7QvD3AP/MNoJdEDLaf8OxYyoWgorQ==",
       "requires": {
         "bail": "^1.0.0",
         "extend": "^3.0.0",
@@ -15616,9 +15621,9 @@
       }
     },
     "url-parse": {
-      "version": "1.5.3",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz",
-      "integrity": "sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ==",
+      "version": "1.5.10",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
+      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
       "requires": {
         "querystringify": "^2.1.1",
         "requires-port": "^1.0.0"
@@ -15629,6 +15634,11 @@
       "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz",
       "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ=="
     },
+    "use-context-selector": {
+      "version": "1.3.9",
+      "resolved": "https://registry.npmjs.org/use-context-selector/-/use-context-selector-1.3.9.tgz",
+      "integrity": "sha512-YgzRyeFjoJXwFn2qLVAuIbV6EQ8DOuzu3SS/eiCxyAyvBhcn02jYSz8c5v22QQU3LW6Ez/Iyo62kKvS7Kdqt3A=="
+    },
     "util": {
       "version": "0.11.1",
       "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz",
@@ -17189,27 +17199,17 @@
       "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg=="
     },
     "xlsx": {
-      "version": "0.17.0",
-      "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.17.0.tgz",
-      "integrity": "sha512-bZ36FSACiAyjoldey1+7it50PMlDp1pcAJrZKcVZHzKd8BC/z6TQ/QAN8onuqcepifqSznR6uKnjPhaGt6ig9A==",
+      "version": "0.17.5",
+      "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.17.5.tgz",
+      "integrity": "sha512-lXNU0TuYsvElzvtI6O7WIVb9Zar1XYw7Xb3VAx2wn8N/n0whBYrCnHMxtFyIiUU1Wjf09WzmLALDfBO5PqTb1g==",
       "requires": {
         "adler-32": "~1.2.0",
         "cfb": "^1.1.4",
-        "codepage": "~1.14.0",
-        "commander": "~2.17.1",
+        "codepage": "~1.15.0",
         "crc-32": "~1.2.0",
-        "exit-on-epipe": "~1.0.1",
-        "fflate": "^0.3.8",
         "ssf": "~0.11.2",
         "wmf": "~1.0.1",
         "word": "~0.3.0"
-      },
-      "dependencies": {
-        "commander": {
-          "version": "2.17.1",
-          "resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz",
-          "integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg=="
-        }
       }
     },
     "xml-name-validator": {

services/frontend/package.json

@@ -6,37 +6,41 @@
     "build": "DISABLE_ESLINT_PLUGIN=true react-scripts build"
   },
   "dependencies": {
-    "@reduxjs/toolkit": "^1.6.2",
-    "@sentry/browser": "^6.9.0",
-    "@sentry/tracing": "^6.11.0",
-    "axios": "^0.21.2",
+    "@reduxjs/toolkit": "^1.8.0",
+    "@sentry/browser": "^6.18.2",
+    "@sentry/tracing": "^6.18.2",
+    "axios": "^0.21.4",
+    "chroma-js": "^2.4.2",
     "formik": "^2.2.9",
-    "highcharts": "^9.1.2",
+    "highcharts": "^9.3.3",
     "highcharts-custom-events": "^3.0.10",
     "highcharts-exporting": "^0.1.7",
     "history": "^4.10.1",
-    "immer": "^9.0.7",
+    "immer": "^9.0.12",
     "lodash": "^4.17.21",
     "moment": "^2.29.1",
     "path-to-regexp": "^0.1.7",
     "piwik-react-router": "^0.12.1",
-    "prop-types": "^15.7.2",
-    "qs": "^6.10.1",
+    "prop-types": "^15.8.1",
+    "qs": "^6.10.3",
     "query-string": "^6.14.1",
     "react": "^16.14.0",
-    "react-datetime": "^3.0.4",
+    "react-datetime": "^3.1.1",
     "react-dom": "^16.14.0",
     "react-highcharts": "^16.1.0",
-    "react-markdown": "^6.0.2",
+    "react-markdown": "^6.0.3",
+    "react-range": "^1.8.12",
     "react-redux": "^7.2.6",
-    "react-router-dom": "^5.2.0",
+    "react-reverse-portal": "^2.1.0",
+    "react-router-dom": "^5.3.0",
     "react-scripts": "4.0.3",
     "react-scroll-to-component": "^1.0.2",
     "semantic-ui-css": "^2.4.1",
-    "semantic-ui-react": "^2.0.3",
-    "unfuck-spa-shibboleth-session": "^3.1.3",
+    "semantic-ui-react": "^2.1.2",
+    "unfuck-spa-shibboleth-session": "^3.1.4",
+    "use-context-selector": "^1.3.9",
     "uuid": "^3.4.0",
-    "xlsx": "^0.17.0"
+    "xlsx": "^0.17.5"
   },
   "proxy": "http://backend:8080",
   "browserslist": {

services/frontend/src/apiConnection/index.js

@@ -107,7 +107,7 @@ export const handleRequest = store => next => async action => {
 // All tags used for invalidating cache must be defined here
 export const RTKApi = createApi({
   reducerPath: 'api',
-  tagTypes: ['StudyGuidanceGroups'],
+  tagTypes: ['StudyGuidanceGroups', 'Semester'],
   baseQuery: fetchBaseQuery({
     baseUrl: apiBasePath,
     prepareHeaders: headers => {

services/frontend/src/common/InfoToolTips/coolDataScience.js

@@ -6,9 +6,9 @@ export default {
   
   **4 vuoden tahdissa** olevat opiskelijat 45op × ((tämä päivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet,
   
-  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu).
+  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - passiivinen).
 
-  **peruutettu**, jos opinto-oikeus merkattu perutuksi
+  **passiivinen**, jos opinto-oikeus on päättynyt tai opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle, eikä opiskelija ole vielä valmistunut. 
     
   **Include only at least once enrolled students** suodattaa pois opiskelijat jotka eivät ole ollenkaan ilmoittautunut. 
   Vakiona opiskelijat jotka eivät ole ilmoittautunut millekkään lukuvuodelle (läsnä tai poissa olevaksi)
@@ -24,9 +24,9 @@ export default {
   
   **4 vuoden tahdissa** olevat opiskelijat 45op × ((tämä päivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet,
   
-  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu).
+  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - passiivinen).
 
-  **peruutettu**, jos opinto-oikeus merkattu perutuksi
+  **passiivinen**, jos opinto-oikeus on päättynyt tai opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle, eikä opiskelija ole vielä valmistunut. 
   
   **Include only at least once enrolled students** suodattaa pois opiskelijat,
   jotka eivät ole ollenkaan ilmoittautunut. Vakiona opiskelijat, jotka eivät ole ilmoittautunut millekkään lukuvuodelle (läsnä tai poissa olevaksi) on sisällytetty laskuihin.
@@ -41,11 +41,11 @@ export default {
   
   **4 vuoden tahdissa** olevat opiskelijat 45op × ((tarkastuspäivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet,
 
-  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu). 
+  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - passiivinen). 
   
   **Tarkastuspäivät** ovat 31.7, 30.11 ja 1.4 joka vuodelle.
 
-  **peruutettu**, jos opinto-oikeus merkattu perutuksi
+  **passiivinen**, jos opinto-oikeus on päättynyt tai opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle, eikä opiskelija ole vielä valmistunut. 
 
   **Include attainments attained before the studyright start** laskee mukaan opiskelijan ennen opinto-oikeuden alkua saadut opintopisteet.
   Vakiona opintopisteet ennen opinto-oikeutta suodatetaan pois.

services/frontend/src/common/InfoToolTips/filters.js

@@ -4,6 +4,11 @@ export default {
     short:
       'Suodata opiskelijoita sen mukaan ovatko he ilmoittautuneet läsnä- tai poissaoleviksi kyseiselle lukukaudelle',
   },
+  studyrightStatus: {
+    label: 'Rajaa opiskelijoita opinto-oikeuden statuksen mukaan',
+    short:
+      'Opinto-oikeus on aktiivinen, mikäli opiskelija on ilmoittautunut läsnä- tai poissaolevaksi tälle lukukaudelle ja opinto-oikeus ei ole vielä päättynyt. \n Opinto-oikeus on passiivinen, jos opiskelija ei ole ilmoittautunut läsnä- tai poissaolevaksi tälle lukukaudelle tai opinto-oikeus on vanhentunut. Valmistuneet opiskelijat on suodatettu pois molemmista kategorioista.',
+  },
   transferred: {
     label: 'Include and exclude students from this program',
     short:
@@ -13,4 +18,9 @@ export default {
     label: null,
     short: 'Rajaa kurssisuorituksia tietyn ajanjakson mukaan. Tämä rajaus ei muuta opiskelijoiden määrää',
   },
+  cherrypicker: {
+    label: null,
+    short:
+      'Rajaa yksittäisiä opiskelijoita pois tilastoista (blocklist) tai näytä vain määritellyt opiskelijat (allowlist). Vain täsmälliset opiskelijanumerot toimivat.',
+  },
 }

services/frontend/src/common/InfoToolTips/index.jsx

@@ -1,5 +1,4 @@
 import PopulationStatistics from './populationStatistics'
-import PopulationOverview from './populationOverview'
 import CoolDataScience from './coolDataScience'
 import CourseStatistics from './courseStatistics'
 import CoursePopulation from './coursePopulation'
@@ -7,7 +6,6 @@ import Studyprogramme from './studyprogramme'
 
 export default {
   PopulationStatistics,
-  PopulationOverview,
   CoolDataScience,
   CourseStatistics,
   CoursePopulation,

services/frontend/src/common/InfoToolTips/populationOverview.js

@@ -1,24 +0,0 @@
-export default {
-  PopulationProgress: `Tämä koontisivu sisältää koulutusohjelman kaikkien opiskelijoiden perustiedot lukuvuosittain. Luvut perustuvat opinto-oikeuksiin kyseisessä ohjelmassa.
-
-  Population progress:
-  Vuosittain ohjelmassa aloittaneet, ohjelman keskeyttäneet / peruneet sekä ohjelmaan siirtyneet opiskelijat. 
-  Huomattavaa on, että sama opiskelija voi olla kahdessa luokassa, jos hän on esimerkiksi aloittanut opinnot jo, mutta keskeyttänyt ne lukuvuoden aikana.
-  Tarkemmat tiedot saat esille klikkaamalla nuolta kunkin lukuvuoden kohdalla.`,
-  YearlyProductivity: `
-  Yearly productivity:
-  Koulutusohjelman kunakin kalenterivuonna tuottamat opintopisteet, valmistuneiden määrän, ohjelman pääaineopiskelijoiden
-  opintopisteet sekä muiden kuin pääaineopiskelijoiden opintopisteet. Mukana on myös kuhunkin koulutusohjelmaan hyväksiluetut opintopisteet.
-  
-  Vanhojen koulutusohjelmien lakkauttaminen ja opiskelijoiden siirtäminen uusiin ohjelmiin voi aiheuttaa tavallista suuremmat opintopistemäärät lukuvuodelle 2020-2021.`,
-  Year: `Lukuvuosi`,
-  Students: `Opiskelijat, jotka ovat koulutusohjelmassa aloittaneet, opinto-oikeutensa peruneet sekä koulutusohjelmaan tai siitä pois siirtyneet.`,
-  Started: `Koulutusohjelmassa kunakin lukuvuonna aloittaneet opiskelijat. EI sisällä ohjelmaan siirtyneitä opiskelijoita.`,
-  Cancelled: `Koulutusohjelmaan oikeuttavan opinto-oikeutensa kunakin lukuvuonna peruttaneet`,
-  Graduated: `Tiettynä lukuvuonna aloittaneista opiskelijoista on valmistunut`,
-  Transferred: `Koulutusohjelmaan kyseisenä lukuvuonna siirtyneet opiskelijat. 
-  
-  Vanhojen koulutusohjelmien lakkauttaminen aiheuttaa tavallista suuremmat siirtoluvut lukuvuodelle 2020-2021.`,
-  Credits: `Kunakin vuonna aloittaneista opiskelijoista ne, jotka ovat ylittäneet tietyn opintopisterajapyykin`,
-  Thesis: `Loppututkielman kirjoittaneiden määrä`,
-}

services/frontend/src/common/InfoToolTips/populationStatistics.js

@@ -1,7 +1,7 @@
 export default {
   Advanced: `**Semesters:**  Jos haluaa tarkastella vain syyslukukaudella tai kevätlukukaudella ilmoittautuneita, haluttu ajanjakso valitaan tästä.
 
-  **Include:** Koulutusohjelmapopulaatiosta on oletusarvoisesti suodatettu pois vaihto-opiskelijat, tutkinnonsuoritusoikeudesta luopuneet ja erillisopinto-oikeudella opiskelevat. Näitä ryhmiä voi sisällyttää mukaan lisäämällä ruksin haluamaansa kohtaan.`,
+  **Include:** Koulutusohjelmapopulaatiosta on oletusarvoisesti suodatettu pois vaihto-opiskelijat ja erillisopinto-oikeudella opiskelevat. Näitä ryhmiä voi sisällyttää mukaan lisäämällä ruksin haluamaansa kohtaan.`,
   QueryCard: `**Valitun koulutusohjelmavuosikurssin perustiedot.**
   
   **Sample size** luku sisältää läsnä- ja poissaolevaksi ilmoittautuneet opiskelijat, muttei "excludes"-ryhmiä, jotka voi määritellä "advanced settings" -kohdasta.
@@ -41,6 +41,12 @@ export default {
   + **Min ja Max** kertovat pienimmän ja suurimman opintopistemäärän, jotka jollain opiskelijoilla kyseisessä populaatiossa on.
 
   Mikäli valintatapatieto kyseiselle populaatiolle on saatavissa, näytetään myös samat statistiikat per valintatapa.  
+
+  **Distribution Development**
+
+  Pylväsdiagrammi kertoo opiskelijoiden jakauman opintopistehaarukoihin kalenterivuosittain, lukuvuosittain tai lukukausittain eriteltynä. Valmistuneita opiskelijoita ei sisällytetä valmistumisen jälkeisiin pylväisiin.
+
+  **Cumulative**-valinnan ollessa käytössä kunkin opiskelijan kaikki aiemmat opintopisteet lasketaan mukaan opintopistemäärään. Muutoin vain pylvästä vastaavalla aikavälillä kirjatut opintopisteet otetaan huomioon.
   `,
   CoursesOfPopulation: `**Courses of population**
 
@@ -60,6 +66,7 @@ export default {
   **Taulukot**
   
   - **pass/fail** - oletusnäkymä, joka sisältää kurssisuoritukset, hylätyt sekä yritykset. Taulukon tiedot voi järjestää sarakkeita (n, after retry, percentage, many times, per student, passed ja attempted) klikkaamalla.
+  - **pass/fail w/ enrollments** - näkymä, joka sisältää suoritustiedot ilmoittautumisten kanssa. Huom.! ilmoittautumistiedot ovat saatavilla vain Sisun käyttöönoton jälkeiseltä ajalta.
   - **grades** - arvosanajakauma.
   - **when passed** - kurssisuorituksen ajankohta. Huom.! *Passed*-sarakkeen kokonaissumma ei välttämättä täsmää puolivuosittaisten lukumäärien summan kanssa johtuen tuplakirjauksista Oodissa.
   `,

services/frontend/src/common/InfoToolTips/studyprogramme.js

@@ -4,11 +4,11 @@ export default {
   Calendar year: Näyttää tilastot kalenterivuosille jaoteltuna, esimerkiksi 1.1.2019-31.12.2019 
   `,
   StudentToggle: `
-  All studyrights included: Valinnassa ovat mukana vaihto-opiskelijat, erillisoikeudella opiskelevat, opintonsa keskeyttäneet opiskelijat, ohjelmaan siirtyneet opiskelijat sekä ohjelmasta pois siirtyneet opiskelijat.
+  All studyrights: Valinnassa ovat mukana vaihto-opiskelijat, erillisoikeudella opiskelevat ja ohjelmaan siirtyneet opiskelijat sekä ohjelmasta pois siirtyneet opiskelijat.
   
   Special studyrights excluded: Valinnasta ja kaikista luvuista on poissuljettu kaikki edellä mainitut erikoisryhmät.
   `,
-  GradToggle: `
+  GraduatedToggle: `
   Graduated included: Valinnassa ovat mukana ohjelmasta jo valmistuneet opiskelijat.
   
   Graduated excluded: Valinnasta on poistettu ohjelmasta jo valmistuneet opiskelijat.
@@ -18,9 +18,7 @@ export default {
 
   **Started studying**: Sisältää kyseisenä vuonna aloittaneet opiskelijat (aloitetut opinto-oikeudet) riippumatta siitä, minä vuonna kyseinen opiskeluoikeus on myönnetty. Opiskelija on myös voinut myöhemmin keskeyttää opintonsa tai siirtyä pois ohjelmasta.\n
   **Graduated**: Sisältää kyseisenä vuonna valmistuneet opiskelijat riippumatta siitä, minä vuonna opiskeluoikeus ohjelmaan on myönnetty. Jaottelu on tehty opinto-oikeuden päättymispäivän (=valmistumispäivän) perusteella.
-  
-  **Cancelled**: Sisältää kyseisenä vuonna perutut opiskeluoikeudet riippumatta siitä, onko kyseinen opiskelija ikinä edes alkanut opiskelemaan ohjelmassa tai opiskelun aloitusvuodesta.
-  
+    
   **Transferred away**: Sisältää kyseisenä vuonna pois ohjelmasta siirtyneet opiskelijat (opinto-oikeudet). Sama opiskelija voi esiintyä tilastoissa useana vuonna, mikäli hän on siirtynyt ohjelmaan ja siitä pois useasti.
   
   **Transferred to**: Sisältää kyseisenä vuonna ohjelmaan siirtyneet opiskelijat (opinto-oikeudet).
@@ -48,10 +46,12 @@ export default {
   **Mean time**: Näyttää kyseisenä vuonna valmistuneiden opiskelijoiden opintojen keston **keskiarvon** kuukausina. Luku sisältää mahdolliset poissaolokuukaudet.
   
   n = **valmistuneet opiskelijat / kaikki ohjelmassa kyseisenä vuonna alkaneet opinto-oikeudet**. Kandiohjelmissa tavoiteaika on 36 kk, maisteriohjelmissa 24 kk`,
-  ProgrammesAfterGraduation: `
-  Sisältää maisteriohjelmat, joissa tämän kandiohjelman opiskelijat ovat aloittaneet valmistuttuaan kandeiksi. Vuosijaottelu on tehty maisteriohjelmassa aloittamisen perusteella.
+  ProgrammesBeforeOrAfter: `
+  **Mikäli valittuna on kandiohjelma:** Sisältää maisteriohjelmat, joissa tämän kandiohjelman opiskelijat ovat aloittaneet valmistuttuaan kandeiksi. Vuosijaottelu on tehty maisteriohjelmassa aloittamisen perusteella.
 
-  Mukana ovat **vain valmistuneet opiskelijat ja vain maisteriohjelmat**.
+  Mukana ovat **vain valmistuneet opiskelijat ja vain maisteriohjelmat.**
+
+  **Mikäli valittuna on maisteriohjelma:** Sisältää kandiohjelmat, joissa tämän ohjelman opiskelijat ovat opiskelleet ennen tuloaan maisteriohjelmaan. Vuosijaottelu on tehty maisteriohjelmassa aloittamisen perusteella.
   `,
   StudytrackOverview: `  
   Kuvaa koulutusohjelman kyseiseen ohjelmaan opinto-oikeudella varustettujen opiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
@@ -59,8 +59,10 @@ export default {
   HUOM!
   **All** Sisältää kaikki opiskelijat, joiden opinto-oikeudella on aloituspäivä kyseisenä vuonna, riippumatta siitä, onko opiskelija aloittanut opiskelemaan ko. vuonna.
   **Started studying** Sisältää kaikki opiskelijat, jotka ovat aloittaneet opiskelemaan koulutusohjelmassa kyseisenä lukuvuonna.
+  **Inactive** Sisältää kaikki opinto-oikeudet, jotka ovat alkaneet kyseisenä vuonna, mutta jotka ovat sittemmin vanhentuneet ilman, että opiskelija on valmistunut ohjelmasta. Sisältää myös opinto-oikeudet,
+  joissa opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle.
   
-  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, tutkinnonsuoritusoikeudesta luopuneet ja erillisopinto-oikeudella opiskelevat.
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, ohjelmaan ja siitä pois siirtyneet opiskelijat sekä erillisopinto-oikeudella opiskelevat.
   Taulukko näyttää myös näistä opiskelijoista miesten, naisten ja suomalaisten osuudet sekä valmistuneiden määrän. 
 
   Yläosan valikosta on mahdollista valita tarkasteluun yhden opintosuunnan opiskelijat. Sekä opiskelijoiden yleistiedot että edistyminen opintopisteittäin
@@ -68,8 +70,14 @@ export default {
   `,
   StudytrackProgress: `
   Kuvaa koulutusohjelman kyseiseen ohjelmaan opinto-oikeudella varustettujen opiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
-  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, tutkinnonsuoritusoikeudesta luopuneet ja erillisopinto-oikeudella opiskelevat.   
+  
+  HUOM! Mukana ovat myös muuhun kuin valittuun koulutusohjelmaan tai opintosuuntaan tehdyt opintopisteet.
+
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, erillisopinto-oikeudella opiskelevat ja ohjelmaan siirtyneet.   
 
   Mikäli yläosan valikosta valittuna on jokin tietty opintosuunta, myös nämä graafit ja taulukot kuvaavat kyseisen opintosuunnan opiskelijoita.    
   `,
+  NonDegree: `
+  Opiskelijat, joilla ei ole tutkinto-oikeutta Helsingin yliopistossa.
+  `,
 }

services/frontend/src/common/hooks.js

@@ -271,6 +271,55 @@ export const useLanguage = () => {
   return language
 }
 
+export const useDebounce = (value, timeout, onChange) => {
+  const [innerValue, setInnerValue] = useState(value)
+  const [dirty, setDirty] = useState(false)
+
+  const timeoutRef = useRef(null)
+
+  useEffect(() => {
+    if (!isEqual(value, innerValue)) {
+      setInnerValue(value)
+      setDirty(false)
+
+      if (timeoutRef.current) {
+        clearTimeout(timeoutRef.current)
+        timeoutRef.current = null
+      }
+    }
+  }, [value])
+
+  const setValue = useCallback(
+    value => {
+      setInnerValue(value)
+      setDirty(true)
+
+      if (timeoutRef.current) {
+        clearTimeout(timeoutRef.current)
+        timeoutRef.current = null
+      }
+
+      timeoutRef.current = setTimeout(() => {
+        onChange(value)
+        setDirty(false)
+      }, timeout)
+    },
+    [innerValue, setInnerValue, timeoutRef, onChange, timeout]
+  )
+
+  const flush = useCallback(() => {
+    if (timeoutRef.current) {
+      clearTimeout(timeoutRef.current)
+      timeoutRef.current = null
+    }
+
+    onChange(innerValue)
+    setDirty(false)
+  }, [timeoutRef, onChange, innerValue])
+
+  return [innerValue, setValue, flush, dirty]
+}
+
 // From: https://www.joshwcomeau.com/snippets/react-hooks/use-toggle/
 export const useToggle = (initialValue = false) => {
   const [value, setValue] = useState(initialValue)

services/frontend/src/common/index.js

@@ -101,6 +101,52 @@ export const getTextIn = (texts, language) => {
   return null
 }
 
+export const getTextInWithOpen = (texts, language, isOpenCourse) => {
+  const prefixes = ['Avoin', 'Öppna', 'Open']
+  if (texts) {
+    const lanText = texts[language] || texts.fi || texts.en || texts.sv || Object.values(texts)[0]
+    const splitText = lanText.split(':')
+    if (prefixes.some(word => splitText[0].startsWith(word))) {
+      splitText.shift()
+    }
+    const newText = [...splitText].join()
+
+    if (isOpenCourse) {
+      switch (language) {
+        case 'fi':
+          return `${prefixes[0]} yo: ${newText}`
+
+        case 'sv':
+          return `${prefixes[1]} uni: ${newText}`
+
+        case 'en':
+          return `${prefixes[2]} uni: ${newText}`
+
+        default:
+          return `${prefixes[0]} yo: ${newText}`
+      }
+    }
+    return newText
+  }
+  return null
+}
+
+export const getUnifyTextIn = unifyCourses => {
+  switch (unifyCourses) {
+    case 'reqularStats':
+      return '(normal)'
+
+    case 'openStats':
+      return '(open)'
+
+    case 'unifyStats':
+      return '(open and normal)'
+
+    default:
+      return ''
+  }
+}
+
 export const cancelablePromise = promise => {
   let hasCanceled = false
 
@@ -160,7 +206,12 @@ export const getNewestProgramme = (studyrights, studentNumber, studentToTargetCo
         elementDetails[srE.code] &&
         elementDetails[srE.code].type === 20 &&
         (studentToTargetCourseDateMap && studentNumber
-          ? moment(studentToTargetCourseDateMap[studentNumber]).isBetween(srE.startdate, srE.enddate, 'day', '[]')
+          ? moment(studentToTargetCourseDateMap[studentNumber]).isBetween(
+              srE.startdate,
+              srE.enddate || moment(),
+              'day',
+              '[]'
+            )
           : true)
     )
     if (studyrightElements.length > 0) {
@@ -210,12 +261,100 @@ export const getHighestGradeOfCourseBetweenRange = (courses, lowerBound, upperBo
   return maxBy(grades, grade => grade.value)
 }
 
+export const getHighestGradeOrEnrollmentOfCourseBetweenRange = (courses, enrollments, lowerBound, upperBound) => {
+  const grade = getHighestGradeOfCourseBetweenRange(courses, lowerBound, upperBound)
+  if (!grade) return enrollments.length ? { grade: 'No grade' } : undefined
+  return grade
+}
+
+const getEarliestStudyRightElement = studyright => {
+  if (!studyright) return null
+  return (
+    studyright.studyright_elements
+      // eslint-disable-next-line camelcase
+      .filter(({ element_detail }) => element_detail.type === 20)
+      .sort((a, b) => new Date(a.startdate) - new Date(b.startdate))[0]
+  )
+}
+
+export const getTargetCreditsForProgramme = code => {
+  if (code === 'MH30_001') return 360
+  if (code === 'MH30_003') return 330
+  if (code === 'MH30_004') return 150
+  if (code === 'MH90_001') return 180
+  if (code.includes('MH')) return 120
+  return 180
+}
+
+const getMonthsForDegree = code => getTargetCreditsForProgramme(code) / (60 / 12)
+
+/**
+ * Get start and end dates for study right element. For bachelor a 3 year target is used
+ * and for masters a 2 year target. Any absences within the study right element extends the end date.
+ * If any absence is in the start of study right element the start date is postponed.
+ */
+export const getStudyRightElementTargetDates = (studyRight, absences = []) => {
+  const studyRightElement = getEarliestStudyRightElement(studyRight)
+  if (!studyRightElement) return []
+  const { code, startdate: sreStartDate } = studyRightElement
+  const months = getMonthsForDegree(code)
+  const end =
+    code.includes('KH') || code.includes('ba') || ['MH30_001', 'MH30_003'].includes(code)
+      ? moment(sreStartDate).add(months, 'months').set('month', 7).endOf('month')
+      : moment(sreStartDate).add(months, 'months')
+
+  if (!absences) return [new Date(sreStartDate), new Date(end)]
+  const absencesWithinStudyRightElement = absences.filter(
+    ({ startdate, enddate }) => startdate >= new Date(sreStartDate).getTime() && enddate <= new Date(end).getTime()
+  )
+
+  if (!absencesWithinStudyRightElement.length) return [new Date(sreStartDate), new Date(end)]
+
+  const absenceInStartOfStudyRight = absencesWithinStudyRightElement.find(
+    ({ startdate }) => new Date(sreStartDate).getTime() === startdate
+  )
+  const absentMonthsDuringStudy = Math.round(
+    absencesWithinStudyRightElement
+      .filter(({ startdate, enddate }) => {
+        if (!absenceInStartOfStudyRight) return true
+        return startdate !== absenceInStartOfStudyRight.startdate && enddate !== absenceInStartOfStudyRight.enddate
+      })
+      .filter(({ startdate }) => startdate < new Date(end).getTime())
+      .reduce((acc, absent) => {
+        const { startdate, enddate } = absent
+        const diff = moment(startdate).diff(moment(enddate), 'days') / 30
+        return acc + Math.abs(diff)
+      }, 0)
+  )
+  const absentMonthsBeforeStudy = absenceInStartOfStudyRight
+    ? Math.round(
+        Math.abs(
+          moment(absenceInStartOfStudyRight.startdate).diff(moment(absenceInStartOfStudyRight.enddate), 'days') / 30
+        )
+      )
+    : 0
+  return [
+    new Date(moment(sreStartDate).add(absentMonthsBeforeStudy, 'months')),
+    new Date(end.add(absentMonthsDuringStudy + absentMonthsBeforeStudy, 'months')),
+  ]
+}
+
 export const validateInputLength = (input, minLength) => input && input.trim().length >= minLength
 
 export const splitByEmptySpace = str => str.replace(/\s\s+/g, ' ').split(' ')
 
 export const isNewHYStudyProgramme = code => !!(code && code.match(/^[A-Z]*[0-9]*_[0-9]*$/))
 
+export const resolveStudyPlan = (studyPlans, studyRight) => {
+  if (!studyRight) return null
+  const { code } =
+    studyRight.studyright_elements
+      .filter(e => e.element_detail.type === 20)
+      .sort((a, b) => new Date(b.startdate) - new Date(a.startdate))[0] || {}
+  if (!code) return null
+  return studyPlans.find(p => p.programme_code === code)
+}
+
 // These are the new Bachelor's programmes in Matlu, that have BH possibility
 export const bachelorHonoursProgrammes = [
   'KH50_001',

services/frontend/src/components/CoolDataScience/ProtoC.jsx

@@ -3,20 +3,26 @@ import React, { useState, useEffect, useCallback, useMemo } from 'react'
 import { connect } from 'react-redux'
 import Highcharts from 'highcharts'
 import ReactHighcharts from 'react-highcharts'
-import { Segment, Loader, Dimmer, Checkbox, Button, Message, Icon } from 'semantic-ui-react'
+import { Menu, Segment, Loader, Dimmer, Checkbox, Button, Message, Icon, Dropdown } from 'semantic-ui-react'
 import _ from 'lodash'
 import HighchartsCustomEvents from 'highcharts-custom-events'
 
 import TSA from '../../common/tsa'
 import InfoToolTips from '../../common/InfoToolTips'
-import { getProtoC, getProtoCProgramme } from '../../redux/coolDataScience'
+import { useGetProtoCQuery, getProtoC, getProtoCProgramme, useGetProtoCProgrammeQuery } from '../../redux/coolDataScience'
 import ReactMarkdown from 'react-markdown'
+import moment from 'moment'
+import RangeSelector from 'components/RangeSelector'
 
 HighchartsCustomEvents(Highcharts)
 
 const ANALYTICS_CATEGORY = 'Trends'
 const sendAnalytics = (action, name, value) => TSA.Matomo.sendEvent(ANALYTICS_CATEGORY, action, name, value)
 
+const upToYear = moment().isBefore({ day: 1, month: 7 })
+  ? moment().year() - 2
+  : moment().year() - 1
+
 const defaultConfig = (pointer = true) => {
   return {
     chart: {
@@ -55,6 +61,9 @@ const defaultConfig = (pointer = true) => {
       reversed: false
     },
     plotOptions: {
+      series: {
+        animation: false
+      },
       area: {
         cursor: pointer ? 'pointer' : undefined,
         fillOpacity: 0.5,
@@ -70,9 +79,8 @@ const defaultConfig = (pointer = true) => {
             function round(x) {
               return Math.round(x * 100) / 100
             }
-            return `${point.index + 1}, ${point.category}, ${point.y}, ${round(point.percentage)}%, ${
-              point.series.name
-            }`
+            return `${point.index + 1}, ${point.category}, ${point.y}, ${round(point.percentage)}%, ${point.series.name
+              }`
           }
         }
       }
@@ -80,7 +88,7 @@ const defaultConfig = (pointer = true) => {
   }
 }
 
-const makeClickableChartConfig = (sortedData, onPointClicked, org) => {
+const makeClickableChartConfig = (sortedData, onPointClicked, org, [startYear, endYear]) => {
   const addPointClickHandler = serie => {
     serie.point = {
       events: {
@@ -124,7 +132,7 @@ const makeClickableChartConfig = (sortedData, onPointClicked, org) => {
   const series = [
     {
       color: '#7f8c8d',
-      name: 'tällä hetkellä peruutettu',
+      name: 'tällä hetkellä passiivinen',
       data: sortedData.map(data => ({
         y: data.currentlyCancelled,
         // pass % of total as z so we can display it in the tooltip
@@ -266,16 +274,15 @@ const makeClickableChartConfig = (sortedData, onPointClicked, org) => {
     },
     yAxis: {
       title: {
-        text: ` ${
-          org ? `2017-2019 aloittaneet uudet kandiopiskelijat<br/>${org.name}` : '% tiedekunnan opiskelijoista'
-        }`
+        text: ` ${org ? `${startYear}-${endYear} aloittaneet uudet kandiopiskelijat<br/>${org.name}` : '% tiedekunnan opiskelijoista'
+          }`
       }
     },
     series
   })
 }
 
-const makeNonClickableChartConfig = programme => {
+const makeNonClickableChartConfig = (programme, [startYear, endYear]) => {
   const addMouseOverHandler = serie => {
     serie.point = {
       events: {
@@ -311,7 +318,7 @@ const makeNonClickableChartConfig = programme => {
   const series = [
     {
       color: '#7f8c8d',
-      name: 'tällä hetkellä peruutettu',
+      name: 'tällä hetkellä passiivinen',
       data: programme.studytracks.map(p => ({
         y: p.currentlyCancelled,
         // pass % of total as z so we can display it in the tooltip
@@ -347,7 +354,7 @@ const makeNonClickableChartConfig = programme => {
 
   return Highcharts.merge(defaultConfig(false), {
     title: {
-      text: `2017-2019 aloittaneet uudet kandiopiskelijat<br/>${programme.name}`
+      text: `${startYear}-${endYear} aloittaneet uudet kandiopiskelijat<br/>${programme.name}`
     },
     xAxis: {
       categories: programme.studytracks.map(data => data.name),
@@ -450,24 +457,29 @@ const sorters = {
   'ei tahdissa': (a, b) =>
     (countNotInTarget(a) + a.currentlyCancelled) / a.totalStudents -
     (countNotInTarget(b) + b.currentlyCancelled) / b.totalStudents,
-  peruutettu: (a, b) => a.currentlyCancelled / a.totalStudents - b.currentlyCancelled / b.totalStudents
+  passiivinen: (a, b) => a.currentlyCancelled / a.totalStudents - b.currentlyCancelled / b.totalStudents
 }
 
-const OrgChart = React.memo(({ orgs, onOrgClicked }) => {
-  return <ReactHighcharts highcharts={Highcharts} config={makeClickableChartConfig(orgs, onOrgClicked)} />
-})
-
-const ProgrammeChart = React.memo(({ programmes, onProgrammeClicked, org }) => {
+const OrgChart = React.memo(({ orgs, onOrgClicked, yearRange }) => {
   return (
-    <ReactHighcharts highcharts={Highcharts} config={makeClickableChartConfig(programmes, onProgrammeClicked, org)} />
+    <ReactHighcharts
+      highcharts={Highcharts}
+      config={makeClickableChartConfig(orgs, onOrgClicked, undefined, yearRange)}
+    />
   )
 })
 
-const StudytrackChart = React.memo(({ programme }) => {
-  return <ReactHighcharts highcharts={Highcharts} config={makeNonClickableChartConfig(programme)} />
+const ProgrammeChart = React.memo(({ programmes, onProgrammeClicked, org, yearRange }) => {
+  return (
+    <ReactHighcharts highcharts={Highcharts} config={makeClickableChartConfig(programmes, onProgrammeClicked, org, yearRange)} />
+  )
 })
 
-const ProgrammeDrilldown = ({ org, sorter, sortDir, onProgrammeClicked }) => {
+const StudytrackChart = React.memo(({ programme, yearRange }) => {
+  return <ReactHighcharts highcharts={Highcharts} config={makeNonClickableChartConfig(programme, yearRange)} />
+})
+
+const ProgrammeDrilldown = ({ org, sorter, sortDir, onProgrammeClicked, yearRange }) => {
   const orgSortedProgrammes = useMemo(() => {
     return { ...org, programmes: [...org.programmes].sort((a, b) => sorters[sorter](a, b) * sortDir) }
   }, [org, sorter, sortDir])
@@ -476,12 +488,13 @@ const ProgrammeDrilldown = ({ org, sorter, sortDir, onProgrammeClicked }) => {
     <ProgrammeChart
       programmes={orgSortedProgrammes.programmes}
       onProgrammeClicked={onProgrammeClicked}
+      yearRange={yearRange}
       org={orgSortedProgrammes}
     />
   )
 }
 
-const StudytrackDrilldown = ({ programme, sorter, sortDir }) => {
+const StudytrackDrilldown = ({ programme, sorter, sortDir, yearRange }) => {
   if (!programme.studytracks || programme.studytracks.length < 1)
     return (
       <Segment>
@@ -491,49 +504,54 @@ const StudytrackDrilldown = ({ programme, sorter, sortDir }) => {
   const programmeSortedStudytracks = useMemo(() => {
     return { ...programme, studytracks: [...programme.studytracks].sort((a, b) => sorters[sorter](a, b) * sortDir) }
   }, [programme, sorter, sortDir])
-  return <StudytrackChart programme={programmeSortedStudytracks} />
+  return <StudytrackChart programme={programmeSortedStudytracks} yearRange={yearRange} />
 }
 
 const ProtoC = ({
   programme,
-  getProtoCDispatch,
-  getProtoCProgrammeDispatch,
-  protoC,
-  protoCProgramme,
-  loadingProtoC,
-  loadingProtoCProgramme
 }) => {
+  const [firstLoad, setFirstLoad] = useState(true)
   const [sorter, setSorter] = useState('3v tahti')
   const [sortDir, setSortDir] = useState(1)
-  const [drilldownOrg, setDrilldownOrg] = useState(null)
-  const [drilldownProgramme, setDrilldownProgramme] = useState(null)
+  const [drilldownOrgCode, setDrilldownOrgCode] = useState(null)
+  const [drilldownProgrammeCode, setDrilldownProgrammeCode] = useState(null)
 
   const [includeOldAttainments, setIncludeOldAttainments] = useState(false)
   const [excludeNonEnrolled, setExcludeNonEnrolled] = useState(false)
 
-  useEffect(() => {
-    const load = async () => {
-      if (!!programme) {
-        getProtoCProgrammeDispatch({
-          includeOldAttainments: includeOldAttainments.toString(),
-          excludeNonEnrolled: excludeNonEnrolled.toString(),
-          code: programme
-        })
-      } else {
-        getProtoCDispatch({
-          includeOldAttainments: includeOldAttainments.toString(),
-          excludeNonEnrolled: excludeNonEnrolled.toString()
-        })
-      }
-    }
-    load()
-  }, [includeOldAttainments, excludeNonEnrolled])
+  const [yearRange, setYearRange] = useState([2017, upToYear])
+  const [startYear, endYear] = yearRange
 
-  useEffect(() => {
-    if (!!programme) {
-      setDrilldownProgramme(protoCProgramme)
-    }
-  }, [protoCProgramme])
+  const { data: protoC, isLoading: loadingProtoC, isFetching: fetchingProtoC } = useGetProtoCQuery({
+    excludeNonEnrolled,
+    includeOldAttainments,
+    startYear,
+    endYear,
+  })
+
+  const { date: protoCProgramme, isLoading: loadingProtoCProgramme, isFetching: fetchingProtoCProgramme } = useGetProtoCProgrammeQuery({
+    includeOldAttainments: includeOldAttainments.toString(),
+    excludeNonEnrolled: excludeNonEnrolled.toString(),
+    code: programme,
+    startYear,
+    endYear,
+  })
+
+  const drilldownOrg = useMemo(
+    () => protoC
+      ? Object.values(protoC).find(org => org.code === drilldownOrgCode)
+      : null,
+    [drilldownOrgCode, protoC],
+  )
+
+  const drilldownProgramme = useMemo(
+    () => protoC
+      ? Object.values(protoC)
+        .find(org => org.code === drilldownOrgCode)
+        ?.programmes?.find?.(prog => prog.code === drilldownProgrammeCode)
+      : null,
+    [drilldownOrgCode, drilldownProgrammeCode, protoC],
+  )
 
   const handleOldAttainmentToggled = useCallback(() => {
     setIncludeOldAttainments(previous => !previous)
@@ -546,13 +564,13 @@ const ProtoC = ({
   }, [])
 
   const handleOrgClicked = useCallback(org => {
-    setDrilldownOrg(org)
-    setDrilldownProgramme(null)
+    setDrilldownOrgCode(org.code)
+    setDrilldownProgrammeCode(null)
     sendAnalytics('C Org drilldown clicked', 'ProtoC')
   }, [])
 
   const handleProgrammeClicked = useCallback(programme => {
-    setDrilldownProgramme(programme)
+    setDrilldownProgrammeCode(programme.code)
     sendAnalytics('C Programme drilldown clicked', 'ProtoC')
   }, [])
 
@@ -577,14 +595,39 @@ const ProtoC = ({
       return a > b ? 1 : -1
     })
 
+  const RangeButtons = () => (
+    <div align="center" style={{ marginTop: '15px' }}>
+      <Menu compact>
+        <Menu.Item>
+          Started between
+        </Menu.Item>
+        <Dropdown
+          className="link item"
+          value={startYear}
+          onChange={(_, { value }) => setYearRange([value, endYear])}
+          options={_.range(2017, upToYear + 1).map((year) => ({ key: year, value: year, text: year, disabled: year > endYear }))}
+        />
+        <Menu.Item>
+          and
+        </Menu.Item>
+        <Dropdown
+          className="link item"
+          value={endYear}
+          onChange={(_, { value }) => setYearRange([startYear, value])}
+          options={_.range(2017, upToYear + 1).map((year) => ({ key: year, value: year, text: year, disabled: year < startYear }))}
+        />
+      </Menu>
+    </div>
+  )
+
   const SorterButtons = () => (
     <div align="center" style={{ marginTop: '15px' }}>
-      <Button.Group>
-        <Button style={{ cursor: 'default' }} active color="black">
+      <Menu compact>
+        <Menu.Item style={{ cursor: 'default' }} active color="black">
           Sort by:
-        </Button>
+        </Menu.Item>
         {sorterNames.map(sorterName => (
-          <Button
+          <Menu.Item
             basic={sorter !== sorterName}
             color={sorter === sorterName ? 'blue' : 'black'}
             key={sorterName}
@@ -595,7 +638,7 @@ const ProtoC = ({
             content={sorterName}
           />
         ))}
-      </Button.Group>
+      </Menu>
     </div>
   )
   // legend and checkboxes, custom legend because highcharts is pita
@@ -606,7 +649,7 @@ const ProtoC = ({
           <Icon style={{ marginLeft: '10px', color: '#6ab04c' }} name="circle" size="small" /> 3v tahdissa
           <Icon style={{ marginLeft: '10px', color: '#f9ca24' }} name="circle" size="small" /> 4v tahdissa
           <Icon style={{ marginLeft: '10px', color: '#ff7979' }} name="circle" size="small" /> ei tahdissa
-          <Icon style={{ marginLeft: '10px', color: '#7f8c8d' }} name="circle" size="small" /> tällä hetkellä peruutettu
+          <Icon style={{ marginLeft: '10px', color: '#7f8c8d' }} name="circle" size="small" /> tällä hetkellä passiivinen
         </span>
       </div>
       <div align="center">
@@ -637,9 +680,9 @@ const ProtoC = ({
       <Segment>
         <SorterButtons />
         <Segment placeholder={loadingProtoC || loadingProtoCProgramme} vertical>
-          <Dimmer inverted active={loadingProtoC || loadingProtoCProgramme} />
-          <Loader active={loadingProtoC || loadingProtoCProgramme} />
-          <StudytrackDrilldown programme={drilldownProgramme} sorter={sorter} sortDir={sortDir} />
+          <Dimmer inverted active={fetchingProtoC || fetchingProtoCProgramme} />
+          <Loader active={fetchingProtoC || fetchingProtoCProgramme} />
+          <StudytrackDrilldown yearRange={yearRange} programme={drilldownProgramme} sorter={sorter} sortDir={sortDir} />
         </Segment>
         <RenderBelowGraph />
       </Segment>
@@ -649,26 +692,37 @@ const ProtoC = ({
   return (
     <Segment>
       <div align="center">
-        <h2>Kandiohjelmat: Suhteellinen tavoiteaikaerittely, 2017-2019 aloittaneet</h2>
+        <h2>Kandiohjelmat: Suhteellinen tavoiteaikaerittely, {startYear}-{endYear} aloittaneet</h2>
       </div>
       <DrilldownMessage />
-      <SorterButtons />
+      <div style={{ display: 'flex', gapX: 15, gapY: 0, justifyContent: 'center', flexFlow: 'wrap' }}>
+        <RangeButtons />
+        <SorterButtons />
+      </div>
       <Segment placeholder={loadingProtoC || loadingProtoCProgramme} vertical>
-        <Dimmer inverted active={loadingProtoC || loadingProtoCProgramme} />
-        <Loader active={loadingProtoC || loadingProtoCProgramme} />
-        {!(loadingProtoC || loadingProtoCProgramme) && protoC && (
-          <OrgChart orgs={sortedOrgs} onOrgClicked={handleOrgClicked} />
-        )}
-        {!(loadingProtoC || loadingProtoCProgramme) && protoC && drilldownOrg && (
+        <Dimmer inverted active={fetchingProtoC || fetchingProtoCProgramme} />
+        <Loader active={fetchingProtoC || fetchingProtoCProgramme} />
+        <OrgChart
+          orgs={sortedOrgs}
+          onOrgClicked={handleOrgClicked}
+          yearRange={yearRange}
+        />
+        {drilldownOrg && (
           <ProgrammeDrilldown
             org={drilldownOrg}
             sorter={sorter}
             sortDir={sortDir}
+            yearRange={yearRange}
             onProgrammeClicked={handleProgrammeClicked}
           />
         )}
         {!(loadingProtoC || loadingProtoCProgramme) && protoC && drilldownProgramme && (
-          <StudytrackDrilldown programme={drilldownProgramme} sorter={sorter} sortDir={sortDir} />
+          <StudytrackDrilldown
+            programme={drilldownProgramme}
+            sorter={sorter}
+            sortDir={sortDir}
+            yearRange={yearRange}
+          />
         )}
       </Segment>
       <RenderBelowGraph />
@@ -679,14 +733,4 @@ const ProtoC = ({
   )
 }
 
-const mapStateToProps = ({ coolDataScience }) => ({
-  protoC: coolDataScience.data.protoC || {},
-  protoCProgramme: coolDataScience.data.protoCProgramme || {},
-  loadingProtoC: coolDataScience.pending.protoC,
-  loadingProtoCProgramme: coolDataScience.pending.protoCProgramme
-})
-
-export default connect(
-  mapStateToProps,
-  { getProtoCDispatch: getProtoC, getProtoCProgrammeDispatch: getProtoCProgramme }
-)(ProtoC)
+export default ProtoC

services/frontend/src/components/CoolDataScience/ProtoC2.jsx

@@ -1,21 +1,23 @@
-import React, { useState, useEffect, useCallback, useMemo } from 'react'
-import PropTypes, { func, bool, shape, string } from 'prop-types'
-import { connect } from 'react-redux'
+import React, { useState, useCallback, useMemo } from 'react'
 import Highcharts from 'highcharts'
 import ReactHighcharts from 'react-highcharts'
-import { Segment, Loader, Dimmer, Checkbox, Button, Message } from 'semantic-ui-react'
+import _ from 'lodash'
+import { Dropdown, Menu, Segment, Loader, Dimmer, Checkbox, Message } from 'semantic-ui-react'
 import ReactMarkdown from 'react-markdown'
 import HighchartsCustomEvents from 'highcharts-custom-events'
 
+import moment from 'moment'
 import TSA from '../../common/tsa'
 import InfoToolTips from '../../common/InfoToolTips'
-import { getProtoC, getProtoCProgramme } from '../../redux/coolDataScience'
+import { useGetProtoCQuery, useGetProtoCProgrammeQuery } from '../../redux/coolDataScience'
 
 HighchartsCustomEvents(Highcharts)
 
 const ANALYTICS_CATEGORY = 'Trends'
 const sendAnalytics = (action, name, value) => TSA.Matomo.sendEvent(ANALYTICS_CATEGORY, action, name, value)
 
+const upToYear = moment().isBefore({ day: 1, month: 7 }) ? moment().year() - 2 : moment().year() - 1
+
 const defaultConfig = () => {
   return {
     chart: {
@@ -111,7 +113,7 @@ const makeConfig = (data, sorter, type = 'column', clickHandler) => {
   const orgSeries = [
     {
       color: '#7f8c8d',
-      name: 'tällä hetkellä peruutettu',
+      name: 'tällä hetkellä passiivinen',
       data: data.map(entry => ({
         y: entry.currentlyCancelled,
         // pass % of total as z so we can display it in the tooltip
@@ -168,7 +170,7 @@ const makeConfig = (data, sorter, type = 'column', clickHandler) => {
             const { chart } = this
             chart.myLabel.destroy()
             const clickedLabel = data.find(entry => entry.name === this.value)
-            if (clickedLabel) clickHandler(clickedLabel.programmes || clickedLabel.studytracks)
+            if (clickedLabel) clickHandler(clickedLabel)
             sendAnalytics('Drilldown clicked', 'ProtoC2')
           },
           mouseover() {
@@ -248,7 +250,7 @@ const makeConfig = (data, sorter, type = 'column', clickHandler) => {
                 // clicked on top-level, drill down
                 sendAnalytics('Org drilldown clicked', 'ProtoC2')
                 const datapoint = data.find(entry => entry.code === point.custom.code)
-                clickHandler(datapoint.programmes || datapoint.studytracks)
+                clickHandler(datapoint)
               }
             },
           },
@@ -264,7 +266,7 @@ const sorters = {
   '4v tahti': (a, b) => a.students4y - b.students4y,
   '3v tahti': (a, b) => a.students3y - b.students3y,
   'ei tahdissa': (a, b) => countNotInTarget(a) + a.currentlyCancelled - (countNotInTarget(b) + b.currentlyCancelled),
-  peruutettu: (a, b) => a.currentlyCancelled - b.currentlyCancelled,
+  passiivinen: (a, b) => a.currentlyCancelled - b.currentlyCancelled,
 }
 
 const ClickableChart = React.memo(({ data, sorter, isSideways, clickHandler }) => {
@@ -290,78 +292,60 @@ NonClickableChart.defaultProps = {
   isSideways: false,
 }
 
-NonClickableChart.propTypes = {
-  data: PropTypes.arrayOf(
-    PropTypes.shape({
-      name: PropTypes.string,
-      code: PropTypes.string,
-      students3y: PropTypes.number,
-      students4y: PropTypes.number,
-      totalStudents: PropTypes.number,
-      currentlyCancelled: PropTypes.number,
-    })
-  ).isRequired,
-  sorter: PropTypes.func.isRequired,
-  isSideways: PropTypes.bool,
-}
-
 ClickableChart.defaultProps = {
   isSideways: false,
 }
 
-ClickableChart.propTypes = {
-  data: PropTypes.arrayOf(
-    PropTypes.shape({
-      name: PropTypes.string,
-      code: PropTypes.string,
-      students3y: PropTypes.number,
-      students4y: PropTypes.number,
-      totalStudents: PropTypes.number,
-      currentlyCancelled: PropTypes.number,
-    })
-  ).isRequired,
-  sorter: PropTypes.func.isRequired,
-  clickHandler: func.isRequired,
-  isSideways: PropTypes.bool,
-}
-
-const ProtoC = ({
-  getProtoCDispatch,
-  getProtoCProgrammeDispatch,
-  data,
-  isLoading,
-  programme = 'KH50_005',
-  protoCProgrammeData,
-  protoCProgrammeLoading,
-}) => {
+const ProtoC = ({ programme = 'KH50_005' }) => {
   const [sorter, setSorter] = useState('3v tahti')
   const [sortDir, setSortDir] = useState(1)
-  const [drilldownOrg, setDrilldownOrg] = useState(null)
-  const [drilldownProgramme, setDrilldownProgramme] = useState(null)
+  const [drilldownOrgCode, setDrilldownOrgCode] = useState(null)
+  const [drilldownProgrammeCode, setDrilldownProgrammeCode] = useState(null)
 
   const [includeOldAttainments, setIncludeOldAttainments] = useState(false)
   const [excludeNonEnrolled, setExcludeNonEnrolled] = useState(false)
 
-  useEffect(() => {
-    if (programme) {
-      getProtoCProgrammeDispatch({
-        includeOldAttainments: includeOldAttainments.toString(),
-        excludeNonEnrolled: excludeNonEnrolled.toString(),
-        code: programme,
-      })
-    } else {
-      getProtoCDispatch({
-        includeOldAttainments: includeOldAttainments.toString(),
-        excludeNonEnrolled: excludeNonEnrolled.toString(),
-      })
-    }
-  }, [includeOldAttainments, excludeNonEnrolled])
+  const [yearRange, setYearRange] = useState([2017, upToYear])
+  const [startYear, endYear] = yearRange
 
-  useEffect(() => {
-    if (programme) {
-      setDrilldownProgramme(protoCProgrammeData.studytracks)
+  const { data, isLoading, isFetching } = useGetProtoCQuery({
+    excludeNonEnrolled,
+    includeOldAttainments,
+    startYear,
+    endYear,
+  })
+
+  const {
+    data: protoCProgrammeData,
+    isLoading: protoCProgrammeLoading,
+    isFetching: protoCProgrammeFetching,
+  } = useGetProtoCProgrammeQuery(
+    {
+      includeOldAttainments: includeOldAttainments.toString(),
+      excludeNonEnrolled: excludeNonEnrolled.toString(),
+      code: programme,
+      startYear,
+      endYear,
+    },
+    {
+      skip: !programme,
     }
-  }, [protoCProgrammeData])
+  )
+
+  const drilldownOrg = useMemo(
+    () => (data ? Object.values(data).find(org => org.code === drilldownOrgCode) : null),
+    [drilldownOrgCode, data]
+  )
+
+  const drilldownProgramme = useMemo(
+    () =>
+      data
+        ? Object.values(data)
+            .find(org => org.code === drilldownOrgCode)
+            ?.programmes?.find?.(prog => prog.code === drilldownProgrammeCode)
+        : null,
+    [drilldownOrgCode, drilldownProgrammeCode, data]
+  )
 
   const handleOldAttainmentToggled = useCallback(() => {
     setIncludeOldAttainments(previous => !previous)
@@ -380,19 +364,21 @@ const ProtoC = ({
   }, [data, currentSorter])
 
   const sortedProgrammes = useMemo(() => {
-    return (drilldownOrg || []).sort(currentSorter)
+    return [...(drilldownOrg?.programmes || [])].sort(currentSorter)
   }, [drilldownOrg, currentSorter])
 
   const sortedStudytracks = useMemo(() => {
-    return (drilldownProgramme || []).sort(currentSorter)
-  }, [drilldownProgramme, currentSorter])
+    const studytracks = programme ? protoCProgrammeData?.studytracks : drilldownProgramme?.studytracks
+
+    return [...(studytracks || [])].sort(currentSorter)
+  }, [programme, drilldownProgramme, protoCProgrammeData, currentSorter])
 
   const drilldownOrgClick = useCallback(org => {
-    setDrilldownOrg(org)
+    setDrilldownOrgCode(org.code)
   })
 
   const drilldownProgrammeClick = useCallback(programme => {
-    setDrilldownProgramme(programme)
+    setDrilldownProgrammeCode(programme.code)
   })
 
   const { CoolDataScience } = InfoToolTips
@@ -423,15 +409,46 @@ const ProtoC = ({
     return (
       <Segment>
         <div align="center">
-          <h2>Prototyyppi: Tavoiteaikaerittely, 2017-2019 aloittaneet</h2>
+          <h2>
+            Prototyyppi: Tavoiteaikaerittely, {startYear}-{endYear} aloittaneet
+          </h2>
         </div>
-        <div align="center" style={{ marginTop: '10px' }}>
-          <Button.Group>
-            <Button style={{ cursor: 'default' }} active color="black">
+        <div
+          align="center"
+          style={{ marginTop: '10px', display: 'flex', gap: '10px 20px', flexFlow: 'wrap', justifyContent: 'center' }}
+        >
+          <Menu compact>
+            <Menu.Item>Started between</Menu.Item>
+            <Dropdown
+              className="link item"
+              value={startYear}
+              onChange={(_, { value }) => setYearRange([value, endYear])}
+              options={_.range(2017, upToYear + 1).map(year => ({
+                key: year,
+                value: year,
+                text: year,
+                disabled: year > endYear,
+              }))}
+            />
+            <Menu.Item>and</Menu.Item>
+            <Dropdown
+              className="link item"
+              value={endYear}
+              onChange={(_, { value }) => setYearRange([startYear, value])}
+              options={_.range(2017, upToYear + 1).map(year => ({
+                key: year,
+                value: year,
+                text: year,
+                disabled: year < startYear,
+              }))}
+            />
+          </Menu>
+          <Menu compact>
+            <Menu.Item style={{ cursor: 'default' }} active color="black">
               Sort by:
-            </Button>
+            </Menu.Item>
             {sorterNames.map(sorterName => (
-              <Button
+              <Menu.Item
                 basic={sorter !== sorterName}
                 color={sorter === sorterName ? 'blue' : 'black'}
                 key={sorterName}
@@ -442,11 +459,11 @@ const ProtoC = ({
                 content={sorterName}
               />
             ))}
-          </Button.Group>
+          </Menu>
         </div>
         <Segment placeholder={isLoading} vertical>
-          <Dimmer inverted active={isLoading} />
-          <Loader active={isLoading} />
+          <Dimmer inverted active />
+          <Loader active />
           {!protoCProgrammeLoading && protoCProgrammeData && (
             <>
               <NonClickableChart data={drilldownProgramme} sorter={currentSorter} isSideways />
@@ -476,19 +493,54 @@ const ProtoC = ({
     )
   }
 
+  const [studytracksLoading, studytracksFetching] = !programme
+    ? [isLoading, isFetching]
+    : [protoCProgrammeLoading, protoCProgrammeFetching]
+
   return (
     <Segment>
       <div align="center">
-        <h2>Kandiohjelmat: Tavoiteaikaerittely, 2017-2019 aloittaneet</h2>
+        <h2>
+          Kandiohjelmat: Tavoiteaikaerittely, {startYear}-{endYear} aloittaneet
+        </h2>
       </div>
       <DrilldownMessage />
-      <div align="center" style={{ marginTop: '10px' }}>
-        <Button.Group>
-          <Button style={{ cursor: 'default' }} active color="black">
+      <div
+        align="center"
+        style={{ marginTop: '10px', display: 'flex', gap: '10px 20px', flexFlow: 'wrap', justifyContent: 'center' }}
+      >
+        <Menu compact>
+          <Menu.Item>Started between</Menu.Item>
+          <Dropdown
+            className="link item"
+            value={startYear}
+            onChange={(_, { value }) => setYearRange([value, endYear])}
+            options={_.range(2017, upToYear + 1).map(year => ({
+              key: year,
+              value: year,
+              text: year,
+              disabled: year > endYear,
+            }))}
+          />
+          <Menu.Item>and</Menu.Item>
+          <Dropdown
+            className="link item"
+            value={endYear}
+            onChange={(_, { value }) => setYearRange([startYear, value])}
+            options={_.range(2017, upToYear + 1).map(year => ({
+              key: year,
+              value: year,
+              text: year,
+              disabled: year < startYear,
+            }))}
+          />
+        </Menu>
+        <Menu compact>
+          <Menu.Item style={{ cursor: 'default' }} active color="black">
             Sort by:
-          </Button>
+          </Menu.Item>
           {sorterNames.map(sorterName => (
-            <Button
+            <Menu.Item
               basic={sorter !== sorterName}
               color={sorter === sorterName ? 'blue' : 'black'}
               key={sorterName}
@@ -499,11 +551,11 @@ const ProtoC = ({
               content={sorterName}
             />
           ))}
-        </Button.Group>
+        </Menu>
       </div>
       <Segment placeholder={isLoading} vertical>
-        <Dimmer inverted active={isLoading} />
-        <Loader active={isLoading} />
+        <Dimmer inverted active={isFetching} />
+        <Loader active={isFetching} />
         {!isLoading && data && (
           <>
             <ClickableChart data={sortedOrgs} sorter={currentSorter} isSideways clickHandler={drilldownOrgClick} />
@@ -515,9 +567,17 @@ const ProtoC = ({
                 clickHandler={drilldownProgrammeClick}
               />
             )}
-            {drilldownProgramme && <NonClickableChart data={sortedStudytracks} sorter={currentSorter} isSideways />}
           </>
         )}
+      </Segment>
+      {drilldownProgramme && (
+        <Segment placeholder={studytracksLoading} vertical>
+          <Dimmer inverted active={studytracksFetching} />
+          <Loader active={studytracksFetching} />
+          <NonClickableChart data={sortedStudytracks} sorter={currentSorter} isSideways />
+        </Segment>
+      )}
+      <Segment>
         <div align="center">
           <Checkbox
             label="Include only at least once enrolled students"
@@ -546,24 +606,4 @@ ProtoC.defaultProps = {
   programme: '',
 }
 
-ProtoC.propTypes = {
-  isLoading: bool.isRequired,
-  data: shape({}).isRequired,
-  getProtoCDispatch: func.isRequired,
-  getProtoCProgrammeDispatch: func.isRequired,
-  protoCProgrammeLoading: bool.isRequired,
-  protoCProgrammeData: shape({}).isRequired,
-  programme: string,
-}
-
-const mapStateToProps = ({ coolDataScience }) => ({
-  data: coolDataScience.data.protoC,
-  protoCProgrammeData: coolDataScience.data.protoCProgramme,
-  isLoading: coolDataScience.pending.protoC,
-  protoCProgrammeLoading: coolDataScience.pending.protoCProgramme,
-})
-
-export default connect(mapStateToProps, {
-  getProtoCDispatch: getProtoC,
-  getProtoCProgrammeDispatch: getProtoCProgramme,
-})(ProtoC)
+export default ProtoC

services/frontend/src/components/CoolDataScience/Status.jsx

@@ -22,7 +22,7 @@ const settingDefinitions = _.map(
     {
       key: 'showByYear',
       type: 'checkbox',
-      defaultValue: false,
+      defaultValue: true,
     },
     {
       key: 'showYearlyValues',

services/frontend/src/components/CoursePopulation/CoursePopulationGradeDist/index.jsx

@@ -3,6 +3,7 @@ import { connect } from 'react-redux'
 import { Progress, Table } from 'semantic-ui-react'
 import { intersection, orderBy } from 'lodash'
 import { shape, bool, arrayOf, string, number } from 'prop-types'
+import moment from 'moment'
 import { getHighestGradeOfCourseBetweenRange } from '../../../common'
 import ExternalGradeFilterToggle from './ExternalGradeFilterToggle'
 
@@ -10,13 +11,21 @@ const CoursePopulationCreditDist = ({ singleCourseStats, pending, selectedStuden
   const [courseGrades, setCourseGrades] = useState([])
 
   useEffect(() => {
-    if (samples && singleCourseStats.alternatives) {
+    if (samples && singleCourseStats?.unifyStats?.alternatives) {
       const filteredGradeArray = []
       const grades = {}
 
       samples.forEach(student => {
         const courses = student.courses.filter(c => codes.includes(c.course_code))
+        const hasEnrollment =
+          student.enrollments?.some(
+            e => codes.includes(e.course_code) && moment(e.enrollment_date_time).isBetween(moment(from), moment(to))
+          ) ?? false
         const highestGrade = getHighestGradeOfCourseBetweenRange(courses, from, to)
+        if (!highestGrade && hasEnrollment) {
+          if (!grades['No grade']) grades['No grade'] = []
+          grades['No grade'].push(student.studentNumber)
+        }
 
         if (highestGrade) {
           if (!grades[highestGrade.grade]) {

services/frontend/src/components/CoursePopulation/index.jsx

@@ -10,7 +10,7 @@ import { getCoursePopulation } from '../../redux/populations'
 import { getSingleCourseStats } from '../../redux/singleCourseStats'
 import { useGetStudentListCourseStatisticsQuery } from '../../redux/populationCourses'
 import { getFaculties } from '../../redux/faculties'
-import { getSemesters } from '../../redux/semesters'
+import { useGetSemestersQuery } from '../../redux/semesters'
 import { getElementDetails } from '../../redux/elementdetails'
 import PopulationStudents from '../PopulationStudents'
 import CoursePopulationGradeDist from './CoursePopulationGradeDist'
@@ -19,7 +19,7 @@ import CoursePopulationCreditGainTable from './CoursePopulationCreditGainTable'
 import CustomPopulationProgrammeDist from '../CustomPopulation/CustomPopulationProgrammeDist'
 import CustomPopulationCourses from '../CustomPopulation/CustomPopulationCourses'
 import ProgressBar from '../ProgressBar'
-import { getStudentToTargetCourseDateMap, getTextIn } from '../../common'
+import { getStudentToTargetCourseDateMap, getTextIn, getUnifyTextIn } from '../../common'
 import { useProgress, useTitle } from '../../common/hooks'
 import infotooltips from '../../common/InfoToolTips'
 import {
@@ -31,11 +31,18 @@ import {
   startYearAtUniFilter,
   programmeFilter,
   studyTrackFilter,
+  cherrypickFilter,
 } from '../FilterView/filters'
 import FilterView from '../FilterView'
 import useLanguage from '../LanguagePicker/useLanguage'
 import { queryParamsFromUrl } from '../../common/query'
 
+const NO_PROGRAMME = {
+  code: '00000',
+  name: { en: 'No programme', fi: 'Ei ohjelmaa' },
+  startdate: '',
+}
+
 const CoursePopulation = ({
   getCoursePopulationDispatch,
   getSingleCourseStatsDispatch,
@@ -44,12 +51,10 @@ const CoursePopulation = ({
   pending,
   history,
   courseData,
-  getSemestersDispatch,
-  semesters,
   getFacultiesDispatch,
+  unifyCourses,
 }) => {
   const { language } = useLanguage()
-
   const [codes, setCodes] = useState([])
   const [headerYears, setYears] = useState('')
   const [dateFrom, setDateFrom] = useState(null)
@@ -75,9 +80,10 @@ const CoursePopulation = ({
     studentNumbers: studentData.students ? studentData.students.map(student => student.studentNumber) : [],
   })
 
+  const { data: semesters = {} } = useGetSemestersQuery()
+
   useEffect(() => {
     getElementDetails()
-    getSemestersDispatch()
   }, [])
 
   useEffect(() => {
@@ -100,9 +106,9 @@ const CoursePopulation = ({
 
   useEffect(() => {
     if (semesters.years && semesters.semesters) {
-      const { coursecodes, from, to, years, years2, separate } = queryParamsFromUrl(history.location)
+      const { coursecodes, from, to, years, years2, separate, unifyCourses } = queryParamsFromUrl(history.location)
       const parsedCourseCodes = JSON.parse(coursecodes)
-      getCoursePopulationDispatch({ coursecodes, from, to, onProgress, separate })
+      getCoursePopulationDispatch({ coursecodes, from, to, onProgress, separate, unifyCourses })
       getSingleCourseStatsDispatch({
         fromYear: from,
         toYear: to,
@@ -124,7 +130,8 @@ const CoursePopulation = ({
     }
   }, [semesters])
 
-  const header = courseData ? `${getTextIn(courseData.name, language)} ${headerYears}` : null
+  const avoin = getUnifyTextIn(unifyCourses)
+  const header = courseData ? `${getTextIn(courseData.name, language)} ${headerYears} ${avoin}` : null
 
   const subHeader = codes.join(', ')
 
@@ -215,6 +222,7 @@ const CoursePopulation = ({
             <CustomPopulationProgrammeDist
               studentToTargetCourseDateMap={studentToTargetCourseDateMap}
               samples={filtered}
+              coursecode={codes}
               selectedStudents={filtered.map(s => s.studentNumber)}
             />
           </div>
@@ -284,6 +292,8 @@ const CoursePopulation = ({
               coursePopulation
               language={language}
               coursecode={codes}
+              from={dateFrom}
+              to={dateTo}
             />
           </div>
         ),
@@ -301,7 +311,7 @@ const CoursePopulation = ({
       .max()
       .value()
 
-    return moment(date).isBetween(sre.startdate, sre.enddate, 'day', '[]')
+    return sre.code === NO_PROGRAMME.code || moment(date).isBetween(sre.startdate, sre.enddate, 'day', '[]')
   }
 
   return (
@@ -309,6 +319,7 @@ const CoursePopulation = ({
       name="CoursePopulation"
       filters={[
         genderFilter,
+        cherrypickFilter,
         ageFilter,
         courseFilter({ courses: courseStatistics?.coursestatistics }),
         creditsEarnedFilter,
@@ -364,12 +375,12 @@ const CustomPopulationCoursesWrapper = props => {
   return <CustomPopulationCourses {...props} courses={courseStatistics} />
 }
 
-const mapStateToProps = ({ singleCourseStats, populations, semesters }) => {
+const mapStateToProps = ({ singleCourseStats, populations, courseSearch }) => {
   return {
     studentData: populations.data,
     pending: populations.pending,
-    courseData: singleCourseStats.stats || {},
-    semesters: semesters.data,
+    courseData: singleCourseStats.stats?.[courseSearch.openOrReqular],
+    unifyCourses: courseSearch.openOrReqular,
     elementDetails: populations?.data?.elementdetails?.data,
   }
 }
@@ -378,7 +389,6 @@ export default withRouter(
   connect(mapStateToProps, {
     getCoursePopulationDispatch: getCoursePopulation,
     getSingleCourseStatsDispatch: getSingleCourseStats,
-    getSemestersDispatch: getSemesters,
     getFacultiesDispatch: getFaculties,
     getElementDetails,
   })(CoursePopulation)

services/frontend/src/components/CourseStatistics/FacultyLevelStatistics/index.jsx

@@ -84,11 +84,13 @@ const CourseTable = ({ course, courseInstance, language, selectedYear }) => {
 
 const FacultyLevelStatistics = () => {
   const { language: activeLanguage } = useLanguage()
+  const openOrReqular = useSelector(state => state.courseSearch.openOrReqular)
   const { courseStats, language } = useSelector(({ courseStats }) => ({
     courseStats: courseStats.data,
     language: activeLanguage,
   }))
-  const yearcodes = uniq(flatten(Object.values(courseStats).map(c => Object.keys(c.facultyStats))))
+
+  const yearcodes = uniq(flatten(Object.values(courseStats).map(c => Object.keys(c[openOrReqular].facultyStats))))
     .sort()
     .reverse()
 
@@ -103,13 +105,13 @@ const FacultyLevelStatistics = () => {
   const yearsCourseStats = Object.values(courseStats)
     .map(course => ({
       course,
-      courseInstance: course.facultyStats[selectedYear],
+      courseInstance: course[openOrReqular].facultyStats[selectedYear],
     }))
     .sort((a, b) => (a.courseInstance === undefined) - (b.courseInstance === undefined))
 
   const courseTables = yearsCourseStats.map(({ course, courseInstance }) => (
     <CourseTable
-      course={course}
+      course={course[openOrReqular]}
       courseInstance={courseInstance}
       language={language}
       key={course.coursecode}

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/Tables/attempts.jsx

@@ -5,18 +5,11 @@ import { Link } from 'react-router-dom'
 import { connect } from 'react-redux'
 import { Header, Icon, Item } from 'semantic-ui-react'
 
+import { row } from 'components/SortableTable'
 import SortableTable from '../../../../SortableTable'
-import { defineCellColor, getGradeSpread, getThesisGradeSpread, isThesisGrades, THESIS_GRADE_KEYS } from '../util'
+import { getGradeSpread, getThesisGradeSpread, isThesisGrades, resolveGrades, getSortableColumn } from '../util'
 
-const getSortableColumn = (key, title, getRowVal, getRowContent) => ({
-  key,
-  title,
-  getRowVal,
-  getRowContent,
-  getCellProps: s => defineCellColor(s),
-})
-
-const getTableData = (stats, notThesisGrades, isRelative) =>
+const getTableData = (stats, useThesisGrades, isRelative) =>
   stats.map(stat => {
     const {
       name,
@@ -27,52 +20,44 @@ const getTableData = (stats, notThesisGrades, isRelative) =>
     } = stat
 
     const attempts = Object.values(grades).reduce((cur, acc) => acc + cur, 0)
-    const gradeSpread = notThesisGrades
-      ? getGradeSpread([grades], isRelative)
-      : getThesisGradeSpread([grades], isRelative)
+    const gradeSpread = useThesisGrades
+      ? getThesisGradeSpread([grades], isRelative)
+      : getGradeSpread([grades], isRelative)
 
-    return {
+    const mapped = {
       name,
       code,
       coursecode,
       passed: stat.attempts.categories.passed,
       failed: stat.attempts.categories.failed,
+      enrollmentsByState: stat.attempts.enrollmentsByState,
+      totalEnrollments: stat.attempts.totalEnrollments,
       passRate: stat.attempts.passRate,
       attempts,
       rowObfuscated,
       ..._.mapValues(gradeSpread, x => x[0]),
     }
+
+    if (mapped.name === 'Total') {
+      return row(mapped, { ignoreFilters: true })
+    }
+
+    return mapped
   })
 
-const includesHTOrTT = stats =>
-  stats.some(({ attempts }) => ['HT', 'TT'].some(grade => Object.keys(attempts.grades).includes(grade)))
-
-const getGradeColumns = (notThesisGrades, addHTAndTT) => {
-  if (!notThesisGrades) return THESIS_GRADE_KEYS.map(k => getSortableColumn(k, k, s => (s.rowObfuscated ? 'NA' : s[k])))
-  const columns = [
-    getSortableColumn('0', '0', s => (s.rowObfuscated ? 'NA' : s['0'])),
-    getSortableColumn('1', '1', s => (s.rowObfuscated ? 'NA' : s['1'])),
-    getSortableColumn('2', '2', s => (s.rowObfuscated ? 'NA' : s['2'])),
-    getSortableColumn('3', '3', s => (s.rowObfuscated ? 'NA' : s['3'])),
-    getSortableColumn('4', '4', s => (s.rowObfuscated ? 'NA' : s['4'])),
-    getSortableColumn('5', '5', s => (s.rowObfuscated ? 'NA' : s['5'])),
-    getSortableColumn('OTHER_PASSED', 'Other passed', s => (s.rowObfuscated ? 'NA' : s['Hyv.'])),
-  ]
-  if (addHTAndTT)
-    columns.splice(
-      6,
-      0,
-      getSortableColumn('HT', 'HT', s => (s.rowObfuscated ? 'NA' : s.HT)),
-      getSortableColumn('TT', 'TT', s => (s.rowObfuscated ? 'NA' : s.TT))
-    )
-  return columns
-}
+const getGradeColumns = grades =>
+  grades.map(({ key, title }) =>
+    getSortableColumn({
+      key,
+      title,
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s[key] || 0),
+    })
+  )
 
 const AttemptsTable = ({
   data: { stats, name },
-  settings: { showGrades },
+  settings: { showGrades, separate },
   alternatives,
-  separate,
   isRelative,
   userHasAccessToAllStats,
   headerVisible = false,
@@ -80,7 +65,7 @@ const AttemptsTable = ({
   const {
     attempts: { grades },
   } = stats[0]
-  const notThesisGrades = !isThesisGrades(grades)
+  const useThesisGrades = isThesisGrades(grades)
 
   const showPopulation = (yearcode, years) => {
     const queryObject = {
@@ -95,46 +80,99 @@ const AttemptsTable = ({
   }
 
   const timeColumn = {
-    ...getSortableColumn(
-      'TIME',
-      'Time',
-      s => s.code,
-      s => (
-        <div>
-          {s.name}
-          {s.name === 'Total' && !userHasAccessToAllStats && <strong>*</strong>}
-          {s.name !== 'Total' && userHasAccessToAllStats && (
+    key: 'TIME-PARENT',
+    merge: true,
+    mergeHeader: true,
+    children: [
+      getSortableColumn({
+        key: 'TIME',
+        title: 'Time',
+        filterType: 'range',
+        cellProps: {},
+        getRowVal: s => s.code + (2011 - 62),
+        getRowExportVal: s => s.name,
+        getRowContent: s => (
+          <div>
+            {s.name}
+            {s.name === 'Total' && !userHasAccessToAllStats && <strong>*</strong>}
+          </div>
+        ),
+      }),
+      {
+        key: 'TIME-ICON',
+        export: false,
+        getRowContent: s =>
+          s.name !== 'Total' &&
+          userHasAccessToAllStats && (
             <Item as={Link} to={showPopulation(s.code, s.name, s)}>
               <Icon name="level up alternate" />
             </Item>
-          )}
-        </div>
-      )
-    ),
+          ),
+      },
+    ],
   }
 
   let columns = [
     timeColumn,
-    getSortableColumn('ATTEMPTS', 'Total attempts', s => (s.rowObfuscated ? '5 or less students' : s.attempts)),
-    getSortableColumn('PASSED', 'Passed', s => (s.rowObfuscated ? 'NA' : s.passed)),
-    getSortableColumn('FAILED', 'Failed', s => (s.rowObfuscated ? 'NA' : s.failed)),
-    getSortableColumn(
-      'PASSRATE',
-      'Pass rate',
-      s => (s.rowObfuscated ? 'NA' : s.passRate),
-      s => (s.rowObfuscated ? 'NA' : `${Number(s.passRate || 0).toFixed(2)} %`)
-    ),
+    getSortableColumn({
+      key: 'ATTEMPTS',
+      title: 'Total attempts',
+      getRowVal: s => (s.rowObfuscated ? '5 or less students' : s.attempts),
+    }),
+    getSortableColumn({
+      key: 'PASSED',
+      title: 'Passed',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.passed),
+    }),
+    getSortableColumn({
+      key: 'FAILED',
+      title: 'Failed',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.failed),
+    }),
+    getSortableColumn({
+      key: 'PASSRATE',
+      title: 'Pass rate',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.passRate),
+      getRowContent: s => (s.rowObfuscated ? 'NA' : `${Number(s.passRate || 0).toFixed(2)} %`),
+    }),
+    getSortableColumn({
+      key: 'TOTAL_ENROLLMENTS',
+      title: 'Total enrollments',
+      helpText: 'All enrollments, including all rejected and aborted states.',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.totalEnrollments),
+    }),
+    getSortableColumn({
+      key: 'ENROLLMENTS_ENROLLED',
+      title: 'Enrolled',
+      helpText: 'All enrollments with enrolled or confirmed state.',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.enrollmentsByState.ENROLLED),
+    }),
+    getSortableColumn({
+      key: 'ENROLLMENTS_REJECTED',
+      title: 'Rejected',
+      helpText: 'All enrollments with rejected state.',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.enrollmentsByState.REJECTED),
+    }),
+    getSortableColumn({
+      key: 'ENROLLMENTS_ABORTED',
+      title: 'Aborted',
+      helpText: 'All enrollments with aborted by student or teacher state.',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.enrollmentsByState.ABORTED),
+    }),
   ]
 
-  if (showGrades) {
+  if (showGrades)
     columns = [
       timeColumn,
-      getSortableColumn('ATTEMPTS', 'Total attempts', s => (s.rowObfuscated ? '5 or less students' : s.attempts)),
-      ...getGradeColumns(notThesisGrades, includesHTOrTT(stats)),
+      getSortableColumn({
+        key: 'ATTEMPTS',
+        title: 'Total attempts',
+        getRowVal: s => (s.rowObfuscated ? '5 or less students' : s.attempts),
+      }),
+      ...getGradeColumns(resolveGrades(stats)),
     ]
-  }
 
-  const data = getTableData(stats, notThesisGrades, isRelative)
+  const data = getTableData(stats, useThesisGrades, isRelative)
 
   return (
     <div>
@@ -144,9 +182,8 @@ const AttemptsTable = ({
         </Header>
       )}
       <SortableTable
-        defaultdescending
-        getRowKey={s => s.code}
-        tableProps={{ celled: true, style: { width: 'auto' } }}
+        title={`Yearly attempt statistics for group ${name}`}
+        defaultSort={['TIME', 'desc']}
         columns={columns}
         data={data}
       />

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/Tables/student.jsx

@@ -4,13 +4,24 @@ import { Link } from 'react-router-dom'
 import { Header, Icon, Item } from 'semantic-ui-react'
 import { connect } from 'react-redux'
 import { uniq } from 'lodash'
-import { shape, string, number, oneOfType, arrayOf, bool } from 'prop-types'
+import { string, object, arrayOf, bool } from 'prop-types'
+import { row } from 'components/SortableTable'
 import SortableTable from '../../../../SortableTable'
-import { defineCellColor } from '../util'
+import { defineCellColor, resolveGrades, getSortableColumn } from '../util'
 
 const formatPercentage = p => `${(p * 100).toFixed(2)} %`
 
-const getColumns = (showDetails, userHasAccessToAllStats, alternatives, separate) => {
+const getGradeColumns = grades =>
+  grades.map(({ key, title }) =>
+    getSortableColumn({
+      key,
+      title,
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.grades[key] || 0),
+      onlyInGradeView: true,
+    })
+  )
+
+const getColumns = (stats, showDetails, showGrades, userHasAccessToAllStats, alternatives, separate, unifyCourses) => {
   const showPopulation = (yearcode, years) => {
     const queryObject = {
       from: yearcode,
@@ -18,6 +29,7 @@ const getColumns = (showDetails, userHasAccessToAllStats, alternatives, separate
       coursecodes: JSON.stringify(uniq(alternatives)),
       years,
       separate,
+      unifyCourses,
     }
     const searchString = qs.stringify(queryObject)
     return `/coursepopulation?${searchString}`
@@ -25,84 +37,179 @@ const getColumns = (showDetails, userHasAccessToAllStats, alternatives, separate
 
   const columns = [
     {
-      key: 'TIME',
-      title: 'Time',
-      getRowVal: s => s.code,
-      getRowContent: s => (
-        <div style={{ whiteSpace: 'nowrap' }}>
-          {s.name}
-          {s.name === 'Total' && !userHasAccessToAllStats && <strong>*</strong>}
-          {s.name !== 'Total' && userHasAccessToAllStats && (
-            <Item as={Link} to={showPopulation(s.code, s.name, s)}>
-              <Icon name="level up alternate" />
-            </Item>
-          )}
-        </div>
-      ),
-      getCellProps: s => defineCellColor(s),
+      key: 'TIME-PARENT',
+      merge: true,
+      mergeHeader: true,
+      children: [
+        {
+          key: 'TIME',
+          title: 'Time',
+          filterType: 'range',
+          getRowVal: s => s.code + (2011 - 62),
+          getRowExportVal: s => s.name,
+          getRowContent: s => (
+            <div style={{ whiteSpace: 'nowrap' }}>
+              {s.name}
+              {s.name === 'Total' && !userHasAccessToAllStats && <strong>*</strong>}
+            </div>
+          ),
+        },
+        {
+          key: 'TIME-ICON',
+          export: false,
+          getRowContent: s => (
+            <>
+              {s.name !== 'Total' && userHasAccessToAllStats && (
+                <Item as={Link} to={showPopulation(s.code, s.name, s)}>
+                  <Icon name="level up alternate" />
+                </Item>
+              )}
+            </>
+          ),
+        },
+      ],
     },
     {
       key: 'TOTAL',
       title: 'Total Students',
-      getRowVal: s => (s.rowObfuscated ? '5 or less students' : s.students.total),
+      helpText: 'Total count of students, including enrolled students with no grade.',
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
+      filterType: 'range',
+      getRowVal: s => (s.rowObfuscated ? 5 : s.students.withEnrollments.total),
+      getRowContent: s => (s.rowObfuscated ? '5 or less students' : s.students.withEnrollments.total),
       getCellProps: s => defineCellColor(s),
     },
+    {
+      key: 'TOTAL_PASSED',
+      title: 'Passed',
+      filterType: 'range',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.totalPassed || 0),
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
+      hideWhenGradesVisible: true,
+    },
+    {
+      key: 'TOTAL_FAILED',
+      title: 'Failed',
+      filterType: 'range',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.totalFailed || 0),
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
+      hideWhenGradesVisible: true,
+    },
+    ...getGradeColumns(resolveGrades(stats)),
+    {
+      key: 'ENROLLMENTS_MISSING_GRADE',
+      title: 'Enrolled no grade',
+      filterType: 'range',
+      helpText: 'Total count of students with a valid enrollment and no passing or failing grade.',
+      getRowVal: s => (s.rowObfuscated ? 5 : s.students.enrolledStudentsWithNoGrade),
+      getRowContent: s => (s.rowObfuscated ? '5 or less students' : s.students.enrolledStudentsWithNoGrade),
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
+    },
     {
       key: 'PASS_RATE',
       title: 'Pass-%',
-      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.passRate),
-      getRowContent: s => (s.rowObfuscated ? 'NA' : formatPercentage(s.students.passRate || 0)),
-      getCellProps: s => defineCellColor(s),
+      getRowVal: s => (s.rowObfuscated ? 0 : s.students.withEnrollments.passRate * 100),
+      getRowContent: s =>
+        s.rowObfuscated ? '5 or less students' : formatPercentage(s.students.withEnrollments.passRate),
+      filterType: 'range',
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
     },
     {
       key: 'PASS_FIRST',
       title: 'On First Attempt',
+      filterType: 'range',
       getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedFirst || 0),
-      getCellProps: s => defineCellColor(s),
-      headerProps: { style: { borderLeft: '0' } },
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
       onlyInDetailedView: true,
     },
     {
       key: 'PASS_EVENTUALLY',
       title: 'Eventually',
+      filterType: 'range',
       getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedEventually || 0),
-      getCellProps: s => defineCellColor(s),
-      headerProps: { style: { borderLeft: '0' } },
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
       onlyInDetailedView: true,
     },
     {
       key: 'FAIL_RATE',
       title: 'Fail-%',
-      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.failRate),
+      filterType: 'range',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : (s.students.failRate || 0) * 100),
       getRowContent: s => (s.rowObfuscated ? 'NA' : formatPercentage(s.students.failRate || 0)),
-      getCellProps: s => defineCellColor(s),
-    },
-    {
-      key: 'NEVER PASSED',
-      title: 'Yet to Pass',
-      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.neverPassed || 0),
-      getCellProps: s => defineCellColor(s),
-      headerProps: { style: { borderLeft: '0' } },
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
       onlyInDetailedView: true,
     },
   ]
 
-  return columns.filter(column => showDetails || !column.onlyInDetailedView)
+  return columns.filter(column => {
+    if (showDetails && column.onlyInDetailedView) return true
+    if (showGrades && column.onlyInGradeView) return true
+    if (showGrades && column.hideWhenGradesVisible) return false
+    return !column.onlyInDetailedView && !column.onlyInGradeView
+  })
 }
 
 const StudentTable = ({
   data: { name, stats },
-  settings: { showDetails },
+  settings: { showDetails, separate, showGrades },
   alternatives,
-  separate,
+  unifyCourses,
   userHasAccessToAllStats,
   headerVisible = false,
 }) => {
   const columns = useMemo(
-    () => getColumns(showDetails, userHasAccessToAllStats, alternatives, separate),
-    [showDetails, userHasAccessToAllStats, alternatives, separate]
+    () => getColumns(stats, showDetails, showGrades, userHasAccessToAllStats, alternatives, separate, unifyCourses),
+    [stats, showDetails, showGrades, userHasAccessToAllStats, alternatives, separate, unifyCourses]
   )
 
+  const data = stats.map(stats => {
+    if (stats.name === 'Total') {
+      return row(stats, { ignoreFilters: true })
+    }
+
+    return stats
+  })
+
   return (
     <div>
       {headerVisible && (
@@ -111,11 +218,13 @@ const StudentTable = ({
         </Header>
       )}
       <SortableTable
+        title={`Student statistics for group ${name}`}
+        defaultSort={['TIME', 'desc']}
         defaultdescending
         getRowKey={s => s.code}
-        tableProps={{ celled: true, fixed: true, style: { width: 'auto' } }}
+        tableProps={{ celled: true, fixed: true }}
         columns={columns}
-        data={stats}
+        data={data}
       />
       {!userHasAccessToAllStats && (
         <span className="totalsDisclaimer">* Years with 5 students or less are NOT included in the total</span>
@@ -125,12 +234,10 @@ const StudentTable = ({
 }
 
 StudentTable.propTypes = {
-  stats: arrayOf(shape({})).isRequired,
-  name: oneOfType([number, string]).isRequired,
+  data: object.isRequired,
   alternatives: arrayOf(string).isRequired,
   separate: bool,
   userHasAccessToAllStats: bool.isRequired,
-  headerVisible: bool.isRequired,
 }
 
 StudentTable.defaultProps = {

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/tables.jsx

@@ -5,57 +5,71 @@ import { viewModeNames } from './util'
 import HelpButton from '../HelpButton'
 import StudentTable from './Tables/student'
 import AttemptsTable from './Tables/attempts'
+import selectors from '../../../../selectors/courseStats'
+import UnifyRadioButtons from '../UnifyRadioButtons'
 
-export const TablesSettings = ({ value, onChange }) => {
-  const { viewMode, showDetails, showGrades } = value
+export const TablesSettings = ({ value, onChange, availableStats, onSeparateChange }) => {
+  const { viewMode, showDetails, showGrades, separate } = value
 
   return (
-    <Menu secondary style={{ marginBottom: 0 }}>
-      {Object.entries(viewModeNames).map(([key, name]) => (
-        <Menu.Item
-          active={viewMode === key}
-          data-cy={`viewMode-${name}`}
-          name={name}
-          onClick={() =>
-            onChange({
-              ...value,
-              viewMode: key,
-            })
-          }
-        />
-      ))}
-      <Menu.Item>
-        <Radio
-          toggle
-          label="Show details"
-          disabled={viewMode !== 'STUDENT'}
-          checked={showDetails}
-          onChange={() => onChange({ ...value, showDetails: !showDetails })}
-        />
-      </Menu.Item>
-      <Menu.Item>
-        <Radio
-          toggle
-          label="Show grades"
-          data-cy="gradeToggle"
-          disabled={viewMode !== 'ATTEMPTS'}
-          checked={showGrades}
-          onChange={() => onChange({ ...value, showGrades: !showGrades })}
-        />
-      </Menu.Item>
-      <Menu.Item>
-        <HelpButton tab="Tables" viewMode={viewMode} />
-      </Menu.Item>
-    </Menu>
+    <div>
+      <Menu style={{ flexWrap: 'wrap' }} secondary>
+        {Object.entries(viewModeNames).map(([key, name]) => (
+          <Menu.Item
+            key={key}
+            active={viewMode === key}
+            data-cy={`viewMode-${name}`}
+            name={name}
+            onClick={() =>
+              onChange({
+                ...value,
+                viewMode: key,
+              })
+            }
+          />
+        ))}
+        <Menu.Item>
+          <Radio
+            toggle
+            label="Show details"
+            disabled={viewMode !== 'STUDENT'}
+            checked={showDetails}
+            onChange={() => onChange({ ...value, showDetails: !showDetails })}
+          />
+        </Menu.Item>
+        <Menu.Item>
+          <Radio
+            toggle
+            label="Show grades"
+            data-cy="gradeToggle"
+            checked={showGrades}
+            onChange={() => onChange({ ...value, showGrades: !showGrades })}
+          />
+        </Menu.Item>
+        <Menu.Item>
+          <Radio
+            toggle
+            label="Separate by semesters"
+            data-cy="separateToggle"
+            checked={separate}
+            onChange={() => onSeparateChange(!separate)}
+          />
+        </Menu.Item>
+
+        <Menu.Item>
+          <HelpButton tab="Tables" viewMode={viewMode} />
+        </Menu.Item>
+      </Menu>
+      <UnifyRadioButtons availableStats={availableStats} />
+    </div>
   )
 }
 
 export const Tables = props => {
-  const alternatives = useSelector(state => state.courseStats.data[state.singleCourseStats.selectedCourse].alternatives)
-
+  const alternatives = useSelector(selectors.getCourseAlternatives)
   const viewModes = { ATTEMPTS: AttemptsTable, STUDENT: StudentTable }
-
   const Content = viewModes[props.settings.viewMode]
+  const openOrReqular = useSelector(state => state.courseSearch.openOrReqular)
 
-  return <Content {...props} alternatives={alternatives} />
+  return <Content {...props} alternatives={alternatives} unifyCourses={openOrReqular} />
 }

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/util.js

@@ -1,4 +1,23 @@
 import { arrayOf, number, oneOfType, shape, string, oneOf } from 'prop-types'
+import { flatten } from 'lodash'
+
+const gradesMap = {
+  0: 0,
+  1: 10,
+  2: 20,
+  3: 30,
+  4: 40,
+  5: 50,
+  'Hyv.': 60,
+  I: 0,
+  A: 1,
+  LUB: 2,
+  NSLA: 3,
+  CL: 4,
+  MCLA: 5,
+  ECLA: 6,
+  L: 7,
+}
 
 export const viewModeNames = {
   STUDENT: 'Students',
@@ -23,12 +42,55 @@ export const viewModeType = oneOf(Object.values(viewModeNames))
 
 export const THESIS_GRADE_KEYS = ['I', 'A', 'NSLA', 'LUB', 'CL', 'MCLA', 'ECLA', 'L']
 
+export const sortGrades = (a, b) => gradesMap[a] - gradesMap[b]
+
 export const isThesisGrades = grades => Object.keys(grades).some(k => THESIS_GRADE_KEYS.includes(k))
 
 export const isThesisSeries = series => series && series.some(s => isThesisGrades(s))
 
 export const absoluteToRelative = all => (p, i) => p / all[i] || 0
 
+export const resolveGrades = stats => {
+  const failedGrades = ['eisa', 'hyl.', 'hyl', '0', 'luop']
+  const otherPassedGrades = ['hyv.', 'hyv']
+
+  const allGrades = [
+    '0',
+    ...flatten(
+      stats.map(({ students }) =>
+        [...Object.keys(students.grades)].map(grade => {
+          const parsedGrade = Number(grade) ? Math.round(Number(grade)).toString() : grade
+          if (failedGrades.includes(parsedGrade.toLowerCase())) return '0'
+          if (parsedGrade === 'LA') return 'LUB' // merge LA and LUB grades
+          return parsedGrade
+        })
+      )
+    ),
+  ]
+
+  // If any of grades 1-5 is present, make sure that full the grade scale is present
+  if (allGrades.filter(grade => ['1', '2', '3', '4', '5'].includes(grade)).length)
+    allGrades.push(...['1', '2', '3', '4', '5'])
+  const grades = [...new Set(allGrades)]
+
+  return grades.sort(sortGrades).map(grade => {
+    if (grade === '0') return { key: grade, title: 'Failed' }
+    if (otherPassedGrades.includes(grade.toLowerCase())) return { key: grade, title: 'Other passed' }
+    return { key: grade, title: grade.charAt(0).toUpperCase() + grade.slice(1) }
+  })
+}
+
+export const getSortableColumn = opts => ({
+  filterType: 'range',
+  cellProps: s => ({
+    style: {
+      textAlign: 'right',
+      color: s.rowObfuscated ? 'gray' : 'inherit',
+    },
+  }),
+  ...opts,
+})
+
 export const getThesisGradeSpread = (series, isRelative) => {
   const thesisGradeAccumulator = {
     L: [],
@@ -39,13 +101,20 @@ export const getThesisGradeSpread = (series, isRelative) => {
     NSLA: [],
     A: [],
     I: [],
+    0: [],
+    1: [],
+    2: [],
+    3: [],
+    4: [],
+    5: [],
   }
   const newSeries = series.reduce(
     (acc, cur, i) => {
       const currentEntries = Object.entries(cur)
 
       currentEntries.forEach(([k, v]) => {
-        acc[k].push(v)
+        const merged = k === 'LA' ? 'LUB' : k
+        acc[merged].push(v)
       })
 
       Object.entries(acc).forEach(([k, v]) => {
@@ -94,7 +163,8 @@ export const getGradeSpread = (series, isRelative) => {
         if (failedKeys.includes(k.toLowerCase())) {
           failed += v
         } else {
-          acc[k].push(v)
+          const parsedGrade = Number(k) ? Math.round(Number(k)) : k
+          acc[parsedGrade].push(v)
         }
       })
       acc[0].push(failed)

services/frontend/src/components/CourseStatistics/ResultTabs/UnifyRadioButtons.jsx

@@ -0,0 +1,60 @@
+import React from 'react'
+import { useSelector, useDispatch } from 'react-redux'
+import { Menu, Radio, Form } from 'semantic-ui-react'
+import { toggleOpenAndReqularCourses } from '../../../redux/coursesearch'
+
+const UnifyRadioButtons = ({ availableStats }) => {
+  const dispatch = useDispatch()
+
+  const openOrReqular = useSelector(state => state.courseSearch.openOrReqular)
+  const toggleUnifyRadioValue = (event, { value }) => {
+    dispatch(toggleOpenAndReqularCourses(value))
+  }
+
+  return (
+    <Menu secondary style={{ marginLeft: '0.5rem' }}>
+      <Form>
+        <Form.Group>
+          <Form.Field>
+            <b>Provider organization:</b>
+          </Form.Field>
+          <Form.Field>
+            <Radio
+              label={availableStats.university ? 'university' : 'university (no data)'}
+              name="radioGroup"
+              value="reqularStats"
+              checked={openOrReqular === 'reqularStats'}
+              disabled={!availableStats.university}
+              onChange={toggleUnifyRadioValue}
+              data-cy="unify_radio_reqular"
+            />
+          </Form.Field>
+          <Form.Field>
+            <Radio
+              label={availableStats.open ? 'open' : 'open (no data)'}
+              name="radioGroup"
+              value="openStats"
+              checked={openOrReqular === 'openStats'}
+              onChange={toggleUnifyRadioValue}
+              disabled={!availableStats.open}
+              data-cy="unify_radio_open"
+            />
+          </Form.Field>
+          <Form.Field>
+            <Radio
+              label={availableStats.unify ? 'both' : 'both (no data)'}
+              name="radioGroup"
+              value="unifyStats"
+              checked={openOrReqular === 'unifyStats'}
+              onChange={toggleUnifyRadioValue}
+              disabled={!availableStats.unify}
+              data-cy="unify_radio_unify"
+            />
+          </Form.Field>
+        </Form.Group>
+      </Form>
+    </Menu>
+  )
+}
+
+export default UnifyRadioButtons

services/frontend/src/components/CourseStatistics/ResultTabs/index.jsx

@@ -1,10 +1,13 @@
 import React, { useState } from 'react'
 import { Tab, Segment, Menu, Icon } from 'semantic-ui-react'
 import { withRouter } from 'react-router-dom'
+import qs from 'query-string'
+import { useDispatch, useSelector } from 'react-redux'
+import { getCourseStats } from 'redux/coursestats'
 import { PassRate, PassRateSettings } from './Panes/passRate'
 import { Distribution, DistributionSettings } from './Panes/distribution'
 import { Tables, TablesSettings } from './Panes/tables'
-import { useTabs } from '../../../common/hooks'
+import { useProgress, useTabs } from '../../../common/hooks'
 
 import './resultTabs.css'
 import TSA from '../../../common/tsa'
@@ -12,15 +15,33 @@ import TSA from '../../../common/tsa'
 const ANALYTICS_CATEGORY = 'Course Statistics'
 const sendAnalytics = (action, name, value) => TSA.Matomo.sendEvent(ANALYTICS_CATEGORY, action, name, value)
 
-const PaneContent = ({ component: Component, settings: SettingsComponent, initialSettings, datasets, ...rest }) => {
+const PaneContent = ({
+  component: Component,
+  settings: SettingsComponent,
+  initialSettings,
+  datasets,
+  availableStats,
+  updateQuery,
+  ...rest
+}) => {
   const [settings, setSettings] = useState(initialSettings)
-  const [splitDirection, setSplitDirection] = useState('column')
+  const [splitDirection, setSplitDirection] = useState('row')
+
+  const toggleSeparate = separate => {
+    setSettings({ ...settings, separate })
+    updateQuery(separate)
+  }
 
   return (
     <Tab.Pane>
       <Segment basic>
         <div style={{ display: 'flex', marginBottom: '2em' }}>
-          <SettingsComponent value={settings} onChange={setSettings} />
+          <SettingsComponent
+            value={settings}
+            onChange={setSettings}
+            onSeparateChange={toggleSeparate}
+            availableStats={availableStats}
+          />
           <div style={{ flexGrow: 1 }} />
           {datasets.filter(i => i).length > 1 && (
             <div style={{ display: 'flex', alignItems: 'center', gap: '1em' }}>
@@ -40,7 +61,7 @@ const PaneContent = ({ component: Component, settings: SettingsComponent, initia
           {datasets
             .filter(i => i)
             .map(data => (
-              <div style={{ flexGrow: 1, flexBasis: 1 }}>
+              <div style={{ flexGrow: 1, flexBasis: 1, width: '100%' }}>
                 <h3>{data.name}</h3>
                 <Component data={data} settings={settings} {...rest} />
               </div>
@@ -51,137 +72,45 @@ const PaneContent = ({ component: Component, settings: SettingsComponent, initia
   )
 }
 
-const ResultTabs = ({ primary, comparison, history }) => {
+const ResultTabs = ({ primary, comparison, history, separate, availableStats, location }) => {
   const [tab, setTab] = useTabs('cs_tab', 0, history)
-  // const [viewMode, setViewMode] = useState(viewModeNames.STUDENT)
-  // const [selectedView, setSelectedView] = useState(false)
-  // const [isRelative, setIsRelative] = useState(false)
-  // const [showGrades, setShowGrades] = useState(false)
   const { userHasAccessToAllStats } = primary
-
-  /* const handleModeChange = newViewMode => {
-    sendAnalytics(`Current view mode '${newViewMode}'`, 'Course statistics')
-    setViewMode(newViewMode)
-  } */
-
-  /* useEffect(() => {
-    const newViewMode = selectedView ? viewModeNames.ATTEMPTS : viewModeNames.STUDENT
-    handleModeChange(newViewMode)
-  }, [selectedView]) */
+  const courseStats = useSelector(({ courseStats }) => courseStats)
+  const { pending: loading } = courseStats
+  const { onProgress } = useProgress(loading)
+  const dispatch = useDispatch()
 
   const handleTabChange = (...params) => {
     const { activeIndex } = params[1]
     const currentTab = params[1].panes[activeIndex]
     sendAnalytics(`Current tab '${currentTab.menuItem.content}'`, 'Course statistics')
     setTab(...params)
-    // setViewMode(resetViewMode ? viewMode : viewModeNames.ATTEMPTS)
   }
 
-  /* const getRadioButton = (firstLabel, secondLabel, value, setValue) => (
-    <div className="toggleContainer">
-      <label className="toggleLabel">{firstLabel}</label>
-      <Radio toggle data-cy="gradeToggle" checked={value} onChange={() => setValue(!value)} />
-      <label className="toggleLabel">{secondLabel}</label>
-    </div>
-  ) */
-
-  /* const renderViewModeSelector = () => {
-    const isTogglePane = tab !== 0
-    const getButtonMenu = () => (
-      <Menu secondary>
-        {Object.values(viewModeNames).map(name => (
-          <Menu.Item key={name} name={name} active={viewMode === name} onClick={() => handleModeChange(name)} />
-        ))}
-        {viewMode === 'Attempts' && getRadioButton('Totals', 'Grade distribution', showGrades, setShowGrades)}
-        {viewMode === 'Attempts' && showGrades && getRadioButton('Absolute', 'Relative', isRelative, setIsRelative)}
-      </Menu>
-    )
-
-    const getToggle = () => {
-      return (
-        <div className="chartToggleContainer">
-          {tab === 1 && getRadioButton('Student', 'Attempts', selectedView, setSelectedView)}
-          {(tab === 2 || comparison) && getRadioButton('Absolute', 'Relative', isRelative, setIsRelative)}
-        </div>
-      )
+  const updateSeparate = separate => {
+    const { courseCodes, ...params } = qs.parse(location.search)
+    const query = {
+      ...params,
+      courseCodes: JSON.parse(courseCodes),
+      separate,
     }
-
-    // Remove "false" and activate infoboxes when the texts for them are ready
-    return (
-      <div className="modeSelectorContainer">
-        {isTogglePane ? getToggle() : getButtonMenu()}
-        <InfoBox content={infotooltips.CourseStatistics[tab][viewMode]} />
-      </div>
-    )
-  } */
-
-  /* const getPanes = () => {
-    return [
-      {
-        menuItem: { key: 'Table', icon: 'table', content: 'Table' },
-        render: () => (
-          <Tables
-            separate={separate}
-            comparison={comparison}
-            primary={primary}
-            viewMode={viewMode}
-            isRelative={isRelative}
-            showGrades={showGrades}
-            userHasAccessToAllStats={userHasAccessToAllStats}
-          />
-        ),
-      },
-      {
-        menuItem: { key: 'pass', icon: 'balance', content: 'Pass rate chart' },
-        render: () => (
-          <PassRate
-            comparison={comparison}
-            primary={primary}
-            viewMode={viewMode}
-            isRelative={isRelative && !!comparison}
-            userHasAccessToAllStats={userHasAccessToAllStats}
-          />
-        ),
-      },
-      {
-        menuItem: { key: 'grade', icon: 'chart bar', content: 'Grade distribution chart' },
-        render: () => (
-          <Distribution
-            comparison={comparison}
-            primary={primary}
-            isRelative={isRelative}
-            userHasAccessToAllStats={userHasAccessToAllStats}
-          />
-        ),
-      },
-    ];
-
-    return paneMenuItems.map(p => {
-      const { menuItem, renderFn } = p
-      return {
-        menuItem,
-        render: () => (
-          <Grid padded="vertically" columns="equal">
-            <Grid.Row className="modeSelectorRow">{renderViewModeSelector()}</Grid.Row>
-            {renderFn()}
-          </Grid>
-        ),
-      }
-    })
-  } */
+    dispatch(getCourseStats(query, onProgress))
+    const queryToString = { ...query, courseCodes: JSON.stringify(query.courseCodes) }
+    history.replace({ search: qs.stringify(queryToString) })
+  }
 
   const paneTypes = [
     {
       label: 'Tables',
       icon: 'table',
-      initialSettings: { showDetails: false, viewMode: 'STUDENT' },
+      initialSettings: { showDetails: false, viewMode: 'STUDENT', separate },
       settings: TablesSettings,
       component: Tables,
     },
     {
       label: 'Pass rate chart',
       icon: 'balance',
-      initialSettings: { viewMode: 'STUDENT' },
+      initialSettings: { viewMode: 'STUDENT', separate },
       settings: PassRateSettings,
       component: PassRate,
     },
@@ -201,18 +130,20 @@ const ResultTabs = ({ primary, comparison, history }) => {
         <PaneContent
           component={Component}
           settings={SettingsComponent}
+          updateQuery={updateSeparate}
           userHasAccessToAllStats={userHasAccessToAllStats}
           initialSettings={initialSettings}
           datasets={[primary, comparison]}
+          availableStats={availableStats}
         />
       ),
     })
   )
 
   return (
-    <div>
+    <Segment loading={loading} basic>
       <Tab id="CourseStatPanes" panes={panes} onTabChange={handleTabChange} activeIndex={tab} />
-    </div>
+    </Segment>
   )
 }
 

services/frontend/src/components/CourseStatistics/SearchForm/YearFilter/index.jsx

@@ -4,32 +4,34 @@ import { Form } from 'semantic-ui-react'
 
 const YearFilter = ({ years, fromYear, toYear, handleChange }) => (
   <>
-    <Form.Group inline>
-      <Form.Dropdown
-        label="From:"
-        name="fromYear"
-        options={toYear ? years.filter(({ value }) => value <= toYear) : years}
-        selection
-        inline
-        placeholder="Select academic year"
-        onChange={handleChange}
-        value={fromYear}
-        selectOnBlur={false}
-        selectOnNavigation={false}
-      />
-      <Form.Dropdown
-        label="To:"
-        name="toYear"
-        options={fromYear ? years.filter(({ value }) => value >= fromYear) : years}
-        inline
-        selection
-        placeholder="Select academic year"
-        onChange={handleChange}
-        value={toYear}
-        selectOnBlur={false}
-        selectOnNavigation={false}
-      />
-    </Form.Group>
+    <Form>
+      <Form.Group inline>
+        <Form.Dropdown
+          label="From:"
+          name="fromYear"
+          options={toYear ? years.filter(({ value }) => value <= toYear) : years}
+          selection
+          inline
+          placeholder="Select academic year"
+          onChange={handleChange}
+          value={fromYear}
+          selectOnBlur={false}
+          selectOnNavigation={false}
+        />
+        <Form.Dropdown
+          label="To:"
+          name="toYear"
+          options={fromYear ? years.filter(({ value }) => value >= fromYear) : years}
+          inline
+          selection
+          placeholder="Select academic year"
+          onChange={handleChange}
+          value={toYear}
+          selectOnBlur={false}
+          selectOnNavigation={false}
+        />
+      </Form.Group>
+    </Form>
   </>
 )
 

services/frontend/src/components/CourseStatistics/SearchForm/index.jsx

@@ -1,11 +1,11 @@
-import React, { useState, useEffect, Fragment } from 'react'
+import React, { useState, useEffect } from 'react'
 import { Segment, Header, Form } from 'semantic-ui-react'
-import { connect } from 'react-redux'
+import { useSelector, useDispatch } from 'react-redux'
 import { withRouter } from 'react-router-dom'
 import qs from 'query-string'
 import { sortBy } from 'lodash'
-import { func, arrayOf, shape, bool } from 'prop-types'
-import { clearCourses, findCoursesV2, toggleUnifyOpenUniCourses } from '../../../redux/coursesearch'
+import { func, shape } from 'prop-types'
+import { clearCourses, findCoursesV2 } from '../../../redux/coursesearch'
 import { getCourseStats, clearCourseStats } from '../../../redux/coursestats'
 import { getCourseSearchResults } from '../../../selectors/courses'
 import { useSearchHistory, usePrevious } from '../../../common/hooks'
@@ -15,7 +15,6 @@ import AutoSubmitSearchInput from '../../AutoSubmitSearchInput'
 import CourseTable from '../CourseTable'
 import SearchHistory from '../../SearchHistory'
 import useLanguage from '../../LanguagePicker/useLanguage'
-import filterCourseSearchResults from './searchFormUtils'
 
 const sendAnalytics = (action, name, value) => TSA.Matomo.sendEvent('Course statistics search', action, name, value)
 
@@ -55,13 +54,18 @@ const useTSASearchResultsHook = (coursesLoading, courseName, courseCode, matchin
 
 const SearchForm = props => {
   const { language } = useLanguage()
+  const dispatch = useDispatch()
+  const isLoading = useSelector(state => state.courseStats.pending)
+  const coursesLoading = useSelector(state => state.courseSearch.pending)
+  const matchingCourses = useSelector(getCourseSearchResults)
   const [state, setState] = useState({
     ...INITIAL,
   })
   const [searchHistory, addItemToSearchHistory, updateItemInSearchHistory] = useSearchHistory('courseSearch', 6)
 
   const { courseName, courseCode, selectedCourses, separate } = state
-  const { coursesLoading, isLoading, matchingCourses, unifyOpenUniCourses } = props
+
+  // const { coursesLoading, isLoading, matchingCourses /* unifyOpenUniCourses */ } = props
 
   const parseQueryFromUrl = () => {
     const { location } = props
@@ -71,7 +75,7 @@ const SearchForm = props => {
       ...rest,
       courseCodes: JSON.parse(courseCodes),
       separate: JSON.parse(separate),
-      unifyOpenUniCourses: JSON.parse(unifyOpenUniCourses || false),
+      // unifyOpenUniCourses: JSON.parse(unifyOpenUniCourses || false),
     }
     return query
   }
@@ -79,14 +83,14 @@ const SearchForm = props => {
   const fetchStatisticsFromUrlParams = () => {
     const query = parseQueryFromUrl()
     setState({ ...state, ...query })
-    props.getCourseStats(query, props.onProgress)
+    dispatch(getCourseStats(query, props.onProgress))
   }
 
   useEffect(() => {
     const { location } = props
     if (!location.search) {
-      props.clearCourses()
-      props.clearCourseStats()
+      dispatch(clearCourses())
+      dispatch(clearCourseStats())
     }
   }, [])
 
@@ -143,7 +147,6 @@ const SearchForm = props => {
     const params = {
       courseCodes: codes,
       separate,
-      unifyOpenUniCourses: props.unifyOpenUniCourses,
     }
     const searchHistoryText = codes.map(code => `${getTextIn(selectedCourses[code].name, language)} ${code}`)
     addItemToSearchHistory({
@@ -158,28 +161,14 @@ const SearchForm = props => {
     const isValidCode = validateInputLength(courseCode, 2)
 
     if (isValidName || isValidCode) {
-      return props.findCoursesV2({ name: courseName, code: courseCode })
+      return dispatch(findCoursesV2({ name: courseName, code: courseCode }))
     }
     if (courseName.length < 5 && courseCode.length < 2) {
-      props.clearCourses()
+      dispatch(clearCourses())
     }
     return Promise.resolve()
   }
 
-  const onToggleSeparateStatistics = () => {
-    const newValue = !state.separate
-    setState({ ...state, separate: newValue })
-    sendAnalytics('Toggle Separate stats for spring & fall', `${newValue}`)
-  }
-
-  const onToggleUnifyOpenUniCoursesCheckbox = () => {
-    const newValue = !unifyOpenUniCourses
-    sendAnalytics('Toggle Unify open university courses', `${newValue}`)
-
-    setState({ ...state, selectedCourses: {} })
-    props.toggleUnifyOpenUniCourses()
-  }
-
   const courses = matchingCourses.filter(c => !selectedCourses[c.code])
 
   const disabled = isLoading || Object.keys(selectedCourses).length === 0
@@ -235,16 +224,6 @@ const SearchForm = props => {
                   data-cy="course-code-input"
                 />
               </Form.Field>
-              <Form.Field width={4} style={{ display: 'flex', flexDirection: 'column' }}>
-                <label>Unify open university courses:</label>
-                <span style={{ flex: 1, display: 'flex', alignItems: 'center' }}>
-                  <Form.Checkbox
-                    name="unifyOpenUniCourses"
-                    onChange={onToggleUnifyOpenUniCoursesCheckbox}
-                    checked={unifyOpenUniCourses}
-                  />
-                </span>
-              </Form.Field>
             </Form.Group>
             <div style={!noSelectedCourses ? searchBoxStyle : null}>
               <CourseTable
@@ -256,26 +235,18 @@ const SearchForm = props => {
                 selectedTable
               />
               {!noSelectedCourses && (
-                <>
-                  <Form.Checkbox
-                    label="Separate statistics for Spring and Fall semesters"
-                    name="separate"
-                    onChange={onToggleSeparateStatistics}
-                    checked={separate}
-                  />
-                  <Form.Button
-                    type="button"
-                    disabled={disabled}
-                    fluid
-                    size="huge"
-                    primary
-                    basic
-                    positive
-                    content="Fetch statistics"
-                    onClick={onSubmitFormClick}
-                    data-cy="fetch-stats-button"
-                  />
-                </>
+                <Form.Button
+                  type="button"
+                  disabled={disabled}
+                  fluid
+                  size="huge"
+                  primary
+                  basic
+                  positive
+                  content="Fetch statistics"
+                  onClick={onSubmitFormClick}
+                  data-cy="fetch-stats-button"
+                />
               )}
             </div>
             <CourseTable
@@ -314,38 +285,9 @@ SearchForm.defaultProps = {
 }
 
 SearchForm.propTypes = {
-  findCoursesV2: func.isRequired,
-  getCourseStats: func.isRequired,
-  clearCourses: func.isRequired,
-  clearCourseStats: func.isRequired,
-  matchingCourses: arrayOf(shape({})).isRequired,
-  isLoading: bool.isRequired,
-  coursesLoading: bool.isRequired,
   history: shape({}).isRequired,
   location: shape({}).isRequired,
-  unifyOpenUniCourses: bool.isRequired,
-  toggleUnifyOpenUniCourses: func.isRequired,
   onProgress: func,
 }
 
-const mapStateToProps = state => {
-  const { courses } = getCourseSearchResults(state)
-  const { pending: courseStatsPending } = state.courseStats
-  const { unifyOpenUniCourses } = state.courseSearch
-  return {
-    matchingCourses: filterCourseSearchResults(courses, unifyOpenUniCourses),
-    isLoading: courseStatsPending,
-    coursesLoading: state.courseSearch.pending,
-    unifyOpenUniCourses,
-  }
-}
-
-export default withRouter(
-  connect(mapStateToProps, {
-    getCourseStats,
-    clearCourses,
-    findCoursesV2,
-    clearCourseStats,
-    toggleUnifyOpenUniCourses,
-  })(SearchForm)
-)
+export default withRouter(SearchForm)

services/frontend/src/components/CourseStatistics/SearchForm/searchFormUtils.js

@@ -1,9 +1,9 @@
-import { orderBy } from 'lodash'
+// import { orderBy } from 'lodash'
 
 // handle special case where the course's actual code starts with "A" so it is mistakenly taken as Open Uni course
-const isAvoin = course => !!course.code.match(/^[A][0-9]|^AY/)
+// const isAvoin = course => !!course.code.match(/^[A][0-9]|^AY/)
 
-const sortAlternatives = alternatives => {
+/* const sortAlternatives = alternatives => {
   const result = orderBy(
     alternatives,
     [
@@ -20,17 +20,18 @@ const sortAlternatives = alternatives => {
   )
 
   return result
-}
+} */
 
 // If special case of Open Uni course that starts with A let's just sort them length, that
 // should be good enough in this case.
-const getAlternatives = course => sortAlternatives(course.alternatives)
 
-const filterCourseSearchResults = (courses, unifyOpenUniCourses) => {
+/* const getAlternatives = course => sortAlternatives(course.alternatives)
+
+const filterCourseSearchResults = courses => {
   const mergedCourses = {}
   const sortedCourses = sortAlternatives(courses)
   sortedCourses.forEach(course => {
-    const groupId = isAvoin(course) && !unifyOpenUniCourses ? course.code : course.subsId
+    const groupId = course.subsId
 
     if (!(course.max_attainment_date && course.min_attainment_date)) {
       return
@@ -64,6 +65,4 @@ const filterCourseSearchResults = (courses, unifyOpenUniCourses) => {
   }))
 
   return result
-}
-
-export default filterCourseSearchResults
+} */