Fix study programme graduation programmes

Update tests?
Fix calculation of non degree and non major credits in study programme
2026-10-13 03:13:40 +03:00 · 2022-07-04 14:13:07 +03:00 · 2022-07-04 13:45:25 +03:00 · 2022-07-04 12:26:40 +03:00 · 2022-06-30 09:28:18 +03:00 · 2022-06-29 15:18:14 +03:00
275 changed files with 17793 additions and 22321 deletions
@@ -1,15 +1,47 @@
 version: 2
 updates:
- package-ecosystem: npm
-  directory: "/"
-  schedule:
-    interval: weekly
-    time: "03:00"
-  open-pull-requests-limit: 10
-  target-branch: "trunk"
- package-ecosystem: "github-actions"
-  directory: "/"
-  schedule:
-    interval: weekly
-    time: "03:00"
-  target-branch: "trunk"
+  - package-ecosystem: npm
+    directory: '/'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'infra'
+  - package-ecosystem: npm
+    directory: '/updater/sis-updater-scheduler'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'updater'
+  - package-ecosystem: npm
+    directory: '/updater/sis-updater-worker'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'updater'
+  - package-ecosystem: npm
+    directory: '/services/backend'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'back'
+  - package-ecosystem: npm
+    directory: '/services/frontend'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'front'
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: weekly
+      time: '03:00'
@@ -19,6 +19,9 @@ jobs:
      - name: 'Build the image'
        uses: docker/build-push-action@v2
        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=production
          context: updater/sis-updater-worker/
          target: production
          push: false
@@ -40,6 +43,9 @@ jobs:
      - name: 'Build the image'
        uses: docker/build-push-action@v2
        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=production
          context: updater/sis-updater-scheduler/
          target: production
          push: false
@@ -81,6 +87,11 @@ jobs:
        env:
          SCHEDULE_IMMEDIATE: hourly
          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run daily jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: daily
+          EXIT_AFTER_IMMEDIATES: yes
      - name: 'Run weekly jobs'
        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
        env:
@@ -128,6 +139,19 @@ jobs:
          push: true
          tags: toska/sis-updater-worker:latest
          target: production
+      - name: 'Create Sentry release'
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: updater
+        with:
+          environment: production
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/static/js'
+          version: ${{ github.sha }}
      - name: 'Send notification to slack'
        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
        with:
@@ -8,7 +8,7 @@ To run Oodikone locally, you will need the following:

 1. Applications:
   - [Docker](https://docs.docker.com/install/) (version 20.10+),
-   - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
+   - [Docker Compose](https://docs.docker.com/compose/install/)(version 1.29+) and
   - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
 2. Access to Toska Docker Hub,
 3. For real data setup, access to Toskas production servers.
@@ -48,6 +48,10 @@ What different CLI options do:

 ## ⌨️ Development

+### Architecture
+
+![Oodikone architecture](Oodikone.png)
+
 ### Basics

 The development environment is entirely configured in the `docker-compose.yml` file located in this repository. The file defines the services for oodikone's two main components: **oodikone** and **updater**.
@@ -20,15 +20,13 @@ describe('Course Statistics tests', () => {

      cy.contains('.tabular.menu a', 'Table').click()
      cy.contains('All')
-      cy.contains('.modeSelectorRow a', 'Attempts').click()
-      cy.contains('.modeSelectorRow a', 'Students').click()
+      cy.cs('viewMode-Attempts').click()
+      cy.cs('viewMode-Students').click()

      cy.contains('.tabular.menu a', 'Pass rate chart').click()
-      cy.get('div.modeSelectorContainer').click()
      cy.contains('svg', 'Pass rate chart')

      cy.contains('.tabular.menu a', 'Grade distribution chart').click()
-      cy.get('div.modeSelectorContainer').click()
      cy.contains('svg', 'Grades')

      cy.contains('a', 'New query').click()
@@ -84,8 +82,8 @@ describe('Course Statistics tests', () => {
      cy.contains('Search for courses')
      cy.get("input[placeholder='Search by a course code']").type(courseCode)
      cy.contains('td', courseCode).click()
-      cy.contains('Separate statistics for Spring and Fall semesters').click()
      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.get('[data-cy=separateToggle]', { force: true }).click({ force: true })
      cy.contains('Syksy 2019')
      cy.contains('Syksy 2017')
      cy.get('tbody').contains('19')
@@ -98,7 +96,7 @@ describe('Course Statistics tests', () => {
      cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')

      cy.contains('Tietokantojen perusteet')
-      cy.contains('Avoin yo: Tietokantojen perusteet')
+      cy.contains('Tietokantojen perusteet')
      cy.contains('A581328')
      cy.contains('AYTKT10004')
      cy.contains('TKT10004, 581328')
@@ -107,46 +105,39 @@ describe('Course Statistics tests', () => {
      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.contains('Search for courses').should('not.exist')

-      cy.contains('TKT10004, 581328 Tietokantojen perusteet')
+      cy.contains('TKT10004, 581328, AYTKT10004, A581328 Tietokantojen perusteet')
      cy.get('.right').click()
      cy.contains('No results')
+    })

+    it('Provider organization toggle works', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
      cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
-      cy.contains('td', /^AYTKT10004/).click()

+      cy.contains('td', /^TKT10004/).click()
      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.contains('Search for courses').should('not.exist')
-      cy.contains('AYTKT10004 Avoin yo: Tietokantojen perusteet')
+
+      cy.contains('TKT10004, 581328, AYTKT10004, A581328 Tietokantojen perusteet')
+      cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.contains('TKT10004, 581328 Tietokantojen perusteet')
    })

    it('Searching course by name displays right courses, 10 credit courses', () => {
      cy.url().should('include', '/coursestatistics')
      cy.contains('Search for courses')
      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
-
      cy.contains('Tietorakenteet ja algoritmit')
-      cy.contains('Avoin yo: Tietorakenteet ja algoritmit')
      cy.contains('AYTKT20001')
      cy.contains('TKT20001, 58131')
      cy.contains('td', /^TKT20001/).click()
-
      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.contains('Search for courses').should('not.exist')

-      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
      cy.get('.right').click()
      cy.contains('No results')
-
-      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
-      cy.contains('td', /^AYTKT20001/).click()
-
-      cy.contains('Fetch statistics').should('be.enabled').click()
-      cy.contains('Search for courses').should('not.exist')
-      cy.contains('AYTKT20001 Avoin yo: Tietorakenteet ja algoritmit')
-
-      cy.get('.right').click()
-      cy.contains('No results')
-      cy.get("input[name='unifyOpenUniCourses']").parent().click()
      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
      cy.contains('td', 'TKT20001, 58131, AYTKT20001').click()

@@ -161,12 +152,11 @@ describe('Course Statistics tests', () => {
      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
      cy.contains('tr', 'TKT20003').click()
      cy.contains('Fetch statistics').should('be.enabled').click()
-
      cy.contains('Käyttöjärjestelmät')
      cy.contains('TKT20003')

-      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
-      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020 (open and normal')
      cy.contains('TKT20003')

      cy.contains('Students (127)').click()
@@ -176,12 +166,12 @@ describe('Course Statistics tests', () => {

    it('Population of course shows grades for each student', () => {
      cy.get("input[placeholder='Search by a course code']").type('TKT20001')
-      cy.contains(/^TKT20001$/).click()
+      cy.contains(/^TKT20001, AYTKT20001$/).click()
      cy.contains('Fetch statistics').should('be.enabled').click()
-      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
-      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
-      cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019')
-      cy.contains('Students (197)').click()
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
+      cy.get('tbody >:nth-child(3) > :nth-child(2) .level').click()
+      cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019 (open and normal)')
+      cy.contains('Students (204)').click()
      cy.contains('td', '010262566').siblings().contains('4')
      cy.contains('td', '010674989').siblings().contains('1')
    })
@@ -191,7 +181,8 @@ describe('Course Statistics tests', () => {
      cy.contains(/^TKT20003$/).click()
      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
-      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
      cy.contains('Language distribution').click()
      cy.contains('td', 'finnish').siblings().contains('122')
@@ -202,7 +193,6 @@ describe('Course Statistics tests', () => {
      beforeEach(() => {
        cy.url().should('include', '/coursestatistics')
        cy.contains('Search for courses')
-        cy.get("input[name='unifyOpenUniCourses']").parent().click()
        cy.get("input[placeholder='Search by a course code']").type('TKT10002')
        cy.contains('td', 'TKT10002, AYTKT10002').click()
        cy.contains('Fetch statistics').should('be.enabled').click()
@@ -214,18 +204,28 @@ describe('Course Statistics tests', () => {
      const yearRange = { from: '2000-2001', to: '2020-2021' }
      const attemptsTableContents = [
        // [time, passed, failed, passrate]
-        ['Total', 511, 486, 25],
+        ['Total', null, 511, 486, 25],
+      ]
+
+      const attemptsTableContentsOpen = [
+        // [time, passed, failed, passrate]
+        ['Total', null, 175, 175, 0],
+      ]
+
+      const attemptsTableContentsReqular = [
+        // [time, passed, failed, passrate]
+        ['Total', null, 312, 298, 14],
      ]

      const gradesTableContents = [
        // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-        ['Total', 511, 25, 11, 17, 21, 59, 365, 13],
-        ['2020-2021', 1, 0, 0, 0, 0, 0, 1, 0],
-        ['2019-2020', 54, 4, 3, 1, 3, 4, 39, 0],
-        ['2018-2019', 154, 7, 2, 6, 5, 15, 119, 0],
-        ['2017-2018', 186, 9, 3, 5, 6, 16, 139, 8],
-        ['2016-2017', 71, 2, 1, 2, 5, 13, 48, 0],
-        ['2015-2016', 21, 1, 1, 1, 0, 7, 8, 3],
+        ['Total', null, 511, 25, 11, 17, 21, 59, 365, 13],
+        ['2020-2021', null, 1, 0, 0, 0, 0, 0, 1, 0],
+        ['2019-2020', null, 54, 4, 3, 1, 3, 4, 39, 0],
+        ['2018-2019', null, 154, 7, 2, 6, 5, 15, 119, 0],
+        ['2017-2018', null, 186, 9, 3, 5, 6, 16, 139, 8],
+        ['2016-2017', null, 71, 2, 1, 2, 5, 13, 48, 0],
+        ['2015-2016', null, 21, 1, 1, 1, 0, 7, 8, 3],
      ]

      it('shows stats', () => {
@@ -252,6 +252,7 @@ describe('Course Statistics tests', () => {
              .eq(trIndex)
              .within(() => {
                values.forEach((value, tdIndex) => {
+                  if (value === null) return
                  cy.get('td').eq(tdIndex).contains(value)
                })
              })
@@ -265,6 +266,7 @@ describe('Course Statistics tests', () => {
              .eq(trIndex)
              .within(() => {
                values.forEach((value, tdIndex) => {
+                  if (value === null) return
                  cy.get('td').eq(tdIndex).contains(value)
                })
              })
@@ -273,7 +275,45 @@ describe('Course Statistics tests', () => {
        })
      })

-      it('after changing time range shows same stats', () => {
+      it('Toggling course provider changes stats correctly', () => {
+        cy.get('[data-cy=unify_radio_reqular]').click()
+        cy.get("div[name='toYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', '2020-2021')
+          cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', yearRange.from)
+          cy.get("div[role='option']").should('have.length', 21)
+        })
+        cy.contains('#CourseStatPanes a.item', 'Table').click()
+        cy.contains('#CourseStatPanes a.item', 'Attempts').click()
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          attemptsTableContentsReqular.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  if (value === null) return
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
+              })
+          })
+        })
+        cy.get('[data-cy=unify_radio_open]').click()
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          attemptsTableContentsOpen.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  if (value === null) return
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
+              })
+          })
+        })
+        cy.get('[data-cy=unify_radio_unify]').click()
+      })
+
+      it('After changing time range shows same stats', () => {
        const newYearRange = { from: '2016-2017', to: '2019-2020' }
        cy.get("div[name='fromYear']")
          .click()
@@ -302,6 +342,22 @@ describe('Course Statistics tests', () => {
        cy.contains('Show population').should('be.enabled')
      })
    })
+
+    it('If no data available, unify toggle is disabled', () => {
+      cy.get("input[placeholder='Search by a course code']").type('AYCSM14111')
+      cy.contains('AYCSM14111').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('AYCSM14111 Avoin yo: Full Stack -websovelluskehitys: React Native')
+      cy.get('[data-cy=unify_radio_reqular]').find('input').should('be.disabled')
+      cy.get('[data-cy=unify_radio_unify]').find('input').should('not.be.disabled')
+    })
+
+    it('Has right to see all the sudents, because course provider is TKT', () => {
+      cy.visit('coursestatistics?courseCodes=%5B%22TKT10004%22%5D&cs_tab=0&separate=false')
+      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
+      cy.contains('Students (154)').click()
+      cy.contains('015224224')
+    })
  })

  it('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
@@ -314,15 +370,17 @@ describe('Course Statistics tests', () => {
    cy.contains('Faculty statistics').should('not.exist')
    cy.contains('Show population').should('not.exist')

+    cy.cs('viewMode-Attempts').click()
+
    const attemptsTableContents = [
      // [time, passed, failed, passrate]
-      ['Total*', 288, 213, 75, '73.96 %'],
-      // ['Total', 295, 216, 79, '73.22 %'],
-      ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
-      //['2020-2021', 5, 2, 3, '40.00 %'],
-      ['2019-2020', 164, 121, 43, '73.78 %'],
-      ['2018-2019', 85, 60, 25, '70.59 %'],
-      ['2017-2018', 39, 32, 7, '82.05 %'],
+      // ['Total', 288, 213, 75, '73.96 %'],
+      // ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
+      ['Total', null, 295, 216, 79, '73.22 %'],
+      ['2020-2021', null, 5, 2, 3, '40.00 %'],
+      ['2019-2020', null, 164, 121, 43, '73.78 %'],
+      ['2018-2019', null, 85, 60, 25, '70.59 %'],
+      ['2017-2018', null, 39, 32, 7, '82.05 %'],
    ]

    cy.contains('#CourseStatPanes a.item', 'Attempts').click()
@@ -332,6 +390,7 @@ describe('Course Statistics tests', () => {
          .eq(trIndex)
          .within(() => {
            values.forEach((value, tdIndex) => {
+              if (value === null) return
              cy.get('td').eq(tdIndex).contains(value)
            })
          })
@@ -17,6 +17,28 @@ const clearSingleDropdownSelection = dataCyAttribute => {
  cy.cs(dataCyAttribute).find('i.clear').click({ force: true })
 }

+const testRangeFilter = (parentEl, min, max, expected) => {
+  cy.cs(parentEl)
+    .cs('range-selector-min')
+    .find('input')
+    .invoke('val')
+    .then(initialMin => {
+      cy.cs(parentEl)
+        .cs('range-selector-max')
+        .find('input')
+        .invoke('val')
+        .then(initialMax => {
+          cy.cs(parentEl).cs('range-selector-min').find('input').type(`{selectall}{backspace}${min}`).blur()
+          cy.cs(parentEl).cs('range-selector-max').find('input').type(`{selectall}{backspace}${max}`).blur()
+
+          checkFilteringResult(expected)
+
+          cy.cs(parentEl).cs('range-selector-min').find('input').type(`{selectall}{backspace}${initialMin}`).blur()
+          cy.cs(parentEl).cs('range-selector-max').find('input').type(`{selectall}{backspace}${initialMax}`).blur()
+        })
+    })
+}
+
 // Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
 // to every it-function. Reason behind using test function wrapper is that Cypresses internal beforeEach / afterEach
 // functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
@@ -47,7 +69,7 @@ const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
 describe('Population Statistics', () => {
  const pathToCSBach2018 =
    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2018'
-  const defaultAmountOfStudents = 148
+  const defaultAmountOfStudents = 159
  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)

  before(() => {
@@ -78,7 +100,10 @@ describe('Population Statistics', () => {
    })
  })

-  it('Study track filter works', () => {
+  // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
+  // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
+  // So fix this
+  it.skip('Study track filter works', () => {
    runTestStepWithPreAndPostParts('StudyTrack', () => {
      const card = cy.cs('StudyTrack-filter-card')
      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
@@ -115,37 +140,29 @@ describe('Population Statistics', () => {
    runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
      cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(0)
-      checkFilteringResult(137)
+      checkFilteringResult(148)
      clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
    })
  })

  it('Credit filter works', () => {
    runTestStepWithPreAndPostParts('CreditsEarned', () => {
-      cy.cs('credit-filter-min').type('50')
-      cy.cs('credit-filter-max').type('150')
-      checkFilteringResult(116)
-      cy.cs('credit-filter-min').clear()
-      cy.cs('credit-filter-max').clear()
+      testRangeFilter('CreditsEarned-filter-card', 50, 150, 118)
    })
  })

  it('Age filter works', () => {
    runTestStepWithPreAndPostParts('Age', () => {
-      cy.cs('ageFilter-min').type('20')
-      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(40)
-      cy.cs('ageFilter-min').find('input').clear()
-      cy.cs('ageFilter-max').find('input').clear()
+      testRangeFilter('Age-filter-card', 20, 40, 47)
    })
  })

  it('Gender filter works', () => {
    runTestStepWithPreAndPostParts('Gender', () => {
      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
-      checkFilteringResult(36)
+      checkFilteringResult(42)
      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
-      checkFilteringResult(112)
+      checkFilteringResult(117)
      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
      checkFilteringResult(0)
      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
@@ -160,7 +177,7 @@ describe('Population Statistics', () => {
      checkFilteringResult(1)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
      cy.cs('startYearAtUni-dropdown').selectFromDropdown(7)
-      checkFilteringResult(141)
+      checkFilteringResult(152)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
    })
  })
@@ -200,11 +217,7 @@ describe('Population Statistics', () => {
      runTestStepWithPreAndPostParts('Age', () => {
        const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
        getCard().cs('option-graduated-true').click()
-        cy.cs('ageFilter-min').type('20')
-        cy.cs('ageFilter-max').type('30')
-        checkFilteringResult(11)
-        cy.cs('ageFilter-min').find('input').clear()
-        cy.cs('ageFilter-max').find('input').clear()
+        testRangeFilter('Age-filter-card', 20, 30, 1)
        getCard().cs('option-all').click()
      })
    })
@@ -213,7 +226,7 @@ describe('Population Statistics', () => {

 describe('Course Statistics', () => {
  const pathToDSAndAlgoSpring2019 =
-    '/coursepopulation?coursecodes=%5B%22TKT20001%22%2C%2258131%22%5D&from=138&separate=true&to=138&years=Kev%C3%A4t%202019'
+    '/coursepopulation?coursecodes=%5B%22TKT20001%22%2C%2258131%22%5D&from=138&separate=true&to=138&years=Kev%C3%A4t%202019&unifyCourses=unifyStats'
  const defaultAmountOfStudents = 118
  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)

@@ -247,7 +260,10 @@ describe('Course Statistics', () => {
    })
  })

-  it('Study track filter works', () => {
+  // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
+  // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
+  // So fix this
+  it.skip('Study track filter works', () => {
    runTestStepWithPreAndPostParts('StudyTrack', () => {
      const card = cy.cs('StudyTrack-filter-card')
      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
@@ -258,11 +274,7 @@ describe('Course Statistics', () => {

  it('Age filter works', () => {
    runTestStepWithPreAndPostParts('Age', () => {
-      cy.cs('ageFilter-min').type('20')
-      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(31)
-      cy.cs('ageFilter-min').find('input').clear()
-      cy.cs('ageFilter-max').find('input').clear()
+      testRangeFilter('Age-filter-card', 20, 40, 33)
    })
  })

@@ -295,11 +307,7 @@ describe('Course Statistics', () => {
    runTestStepWithPreAndPostParts('Grade', () => {
      runTestStepWithPreAndPostParts('Age', () => {
        cy.cs('gradeFilter-3').click()
-        cy.cs('ageFilter-min').type('20')
-        cy.cs('ageFilter-max').type('30')
-        checkFilteringResult(1)
-        cy.cs('ageFilter-min').find('input').clear()
-        cy.cs('ageFilter-max').find('input').clear()
+        testRangeFilter('Age-filter-card', 20, 30, 1)
        cy.cs('gradeFilter-3').click()
      })
    })
@@ -323,11 +331,7 @@ describe('Custom Population Statistics', () => {

  it('Age filter works', () => {
    runTestStepWithPreAndPostParts('Age', () => {
-      cy.cs('ageFilter-min').type('20')
-      cy.cs('ageFilter-max').type('30')
-      checkFilteringResult(1)
-      cy.cs('ageFilter-min').find('input').clear()
-      cy.cs('ageFilter-max').find('input').clear()
+      testRangeFilter('Age-filter-card', 20, 30, 1)
    })
  })

@@ -3,15 +3,15 @@
 const moment = require('moment')
 const _ = require('lodash')

-const setPopStatsUntil = (until, includeSettings = []) => {
-  cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
-  includeSettings.forEach(setting => {
-    cy.contains('Advanced settings').parent().siblings().contains(setting).click()
-  })
-  cy.get('.adv-stats-until > .form-control').click().clear().type(until)
-  cy.contains('Fetch population with new settings').click()
-  cy.contains('Advanced settings')
-}
+// const setPopStatsUntil = (until, includeSettings = []) => {
+//   cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
+//   includeSettings.forEach(setting => {
+//     cy.contains('Advanced settings').parent().siblings().contains(setting).click()
+//   })
+//   cy.get('.adv-stats-until > .form-control').click().clear().type(until)
+//   cy.contains('Fetch population with new settings').click()
+//   cy.contains('Advanced settings')
+// }

 describe('Population Statistics tests', () => {
  const pathToCSBach2017 =
@@ -65,13 +65,14 @@ describe('Population Statistics tests', () => {
    it('Population statistics is usable on general level', () => {
      cy.visit(pathToCSBach2017)
      cy.cs('filtered-students')
-      setPopStatsUntil('toukokuu 2020')
+      // This advanced settings does not work correctly
+      // see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3552
+      // setPopStatsUntil('toukokuu 2020')

      cy.get('.card').within(() => {
        cy.contains('Tietojenkäsittelytieteen kandiohjelma')
-        cy.contains('Sample size: 149 students')
+        cy.contains('Sample size: 170 students')
        cy.contains('Excludes exchange students')
-        cy.contains("Excludes students who haven't enrolled present nor absent")
        cy.contains('Excludes students with non-degree study right')
        cy.contains('Excludes students who have transferred out of this programme')
      })
@@ -80,17 +81,17 @@ describe('Population Statistics tests', () => {

      cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(15000) // a bit hacky way, wait until ui is ready
-      cy.get('[data-cy=expand-TKT1]').click()
-      cy.get('[data-cy=coursestats-link-TKT10002]').click()
+      cy.wait(25000) // a bit hacky way, wait until ui is ready
+      cy.get('[data-cy=toggle-group-module-TKT1]').click()
+      cy.contains('td', 'TKT10002').siblings().find('i.level.up').click()
      cy.wait('@coursePage')
      cy.url().should('include', '/coursestatistics')
-      cy.contains('TKT10002, 581325 Ohjelmoinnin perusteet')
+      cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
    })

    it("Empty 'tags' tab has a link to the page where tags can be created", () => {
      cy.visit(pathToCSBach2017)
-      cy.contains('Students (149)').click()
+      cy.contains('Students (170)').click()
      cy.get('[data-cy=student-table-tabs]').contains('Tags').click()
      cy.contains('No tags defined. You can define them here.').find('a').click()
      cy.contains('Tags').click()
@@ -119,12 +120,11 @@ describe('Population Statistics tests', () => {

      cy.contains('Fetch population').click()

-      cy.contains('Credit accumulation (for 149 students)')
+      cy.contains('Credit accumulation (for 170 students)')

-      // spring + fall and include cancelled
+      // spring + fall
      cy.get('[data-cy=advanced-toggle]').click()
      cy.cs('toggle-spring').click()
-      cy.cs('toggle-cancelled').click()
      cy.contains('Fetch population').click()

      cy.contains('Credit accumulation (for 170 students)')
@@ -133,6 +133,7 @@ describe('Population Statistics tests', () => {
    it('Credit Statistics, Credits Gained tab works', () => {
      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
      cy.contains('Credit statistics').click()
+      cy.contains('Credits Gained').click()
      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the population')

      const months = Math.ceil(moment.duration(moment().diff(moment('2017-08-1'))).asMonths())
@@ -140,7 +141,7 @@ describe('Population Statistics tests', () => {
      const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
      const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))

-      cy.get('.credits-gained-table').should('contain', '(n=149)')
+      cy.get('.credits-gained-table').should('contain', '(n=170)')

      for (const [start, end] of ranges) {
        let value = 'credits'
@@ -159,7 +160,7 @@ describe('Population Statistics tests', () => {
      cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
        const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)

-        expect(sum).to.equal(149)
+        expect(sum).to.equal(170)
      })

      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('not.exist')
@@ -170,6 +171,7 @@ describe('Population Statistics tests', () => {
      cy.cs('filtered-students')
      cy.contains('Credit statistics')
      cy.get("[data-cy='credit-statistics']").click()
+      cy.contains('Credits Gained').click()
      cy.get('.credits-gained-divider').click()
      cy.get("[data-cy='credits-gained-table-Avoin väylä']").should('exist')
      cy.get("[data-cy='credits-gained-table-Yhteispisteet']").should('exist')
@@ -182,10 +184,10 @@ describe('Population Statistics tests', () => {
      cy.contains('Credit statistics').click({ force: true })
      cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()

-      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
-      cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
-      cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
-      cy.get("[data-cy='credit-stats-min']").should('contain', '11')
+      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 170 Students')
+      cy.get("[data-cy='credit-stats-mean']").should('contain', '114.75')
+      cy.get("[data-cy='credit-stats-stdev']").should('contain', '65.53')
+      cy.get("[data-cy='credit-stats-min']").should('contain', '5')
      cy.get("[data-cy='credit-stats-max']").should('contain', '314')
    })
  })
@@ -198,7 +200,7 @@ describe('Population Statistics tests', () => {
      const existing = '010113437'
      const nonExisting = '66666666'
      cy.visit(pathToCSBach2017)
-      cy.contains('Students (149)').click()
+      cy.contains('Students (170)').click()
      cy.contains(existing)
      cy.contains(nonExisting).should('not.exist')
      cy.contains('button', 'Check studentnumbers').click()
@@ -11,6 +11,7 @@ describe('Student Statistics tests', () => {
    firstnames: 'Matti Jaakko',
    lastname: 'Nieminen',
    studentnumber: '010654019',
+    sis_person_id: 'hy-hlo-84830887',
  }

  it('Student statistics search form is usable', () => {
@@ -37,9 +38,9 @@ describe('Student Statistics tests', () => {
  it('Can get student specific page by clicking student', () => {
    cy.url().should('include', '/students')
    cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
    cy.contains('Started: 01.08.2017')
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 01.08.2025')
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 31.07.2025')
    cy.contains(student.lastname).should('not.exist')
    cy.contains(student.firstnames).should('not.exist')

@@ -54,7 +55,7 @@ describe('Student Statistics tests', () => {

  it('Can get back to search menu', () => {
    cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
    cy.get('.remove').click()
    cy.contains('Student number').should('not.exist')
    cy.contains('Credits').should('not.exist')
@@ -62,7 +63,7 @@ describe('Student Statistics tests', () => {

  it('Can jump to course', () => {
    cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
    cy.contains('Introduction to Machine Learning (DATA11002)')
      .siblings()
      .within(() => {
@@ -72,6 +73,14 @@ describe('Student Statistics tests', () => {
    cy.contains('Introduction to Machine Learning')
  })

+  it('Has correct Sisu link', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains('td a', student.studentnumber).click()
+    cy.get('[data-cy=sisulink] > a')
+      .should('have.attr', 'href')
+      .and('include', 'https://sis-helsinki.funidata.fi/tutor/role/staff/student/hy-hlo-84830887/basic/basic-info')
+  })
+
  it('Searching with bad inputs doesnt yield results', () => {
    cy.get('.prompt').type('SWAG LITTINEN')
    cy.contains('Student number').should('not.exist')
@@ -82,7 +91,7 @@ describe('Student Statistics tests', () => {

  it('Can jump to population page', () => {
    cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
    cy.get('i.level.up.alternate.icon').eq(0).click()
    cy.contains('Population statistics')
    cy.contains('Tietojenkäsittelytieteen kandiohjelma')
@@ -1,172 +1,447 @@
 /// <reference types="Cypress" />

 const deleteTag = name => {
-  cy.contains(name).siblings().contains('Delete').click()
+  cy.contains('td', name).siblings().contains('Delete').click()
  cy.contains('Are you sure you want to delete tag')
  cy.contains('Confirm').click()
-  cy.contains(name).should('not.exist')
+  cy.contains('td', name).should('not.exist')
+}
+
+const getEmptyYears = isAcademicYear => {
+  const thisYear = new Date().getFullYear()
+  const years = []
+  for (let year = thisYear; year > 2021; year--) {
+    if (isAcademicYear) {
+      years.push(`${year} - ${year + 1}`)
+    } else {
+      years.push(year)
+    }
+  }
+  return years
 }

 describe('Studyprogramme overview', () => {
-  // check is used by couple of tests
-  const checkProgressAndProductivity = (user = 'basic') => {
-    // Column amount depends on user. Note that with dev user, you'll see even more
-    // stuff than with admin or basic.
-    const CSBachPopulationProgress2018ForAdmin = [
-      '162',
-      '44 (27%)',
-      '118 (72%)',
-      '161 (99%)',
-      '161',
-      '1',
-      '11',
-      '11',
-      '152',
-      '125',
-      '78',
-      '42',
-      '16',
-    ]
-    const CSBachPopulationProgress2018ForBasic = [
-      '162',
-      '44 (27%)',
-      '118 (72%)',
-      '161 (99%)',
-      '161',
-      '1',
-      '11',
-      '11',
-      '152',
-      '125',
-      '78',
-      '42',
-      '16',
-    ] // cancelled and transferred from are missing
-
-    const population = user === 'basic' ? CSBachPopulationProgress2018ForBasic : CSBachPopulationProgress2018ForAdmin
-    cy.contains('Population progress')
-    cy.contains('Yearly productivity')
-
-    cy.contains('2018-2019')
-      .siblings()
-      .each((elem, index) => {
-        cy.wrap(elem).contains(population[index])
-      })
-    const populationproductivity2019 = ['7388.0', '17', '7388.00', '0.00', '349.00']
-    cy.get('table')
-      .eq(1)
-      .contains('2019')
-      .siblings()
-      .each((elem, index) => {
-        cy.wrap(elem).contains(populationproductivity2019[index])
-      })
-  }
-
-  describe('when opening programme page with admin user', () => {
+  /* Basic information overview -tests*/
+  describe('Basic information -view works for basic user', () => {
    beforeEach(() => {
-      cy.init('/study-programme/KH50_005', 'admin')
+      cy.init('/study-programme')
+      cy.viewport(1536, 960)
    })

-    it('progress should not be recalculating when opened for the first time', () => {
-      cy.contains('Recalculating').should('not.exist')
-      cy.contains('Last updated').should('exist')
+    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
+    it('Basic information -tab loads', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('[data-cy=Section-StudentsOfTheStudyprogramme]')
+      cy.get('[data-cy=Section-CreditsProducedByTheStudyprogramme]')
+      cy.get('[data-cy=Section-GraduatedAndThesisWritersOfTheProgramme]')
+      cy.get('[data-cy=Section-ProgrammesBeforeOrAfter]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
    })

-    // Following vars are used to compare values before and after stats calculation
-    // Taken from https://docs.cypress.io/api/commands/should#Compare-text-values-of-two-elements
-    const normalizeText = s => s.replace(/\s/g, '').toLowerCase()
-    let originalProgressCalculatedText
-    let originalProductivityCalculatedText
-    const doRecalculation = () => {
-      // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
-      cy.wait(5000)
-      cy.get('.attached').contains('Admin').click()
-      cy.contains('recalculate productivity').click()
-      cy.contains('recalculate throughput').click()
-      cy.get('.attached').contains('Overview').click() // TODO: add data-cy tags for tabs
-      // Wait to "recalculating" to disappear, TODO: change this to cy.intercept
-      cy.wait(5000)
-      cy.reload()
-    }
+    it('Basic information contains correct students', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Started, Graduated, Transferred Away, Transferred to]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        [2021, 0, 0, 1, 0],
+        [2020, 12, 36, 3, 2],
+        [2019, 87, 17, 0, 1],
+        [2018, 161, 1, 0, 0],
+        [2017, 171, 0, 0, 0],
+      ]

-    it('renders progress and productivity tables with calculated status', () => {
-      doRecalculation()
-
-      // check contains also study programme options for admin
-      cy.contains('Study programme options')
-      cy.get('table').should('have.length', 3)
-
-      checkProgressAndProductivity('admin')
-
-      // Grab update dates to be compared later
-      cy.cs('throughputUpdateStatus')
-        .invoke('text')
-        .then(text => {
-          originalProgressCalculatedText = normalizeText(text)
-          expect(originalProgressCalculatedText).not.to.contain('recalculating')
-          expect(originalProgressCalculatedText).not.to.contain('refresh')
-        })
-
-      cy.cs('productivityUpdateStatus')
-        .invoke('text')
-        .then(text => {
-          originalProductivityCalculatedText = normalizeText(text)
-          expect(originalProductivityCalculatedText).not.to.contain('recalculating')
-          expect(originalProgressCalculatedText).not.to.contain('refresh')
-        })
+      cy.checkTableStats(tableContents, 'StudentsOfTheStudyprogramme')
    })

-    it('renders progress and productivity tables with calculated status after recalculating stats again', () => {
-      doRecalculation()
-      checkProgressAndProductivity('admin')
+    it('Basic information contains correct credits', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Total, Major credits, Non-major credits, Non-degree credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0, 0, 0]),
+        [2021, 0, 0, 0, 0, 0],
+        [2020, 5860, 5591, 102, 0, 167],
+        [2019, 8696, 7737, 587, 23, 349],
+        [2018, 7818, 6584, 857, 105, 272],
+        [2017, 3085, 2492, 158, 47, 388],
+      ]

-      // Check new calculation statuses are reported
-      const newProgressCalculatedTextElement = cy.cs('throughputUpdateStatus').invoke('text')
-      const newProductivityCalculatedTextElement = cy.cs('productivityUpdateStatus').invoke('text')
+      cy.checkTableStats(tableContents, 'CreditsProducedByTheStudyprogramme')
+    })

-      newProgressCalculatedTextElement.should(text => {
-        const newProgressCalculatedText = normalizeText(text)
-        expect(newProgressCalculatedText).not.to.contain('recalculating')
-        expect(newProgressCalculatedText).not.to.contain('refresh')
-        expect(newProgressCalculatedText).not.to.contain(originalProgressCalculatedText)
+    it('Basic information contains correct thesis writers and graduates', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Graduated, Wrote thesis]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 36, 35],
+        [2019, 17, 25],
+        [2018, 1, 5],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(tableContents, 'GraduatedAndThesisWritersOfTheProgramme')
+    })
+
+    it.skip('Special studyrights can be excluded and basic data changes accordingly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen maisteriohjelma').click()
+      cy.get('[data-cy=StudentToggle]').click()
+      const years = getEmptyYears()
+      const studentTableContents = [
+        // [Year, Started, Graduated]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 22, 0],
+        [2019, 12, 0],
+        [2018, 0, 0],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
+
+      const creditTableContents = [
+        // [Year, Total, Major students credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0]),
+        [2021, 0, 0, 0],
+        [2020, 117.5, 0, 7.5],
+        [2019, 67, 0, 0],
+        [2018, 0, 0, 0],
+        [2017, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+
+      const graduatedTableContents = [
+        // [Year, Graduated, Wrote thesis]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 0, 0],
+        [2019, 0, 0],
+        [2018, 0, 0],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(graduatedTableContents, 'GraduatedAndThesisWritersOfTheProgramme')
+      cy.get('[data-cy=StudentToggle]').click()
+    })
+
+    it('Year can be changed to academic year, and data changes accordingly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen maisteriohjelma').click()
+      cy.get('[data-cy=YearToggle]').click()
+      const isAcademicYear = true
+      const years = getEmptyYears(isAcademicYear)
+      const studentTableContents = [
+        // [Year, Started, Graduated, Transferred away, Transferred to]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0],
+        ['2020 - 2021', 5, 0, 0, 0],
+        ['2019 - 2020', 27, 0, 0, 0],
+        ['2018 - 2019', 4, 0, 0, 0],
+        ['2017 - 2018', 0, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
+
+      const creditTableContents = [
+        // [Year, Total, Major students credits, Non-major students credits, Non-degree credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0],
+        ['2020 - 2021', 0, 0, 0, 0],
+        ['2019 - 2020', 338.5, 177, 154, 0, 7.5],
+        ['2018 - 2019', 173, 0, 173, 0, 0],
+        ['2017 - 2018', 42, 0, 42, 0, 0],
+      ]
+
+      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+      cy.get('[data-cy=YearToggle]').click()
+    })
+
+    it('Basic information graphs render', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+
+      cy.get('[data-cy=Graph-StudentsOfTheStudyprogramme')
+        .should('contain', 'Started')
+        .should('contain', 'Graduated')
+        .should('contain', 'Transferred away')
+
+      cy.get('[data-cy=Graph-CreditsProducedByTheStudyprogramme')
+        .should('contain', 'Major students credits')
+        .should('contain', 'Non-major students credits')
+        .should('contain', 'Transferred credits')
+        .should('contain', 857)
+        .should('contain', 587)
+
+      cy.get('[data-cy=Graph-GraduatedAndThesisWritersOfTheProgramme')
+        .should('contain', 'Graduated students')
+        .should('contain', 'Wrote thesis')
+        .should('contain', 1)
+        .should('contain', 5)
+        .should('contain', 17)
+        .should('contain', 25)
+        .should('contain', 36)
+        .should('contain', 35)
+
+      cy.get('[data-cy=Graph-2019-AverageGraduationTimes]')
+        .should('contain', '2019')
+        .should('contain', 'n = 17')
+        .should('contain', '23 kk')
+
+      cy.get('[data-cy=Graph-ProgrammesBeforeOrAfter')
+        .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
+        .should('contain', 'Datatieteen maisteriohjelma')
+        .should('contain', 'Matematiikan ja tilastotieteen maisteriohjelma')
+        .should('contain', 1)
+        .should('contain', 4)
+        .should('contain', 13)
+    })
+  })
+
+  /* Studytrack overview -tests*/
+  describe('Studytrack overview works for basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme')
+    })
+
+    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
+    it('Studytracks and student populations -tab loads', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      cy.get('[data-cy=Section-StudytrackOverview]')
+      cy.get('[data-cy=Section-StudytrackProgress]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
+    })
+
+    it('Students of the studyprogramme are shown correctly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      const tableContents = [
+        // [Year, All, Started studying, Currently enrolled, Absent, Cancelled, Graduated, Men, Women, Finnish]
+        [
+          '2020 - 2021',
+          12,
+          '100.0 %',
+          10,
+          '83.3 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          10,
+          '83.3 %',
+          2,
+          '16.7 %',
+          12,
+          '100.0 %',
+        ],
+        [
+          '2019 - 2020',
+          86,
+          '100.0 %',
+          86,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          1,
+          '1.2 %',
+          65,
+          '75.6 %',
+          21,
+          '24.4 %',
+          86,
+          '100.0 %',
+        ],
+        [
+          '2018 - 2019',
+          162,
+          '100.0 %',
+          161,
+          '99.4 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          11,
+          '6.8 %',
+          118,
+          '72.8 %',
+          44,
+          '27.2 %',
+          161,
+          '99.4 %',
+        ],
+        [
+          '2017 - 2018',
+          171,
+          '100.0 %',
+          171,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          3,
+          '1.8 %',
+          42,
+          '24.6 %',
+          135,
+          '78.9 %',
+          36,
+          '21.1 %',
+          166,
+          '97.1 %',
+        ],
+        [
+          'Total',
+          431,
+          '100.0 %',
+          431,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          3,
+          '0.7 %',
+          54,
+          '12.5 %',
+          328,
+          '76.1 %',
+          103,
+          '23.9 %',
+          425,
+          '98.6 %',
+        ],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudytrackOverview')
+    })
+
+    it('Student progress data is shown correctly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      const tableContents = [
+        ['2020 - 2021', 12, 5, 2, 2, 0, 2, 0, 1],
+        ['2019 - 2020', 86, 13, 15, 32, 11, 2, 0, 13],
+        ['2018 - 2019', 162, 7, 16, 35, 33, 27, 18, 26],
+        ['2017 - 2018', 171, 10, 22, 18, 22, 24, 27, 48],
+        ['Total', 431, 35, 55, 87, 66, 55, 45, 88],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudytrackProgress')
+    })
+
+    it('Studytrack overview graphs render', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+
+      cy.get('[data-cy=Graph-StudytrackProgress]')
+        .should('contain', 'Less than 30 credits')
+        .should('contain', '30-59 credits')
+        .should('contain', 'More than 180 credits')
+        .should('contain', '5.8%') // The percentage for less than 30 credits in 2017-2018, to check that the graph renders
+        .should('contain', '4.3%') // The percentage for less than 30 credits in 2018-2019
+        .should('contain', '15.1%') // The percentage for less than 30 credits in 2019-2020
+
+      cy.get('[data-cy=Graph-2018-AverageGraduationTimes]')
+        .should('contain', '2018 - 2019')
+        .should('contain', 'n = 11 / 162')
+        .should('contain', '22 kk')
+    })
+  })
+
+  /* Programme courses -tests*/
+  describe('Programme courses works for basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme')
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Programme courses').click()
+    })
+
+    it('content loads', () => {
+      cy.get('[data-cy=CoursesYearFilter]')
+      cy.get('[data-cy=CourseTabs]')
+    })
+
+    it('time range selection works', () => {
+      cy.get('[data-cy=fromYear]').click().contains('2018').click()
+
+      cy.get('[data-cy=toYear').click().contains('2019').click()
+
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('684')
+      })
+    })
+
+    it('calendar year -> academic year toggle works', () => {
+      cy.get('[data-cy=fromYear]').click().contains('2018').click()
+
+      cy.get('[data-cy=toYear').click().contains('2019').click()
+
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('684')
      })

-      newProductivityCalculatedTextElement.should(text => {
-        const newProductivityCalculatedText = normalizeText(text)
-        expect(newProductivityCalculatedText).not.to.contain('recalculating')
-        expect(newProductivityCalculatedText).not.to.contain('refresh')
-        expect(newProductivityCalculatedText).not.to.contain(originalProgressCalculatedText)
+      cy.get('[data-cy=calendarAcademicYearToggle]').first().click()
+
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('772')
+      })
+    })
+
+    it('contains correct courses in alphabetical order', () => {
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('Aineopintojen harjoitustyö: Tietokantasovellus')
+        cy.get('tr').eq(-1).contains('Äidinkielinen viestintä')
+      })
+    })
+
+    it('different sorting options work', () => {
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        // Test sorting by different columns
+        cy.get('th').eq(0).click()
+        cy.get('tr').eq(1).contains('Tietojenkäsittelytieteen kisälliopetus: Java-ohjelmointi')
+        cy.get('th').eq(1).click()
+        cy.get('tr').eq(1).contains('Äidinkielinen viestintä')
+        cy.get('th').eq(2).click()
+        cy.get('tr').eq(1).contains('Tietorakenteet ja algoritmit')
+      })
+    })
+
+    it('show credits -> show students toggle works', () => {
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('1068')
+      })
+
+      cy.get('[data-cy=creditsStudentsToggle]').first().click()
+
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('267')
      })
    })
  })

-  describe('when opening programme page with basic user', () => {
+  /* Tag-tests*/
+  describe('Basic information -view works for basic user', () => {
    beforeEach(() => {
-      cy.init('/study-programme/KH50_005')
-    })
-
-    it('renders progress and productivity tables with previosly calculated values', () => {
-      // check doesn't contain study programme options for admin
-      cy.contains('Study programme options').should('not.exist')
-      cy.get('table').should('have.length', 2)
-      checkProgressAndProductivity()
-    })
-
-    it('can open Thesis page', () => {
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.contains('Thesis Courses').click()
-      cy.contains('Add thesis course').click()
-      cy.contains('No results')
-    })
-
-    it('can move to Population statistics page by clickin', () => {
-      cy.get('i.level.up.alternate.icon').eq(0).click()
-      cy.contains('Students (10)')
+      cy.init('/study-programme')
    })

    it('can create and delete tags for population', () => {
      const name = `tag-${new Date().getTime()}`
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
      cy.get('.attached').contains('Tags').click()
      cy.get('.tagNameSelectInput > .ui > input').type(name)
      cy.get('.yearSelectInput').type('2018')
@@ -178,7 +453,7 @@ describe('Studyprogramme overview', () => {

    it('can create personal tags', () => {
      const name = `tag-${new Date().getTime()}`
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
      cy.get('.attached').contains('Tags').click()
      cy.get('.tagNameSelectInput > .ui > input').type(name)
      cy.get('.yearSelectInput').type('2018')
@@ -194,7 +469,7 @@ describe('Studyprogramme overview', () => {

      const student = '010113437'

-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
      cy.get('.attached').contains('Tags').click()
      cy.get('.tagNameSelectInput > .ui > input').type(name)
      cy.get('.yearSelectInput').type('2018')
@@ -202,7 +477,7 @@ describe('Studyprogramme overview', () => {
      cy.contains(name)

      cy.contains('Add tags to students').click()
-      cy.get('.form > .field > .dropdown').click().get('.ui > .search').type(name).click()
+      cy.get('.form > .field > .dropdown').click().get('.ui > input.search').type(name).click()

      cy.get('.form > .field > .dropdown > .visible').contains(name).click()

@@ -211,7 +486,7 @@ describe('Studyprogramme overview', () => {

      cy.contains('Student statistics').click()
      cy.get('.prompt').type(student)
-      cy.contains(student).click()
+      cy.contains('a', student).click()
      cy.contains(name)

      cy.go('back')
@@ -221,7 +496,7 @@ describe('Studyprogramme overview', () => {

      cy.contains('Student statistics').click()
      cy.get('.prompt').type(student)
-      cy.contains(student).click()
+      cy.contains('a', student).click()
      cy.contains(name).should('not.exist')
    })
  })
@@ -16,7 +16,7 @@ describe('Teachers page tests', () => {
    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
    cy.cs('course-providers').children('.icon').click()
    cy.get('.form > .fluid').click()
-    cy.contains('Name')
+    cy.contains('Teacher')
    cy.contains(teacher1)
    cy.contains(teacher2)
  })
@@ -32,21 +32,13 @@ describe('Teachers page tests', () => {
    cy.url().should('include', '/teachers')
    cy.get('.borderless > :nth-child(3)').click()
    cy.get('.prompt').type(teacher1.split()[0])
-    cy.contains(teacher1).click()
+    cy.contains('td', teacher1).click()
    cy.contains('Aineopintojen harjoitustyö: Tietokantasovellus')
  })

  it('Check leaderboad works', () => {
    cy.get('.borderless > :nth-child(2)').click({ force: true })
    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.contains('2019-2020').click({ force: true })
-    cy.contains('Recalculate this year').click({ force: true })
-    cy.wait(5000)
-    cy.reload()
-    cy.contains('2019-2020').click({ force: true })
-    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.contains('Passed')
-    cy.contains('Ahokas Jarmo Ilkka')
+    cy.contains('2020-2021').click({ force: true })
  })
 })
@@ -39,9 +39,7 @@ describe('Users tests', () => {

    it("mocking normal user shows only the mocked user's programmes", () => {
      cy.contains('mocking').should('not.exist')
-      cy.contains('tr', 'basic').within(() => {
-        cy.contains('.button', 'Edit').click()
-      })
+      cy.cs('user-edit-button-basic').click()
      cy.get('i.spy').click()
      cy.contains('mocking as basic')
      cy.wait(1000)
@@ -104,3 +104,18 @@ Cypress.Commands.add('selectStudyProgramme', name => {
  cy.cs('select-study-programme').click().children().contains(name).click()
  cy.contains('See population').click()
 })
+
+Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {
+  cy.get(`[data-cy=Table-${tableName}]>tbody`).within(() => {
+    correctStats.forEach((values, trIndex) => {
+      cy.get('tr')
+        .eq(trIndex)
+        .within(() => {
+          values.forEach((value, tdIndex) => {
+            cy.get('td').eq(tdIndex).contains(value)
+          })
+        })
+    })
+    cy.get('tr').should('have.length', correctStats.length)
+  })
+})
@@ -14,3 +14,13 @@
 // ***********************************************************

 import './commands'
+
+// This error is benign and seems to be only present in Cypress
+const resizeObserverLoopErrRe = /^[^(ResizeObserver loop limit exceeded)]/
+
+Cypress.on('uncaught:exception', err => {
+  /* returning false here prevents Cypress from failing the test */
+  if (resizeObserverLoopErrRe.test(err.message)) {
+    return false
+  }
+})
@@ -28,24 +28,24 @@
  },
  "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
  "devDependencies": {
-    "@babel/eslint-parser": "^7.15.0",
-    "@babel/plugin-syntax-jsx": "^7.14.5",
-    "concurrently": "^6.2.1",
-    "cypress": "^8.3.1",
+    "@babel/eslint-parser": "^7.17.0",
+    "@babel/plugin-syntax-jsx": "^7.16.7",
+    "concurrently": "^6.5.1",
+    "cypress": "^8.7.0",
    "eslint": "^7.32.0",
    "eslint-config-airbnb": "^18.2.1",
-    "eslint-config-prettier": "^8.3.0",
+    "eslint-config-prettier": "^8.5.0",
    "eslint-plugin-babel": "^5.3.1",
-    "eslint-plugin-cypress": "^2.11.3",
-    "eslint-plugin-import": "^2.24.2",
-    "eslint-plugin-jsx-a11y": "^6.4.1",
+    "eslint-plugin-cypress": "^2.12.1",
+    "eslint-plugin-import": "^2.25.4",
+    "eslint-plugin-jsx-a11y": "^6.5.1",
    "eslint-plugin-prefer-object-spread": "^1.2.1",
    "eslint-plugin-prettier": "^3.4.1",
-    "eslint-plugin-react": "^7.24.0",
-    "eslint-plugin-react-hooks": "^4.2.0",
-    "husky": "^7.0.2",
-    "lint-staged": "^11.1.2",
-    "prettier": "^2.3.2",
+    "eslint-plugin-react": "^7.29.3",
+    "eslint-plugin-react-hooks": "^4.3.0",
+    "husky": "^7.0.4",
+    "lint-staged": "^11.2.6",
+    "prettier": "^2.5.1",
    "stylelint": "^13.13.1",
    "stylelint-config-prettier": "^8.0.2",
    "stylelint-config-standard": "^22.0.0",
@@ -3,10 +3,15 @@
  "requires": true,
  "lockfileVersion": 1,
  "dependencies": {
+    "@colors/colors": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz",
+      "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ=="
+    },
    "@dabh/diagnostics": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz",
-      "integrity": "sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==",
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz",
+      "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==",
      "requires": {
        "colorspace": "1.1.x",
        "enabled": "2.0.x",
@@ -76,48 +81,48 @@
      }
    },
    "@sentry/tracing": {
-      "version": "6.11.0",
-      "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.11.0.tgz",
-      "integrity": "sha512-9VA1/SY++WeoMQI4K6n/sYgIdRtCu9NLWqmGqu/5kbOtESYFgAt1DqSyqGCr00ZjQiC2s7tkDkTNZb38K6KytQ==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.18.2.tgz",
+      "integrity": "sha512-hg6NLqrqJ5sUPTyWEQ2RqdnhQVnyLtx8II0IyWxQLDWD8UCe3Mu6G7mroDtakPWcP+lWz6OnKfMEfuhMcxR8fw==",
      "requires": {
-        "@sentry/hub": "6.11.0",
-        "@sentry/minimal": "6.11.0",
-        "@sentry/types": "6.11.0",
-        "@sentry/utils": "6.11.0",
+        "@sentry/hub": "6.18.2",
+        "@sentry/minimal": "6.18.2",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
        "tslib": "^1.9.3"
      },
      "dependencies": {
        "@sentry/hub": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.11.0.tgz",
-          "integrity": "sha512-pT9hf+ZJfVFpoZopoC+yJmFNclr4NPqPcl2cgguqCHb69DklD1NxgBNWK8D6X05qjnNFDF991U6t1mxP9HrGuw==",
+          "version": "6.18.2",
+          "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.18.2.tgz",
+          "integrity": "sha512-d0AugekMkbnN12b4EXMjseJxtLPc9S20DGobCPUb4oAQT6S2oDQEj1jwP6PQ5vtgyy+GMYWxBMgqAQ4pjVYISQ==",
          "requires": {
-            "@sentry/types": "6.11.0",
-            "@sentry/utils": "6.11.0",
+            "@sentry/types": "6.18.2",
+            "@sentry/utils": "6.18.2",
            "tslib": "^1.9.3"
          }
        },
        "@sentry/minimal": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.11.0.tgz",
-          "integrity": "sha512-XkZ7qrdlGp4IM/gjGxf1Q575yIbl5RvPbg+WFeekpo16Ufvzx37Mr8c2xsZaWosISVyE6eyFpooORjUlzy8EDw==",
+          "version": "6.18.2",
+          "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.18.2.tgz",
+          "integrity": "sha512-n7KYuo34W2LxE+3dnZ47of7XHuORINCnXq66XH72eoj67tf0XeWbIhEJrYGmoLRyRfoCYYrBLWiDl/uTjLzrzQ==",
          "requires": {
-            "@sentry/hub": "6.11.0",
-            "@sentry/types": "6.11.0",
+            "@sentry/hub": "6.18.2",
+            "@sentry/types": "6.18.2",
            "tslib": "^1.9.3"
          }
        },
        "@sentry/types": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.11.0.tgz",
-          "integrity": "sha512-gm5H9eZhL6bsIy/h3T+/Fzzz2vINhHhqd92CjHle3w7uXdTdFV98i2pDpErBGNTSNzbntqOMifYEB5ENtZAvcg=="
+          "version": "6.18.2",
+          "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.18.2.tgz",
+          "integrity": "sha512-WzpJf/Q5aORTzrSwer/As1NlO90dBAQpaHV2ikDDKqOyMWEgjKb5/4gh59p9gH8JMMnLetP1AvQel0fOj5UnUw=="
        },
        "@sentry/utils": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.11.0.tgz",
-          "integrity": "sha512-IOvyFHcnbRQxa++jO+ZUzRvFHEJ1cZjrBIQaNVc0IYF0twUOB5PTP6joTcix38ldaLeapaPZ9LGfudbvYvxkdg==",
+          "version": "6.18.2",
+          "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.18.2.tgz",
+          "integrity": "sha512-EC619jesknyu4xpwud5WC/5odYLz6JUy7OSFy5405PpdGeh/m8XUvuJAx4zDx0Iz/Mlk0S1Md+ZcQwqkv39dkw==",
          "requires": {
-            "@sentry/types": "6.11.0",
+            "@sentry/types": "6.18.2",
            "tslib": "^1.9.3"
          }
        }
@@ -137,10 +142,28 @@
        "tslib": "^1.9.3"
      }
    },
+    "@types/debug": {
+      "version": "4.1.7",
+      "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz",
+      "integrity": "sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==",
+      "requires": {
+        "@types/ms": "*"
+      }
+    },
+    "@types/ms": {
+      "version": "0.7.31",
+      "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz",
+      "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA=="
+    },
    "@types/node": {
-      "version": "14.14.39",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.39.tgz",
-      "integrity": "sha512-Qipn7rfTxGEDqZiezH+wxqWYR8vcXq5LRpZrETD19Gs4o8LbklbmqotSUsMU+s5G3PJwMRDfNEYoxrcBwIxOuw=="
+      "version": "17.0.21",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz",
+      "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ=="
+    },
+    "@types/validator": {
+      "version": "13.7.1",
+      "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.7.1.tgz",
+      "integrity": "sha512-I6OUIZ5cYRk5lp14xSOAiXjWrfVoMZVjDuevBYgQDYzZIjsf2CAISpEcXOkFAtpAHbmWIDLcZObejqny/9xq5Q=="
    },
    "abbrev": {
      "version": "1.1.1",
@@ -149,12 +172,12 @@
      "dev": true
    },
    "accepts": {
-      "version": "1.3.7",
-      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz",
-      "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==",
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
+      "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==",
      "requires": {
-        "mime-types": "~2.1.24",
-        "negotiator": "0.6.2"
+        "mime-types": "~2.1.34",
+        "negotiator": "0.6.3"
      }
    },
    "agent-base": {
@@ -165,11 +188,6 @@
        "debug": "4"
      }
    },
-    "any-promise": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
-      "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8="
-    },
    "arr-diff": {
      "version": "4.0.0",
      "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
@@ -206,9 +224,9 @@
      "dev": true
    },
    "async": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/async/-/async-3.2.1.tgz",
-      "integrity": "sha512-XdD5lRO/87udXCMC9meWdYiR+Nq6ZjUfXidViUZGu2F1MO4T3XwZ1et0hb2++BgLfhyJwy44BGB/yx80ABx8hg=="
+      "version": "3.2.3",
+      "resolved": "https://registry.npmjs.org/async/-/async-3.2.3.tgz",
+      "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g=="
    },
    "async-each": {
      "version": "1.0.3",
@@ -217,9 +235,9 @@
      "dev": true
    },
    "async-lock": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.3.0.tgz",
-      "integrity": "sha512-8A7SkiisnEgME2zEedtDYPxUPzdv3x//E7n5IFktPAtMYSEAV7eNJF0rMwrVyUFj6d/8rgajLantbjcNRQYXIg=="
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.3.1.tgz",
+      "integrity": "sha512-zK7xap9UnttfbE23JmcrNIyueAn6jWshihJqA33U/hEnKprF/lVGBDsBv/bqLm2YMMl1DnpHhUY044eA0t1TUw=="
    },
    "atob": {
      "version": "2.1.2",
@@ -228,9 +246,9 @@
      "dev": true
    },
    "axios": {
-      "version": "0.21.2",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.2.tgz",
-      "integrity": "sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg==",
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
      "requires": {
        "follow-redirects": "^1.14.0"
      }
@@ -291,20 +309,20 @@
      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
    },
    "body-parser": {
-      "version": "1.19.0",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
-      "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
+      "version": "1.19.2",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz",
+      "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==",
      "requires": {
-        "bytes": "3.1.0",
+        "bytes": "3.1.2",
        "content-type": "~1.0.4",
        "debug": "2.6.9",
        "depd": "~1.1.2",
-        "http-errors": "1.7.2",
+        "http-errors": "1.8.1",
        "iconv-lite": "0.4.24",
        "on-finished": "~2.3.0",
-        "qs": "6.7.0",
-        "raw-body": "2.4.0",
-        "type-is": "~1.6.17"
+        "qs": "6.9.7",
+        "raw-body": "2.4.3",
+        "type-is": "~1.6.18"
      },
      "dependencies": {
        "debug": {
@@ -338,9 +356,9 @@
      "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw=="
    },
    "bytes": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
-      "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg=="
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
+      "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="
    },
    "cache-base": {
      "version": "1.0.1",
@@ -514,12 +532,12 @@
      }
    },
    "color": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz",
-      "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==",
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz",
+      "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==",
      "requires": {
-        "color-convert": "^1.9.1",
-        "color-string": "^1.5.2"
+        "color-convert": "^1.9.3",
+        "color-string": "^1.6.0"
      }
    },
    "color-convert": {
@@ -536,25 +554,20 @@
      "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
    },
    "color-string": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.6.0.tgz",
-      "integrity": "sha512-c/hGS+kRWJutUBEngKKmk4iH3sD59MBkoxVapS/0wgpCz2u7XsNloxknyvBhzwEs1IbV36D9PwqLPJ2DTu3vMA==",
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.0.tgz",
+      "integrity": "sha512-9Mrz2AQLefkH1UvASKj6v6hj/7eWgjnT/cVsR8CumieLoT+g900exWeNogqtweI8dxloXN9BDQTYro1oWu/5CQ==",
      "requires": {
        "color-name": "^1.0.0",
        "simple-swizzle": "^0.2.2"
      }
    },
-    "colors": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz",
-      "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA=="
-    },
    "colorspace": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz",
-      "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==",
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz",
+      "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==",
      "requires": {
-        "color": "3.0.x",
+        "color": "^3.1.3",
        "text-hex": "1.0.x"
      }
    },
@@ -571,11 +584,18 @@
      "dev": true
    },
    "content-disposition": {
-      "version": "0.5.3",
-      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
-      "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==",
+      "version": "0.5.4",
+      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
+      "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==",
      "requires": {
-        "safe-buffer": "5.1.2"
+        "safe-buffer": "5.2.1"
+      },
+      "dependencies": {
+        "safe-buffer": {
+          "version": "5.2.1",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+          "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
+        }
      }
    },
    "content-type": {
@@ -602,7 +622,8 @@
    "core-util-is": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
-      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
+      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
+      "dev": true
    },
    "cors": {
      "version": "2.8.5",
@@ -838,16 +859,16 @@
      }
    },
    "express": {
-      "version": "4.17.1",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz",
-      "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==",
+      "version": "4.17.3",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz",
+      "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==",
      "requires": {
-        "accepts": "~1.3.7",
+        "accepts": "~1.3.8",
        "array-flatten": "1.1.1",
-        "body-parser": "1.19.0",
-        "content-disposition": "0.5.3",
+        "body-parser": "1.19.2",
+        "content-disposition": "0.5.4",
        "content-type": "~1.0.4",
-        "cookie": "0.4.0",
+        "cookie": "0.4.2",
        "cookie-signature": "1.0.6",
        "debug": "2.6.9",
        "depd": "~1.1.2",
@@ -861,13 +882,13 @@
        "on-finished": "~2.3.0",
        "parseurl": "~1.3.3",
        "path-to-regexp": "0.1.7",
-        "proxy-addr": "~2.0.5",
-        "qs": "6.7.0",
+        "proxy-addr": "~2.0.7",
+        "qs": "6.9.7",
        "range-parser": "~1.2.1",
-        "safe-buffer": "5.1.2",
-        "send": "0.17.1",
-        "serve-static": "1.14.1",
-        "setprototypeof": "1.1.1",
+        "safe-buffer": "5.2.1",
+        "send": "0.17.2",
+        "serve-static": "1.14.2",
+        "setprototypeof": "1.2.0",
        "statuses": "~1.5.0",
        "type-is": "~1.6.18",
        "utils-merge": "1.0.1",
@@ -875,9 +896,9 @@
      },
      "dependencies": {
        "cookie": {
-          "version": "0.4.0",
-          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz",
-          "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg=="
+          "version": "0.4.2",
+          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz",
+          "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA=="
        },
        "debug": {
          "version": "2.6.9",
@@ -891,6 +912,11 @@
          "version": "2.0.0",
          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+        },
+        "safe-buffer": {
+          "version": "5.2.1",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+          "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
        }
      }
    },
@@ -951,11 +977,6 @@
        }
      }
    },
-    "fast-safe-stringify": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz",
-      "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA=="
-    },
    "fecha": {
      "version": "4.2.1",
      "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.1.tgz",
@@ -1003,9 +1024,9 @@
      "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="
    },
    "follow-redirects": {
-      "version": "1.14.3",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.3.tgz",
-      "integrity": "sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw=="
+      "version": "1.14.9",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
+      "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w=="
    },
    "for-in": {
      "version": "1.0.2",
@@ -1014,9 +1035,9 @@
      "dev": true
    },
    "forwarded": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
-      "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
+      "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="
    },
    "fragment-cache": {
      "version": "0.2.1",
@@ -1106,22 +1127,15 @@
      }
    },
    "http-errors": {
-      "version": "1.7.2",
-      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
-      "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz",
+      "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==",
      "requires": {
        "depd": "~1.1.2",
-        "inherits": "2.0.3",
-        "setprototypeof": "1.1.1",
+        "inherits": "2.0.4",
+        "setprototypeof": "1.2.0",
        "statuses": ">= 1.5.0 < 2",
-        "toidentifier": "1.0.0"
-      },
-      "dependencies": {
-        "inherits": {
-          "version": "2.0.3",
-          "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
-          "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
-        }
+        "toidentifier": "1.0.1"
      }
    },
    "https-proxy-agent": {
@@ -1160,9 +1174,9 @@
      "dev": true
    },
    "inflection": {
-      "version": "1.13.1",
-      "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.1.tgz",
-      "integrity": "sha512-dldYtl2WlN0QDkIDtg8+xFwOS2Tbmp12t1cHa5/YClU6ZQjTFm7B66UcVbh9NQB+HvT5BAd2t5+yKsBkw5pcqA=="
+      "version": "1.13.2",
+      "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.2.tgz",
+      "integrity": "sha512-cmZlljCRTBFouT8UzMzrGcVEvkv6D/wBdcdKG7J1QH5cXjtU75Dm+P27v9EKu/Y43UYyCJd1WC4zLebRrC8NBw=="
    },
    "inherits": {
      "version": "2.0.4",
@@ -1303,7 +1317,8 @@
    "isarray": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
+      "dev": true
    },
    "isexe": {
      "version": "2.0.0",
@@ -1334,14 +1349,14 @@
      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
    },
    "logform": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz",
-      "integrity": "sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg==",
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/logform/-/logform-2.4.0.tgz",
+      "integrity": "sha512-CPSJw4ftjf517EhXZGGvTHHkYobo7ZCc0kvwUoOYcjfR2UVrI66RHj8MCrfAdEitdmFqbu2BYdYs8FHHZSb6iw==",
      "requires": {
-        "colors": "^1.2.1",
-        "fast-safe-stringify": "^2.0.4",
+        "@colors/colors": "1.5.0",
        "fecha": "^4.2.0",
        "ms": "^2.1.1",
+        "safe-stable-stringify": "^2.3.1",
        "triple-beam": "^1.3.0"
      }
    },
@@ -1400,16 +1415,16 @@
      "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
    },
    "mime-db": {
-      "version": "1.47.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz",
-      "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw=="
+      "version": "1.51.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz",
+      "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g=="
    },
    "mime-types": {
-      "version": "2.1.30",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz",
-      "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==",
+      "version": "2.1.34",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz",
+      "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==",
      "requires": {
-        "mime-db": "1.47.0"
+        "mime-db": "1.51.0"
      }
    },
    "minimatch": {
@@ -1443,9 +1458,9 @@
      "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ=="
    },
    "moment-timezone": {
-      "version": "0.5.33",
-      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz",
-      "integrity": "sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==",
+      "version": "0.5.34",
+      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.34.tgz",
+      "integrity": "sha512-3zAEHh2hKUs3EXLESx/wsgw6IQdusOT8Bxm3D9UrHPQR7zlMmzwybC8zHEM1tQ4LJwP7fcxrWr8tuBg05fFCbg==",
      "requires": {
        "moment": ">= 2.9.0"
      }
@@ -1514,9 +1529,9 @@
      }
    },
    "negotiator": {
-      "version": "0.6.2",
-      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
-      "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
+      "version": "0.6.3",
+      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
+      "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="
    },
    "nodemon": {
      "version": "1.19.4",
@@ -2205,14 +2220,14 @@
      "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
    },
    "pg": {
-      "version": "8.7.1",
-      "resolved": "https://registry.npmjs.org/pg/-/pg-8.7.1.tgz",
-      "integrity": "sha512-7bdYcv7V6U3KAtWjpQJJBww0UEsWuh4yQ/EjNf2HeO/NnvKjpvhEIe/A/TleP6wtmSKnUnghs5A9jUoK6iDdkA==",
+      "version": "8.7.3",
+      "resolved": "https://registry.npmjs.org/pg/-/pg-8.7.3.tgz",
+      "integrity": "sha512-HPmH4GH4H3AOprDJOazoIcpI49XFsHCe8xlrjHkWiapdbHK+HLtbm/GQzXYAZwmPju/kzKhjaSfMACG+8cgJcw==",
      "requires": {
        "buffer-writer": "2.0.0",
        "packet-reader": "1.0.0",
        "pg-connection-string": "^2.5.0",
-        "pg-pool": "^3.4.1",
+        "pg-pool": "^3.5.1",
        "pg-protocol": "^1.5.0",
        "pg-types": "^2.1.0",
        "pgpass": "1.x"
@@ -2229,9 +2244,9 @@
      "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="
    },
    "pg-pool": {
-      "version": "3.4.1",
-      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.4.1.tgz",
-      "integrity": "sha512-TVHxR/gf3MeJRvchgNHxsYsTCHQ+4wm3VIHSS19z8NC0+gioEhq1okDY1sm/TYbfoP6JLFx01s0ShvZ3puP/iQ=="
+      "version": "3.5.1",
+      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.1.tgz",
+      "integrity": "sha512-6iCR0wVrro6OOHFsyavV+i6KYL4lVNyYAB9RD18w66xSzN+d8b66HiwuP30Gp1SH5O9T82fckkzsRjlrhD0ioQ=="
    },
    "pg-protocol": {
      "version": "1.5.0",
@@ -2251,11 +2266,11 @@
      }
    },
    "pgpass": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.4.tgz",
-      "integrity": "sha512-YmuA56alyBq7M59vxVBfPJrGSozru8QAdoNlWuW3cz8l+UX3cWge0vTvjKhsSHSJpo3Bom8/Mm6hf0TR5GY0+w==",
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+      "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
      "requires": {
-        "split2": "^3.1.1"
+        "split2": "^4.1.0"
      }
    },
    "posix-character-classes": {
@@ -2290,14 +2305,15 @@
    "process-nextick-args": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
-      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
+      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
+      "dev": true
    },
    "proxy-addr": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz",
-      "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==",
+      "version": "2.0.7",
+      "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
+      "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==",
      "requires": {
-        "forwarded": "~0.1.2",
+        "forwarded": "0.2.0",
        "ipaddr.js": "1.9.1"
      }
    },
@@ -2314,9 +2330,9 @@
      "dev": true
    },
    "qs": {
-      "version": "6.7.0",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
-      "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ=="
+      "version": "6.9.7",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz",
+      "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw=="
    },
    "range-parser": {
      "version": "1.2.1",
@@ -2324,12 +2340,12 @@
      "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
    },
    "raw-body": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
-      "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
+      "version": "2.4.3",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz",
+      "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==",
      "requires": {
-        "bytes": "3.1.0",
-        "http-errors": "1.7.2",
+        "bytes": "3.1.2",
+        "http-errors": "1.8.1",
        "iconv-lite": "0.4.24",
        "unpipe": "1.0.0"
      }
@@ -2350,6 +2366,7 @@
      "version": "2.3.7",
      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
      "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
+      "dev": true,
      "requires": {
        "core-util-is": "~1.0.0",
        "inherits": "~2.0.3",
@@ -2430,12 +2447,9 @@
      "dev": true
    },
    "retry-as-promised": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-3.2.0.tgz",
-      "integrity": "sha512-CybGs60B7oYU/qSQ6kuaFmRd9sTZ6oXSc0toqePvV74Ac6/IFZSI1ReFQmtCN+uvW1Mtqdwpvt/LGOiCBAY2Mg==",
-      "requires": {
-        "any-promise": "^1.3.0"
-      }
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-5.0.0.tgz",
+      "integrity": "sha512-6S+5LvtTl2ggBumk04hBo/4Uf6fRJUwIgunGZ7CYEBCeufGFW1Pu6ucUf/UskHeWOIsUcLOGLFXPig5tR5V1nA=="
    },
    "safe-buffer": {
      "version": "5.1.2",
@@ -2451,11 +2465,39 @@
        "ret": "~0.1.10"
      }
    },
+    "safe-stable-stringify": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.3.1.tgz",
+      "integrity": "sha512-kYBSfT+troD9cDA85VDnHZ1rpHC50O0g1e6WlGHVCz/g+JS+9WKLj+XwFYyR8UbrZN8ll9HUpDAAddY58MGisg=="
+    },
    "safer-buffer": {
      "version": "2.1.2",
      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
    },
+    "semver": {
+      "version": "7.3.5",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
+      "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
+      "requires": {
+        "lru-cache": "^6.0.0"
+      },
+      "dependencies": {
+        "lru-cache": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+          "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+          "requires": {
+            "yallist": "^4.0.0"
+          }
+        },
+        "yallist": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
+        }
+      }
+    },
    "semver-diff": {
      "version": "2.1.0",
      "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-2.1.0.tgz",
@@ -2474,9 +2516,9 @@
      }
    },
    "send": {
-      "version": "0.17.1",
-      "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz",
-      "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==",
+      "version": "0.17.2",
+      "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz",
+      "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==",
      "requires": {
        "debug": "2.6.9",
        "depd": "~1.1.2",
@@ -2485,9 +2527,9 @@
        "escape-html": "~1.0.3",
        "etag": "~1.8.1",
        "fresh": "0.5.2",
-        "http-errors": "~1.7.2",
+        "http-errors": "1.8.1",
        "mime": "1.6.0",
-        "ms": "2.1.1",
+        "ms": "2.1.3",
        "on-finished": "~2.3.0",
        "range-parser": "~1.2.1",
        "statuses": "~1.5.0"
@@ -2507,76 +2549,66 @@
              "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
            }
          }
-        },
-        "ms": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
-          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
        }
      }
    },
    "sequelize": {
-      "version": "6.6.5",
-      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.6.5.tgz",
-      "integrity": "sha512-QyRrJrDRiwuiILqTMHUA1yWOPIL12KlfmgZ3hnzQwbMvp2vJ6fzu9bYJQB+qPMosck4mBUggY4Cjoc6Et8FBIQ==",
+      "version": "6.17.0",
+      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.17.0.tgz",
+      "integrity": "sha512-AZus+0YZDq91Zg0hzDaO5atTzHgJruI23V8nBlAhkLuI81Z53nSRdAe/4R1A6vGOZ/RfCLP9idF4tfQnoAsM5A==",
      "requires": {
-        "debug": "^4.1.1",
-        "dottie": "^2.0.0",
-        "inflection": "1.13.1",
-        "lodash": "^4.17.20",
-        "moment": "^2.26.0",
-        "moment-timezone": "^0.5.31",
-        "retry-as-promised": "^3.2.0",
-        "semver": "^7.3.2",
-        "sequelize-pool": "^6.0.0",
+        "@types/debug": "^4.1.7",
+        "@types/validator": "^13.7.1",
+        "debug": "^4.3.3",
+        "dottie": "^2.0.2",
+        "inflection": "^1.13.2",
+        "lodash": "^4.17.21",
+        "moment": "^2.29.1",
+        "moment-timezone": "^0.5.34",
+        "pg-connection-string": "^2.5.0",
+        "retry-as-promised": "^5.0.0",
+        "semver": "^7.3.5",
+        "sequelize-pool": "^7.1.0",
        "toposort-class": "^1.0.1",
-        "uuid": "^8.1.0",
-        "validator": "^13.6.0",
+        "uuid": "^8.3.2",
+        "validator": "^13.7.0",
        "wkx": "^0.5.0"
      },
      "dependencies": {
-        "lru-cache": {
-          "version": "6.0.0",
-          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
-          "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+        "debug": {
+          "version": "4.3.3",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz",
+          "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==",
          "requires": {
-            "yallist": "^4.0.0"
+            "ms": "2.1.2"
          }
        },
-        "semver": {
-          "version": "7.3.5",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
-          "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
-          "requires": {
-            "lru-cache": "^6.0.0"
-          }
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
        },
        "uuid": {
          "version": "8.3.2",
          "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
          "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
-        },
-        "yallist": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
        }
      }
    },
    "sequelize-pool": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-6.1.0.tgz",
-      "integrity": "sha512-4YwEw3ZgK/tY/so+GfnSgXkdwIJJ1I32uZJztIEgZeAO6HMgj64OzySbWLgxj+tXhZCJnzRfkY9gINw8Ft8ZMg=="
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-7.1.0.tgz",
+      "integrity": "sha512-G9c0qlIWQSK29pR/5U2JF5dDQeqqHRragoyahj/Nx4KOOQ3CPPfzxnfqFPCSB7x5UgjOgnZ61nSxz+fjDpRlJg=="
    },
    "serve-static": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",
-      "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==",
+      "version": "1.14.2",
+      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz",
+      "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==",
      "requires": {
        "encodeurl": "~1.0.2",
        "escape-html": "~1.0.3",
        "parseurl": "~1.3.3",
-        "send": "0.17.1"
+        "send": "0.17.2"
      }
    },
    "set-value": {
@@ -2618,9 +2650,9 @@
      }
    },
    "setprototypeof": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
-      "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw=="
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
+      "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
    },
    "shebang-command": {
      "version": "1.2.0",
@@ -2842,24 +2874,9 @@
      }
    },
    "split2": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz",
-      "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==",
-      "requires": {
-        "readable-stream": "^3.0.0"
-      },
-      "dependencies": {
-        "readable-stream": {
-          "version": "3.6.0",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
-          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
-          "requires": {
-            "inherits": "^2.0.3",
-            "string_decoder": "^1.1.1",
-            "util-deprecate": "^1.0.1"
-          }
-        }
-      }
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz",
+      "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ=="
    },
    "stack-trace": {
      "version": "0.0.10",
@@ -3072,9 +3089,9 @@
      }
    },
    "toidentifier": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
-      "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw=="
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
+      "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="
    },
    "toposort-class": {
      "version": "1.0.1",
@@ -3323,19 +3340,20 @@
      }
    },
    "winston": {
-      "version": "3.3.3",
-      "resolved": "https://registry.npmjs.org/winston/-/winston-3.3.3.tgz",
-      "integrity": "sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw==",
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/winston/-/winston-3.6.0.tgz",
+      "integrity": "sha512-9j8T75p+bcN6D00sF/zjFVmPp+t8KMPB1MzbbzYjeN9VWxdsYnTB40TkbNUEXAmILEfChMvAMgidlX64OG3p6w==",
      "requires": {
        "@dabh/diagnostics": "^2.0.2",
-        "async": "^3.1.0",
+        "async": "^3.2.3",
        "is-stream": "^2.0.0",
-        "logform": "^2.2.0",
+        "logform": "^2.4.0",
        "one-time": "^1.0.0",
        "readable-stream": "^3.4.0",
+        "safe-stable-stringify": "^2.3.1",
        "stack-trace": "0.0.x",
        "triple-beam": "^1.3.0",
-        "winston-transport": "^4.4.0"
+        "winston-transport": "^4.5.0"
      },
      "dependencies": {
        "is-stream": {
@@ -3356,12 +3374,25 @@
      }
    },
    "winston-transport": {
-      "version": "4.4.0",
-      "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz",
-      "integrity": "sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw==",
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.5.0.tgz",
+      "integrity": "sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q==",
      "requires": {
-        "readable-stream": "^2.3.7",
-        "triple-beam": "^1.2.0"
+        "logform": "^2.3.2",
+        "readable-stream": "^3.6.0",
+        "triple-beam": "^1.3.0"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "3.6.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
+          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
+          "requires": {
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
+          }
+        }
      }
    },
    "wkx": {
@@ -8,27 +8,27 @@
  },
  "dependencies": {
    "@sentry/node": "^5.30.0",
-    "@sentry/tracing": "^6.11.0",
-    "async": "^3.2.1",
-    "async-lock": "^1.3.0",
-    "axios": "^0.21.2",
+    "@sentry/tracing": "^6.18.2",
+    "async": "^3.2.3",
+    "async-lock": "^1.3.1",
+    "axios": "^0.21.4",
    "bluebird": "^3.7.2",
    "cors": "^2.8.5",
    "cron": "^1.8.2",
-    "express": "^4.17.1",
+    "express": "^4.17.3",
    "express-async-errors": "^3.1.1",
    "lodash": "^4.17.21",
    "lru-cache": "^5.1.1",
    "moment": "^2.29.1",
-    "moment-timezone": "^0.5.33",
+    "moment-timezone": "^0.5.34",
    "morgan": "^1.10.0",
-    "pg": "^8.7.1",
+    "pg": "^8.7.3",
    "redis": "^3.1.2",
-    "sequelize": "^6.6.5",
+    "sequelize": "^6.17.0",
    "umzug": "^2.3.0",
    "unfuck-utf8-headers-middleware": "^1.0.1",
    "uuid": "^3.4.0",
-    "winston": "^3.3.3"
+    "winston": "^3.6.0"
  },
  "devDependencies": {
    "nodemon": "^1.19.4"
@@ -16,7 +16,7 @@ class DbConnection extends EventEmitter {
      pool: {
        max: 25,
        min: 0,
-        acquire: 10000,
+        acquire: 30000,
        idle: 300000000,
      },
      logging: false,
@@ -0,0 +1,6 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('thesis_courses')
+  },
+  down: async () => {},
+}
@@ -1,19 +1,10 @@
 const { CronJob } = require('cron')
 const moment = require('moment')
-const {
-  refreshProtoC,
-  refreshStatus,
-  refreshStatusGraduated,
-  refreshUber,
-  refreshProtoCProgramme,
-  getStartYears,
-} = require('./services/trends')
+const { refreshStatus, refreshStatusGraduated, refreshUber, getStartYears } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
-const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studyprogramme')
 const { updateBasicView, updateStudytrackView } = require('./services/studyprogrammeUpdates')
 const { findAndSaveTeachers } = require('./services/topteachers')
-const { setProductivity, setThroughput, patchProductivity, patchThroughput } = require('./services/analyticsService')
 const { isProduction } = require('./conf-backend')
 const { getCurrentSemester } = require('./services/semesters')
 const logger = require('./util/logger')
@@ -25,52 +16,11 @@ const refreshStudyrightAssociations = async () => {
  logger.info('Studyright associations refreshed!')
 }

-const refreshOverview = async () => {
-  logger.info('Refreshing throughput and productivity for programmes...')
-  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH') || code.includes('MH'))
-  let ready = 0
-  for (const code of codes) {
-    let programmeStatsSince = new Date('2017-07-31')
-    if (code.includes('MH') || code.includes('KH')) {
-      programmeStatsSince = new Date('2017-07-31')
-    } else {
-      programmeStatsSince = new Date('2000-07-31')
-    }
-    try {
-      await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-      const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
-      await setThroughput(data)
-    } catch (e) {
-      logger.error({ message: `failed to recalculate throughput for ${code}`, meta: e })
-      try {
-        await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-      } catch (e) {
-        logger.error({ message: 'failed to update throughtput status to error', meta: e })
-      }
-    }
-    try {
-      await patchProductivity({ [code]: { status: 'RECALCULATING' } })
-      const data = await productivityStatsForStudytrack(code, programmeStatsSince)
-      await setProductivity(data)
-    } catch (e) {
-      logger.error({ message: `failed to recalculate productivity for ${code}`, meta: e })
-      try {
-        await patchProductivity({
-          [code]: { status: 'RECALCULATION ERRORED' },
-        })
-      } catch (e) {
-        logger.error({ message: 'failed to update productivity status to error', meta: e })
-      }
-    }
-    ready += 1
-    logger.info(`${ready}/${codes.length} programmes done`)
-  }
-  logger.info('Throughput and productivity for programmes refreshed')
-}
-
 const refreshNewOverviews = async () => {
  logger.info('Refreshing studyprogramme and studytrack overview statistics for all programmes')
-  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH') || code.includes('MH'))
+  const codes = (await getAllProgrammes())
+    .map(p => p.code)
+    .filter(code => code.includes('KH') || code.includes('MH') || /^(T)[0-9]{6}$/.test(code))
  let ready = 0
  for (const code of codes) {
    try {
@@ -91,19 +41,6 @@ const refreshTeacherLeaderboard = async () => {
  await findAndSaveTeachers(currentSemestersYearCode - 1, currentSemestersYearCode)
 }

-const refreshProtoCtoRedis = async () => {
-  logger.info('Refreshing CDS ProtoC')
-  const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false' }
-  const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false' }
-  const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true' }
-  const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true' }
-  await refreshProtoC(defaultQuery)
-  await refreshProtoC(onlyOld)
-  await refreshProtoC(onlyEnr)
-  await refreshProtoC(bothToggles)
-  logger.info('Refreshing CDS ProtoC doned')
-}
-
 const refreshStatusToRedis = async () => {
  const unixMillis = moment().valueOf()
  const date = new Date(Number(unixMillis))
@@ -140,32 +77,8 @@ const refreshUberToRedis = async () => {
  logger.info(`Refreshing CDS Uber data doned`)
 }

-const refreshProtoCProgrammeToRedis = async () => {
-  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH'))
-  for (const code of codes) {
-    logger.info(`Refreshing ProtoCProgramme code ${code}`)
-    const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
-    const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
-    const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
-    const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true', code }
-    const queries = [defaultQuery, onlyOld, onlyEnr, bothToggles]
-    for (const query of queries) {
-      try {
-        await refreshProtoCProgramme(query)
-      } catch (e) {
-        logger.error({
-          message: `Error when refreshing ProtocProgramme code ${code}, query ${Object.keys({ query })[0]}`,
-          meta: e,
-        }) // object.keys is hax to print query variable name
-      }
-    }
-    logger.info(`Refreshing ProtoCProgramme code ${code} doned`)
-  }
-  logger.info(`Refreshing protocprogramme doned`)
-}
-
 const refreshStatistics = async () => {
-  const statfuncs = [refreshStudyrightAssociations, refreshOverview, refreshTeacherLeaderboard]
+  const statfuncs = [refreshStudyrightAssociations, refreshTeacherLeaderboard]
  logger.info('Refreshing statistics')
  for (const func of statfuncs) {
    await func()
@@ -174,7 +87,7 @@ const refreshStatistics = async () => {
 }

 const refreshTrends = async () => {
-  const trendfuncs = [refreshProtoCtoRedis, refreshStatusToRedis, refreshUberToRedis, refreshProtoCProgrammeToRedis]
+  const trendfuncs = [refreshStatusToRedis, refreshUberToRedis]
  logger.info('Refreshing trends')
  for (const func of trendfuncs) {
    await func()
@@ -50,6 +50,10 @@ Course.init(
    course_unit_type: {
      type: STRING,
    },
+    mainCourseCode: {
+      field: 'main_course_code',
+      type: STRING,
+    },
  },
  {
    underscored: false,
@@ -0,0 +1,69 @@
+const { Model, DATE, INTEGER, STRING, BOOLEAN } = require('sequelize')
+const { dbConnections } = require('../database/connection')
+
+class Enrollment extends Model {}
+
+Enrollment.init(
+  {
+    id: {
+      primaryKey: true,
+      type: STRING,
+    },
+    studentnumber: {
+      type: STRING,
+      references: {
+        model: 'student',
+        key: 'studentnumber',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+    },
+    course_code: {
+      type: STRING,
+    },
+    state: {
+      type: STRING,
+    },
+    enrollment_date_time: {
+      type: DATE,
+    },
+    course_id: {
+      type: STRING,
+      references: {
+        model: 'course',
+        key: 'id',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+    },
+    semester_composite: {
+      type: STRING,
+      references: {
+        model: 'semesters',
+        key: 'composite',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+    },
+    semestercode: {
+      type: INTEGER,
+    },
+    createdAt: {
+      type: DATE,
+    },
+    updatedAt: {
+      type: DATE,
+    },
+    is_open: {
+      type: BOOLEAN,
+    },
+  },
+  {
+    underscored: false,
+    sequelize: dbConnections.sequelize,
+    modelName: 'enrollment',
+    tableName: 'enrollment',
+  }
+)
+
+module.exports = Enrollment
@@ -10,6 +10,7 @@ const CreditType = require('./creditType')
 const Credit = require('./credit')
 const CreditTeacher = require('./creditTeacher')
 const ElementDetail = require('./elementDetail')
+const Studyplan = require('./studyplan')
 const StudyrightExtent = require('./studyrightExtent')
 const Studyright = require('./studyright')
 const StudyrightElement = require('./studyrightElement')
@@ -17,6 +18,7 @@ const Transfer = require('./transfer')
 const ProgrammeModule = require('./programmeModule')
 const ProgrammeModuleChild = require('./programmeModuleChild')
 const ExcludedCourse = require('./excludedCourse')
+const Enrollment = require('./enrollment')

 CourseType.hasMany(Course, { foreignKey: 'coursetypecode', sourceKey: 'coursetypecode' })
 Course.belongsTo(CourseType, { foreignKey: 'coursetypecode', targetKey: 'coursetypecode' })
@@ -62,6 +64,9 @@ Teacher.belongsToMany(Credit, { through: CreditTeacher, foreignKey: 'teacher_id'
 Organization.hasMany(Studyright, { foreignKey: 'facultyCode', sourceKey: 'code' })
 Studyright.belongsTo(Organization, { foreignKey: 'facultyCode', targetKey: 'code' })

+Studyplan.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
+Student.hasMany(Studyplan, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
+
 Studyright.belongsTo(Student, { foreignKey: 'studentStudentnumber', targetKey: 'studentnumber' })
 Student.hasMany(Studyright, { foreignKey: 'studentStudentnumber', sourceKey: 'studentnumber' })

@@ -88,6 +93,12 @@ Studyright.hasMany(Transfer, { foreignKey: 'studyrightid', sourceKey: 'studyrigh
 Transfer.belongsTo(ElementDetail, { as: 'source', foreignKey: 'sourcecode' })
 Transfer.belongsTo(ElementDetail, { as: 'target', foreignKey: 'targetcode' })

+Enrollment.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
+Student.hasMany(Enrollment, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
+Enrollment.belongsTo(Course, { foreignKey: 'course_id' })
+Course.hasMany(Enrollment, { foreignKey: 'course_id' })
+Enrollment.belongsTo(Semester, { foreignKey: { name: 'semester_composite', allowNull: false } })
+
 ProgrammeModule.belongsToMany(ProgrammeModule, {
  as: 'parents',
  through: ProgrammeModuleChild,
@@ -114,6 +125,7 @@ module.exports = {
  Credit,
  CreditTeacher,
  ElementDetail,
+  Studyplan,
  StudyrightExtent,
  Studyright,
  StudyrightElement,
@@ -121,4 +133,5 @@ module.exports = {
  ProgrammeModule,
  ProgrammeModuleChild,
  ExcludedCourse,
+  Enrollment,
 }
@@ -104,20 +104,6 @@ const MandatoryCourseLabels = sequelizeKone.define('mandatory_course_labels', {
  },
 })

-const ThesisCourse = sequelizeKone.define('thesis_courses', {
-  programmeCode: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  courseCode: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  thesisType: {
-    type: Sequelize.ENUM([ThesisTypeEnums.BACHELOR, ThesisTypeEnums.MASTER]),
-  },
-})
-
 const CustomPopulationSearch = sequelizeKone.define('custom_population_searches', {
  id: {
    primaryKey: true,
@@ -173,7 +159,6 @@ module.exports = {
  MandatoryCourse,
  TagStudent,
  Tag,
-  ThesisCourse,
  ThesisTypeEnums,
  MandatoryCourseLabels,
  CustomPopulationSearch,
@@ -0,0 +1,48 @@
+const { Model, DATE, ARRAY, STRING, INTEGER } = require('sequelize')
+const { dbConnections } = require('../database/connection')
+
+class Studyplan extends Model {}
+
+Studyplan.init(
+  {
+    id: {
+      primaryKey: true,
+      type: STRING,
+      allowNull: false,
+    },
+    studentnumber: {
+      type: STRING,
+      references: {
+        model: 'student',
+        key: 'studentnumber',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+      allowNull: false,
+    },
+    programme_code: {
+      type: STRING,
+      allowNull: false,
+    },
+    included_courses: {
+      type: ARRAY(STRING),
+    },
+    completed_credits: {
+      type: INTEGER,
+    },
+    createdAt: {
+      type: DATE,
+    },
+    updatedAt: {
+      type: DATE,
+    },
+  },
+  {
+    underscored: false,
+    sequelize: dbConnections.sequelize,
+    modelName: 'studyplan',
+    tableName: 'studyplan',
+  }
+)
+
+module.exports = Studyplan
@@ -9,9 +9,6 @@ Studyright.init(
      primaryKey: true,
      type: STRING,
    },
-    canceldate: {
-      type: DATE,
-    },
    startdate: {
      type: DATE,
    },
@@ -27,6 +24,9 @@ Studyright.init(
    graduated: {
      type: INTEGER,
    },
+    active: {
+      type: INTEGER,
+    },
    // irtisanomisperuste
    studentStudentnumber: {
      type: STRING,
@@ -1,41 +1,69 @@
 /* eslint no-console: 0 */

-// script needs rapo certs, so you need to run this in production pannu
-//
-// possible workflows:
-// - go to oodikone pannu, then "docker exec -it backend sh", then run this script with "npm
-// run rapodiff". Modify with vim / nano until you're happy, then copy changes to local version and commit.
-// - or bind mount a file to backend container in pannu, overriding this file. Modify that file
-// inside or outside container, however you want. Keep file in pannu, or put it to pannu's git repo (oodikone-server-setup).
+/*
+  depends on nodeproxy running in importer.cs.helsinki.fi, and env RAPO_NODEPROXY (see importer for a proper value)
+  needs also acces to importer db api, uses env IMPORTER_API
+
+  known differences in APIs
+    - rapo does not show students that have graduated (it however takes some days for rapo to notice a graduation)
+    - rapo does not handle the starting year of transfers properly (shows in old staring population)
+    - rapo show some students incorrectly passive, see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3521
+*/

 const axios = require('axios').default
-const fs = require('fs')
-const https = require('https')
+
 const _ = require('lodash')

 const { optimizedStatisticsOf } = require('./services/populations')
+const { Student, Transfer, StudyrightElement, Studyright } = require('./models')

-const { RAPO_CERT_PATH, RAPO_KEY_PATH, RAPO_API_KEY } = process.env
+const token = process.env.RAPO_NODEPROXY

-const agent = new https.Agent({
-  cert: fs.readFileSync(RAPO_CERT_PATH, 'utf8'),
-  key: fs.readFileSync(RAPO_KEY_PATH, 'utf8'),
-})
+const getFromRapo = async (urlStart, urlEnd) => {
+  const { data: rapoStuffLasna } = await axios.get(urlStart)
+  const { data: rapoStuffLasnaMinus } = await axios.get(urlEnd)

-const api = axios.create({
-  baseURL: 'https://gw.api.helsinki.fi/secure/rapo/report/',
-  headers: { 'X-Api-Key': RAPO_API_KEY },
-  httpsAgent: agent,
-})
+  const rapoStudents = _.difference(
+    rapoStuffLasna.map(s => s.opiskelijanumero),
+    rapoStuffLasnaMinus.map(s => s.opiskelijanumero)
+  )

-const main = async () => {
-  const facultyCode = 'H50'
-  const programme = 'KH50_005'
-  const studyrightStartDate = '2020-08-01'
-  const presence = 'LÄSNÄOLEVA'
+  return rapoStudents
+}

-  const months = 12 // some amount of months, dunno
-  const year = 2020
+const known_problems_in_oodikone = {
+  KH30_001: ['013896191'],
+  KH40_003: ['013200457'],
+  KH55_001: ['013195834', '015119708', '015138336'],
+}
+
+const known_problems_in_rapo = {
+  KH55_001: ['015160142', '015135575', '015119708', '015138336'],
+}
+
+const getFromOodikone = async (programme, year) => {
+  const isNotAmongKnownProblems = student => {
+    const hasProblems = known_problems_in_oodikone[programme] ? known_problems_in_oodikone[programme] : []
+    return !hasProblems.includes(student.studentNumber)
+  }
+
+  const theStudyrights = student => {
+    const theOne = s => s.studyright_elements.find(e => e.code === programme)
+    // in some rare cases student may have more than one studyright for same programme
+    return student.studyrights.filter(theOne)
+  }
+
+  const notGraduated = student => {
+    const graduation = theStudyrights(student).filter(r => r.graduated === 1)
+    return graduation.length === 0
+  }
+
+  const notTransferred = student => {
+    const transfer = student.transfers.find(t => t.targetcode === programme)
+    return transfer === undefined
+  }
+
+  const months = 12

  const oodikoneQuery = {
    semesters: ['FALL', 'SPRING'],
@@ -44,21 +72,292 @@ const main = async () => {
    year,
  }

-  const oodikoneStuff = await optimizedStatisticsOf(oodikoneQuery)
-  const oodikoneStudents = oodikoneStuff.students.map(s => s.studentNumber)
+  const oodikoneStuff = (await optimizedStatisticsOf(oodikoneQuery)).students

-  const rapoUrl = encodeURI(
-    `studyrights?faculty=${facultyCode}&presence=${presence}&education=${programme}&studyrightStartDate=${studyrightStartDate}`
+  return oodikoneStuff
+    .filter(notGraduated)
+    .filter(notTransferred)
+    .filter(isNotAmongKnownProblems)
+    .map(s => s.studentNumber)
+}
+
+const isAcuallyActive = async student => {
+  const currentSemester = 144
+  const tokenImporterApi = process.env.IMPORTER_API
+
+  const url = `https://importer.cs.helsinki.fi/api/importer/students/${student}/rapo_semester_enrollments?token=${tokenImporterApi}`
+  const { data } = await axios.get(url)
+  const terms = data.reduce((all, s) => all.concat(s.terms), []).filter(t => t.semester_code === currentSemester)
+  const active = terms.filter(t => t.type !== 'NEGLECTED')
+  const negl = terms.filter(t => t.type === 'NEGLECTED')
+  return active.length > 0 && negl.length > 0
+}
+
+const hasTransferred = async (student, programme) => {
+  const data = await Student.findByPk(student, {
+    include: [
+      {
+        model: Transfer,
+      },
+    ],
+  })
+
+  const transferTo = data.transfers.find(t => t.targetcode === programme)
+  return transferTo !== undefined
+}
+
+const graduatedVeryRecently = async (student, programme) => {
+  const data = await Student.findByPk(student, {
+    include: [
+      {
+        model: Studyright,
+        include: [
+          {
+            model: StudyrightElement,
+            required: true,
+            where: {
+              code: programme,
+            },
+          },
+        ],
+        where: {
+          graduated: 1,
+        },
+      },
+    ],
+  })
+
+  if (!data || data.studyrights.length === 0) {
+    return false
+  }
+
+  const graduated = new Date(data.studyrights[0].enddate)
+  const weekAgo = new Date(new Date().getTime() - 7 * 24 * 60 * 60 * 1000)
+  return graduated > weekAgo
+}
+
+const programme_diff_year = async (programme, facultyCode, year, brief = false) => {
+  const oodikoneStudents = await getFromOodikone(programme, year)
+
+  const studyrightStartDate = `${year}-08-01`
+  const studyrightEndDate = `${year + 1}-08-01`
+
+  const rapoUrlStart = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+    )
+  const rapoUrlEnd = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+    )
+
+  const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
+  const rapoStudentsPoissa = await getFromRapo(rapoUrlStart('Poissa'), rapoUrlEnd('Poissa'))
+  const rapoStudentPoissaLaki = await getFromRapo(
+    rapoUrlStart('Poissa ei kuluta opintoaikaa'),
+    rapoUrlEnd('Poissa ei kuluta opintoaikaa')
  )

-  const { data: rapoStuff } = await api.get(rapoUrl)
-  const rapoStudents = rapoStuff.map(s => s.opiskelijanumero)
+  const isNotAmongKnownProblems = student => {
+    const hasProblems = known_problems_in_rapo[programme] ? known_problems_in_rapo[programme] : []
+    return !hasProblems.includes(student.studentNumber)
+  }
+
+  const rapoStudents = rapoStudentsLasna
+    .concat(rapoStudentsPoissa)
+    .concat(rapoStudentPoissaLaki)
+    .filter(isNotAmongKnownProblems)

  const onlyInOodikone = _.difference(oodikoneStudents, rapoStudents)
  const onlyInRapo = _.difference(rapoStudents, oodikoneStudents)

-  console.log('Only in Oodikone:', onlyInOodikone)
-  console.log('Only in Rapo:', onlyInRapo)
+  const wrongMissingInRapo = []
+  for (let student of onlyInOodikone) {
+    const status = await isAcuallyActive(student)
+    if (status) {
+      wrongMissingInRapo.push(student)
+    }
+  }
+
+  // these are in wrong RAPO-population
+  const transferred = []
+  for (let student of onlyInRapo) {
+    const isTransfer = await hasTransferred(student, programme)
+    if (isTransfer) {
+      transferred.push(student)
+    } else {
+      const isRecentGraduation = await graduatedVeryRecently(student, programme)
+      if (isRecentGraduation) {
+        transferred.push(student)
+      }
+    }
+  }
+
+  const checkOodikoneOnly = _.difference(onlyInOodikone, wrongMissingInRapo)
+  const checkRapoOnly = _.difference(onlyInRapo, transferred)
+
+  if (!brief) {
+    console.log('vuosi', year)
+    console.log('')
+    console.log('oodikone', oodikoneStudents.length, ' transferred', transferred.length)
+    console.log('rapo', rapoStudents.length, ' activeness wrong rapo', wrongMissingInRapo.length)
+    console.log('only rapo', checkRapoOnly.length)
+    console.log('only oodikone', checkOodikoneOnly.length)
+
+    console.log('')
+
+    if (checkRapoOnly.length > 0) {
+      console.log('Only in Rapo:')
+      console.log(checkRapoOnly.join('\n'))
+    }
+
+    if (checkOodikoneOnly.length > 0) {
+      console.log('Only in Oodikone:')
+      console.log(checkOodikoneOnly.join('\n'))
+    }
+  }
+
+  if (checkRapoOnly.length + checkOodikoneOnly.length) {
+    console.log('******************************', year)
+    if (checkRapoOnly.length > 0) {
+      console.log('Only in Rapo:')
+      console.log(checkRapoOnly.join('\n'))
+    }
+
+    if (checkOodikoneOnly.length > 0) {
+      console.log('Only in Oodikone:')
+      console.log(checkOodikoneOnly.join('\n'))
+    }
+  }
+
+  if (!brief) {
+    console.log('')
+  }
+}
+
+const programme_diff = async (programme, faculty, brief) => {
+  console.log()
+  console.log(programme)
+  console.log('--------', '\n')
+
+  await programme_diff_year(programme, faculty, 2017, brief)
+  await programme_diff_year(programme, faculty, 2018, brief)
+  await programme_diff_year(programme, faculty, 2019, brief)
+  await programme_diff_year(programme, faculty, 2020, brief)
+  await programme_diff_year(programme, faculty, 2021, brief)
+}
+
+const faculty_diff = async (faculty, programmes, brief) => {
+  console.log('\n')
+  console.log(faculty)
+  console.log('==')
+  for (let programme of programmes) {
+    await programme_diff(programme, faculty, brief)
+  }
+}
+
+// eslint-disable-next-line no-unused-vars
+const msc_programme_diff_year = async (programme, facultyCode, year, brief = false) => {
+  const oodikoneStudents = await getFromOodikone(programme, year)
+
+  const studyrightStartDate = `${year}-08-01`
+  const studyrightEndDate = `${year + 1}-08-01`
+
+  const rapoUrlStart = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+    )
+  const rapoUrlEnd = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+    )
+
+  const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
+  const rapoStudentsPoissa = await getFromRapo(rapoUrlStart('Poissa'), rapoUrlEnd('Poissa'))
+  const rapoStudentPoissaLaki = await getFromRapo(
+    rapoUrlStart('Poissa ei kuluta opintoaikaa'),
+    rapoUrlEnd('Poissa ei kuluta opintoaikaa')
+  )
+
+  const rapoStudents = rapoStudentsLasna.concat(rapoStudentsPoissa).concat(rapoStudentPoissaLaki)
+
+  const onlyInOodikone = _.difference(oodikoneStudents, rapoStudents)
+  const onlyInRapo = _.difference(rapoStudents, oodikoneStudents)
+
+  const wrongInRapo = []
+  for (let student of onlyInOodikone) {
+    const status = await isAcuallyActive(student)
+    if (status) {
+      //wrongInRapo.push(student)
+    }
+  }
+
+  // in wrong RAPO population
+  const transferred = []
+  for (let student of onlyInRapo) {
+    const status = await hasTransferred(student, programme)
+    if (status) {
+      transferred.push(student)
+    }
+  }
+
+  const checkOodikoneOnly = _.difference(onlyInOodikone, wrongInRapo)
+  const checkRapoOnly = _.difference(onlyInRapo, transferred)
+  const both = _.intersection(oodikoneStudents, rapoStudents)
+
+  console.log('vuosi', year)
+
+  if (!brief) {
+    console.log('')
+    console.log('oodikone', oodikoneStudents.length, ' transferred', transferred.length)
+    console.log('rapo', rapoStudents.length, ' activeness wrong rapo', wrongInRapo.length)
+    console.log('both', both.length)
+    console.log('only rapo', checkRapoOnly.length)
+    console.log('only oodikone', checkOodikoneOnly.length)
+
+    console.log('')
+
+    if (checkRapoOnly.length > 0) {
+      console.log('Only in Rapo:')
+      console.log(checkRapoOnly.join('\n'))
+    }
+
+    if (checkOodikoneOnly.length > 0) {
+      console.log('Only in Oodikone:')
+      console.log(checkOodikoneOnly.join('\n'))
+    }
+  }
+
+  if (checkRapoOnly.length + checkOodikoneOnly.length) {
+    console.log('******************************************************')
+  }
+
+  if (!brief) {
+    console.log('')
+  }
+}
+
+const bsc_priogrammes_of_faculties = {
+  H10: ['KH10_001'],
+  H20: ['KH20_001'],
+  H30: ['KH30_001', 'KH30_002'],
+  H40: ['KH40_001', 'KH40_002', 'KH40_003', 'KH40_004', 'KH40_005', 'KH40_006'],
+  H50: ['KH50_001', 'KH50_002', 'KH50_003', 'KH50_004', 'KH50_005', 'KH50_006', 'KH50_007', 'KH50_008'],
+  H55: ['KH55_001'],
+  H60: ['KH60_001'],
+  H70: ['KH70_001', 'KH70_002', 'KH70_003', 'KH70_004'],
+  H74: ['KH74_001'],
+  H80: ['KH80_001', 'KH80_002', 'KH80_003', 'KH80_004'],
+  H90: ['KH90_001'],
+}
+
+const main = async () => {
+  for (let faculty of Object.keys(bsc_priogrammes_of_faculties)) {
+    await faculty_diff(faculty, bsc_priogrammes_of_faculties[faculty], true)
+  }
+
+  //await programme_diff_year('KH55_001', 'H55', 2021, true)
+  //await programme_diff('KH60_001', 'H60', true)
 }

 main().then(() => process.exit(0))
@@ -1,23 +1,6 @@
 const router = require('express').Router()
 const MandatoryCourses = require('../services/mandatoryCourses')
-const {
-  productivityStatsForStudytrack,
-  throughputStatsForStudytrack,
-  optionData,
-} = require('../services/studyprogramme')
-const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
-const {
-  getProductivity,
-  setProductivity,
-  getThroughput,
-  setThroughput,
-  patchProductivity,
-  patchThroughput,
-} = require('../services/analyticsService')
 const { getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
-const logger = require('../util/logger')
-
-const programmeStatsSince = new Date('2017-07-31')

 router.get('/elementdetails/all', async (req, res) => {
  const elementdetails = await getAllElementDetails()
@@ -29,22 +12,6 @@ router.get('/studyprogrammes', async (req, res) => {
  res.json(studyrights)
 })

-router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
-  const code = req.params.id
-
-  let level
-  if (code.includes('MH')) {
-    level = 'MSC'
-  } else if (code.includes('KH')) {
-    level = 'BSC'
-  } else {
-    return res.json([])
-  }
-
-  const data = await optionData(new Date('2017-07-31'), new Date(), code, level)
-  res.json(data)
-})
-
 router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
  if (req.params.id) {
    const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
@@ -54,138 +21,4 @@ router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
  }
 })

-router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    let data = null
-    try {
-      data = await getProductivity(code)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} productivity`)
-    }
-    if (!data) {
-      try {
-        const stats = await productivityStatsForStudytrack(code, programmeStatsSince)
-        data = await setProductivity(stats)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      }
-    }
-    return res.json(data)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-  res.status(200).end()
-
-  try {
-    if (code.includes('MH') || code.includes('KH')) {
-      const data = await productivityStatsForStudytrack(code, new Date('2017-07-31'))
-      await setProductivity(data)
-    } else {
-      const data = await productivityStatsForStudytrack(code, new Date('2000-07-31'))
-      await setProductivity(data)
-    }
-  } catch (e) {
-    try {
-      await patchProductivity({
-        [code]: { status: 'RECALCULATION ERRORED' },
-      })
-    } catch (e) {
-      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      return
-    }
-
-    logger.error(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-  }
-  logger.info(`Produdtiviy stats recalculation for studyprogramme ${code} done`)
-})
-
-router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    const thesisPromise = findProgrammeTheses(req.params.id)
-
-    let data = null
-    try {
-      data = await getThroughput(code)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} throughput`)
-    }
-    if (!data) {
-      try {
-        const result = await throughputStatsForStudytrack(req.params.id, programmeStatsSince.getFullYear())
-        data = await setThroughput(result)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      }
-    }
-    return res.json({ ...data, thesis: await thesisPromise })
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-  res.status(200).end()
-
-  try {
-    if (code.includes('MH') || code.includes('KH')) {
-      const data = await throughputStatsForStudytrack(code, 2017)
-      await setThroughput(data)
-    } else {
-      const data = await throughputStatsForStudytrack(code, 2000)
-      await setThroughput(data)
-    }
-  } catch (e) {
-    try {
-      await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-    } catch (e) {
-      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      return
-    }
-    logger.error(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
-  }
-  logger.info(`Throughput stats recalculation for studyprogramme ${code} done`)
-})
-
-router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {
-  const { id } = req.params
-  if (id) {
-    const thesis = await findProgrammeTheses(id)
-    res.json(thesis)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.post('/v2/studyprogrammes/:id/thesis', async (req, res) => {
-  const { id } = req.params
-  const { course, thesisType } = req.body
-  if (id && course && thesisType) {
-    const thesis = await createThesisCourse(id, course, thesisType)
-    res.status(201).json(thesis)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
-  const { id, course } = req.params
-  if (id && course) {
-    const deleted = await deleteThesisCourse(id, course)
-    res.status(204).json(deleted)
-  } else {
-    res.status(422).end()
-  }
-})
-
 module.exports = router
@@ -8,6 +8,8 @@ const StudyrightService = require('../services/studyrights')
 const TagService = require('../services/tags')
 const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
+const { mapToProviders } = require('../util/utils')
+
 const { ApplicationError } = require('../util/customErrors')

 const filterPersonalTags = (population, userId) => {
@@ -141,7 +143,7 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
 router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
  const {
    user: { isAdmin, studentsUserCanAccess },
-    body: { studentnumberlist: studentnumbersInReq },
+    body: { studentnumberlist: studentnumbersInReq, studentCountLimit, ...query },
  } = req

  if (!studentnumbersInReq) {
@@ -152,10 +154,11 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r

  const result = await Population.bottlenecksOf(
    {
-      year: 1900,
-      studyRights: [],
-      semesters: ['FALL', 'SPRING'],
-      months: 10000,
+      year: query?.year ?? 1900,
+      studyRights: query?.studyRights ?? [],
+      semesters: query?.semesters ?? ['FALL', 'SPRING'],
+      months: query?.months ?? 10000,
+      studentCountLimit,
    },
    studentnumberlist
  )
@@ -274,9 +277,9 @@ router.get('/v3/populationstatisticsbytag', async (req, res) => {
 })

 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
-  const { coursecodes, from, to, separate: sep } = req.query
+  const { coursecodes, from, to, separate: sep, unifyCourses } = req.query
  const {
-    user: { id, isAdmin, studentsUserCanAccess: allStudentsUserCanAccess },
+    user: { id, isAdmin, studentsUserCanAccess: allStudentsUserCanAccess, rights },
  } = req
  const separate = sep ? JSON.parse(sep) : false

@@ -293,7 +296,13 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
  }

  const semesters = ['FALL', 'SPRING']
-  const studentnumbers = await Student.findByCourseAndSemesters(JSON.parse(coursecodes), from, to, separate)
+  const studentnumbers = await Student.findByCourseAndSemesters(
+    JSON.parse(coursecodes),
+    from,
+    to,
+    separate,
+    unifyCourses
+  )
  const result = await Population.optimizedStatisticsOf(
    {
      // Useless, because studentnumbers are already filtered above by from & to.
@@ -305,10 +314,15 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
    },
    studentnumbers
  )
+  let courseproviders = []
+  if (!isAdmin) {
+    courseproviders = await CourseService.getCourseProvidersForCourses(JSON.parse(coursecodes))
+  }
+  const rightsMappedToProviders = mapToProviders(rights)
+  const found = courseproviders.some(r => rightsMappedToProviders.includes(r))

-  const studentsUserCanAccess = isAdmin
-    ? new Set(studentnumbers)
-    : new Set(_.intersection(studentnumbers, allStudentsUserCanAccess))
+  const studentsUserCanAccess =
+    isAdmin || found ? new Set(studentnumbers) : new Set(_.intersection(studentnumbers, allStudentsUserCanAccess))

  const randomHash = crypto.randomBytes(12).toString('hex')
  const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
@@ -381,8 +395,23 @@ router.get('/v3/populationstatistics/studyprogrammes/unfiltered', async (req, re

 router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
  const { courseCodes } = req.query
-  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(courseCodes))
-  return res.json(maxYearsToCreatePopulationFrom)
+  const maxYearsToCreatePopulationFromOpen = await CourseService.maxYearsToCreatePopulationFrom(
+    JSON.parse(courseCodes),
+    'openStats'
+  )
+  const maxYearsToCreatePopulationFromUni = await CourseService.maxYearsToCreatePopulationFrom(
+    JSON.parse(courseCodes),
+    'reqularStats'
+  )
+  const maxYearsToCreatePopulationFromBoth = await CourseService.maxYearsToCreatePopulationFrom(
+    JSON.parse(courseCodes),
+    'unifyStats'
+  )
+  return res.json({
+    openCourses: maxYearsToCreatePopulationFromOpen,
+    uniCourses: maxYearsToCreatePopulationFromUni,
+    unifyCourses: maxYearsToCreatePopulationFromBoth,
+  })
 })

 module.exports = router
@@ -1,10 +1,9 @@
 const router = require('express').Router()
-const {
-  getBasicStatsForStudytrack,
-  getCreditStatsForStudytrack,
-  getGraduationStatsForStudytrack,
-} = require('../services/studyprogrammeStats')
+const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics')
+const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
+const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
 const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
+const { getStudyprogrammeCoursesForStudytrack } = require('../services/studyprogrammeCourses')
 const {
  getBasicStats,
  setBasicStats,
@@ -21,11 +20,12 @@ const logger = require('../util/logger')
 router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
  const code = req.params.id
  const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups

  if (code) {
    let data = null
    try {
-      data = await getBasicStats(code, yearType)
+      data = await getBasicStats(code, yearType, specialGroups)
    } catch (e) {
      logger.error(`Failed to get code ${code} basic stats: ${e}`)
    }
@@ -33,9 +33,12 @@ router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
      try {
        const result = await getBasicStatsForStudytrack({
          studyprogramme: req.params.id,
-          yearType,
+          settings: {
+            isAcademicYear: yearType === 'ACADEMIC_YEAR',
+            includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
+          },
        })
-        data = await setBasicStats(result, yearType)
+        data = await setBasicStats(result, yearType, specialGroups)
      } catch (e) {
        logger.error(`Failed to update code ${code} basic stats: ${e}`)
      }
@@ -49,11 +52,13 @@ router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
 router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
  const code = req.params.id
  const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups

  if (code) {
    let data = null
    try {
-      data = await getCreditStats(code, yearType)
+      data = await getCreditStats(code, yearType, specialGroups)
+      data = null
    } catch (e) {
      logger.error(`Failed to get code ${code} credit stats`)
    }
@@ -61,9 +66,12 @@ router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
      try {
        const result = await getCreditStatsForStudytrack({
          studyprogramme: req.params.id,
-          yearType,
+          settings: {
+            isAcademicYear: yearType === 'ACADEMIC_YEAR',
+            includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
+          },
        })
-        data = await setCreditStats(result, yearType)
+        data = await setCreditStats(result, yearType, specialGroups)
      } catch (e) {
        logger.error(`Failed to update code ${code} credit stats`)
      }
@@ -77,11 +85,12 @@ router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
 router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
  const code = req.params.id
  const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups

  if (code) {
    let data = null
    try {
-      data = await getGraduationStats(code, yearType)
+      data = await getGraduationStats(code, yearType, specialGroups)
    } catch (e) {
      logger.error(`Failed to get code ${code} graduation stats`)
    }
@@ -89,9 +98,12 @@ router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
      try {
        const result = await getGraduationStatsForStudytrack({
          studyprogramme: req.params.id,
-          yearType,
+          settings: {
+            isAcademicYear: yearType === 'ACADEMIC_YEAR',
+            includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
+          },
        })
-        data = await setGraduationStats(result, yearType)
+        data = await setGraduationStats(result, yearType, specialGroups)
      } catch (e) {
        logger.error(`Failed to update code ${code} graduation stats`)
      }
@@ -102,20 +114,42 @@ router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
  }
 })

+router.get('/v2/studyprogrammes/:id/coursestats', async (req, res) => {
+  const code = req.params.id
+  const showByYear = req.query.academicyear
+  const date = new Date()
+  date.setHours(23, 59, 59, 999)
+
+  try {
+    const data = await getStudyprogrammeCoursesForStudytrack(date.getTime(), code, showByYear)
+    return res.json(data)
+  } catch (e) {
+    logger.error(`Failed to get code ${code} programme courses stats: ${e}`)
+  }
+})
+
 router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
  const code = req.params.id
+  const graduated = req.query?.graduated
+  const specialGroups = req.query?.special_groups

  if (code) {
    let data = null
    try {
-      data = await getStudytrackStats(code)
+      data = await getStudytrackStats(code, graduated, specialGroups)
    } catch (e) {
      logger.error(`Failed to get code ${code} studytrack stats: ${e}`)
    }
    if (!data) {
      try {
-        const result = await getStudytrackStatsForStudyprogramme({ studyprogramme: req.params.id })
-        data = await setStudytrackStats(result)
+        const result = await getStudytrackStatsForStudyprogramme({
+          studyprogramme: code,
+          settings: {
+            graduated: graduated === 'GRADUATED_INCLUDED',
+            specialGroups: specialGroups === 'SPECIAL_INCLUDED',
+          },
+        })
+        data = await setStudytrackStats(result, graduated, specialGroups)
      } catch (e) {
        logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
      }
@@ -1,183 +1,104 @@
 const { redisClient } = require('../services/redis')
 const moment = require('moment')

-const createRedisKeyForProductivity = id => `PRODUCTIVITY_${id}`
-const createRedisKeyForThroughput = id => `THROUGHPUT_${id}`
-const createRedisKeyForBasicStats = (id, yearType) => `BASIC_STATS_${id}_${yearType}`
-const createRedisKeyForCreditStats = (id, yearType) => `CREDIT_STATS_${id}_${yearType}`
-const createRedisKeyForGraduationStats = (id, yearType) => `GRADUATION_STATS_${id}_${yearType}`
-const createRedisKeyForStudytrackStats = id => `STUDYTRACK_STATS_${id}`
+// Only new bachelor, masters and doctoral programmes get their data updated in redis every night, use redis for them
+const isUpdatedNewProgramme = code => code.includes('KH') || code.includes('MH') || /^(T)[0-9]{6}$/.test(code)
+const createRedisKeyForBasicStats = (id, yearType, specialGroups) => `BASIC_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForCreditStats = (id, yearType, specialGroups) => `CREDIT_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForGraduationStats = (id, yearType, specialGroups) =>
+  `GRADUATION_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForStudytrackStats = (id, graduated, specialGroups) =>
+  `STUDYTRACK_STATS_${id}_${graduated}_${specialGroups}`

-const getProductivity = async id => {
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return {
-    [id]: JSON.parse(dataFromRedis),
-  }
-}
-
-const setProductivity = async data => {
-  const { id } = data
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: dataToRedis,
-  }
-}
-
-const patchProductivity = async data => {
-  const { id } = data
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataFromRedis = JSON.parse(await redisClient.getAsync(redisKey))
-  const patchedData = {
-    ...dataFromRedis,
-    ...data,
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(patchedData))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: patchedData,
-  }
-}
-
-const getThroughput = async id => {
-  const redisKey = createRedisKeyForThroughput(id)
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return {
-    [id]: JSON.parse(dataFromRedis),
-  }
-}
-
-const setThroughput = async data => {
-  const { id, data: dataToSet } = data
-  const redisKey = createRedisKeyForThroughput(id)
-  const dataToRedis = {
-    data: dataToSet.years,
-    totals: dataToSet.totals,
-    stTotals: dataToSet.stTotals,
-    status: 'DONE',
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: dataToRedis,
-  }
-}
-
-const patchThroughput = async data => {
-  const [id, dataToPatch] = Object.entries(data)[0]
-  const redisKey = createRedisKeyForThroughput(id)
-  const { status } = dataToPatch
-  const dataFromRedis = JSON.parse(await redisClient.getAsync(redisKey))
-  const patchedData = {
-    ...dataFromRedis,
-    status,
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(patchedData))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: patchedData,
-  }
-}
-
-const getBasicStats = async (id, yearType) => {
-  const redisKey = createRedisKeyForBasicStats(id, yearType)
+const getBasicStats = async (id, yearType, specialGroups) => {
+  if (!isUpdatedNewProgramme(id)) return null
+  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
  const dataFromRedis = await redisClient.getAsync(redisKey)
  if (!dataFromRedis) return null
  return JSON.parse(dataFromRedis)
 }

-const setBasicStats = async (data, yearType) => {
+const setBasicStats = async (data, yearType, specialGroups) => {
  const { id } = data
-  const redisKey = createRedisKeyForBasicStats(id, yearType)
+  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
  const dataToRedis = {
    ...data,
    status: 'DONE',
    lastUpdated: moment().format(),
  }
+  if (!isUpdatedNewProgramme(id)) return dataToRedis
  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
  if (setOperationStatus !== 'OK') return null
  return dataToRedis
 }

-const getCreditStats = async (id, yearType) => {
-  const redisKey = createRedisKeyForCreditStats(id, yearType)
+const getCreditStats = async (id, yearType, specialGroups) => {
+  if (!isUpdatedNewProgramme(id)) return null
+  const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
  const dataFromRedis = await redisClient.getAsync(redisKey)
  if (!dataFromRedis) return null
  return JSON.parse(dataFromRedis)
 }

-const setCreditStats = async (data, yearType) => {
+const setCreditStats = async (data, yearType, specialGroups) => {
  const { id } = data
-  const redisKey = createRedisKeyForCreditStats(id, yearType)
+  const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
  const dataToRedis = {
    ...data,
    status: 'DONE',
    lastUpdated: moment().format(),
  }
+  if (!isUpdatedNewProgramme(id)) return dataToRedis
  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
  if (setOperationStatus !== 'OK') return null
  return dataToRedis
 }

-const getGraduationStats = async (id, yearType) => {
-  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+const getGraduationStats = async (id, yearType, specialGroups) => {
+  if (!isUpdatedNewProgramme(id)) return null
+  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
  const dataFromRedis = await redisClient.getAsync(redisKey)
  if (!dataFromRedis) return null
  return JSON.parse(dataFromRedis)
 }

-const setGraduationStats = async (data, yearType) => {
+const setGraduationStats = async (data, yearType, specialGroups) => {
  const { id } = data
-  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
  const dataToRedis = {
    ...data,
    status: 'DONE',
    lastUpdated: moment().format(),
  }
+  if (!isUpdatedNewProgramme(id)) return dataToRedis
  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
  if (setOperationStatus !== 'OK') return null
  return dataToRedis
 }

-const getStudytrackStats = async id => {
-  const redisKey = createRedisKeyForStudytrackStats(id)
+const getStudytrackStats = async (id, graduated, specialGroups) => {
+  if (!isUpdatedNewProgramme(id)) return null
+  const redisKey = createRedisKeyForStudytrackStats(id, graduated, specialGroups)
  const dataFromRedis = await redisClient.getAsync(redisKey)
  if (!dataFromRedis) return null
  return JSON.parse(dataFromRedis)
 }

-const setStudytrackStats = async data => {
+const setStudytrackStats = async (data, graduated, specialGroups) => {
  const { id } = data
-  const redisKey = createRedisKeyForStudytrackStats(id)
+  const redisKey = createRedisKeyForStudytrackStats(id, graduated, specialGroups)
  const dataToRedis = {
    ...data,
    status: 'DONE',
    lastUpdated: moment().format(),
  }
+  if (!isUpdatedNewProgramme(id)) return dataToRedis
  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
  if (setOperationStatus !== 'OK') return null
  return dataToRedis
 }

 module.exports = {
-  getProductivity,
-  setProductivity,
-  patchProductivity,
-  getThroughput,
-  setThroughput,
-  patchThroughput,
  getBasicStats,
  setBasicStats,
  getCreditStats,
@@ -40,18 +40,22 @@ class CourseStatsCounter {
      triedOfPopulation: undefined,
      passingSemesters: this.initializePassingSemesters(),
    }
+    this.enrollments = {
+      enrolledNoGrade: new Set(),
+      semesters: this.initializePassingSemesters({}),
+    }
    this.grades = {}
  }

-  initializePassingSemesters() {
+  initializePassingSemesters(initialValue = 0) {
    const passingSemesters = {
-      BEFORE: 0,
-      LATER: 0,
+      BEFORE: initialValue,
+      LATER: initialValue,
    }

    for (let i = 0; i < 7; i++) {
-      passingSemesters[fall[i]] = 0
-      passingSemesters[spring[i]] = 0
+      passingSemesters[fall[i]] = initialValue
+      passingSemesters[spring[i]] = initialValue
    }

    return passingSemesters
@@ -91,6 +95,13 @@ class CourseStatsCounter {
    }
  }

+  markEnrollment(studentnumber, state, semester) {
+    if (!this.enrollments[state]) this.enrollments[state] = new Set()
+    this.enrollments[state].add(studentnumber)
+    if (!this.enrollments.semesters[semester][state]) this.enrollments.semesters[semester][state] = new Set()
+    this.enrollments.semesters[semester][state].add(studentnumber)
+  }
+
  failedBefore(studentnumber) {
    return this.students.failed[studentnumber] !== undefined
  }
@@ -173,6 +184,10 @@ class CourseStatsCounter {
  getFinalStats() {
    const stats = this.stats
    const students = this.students
+    const allStudents = Object.keys(students.all)
+    const totalEnrolledNoGrade = [...(this.enrollments.ENROLLED || [])].filter(
+      student => !allStudents.includes(student)
+    ).length

    stats.students = lengthOf(students.all)
    stats.passed = lengthOf(students.passed)
@@ -185,6 +200,9 @@ class CourseStatsCounter {
    stats.triedOfPopulation = percentageOf(stats.students, this.studentsInTotal)
    stats.perStudent = stats.attempts / (stats.passed + stats.failed)
    stats.passingSemestersCumulative = this.getPassingSemestersCumulative()
+    stats.totalStudents = stats.students + totalEnrolledNoGrade
+    stats.totalEnrolledNoGrade = totalEnrolledNoGrade
+    stats.percentageWithEnrollments = percentageOf(stats.passed, stats.totalStudents)

    return {
      stats,
@@ -8,12 +8,23 @@ class CourseYearlyStatsCounter {
  }

  initProgramme(code, name) {
-    this.programmes[code] = { name, students: {}, passed: {}, credits: {} }
+    this.programmes[code] = {
+      name,
+      students: {},
+      passed: {},
+      credits: {},
+    }
  }

  initFacultyYear(code) {
    const year = `${1949 + Number(code)}-${1950 + Number(code)}`
-    this.facultyStats[code] = { year, allStudents: [], allPassed: [], faculties: {}, allCredits: 0 }
+    this.facultyStats[code] = {
+      year,
+      allStudents: [],
+      allPassed: [],
+      faculties: {},
+      allCredits: 0,
+    }
  }

  initFaculty(yearcode, faculty_code, organization) {
@@ -43,8 +54,11 @@ class CourseYearlyStatsCounter {
          passedEventually: [],
          neverPassed: [],
        },
+        grades: {},
        studentnumbers: [],
      },
+      enrollments: [],
+      allEnrollments: [],
      yearcode,
    }
  }
@@ -97,11 +111,41 @@ class CourseYearlyStatsCounter {
    })
  }

+  markStudyProgrammesEnrollment(studentnumber, programmes, yearcode, state, enrollment_date_time) {
+    programmes.forEach(({ code, name, faculty_code, organization }) => {
+      this.markStudyProgrammeEnrollment(
+        code,
+        name,
+        studentnumber,
+        yearcode,
+        state,
+        enrollment_date_time,
+        faculty_code,
+        organization
+      )
+    })
+  }
+
  markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode) {
    if (!this.groups[groupcode]) {
      this.initGroup(groupcode, groupname, coursecode, yearcode)
    }
    this.markCreditToAttempts(studentnumber, passed, grade, groupcode)
+    this.markBestEffortGrade(studentnumber, passed, grade, groupcode)
+  }
+
+  markEnrollmentToGroup(studentnumber, state, enrollment_date_time, groupcode, groupname, coursecode, yearcode) {
+    if (!this.groups[groupcode]) this.initGroup(groupcode, groupname, coursecode, yearcode)
+
+    const enrollment = { studentnumber, state, enrollment_date_time }
+    this.groups[groupcode].allEnrollments.push(enrollment)
+    const oldEnrollment = this.groups[groupcode].enrollments.find(e => e.studentnumber === studentnumber)
+    if (!oldEnrollment) return this.groups[groupcode].enrollments.push({ studentnumber, state, enrollment_date_time })
+    if (oldEnrollment.state === 'ENROLLED' || oldEnrollment.state === 'CONFIRMED') return
+    if (state !== 'ENROLLED' || state !== 'CONFIRMED') return
+    this.groups[groupcode].enrollments = this.groups[groupcode].enrollments
+      .filter(e => e.studentnumber !== studentnumber)
+      .concat([enrollment])
  }

  markCreditToAttempts(studentnumber, passed, grade, groupcode) {
@@ -118,6 +162,19 @@ class CourseYearlyStatsCounter {
    }
  }

+  markBestEffortGrade(studentnumber, passed, grade, groupcode) {
+    const current = this.groups[groupcode].students.grades[studentnumber]
+
+    if (!current) {
+      this.groups[groupcode].students.grades[studentnumber] = { grade, passed }
+      return
+    }
+
+    if (current.passed && !passed) return
+    if (current.passed && Number(current.grade) >= Number(grade)) return
+    this.groups[groupcode].students.grades[studentnumber] = { grade, passed }
+  }
+
  markCreditToStudentCategories(studentnumber, passed, attainment_date, groupcode) {
    if (!this.students.has(studentnumber)) {
      if (passed) {
@@ -183,8 +240,17 @@ class CourseYearlyStatsCounter {
    }

    const groupStatistics = Object.values(this.groups).map(({ ...rest }) => {
+      const { students } = rest
+      const grades = {}
+      Object.keys(students.grades).forEach(student => {
+        const { grade, passed } = students.grades[student]
+        const parsed = passed ? grade : '0'
+        if (!grades[parsed]) grades[parsed] = []
+        grades[parsed].push(student)
+      })
      const normalStats = {
        ...rest,
+        students: { ...students, grades },
      }

      if (anonymizationSalt && normalStats.students.studentnumbers.length < 6) {
@@ -8,12 +8,14 @@ const {
  Studyright,
  Semester,
  Organization,
+  Enrollment,
 } = require('../models')

 const { parseCredit } = require('./parseCredits')
 const Op = Sequelize.Op
 const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
 const { sortMainCode, getSortRank } = require('../util/utils')
+const { parseEnrollment } = require('./parseEnrollments')

 const byNameOrCode = (searchTerm, language) =>
  Course.findAll({
@@ -59,10 +61,12 @@ const findOneByCode = code => {
  })
 }

-const creditsForCourses = async (codes, anonymizationSalt, unifyOpenUniCourses) => {
+const creditsForCourses = async (codes, anonymizationSalt, unification) => {
  let is_open = false

-  if (unifyOpenUniCourses) {
+  if (unification === 'open') is_open = true
+
+  if (unification === 'unify') {
    is_open = {
      [Op.in]: [false, true],
    }
@@ -104,6 +108,11 @@ const creditsForCourses = async (codes, anonymizationSalt, unifyOpenUniCourses)
      {
        model: Semester,
        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+        where: {
+          startdate: {
+            [Op.lte]: new Date(),
+          },
+        },
      },
    ],
    where: {
@@ -122,6 +131,75 @@ const creditsForCourses = async (codes, anonymizationSalt, unifyOpenUniCourses)
  return parsedCredits
 }

+const enrollmentsForCourses = async (codes, anonymizationSalt, unification) => {
+  let is_open = false
+
+  if (unification === 'open') is_open = true
+
+  if (unification === 'unify') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
+  const enrollments = await Enrollment.findAll({
+    include: [
+      {
+        model: Student,
+        attributes: ['studentnumber'],
+        include: {
+          model: StudyrightElement,
+          attributes: ['code', 'startdate'],
+          include: [
+            {
+              model: ElementDetail,
+              attributes: ['name', 'type'],
+              where: {
+                type: {
+                  [Op.eq]: 20,
+                },
+              },
+            },
+            {
+              model: Studyright,
+              attributes: ['prioritycode', 'faculty_code'],
+              where: {
+                prioritycode: {
+                  [Op.eq]: 1,
+                },
+              },
+              include: {
+                model: Organization,
+              },
+            },
+          ],
+        },
+      },
+      {
+        model: Semester,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+        where: {
+          startdate: {
+            [Op.lte]: new Date(),
+          },
+        },
+      },
+    ],
+    where: {
+      course_code: {
+        [Op.in]: codes,
+      },
+      studentnumber: {
+        [Op.ne]: null,
+      },
+      enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
+      state: ['ENROLLED', 'CONFIRMED'],
+      [Op.or]: [{ is_open }, { is_open: null }],
+    },
+  })
+  return enrollments.map(enrollment => parseEnrollment(enrollment, anonymizationSalt))
+}
+
 const bySearchTerm = async (term, language) => {
  const formatCourse = course => ({
    name: course.name[language],
@@ -162,23 +240,20 @@ const allCodeAlternatives = async code => {
  return sortMainCode([...course.substitutions, code])
 }

-const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anonymizationSalt) => {
+const yearlyStatsOfNew = async (coursecode, separate, unification, anonymizationSalt) => {
  const courseForSubs = await Course.findOne({
    where: { code: coursecode },
  })

  let codes = courseForSubs.substitutions ? sortMainCode([...courseForSubs.substitutions, coursecode]) : [coursecode]

-  if (isOpenUniCourseCode(coursecode) && !unifyOpenUniCourses) {
-    codes = [coursecode]
+  if (unification === 'reqular') {
+    codes = codes.filter(course => !isOpenUniCourseCode(course))
  }

-  if (!isOpenUniCourseCode(coursecode) && !unifyOpenUniCourses) {
-    codes = codes.filter(code => !code.match(/^[A][0-9]|^AY/))
-  }
-
-  const [credits, course] = await Promise.all([
-    creditsForCourses(codes, anonymizationSalt, unifyOpenUniCourses),
+  const [credits, enrollments, course] = await Promise.all([
+    creditsForCourses(codes, anonymizationSalt, unification),
+    enrollmentsForCourses(codes, anonymizationSalt, unification),
    Course.findOne({
      where: {
        code: coursecode,
@@ -233,7 +308,27 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
    counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode)
    counter.markCreditToStudentCategories(studentnumber, passed, attainment_date, groupcode)
  }
+
+  enrollments.forEach(enrollment => {
+    const { studentnumber, semestercode, semestername, yearcode, yearname, coursecode, state, enrollment_date_time } =
+      enrollment
+
+    const groupcode = separate ? semestercode : yearcode
+    const groupname = separate ? semestername : yearname
+
+    counter.markEnrollmentToGroup(
+      studentnumber,
+      state,
+      enrollment_date_time,
+      groupcode,
+      groupname,
+      coursecode,
+      yearcode
+    )
+  })
+
  const statistics = counter.getFinalStatistics(anonymizationSalt)
+
  return {
    ...statistics,
    coursecode,
@@ -242,7 +337,17 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
  }
 }

-const maxYearsToCreatePopulationFrom = async coursecodes => {
+const maxYearsToCreatePopulationFrom = async (coursecodes, unifyCourses) => {
+  let is_open = false
+
+  if (unifyCourses === 'openStats') is_open = true
+
+  if (unifyCourses === 'unifyStats') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
  const maxAttainmentDate = new Date(
    Math.max(
      ...(
@@ -257,6 +362,7 @@ const maxYearsToCreatePopulationFrom = async coursecodes => {
      ).map(c => new Date(c.max_attainment_date).getTime())
    )
  )
+
  const attainmentThreshold = new Date(maxAttainmentDate.getFullYear(), 0, 1)
  attainmentThreshold.setFullYear(attainmentThreshold.getFullYear() - 6)

@@ -268,6 +374,7 @@ const maxYearsToCreatePopulationFrom = async coursecodes => {
      attainment_date: {
        [Op.gt]: attainmentThreshold,
      },
+      is_open,
    },
    order: [['attainment_date', 'ASC']],
  })
@@ -291,11 +398,18 @@ const maxYearsToCreatePopulationFrom = async coursecodes => {
  return maxYearsToCreatePopulationFrom
 }

-const courseYearlyStats = async (coursecodes, separate, unifyOpenUniCourses, anonymizationSalt) => {
-  const stats = await Promise.all(
-    coursecodes.map(code => yearlyStatsOfNew(code, separate, unifyOpenUniCourses, anonymizationSalt))
+const courseYearlyStats = async (coursecodes, separate, anonymizationSalt) => {
+  const statsReqular = await Promise.all(
+    coursecodes.map(async code => {
+      const unifyStats = await yearlyStatsOfNew(code, separate, 'unify', anonymizationSalt)
+      const reqularStats = await yearlyStatsOfNew(code, separate, 'reqular', anonymizationSalt)
+      const openStats = await yearlyStatsOfNew(code, separate, 'open', anonymizationSalt)
+
+      return { unifyStats, reqularStats, openStats }
+    })
  )
-  return stats
+
+  return statsReqular
 }

 const nameLikeTerm = name => {
@@ -381,6 +495,23 @@ const byNameAndOrCodeLike = async (name, code) => {
  return { courses }
 }

+const getCourseProvidersForCourses = async codes => {
+  return (
+    await Organization.findAll({
+      attributes: ['code'],
+      include: {
+        model: Course,
+        where: {
+          code: {
+            [Op.in]: codes,
+          },
+        },
+      },
+      raw: true,
+    })
+  ).map(({ code }) => code)
+}
+
 module.exports = {
  byCode,
  byName,
@@ -393,4 +524,5 @@ module.exports = {
  unifyOpenUniversity,
  allCodeAlternatives,
  findOneByCode,
+  getCourseProvidersForCourses,
 }
@@ -3,7 +3,7 @@ const { isProduction, pateToken } = require('../conf-backend')
 const { ApplicationError } = require('../util/customErrors')

 const pateClient = axios.create({
-  baseURL: 'https://pate.toska.cs.helsinki.fi',
+  baseURL: 'https://importer.cs.helsinki.fi/api/pate/',
  params: {
    token: pateToken,
  },
@@ -22,7 +22,7 @@ const baseSettings = {
 const sendEmail = async (options = {}) => {
  if (!isProduction) throw new ApplicationError('Email sending is disabled in development mode.')
  if (!pateToken) throw new ApplicationError('Email sending failed because pate token is missing.')
-  await pateClient.post('/', options)
+  return await pateClient.post('/', options)
 }

 const sendFeedbackToToska = async ({ feedbackContent, user }) => {
@@ -1,366 +0,0 @@
-const sequelize = require('sequelize')
-const { Op } = sequelize
-const {
-  Credit,
-  Course,
-  Organization,
-  Studyright,
-  StudyrightElement,
-  ElementDetail,
-  Transfer,
-  Student,
-  SemesterEnrollment,
-  Semester,
-} = require('../models')
-const { formatStudyright, formatStudent } = require('./studyprogrammeHelpers')
-
-const whereStudents = studentnumbers => {
-  return studentnumbers ? studentnumbers : { [Op.not]: null }
-}
-
-const studytrackStudents = async studentnumbers =>
-  (
-    await Student.findAll({
-      where: {
-        studentnumber: {
-          [Op.in]: studentnumbers,
-        },
-      },
-    })
-  ).map(formatStudent)
-
-const enrolledStudyrights = async studytrack => {
-  const studyrights = await Studyright.findAll({
-    attributes: ['studyrightid', 'studystartdate', 'enddate', 'prioritycode'],
-    include: [
-      {
-        model: StudyrightElement,
-        required: true,
-        where: { code: studytrack },
-      },
-      {
-        model: Student,
-        attributes: ['studentnumber'],
-        required: true,
-      },
-    ],
-  })
-
-  const studentnumbers = studyrights.map(s => s.student.studentnumber)
-
-  const students = await Student.findAll({
-    attributes: ['studentnumber'],
-    include: [
-      {
-        model: Studyright,
-        include: [
-          {
-            model: StudyrightElement,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        ],
-        attributes: ['studyrightid'],
-      },
-      {
-        model: SemesterEnrollment,
-        attributes: ['semestercode'],
-        include: [
-          {
-            model: Semester,
-            required: true,
-          },
-        ],
-        where: {
-          enrollmenttype: 1,
-        },
-      },
-    ],
-    where: {
-      studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-    },
-  })
-
-  let enrollments = new Set()
-
-  students.forEach(student => {
-    student.semester_enrollments.forEach(e => {
-      const enrollmentStartDate = e.semester.startdate
-      const enrollmentEndDate = e.semester.enddate
-      const correctStudyRight = studyrights.find(
-        s =>
-          student.studentnumber === s.student.studentnumber &&
-          s.studystartdate <= enrollmentStartDate && // Studying in the programme should have started on the same day or earlier than the enrollment
-          s.enddate >= enrollmentEndDate && // Studying in the programme should have ended on the same day or later than the enrollment
-          (!s.canceldate || s.canceldate > enrollmentStartDate) && // Studyright should not be canceled OR the possible canceldate should be after the enrollmentStartDate
-          (s.prioritycode === 1 || s.prioritycode === 30) // Studyright should be primary or the student should have graduated
-      )
-      if (correctStudyRight) {
-        enrollments.add({
-          studentnumber: student.studentnumber,
-          startdate: e.semester.startdate,
-        })
-      }
-    })
-  })
-
-  return enrollments
-}
-
-const allStudyrights = async (studytrack, since, studentnumbers) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        student_studentnumber: whereStudents(studentnumbers),
-      },
-    })
-  ).map(formatStudyright)
-
-const startedStudyrights = async (studytrack, since, studentnumbers) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        studystartdate: {
-          [Op.gte]: since,
-        },
-        student_studentnumber: whereStudents(studentnumbers),
-      },
-    })
-  ).map(formatStudyright)
-
-const graduatedStudyRights = async (studytrack, since, studentnumbers) =>
-  await Studyright.findAll({
-    include: {
-      model: StudyrightElement,
-      required: true,
-      include: {
-        model: ElementDetail,
-        required: true,
-        where: {
-          code: studytrack,
-        },
-      },
-    },
-    where: {
-      graduated: 1,
-      enddate: {
-        [Op.gte]: since,
-      },
-      student_studentnumber: whereStudents(studentnumbers),
-    },
-  })
-
-const cancelledStudyRights = async (studytrack, since) => {
-  return await Studyright.findAll({
-    include: {
-      model: StudyrightElement,
-      required: true,
-      where: {
-        code: {
-          [Op.eq]: studytrack,
-        },
-      },
-    },
-    where: {
-      canceldate: {
-        [Op.gte]: since,
-      },
-    },
-  })
-}
-
-const transfersAway = async (studytrack, since) => {
-  return await Transfer.findAll({
-    where: {
-      transferdate: {
-        [Op.gte]: since,
-      },
-      sourcecode: studytrack,
-    },
-    distinct: true,
-    col: 'studentnumber',
-  })
-}
-
-const transfersTo = async (studytrack, since) => {
-  return await Transfer.findAll({
-    where: {
-      transferdate: {
-        [Op.gte]: since,
-      },
-      targetcode: studytrack,
-    },
-    distinct: true,
-    col: 'studentnumber',
-  })
-}
-
-const getProgrammesStudents = async studyprogramme =>
-  (
-    await Studyright.findAll({
-      attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode'],
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          where: {
-            code: {
-              [Op.in]: [studyprogramme],
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-    })
-  ).map(formatStudyright)
-
-const getCreditsForStudyProgramme = async (provider, since) =>
-  await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
-      where: {
-        is_study_module: false,
-      },
-      include: {
-        model: Organization,
-        required: true,
-        where: {
-          code: provider,
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.notIn]: [10, 9, 7],
-      },
-      isStudyModule: {
-        [Op.not]: true,
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-    },
-  })
-
-const getTransferredCredits = async (provider, since) =>
-  await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'credittypecode', 'student_studentnumber'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
-      where: {
-        is_study_module: false,
-      },
-      include: {
-        model: Organization,
-        required: true,
-        where: {
-          code: provider,
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.eq]: [9],
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-    },
-  })
-
-const getThesisCredits = async (provider, since, thesisType) =>
-  await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
-      where: {
-        course_unit_type: {
-          [Op.in]: thesisType,
-        },
-      },
-      include: {
-        model: Organization,
-        required: true,
-        where: {
-          code: provider,
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.notIn]: [10, 9, 7],
-      },
-      isStudyModule: {
-        [Op.not]: true,
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-    },
-  })
-
-module.exports = {
-  studytrackStudents,
-  enrolledStudyrights,
-  allStudyrights,
-  startedStudyrights,
-  graduatedStudyRights,
-  cancelledStudyRights,
-  transfersAway,
-  transfersTo,
-  getProgrammesStudents,
-  getCreditsForStudyProgramme,
-  getTransferredCredits,
-  getThesisCredits,
-}
@@ -65,4 +65,4 @@ const parseCredit = (credit, anonymizationSalt) => {
  }
 }

-module.exports = { parseCredit }
+module.exports = { parseCredit, formatStudyrightElement }
@@ -0,0 +1,48 @@
+const crypto = require('crypto')
+const { formatStudyrightElement } = require('./parseCredits')
+
+const parseEnrollment = (enrollment, anonymizationSalt) => {
+  const { student, semester, state, enrollment_date_time, course_code } = enrollment
+  const { studentnumber, studyright_elements } = student
+  const { yearcode, yearname, semestercode, name: semestername } = semester
+
+  if (anonymizationSalt) {
+    const anonymizedStudentNumber = crypto
+      .createHash('sha256')
+      .update(`${studentnumber}${anonymizationSalt}`)
+      .digest('hex')
+
+    const anonymizedStudent = {
+      studentnumber: anonymizedStudentNumber,
+    }
+
+    return {
+      obfuscated: true,
+      student: anonymizedStudent,
+      yearcode,
+      yearname,
+      semestercode,
+      semestername,
+      coursecode: course_code,
+      state,
+      enrollment_date_time,
+      studentnumber: anonymizedStudentNumber,
+      programmes: studyright_elements.map(formatStudyrightElement),
+    }
+  }
+
+  return {
+    student,
+    yearcode,
+    yearname,
+    semestercode,
+    semestername,
+    coursecode: course_code,
+    state,
+    enrollment_date_time,
+    studentnumber,
+    programmes: studyright_elements.map(formatStudyrightElement),
+  }
+}
+
+module.exports = { parseEnrollment }
@@ -1,7 +1,7 @@
-const { Op } = require('sequelize')
+const Sequelize = require('sequelize')
+const { Op } = Sequelize
 const moment = require('moment')
-const { sortBy } = require('lodash')
-const { sortMainCode } = require('../util/utils')
+const { sortBy, keyBy } = require('lodash')

 const {
  Student,
@@ -10,11 +10,12 @@ const {
  Studyright,
  StudyrightExtent,
  ElementDetail,
-  CourseType,
+  Studyplan,
  SemesterEnrollment,
  Semester,
  Transfer,
  StudyrightElement,
+  Enrollment,
 } = require('../models')
 const {
  dbConnections: { sequelize },
@@ -47,6 +48,7 @@ const formatStudentForPopulationStatistics = (
    abbreviatedname,
    email,
    studyrights,
+    studyplans,
    semester_enrollments,
    transfers,
    updatedAt,
@@ -58,7 +60,9 @@ const formatStudentForPopulationStatistics = (
    tags,
    option,
    birthdate,
+    sis_person_id,
  },
+  enrollments,
  credits,
  startDate,
  endDate,
@@ -93,6 +97,7 @@ const formatStudentForPopulationStatistics = (
    studentNumber: studentnumber,
    credits: creditcount || 0,
    courses: credits[studentnumber] ? credits[studentnumber].map(toCourse) : [],
+    enrollments: enrollments[studentnumber],
    name: abbreviatedname,
    transfers: transfers || [],
    gender_code,
@@ -105,6 +110,8 @@ const formatStudentForPopulationStatistics = (
    starting: moment(started).isBetween(startDateMoment, endDateMoment, null, '[]'),
    option,
    birthdate,
+    studyplans,
+    sis_person_id,
  }
 }

@@ -140,10 +147,24 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
  if (tag) studentnumbers = studentnumbersWithTag

  const attainmentDateFrom = tag ? moment(startDate).year(tag.year) : startDate
+  const studyPlans = await Studyplan.findAll({
+    where: { studentnumber: studentnumbers },
+    attributes: ['included_courses'],
+    raw: true,
+  })
+  const studyPlanCourses = Array.from(new Set([...studyPlans.map(plan => plan.included_courses)].flat()))
+
  const creditsOfStudentOther = {
-    attainment_date: {
-      [Op.between]: [attainmentDateFrom, endDate],
-    },
+    [Op.or]: [
+      {
+        attainment_date: {
+          [Op.between]: [attainmentDateFrom, endDate],
+        },
+      },
+      {
+        course_code: studyPlanCourses,
+      },
+    ],
    student_studentnumber: {
      [Op.in]: studentnumbers,
    },
@@ -162,7 +183,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
      },
      {
        course_code: {
-          [Op.in]: ['375063', '339101'],
+          [Op.in]: ['375063', '339101'].concat(studyPlanCourses),
        },
      },
    ],
@@ -173,8 +194,9 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
    : creditsOfStudentOther

  if (studentnumbers.length === 0)
-    return { students: [], credits: [], extents: [], semesters: [], elementdetails: [], courses: [] }
-  const [courses, students, credits, extents, semesters, elementdetails] = await Promise.all([
+    return { students: [], enrollments: [], credits: [], extents: [], semesters: [], elementdetails: [], courses: [] }
+
+  const [courses, enrollments, students, credits, extents, semesters, elementdetails] = await Promise.all([
    Course.findAll({
      attributes: [sequelize.literal('DISTINCT ON("code") code'), 'name', 'coursetypecode'],
      include: [
@@ -186,6 +208,19 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
      ],
      raw: true,
    }),
+    Enrollment.findAll({
+      attributes: ['course_code', 'state', 'enrollment_date_time', 'studentnumber'],
+      where: {
+        enrollment_date_time: {
+          [Op.between]: [attainmentDateFrom, endDate],
+        },
+        studentnumber: {
+          [Op.in]: studentnumbers,
+        },
+        state: ['ENROLLED', 'CONFIRMED'],
+      },
+      raw: true,
+    }),
    Student.findAll({
      attributes: [
        'firstnames',
@@ -199,6 +234,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
        'updatedAt',
        'gender_code',
        'birthdate',
+        'sis_person_id',
        /* 'gender_fi',
        'gender_sv',
        'gender_en' */
@@ -217,7 +253,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
            'enddate',
            'extentcode',
            'graduated',
-            'canceldate',
+            'active',
            'prioritycode',
            'faculty_code',
            'studystartdate',
@@ -250,6 +286,11 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
            },
          },
        },
+        {
+          model: Studyplan,
+          attributes: ['included_courses', 'programme_code', 'completed_credits'],
+          separate: true,
+        },
      ],
      where: {
        studentnumber: {
@@ -330,7 +371,7 @@ EXISTS (SELECT 1 FROM studyright_elements WHERE studentnumber IN (:studentnumber
    student.tags = studentNumberToTags[student.studentnumber] || []
  })

-  return { students, credits, extents, semesters, elementdetails, courses }
+  return { students, enrollments, credits, extents, semesters, elementdetails, courses }
 }

 const count = (column, count, distinct = false) => {
@@ -361,10 +402,11 @@ const studentnumbersWithAllStudyrightElements = async (
  startDate,
  endDate,
  exchangeStudents,
-  cancelledStudents,
  nondegreeStudents,
-  transferredStudents,
-  tag
+  transferredOutStudents,
+  tag,
+  transferredToStudents,
+  graduatedStudents
 ) => {
  // eslint-disable-line

@@ -383,9 +425,6 @@ const studentnumbersWithAllStudyrightElements = async (
  if (!nondegreeStudents) {
    filteredExtents.push(33, 99, 14, 13)
  }
-  if (!cancelledStudents) {
-    studyrightWhere.canceldate = null
-  }

  let studentWhere = {}
  if (tag) {
@@ -403,7 +442,7 @@ const studentnumbersWithAllStudyrightElements = async (
  }

  const students = await Studyright.findAll({
-    attributes: ['student_studentnumber', 'graduated'],
+    attributes: ['student_studentnumber', 'graduated', 'enddate'],
    include: {
      model: StudyrightElement,
      attributes: [],
@@ -444,7 +483,7 @@ const studentnumbersWithAllStudyrightElements = async (
    raw: true,
  })

-  const studentnumbers = [...new Set(students.map(s => s.student_studentnumber))]
+  let studentnumbers = [...new Set(students.map(s => s.student_studentnumber))]

  // bit hacky solution, but this is used to filter out studentnumbers who have since changed studytracks
  const rights = await Studyright.findAll({
@@ -491,17 +530,21 @@ const studentnumbersWithAllStudyrightElements = async (
    return acc
  }, {})

+  // Take the newest studytrack primarily by latest starting date in the track, secondarily by the latest enddate
  const filteredStudentnumbers = studentnumbers.filter(studentnumber => {
-    const newestStudytrack = sortBy(formattedStudytracks[studentnumber], ['enddate', 'startdate']).reverse()[0]
+    const newestStudytrack = sortBy(formattedStudytracks[studentnumber], ['startdate', 'enddate']).reverse()[0]
    if (!newestStudytrack) return false
    return studyRights.includes(newestStudytrack.code)
  })

-  const studentnumberlist = filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers
+  // Use the filtered list, if the search includes studytracks
+  // Then the studyrights length is > 1, which means that there is [studyright, studytrack].
+  // When searching only for studyprogramme, there is [studyright]
+  let studentnumberlist = studyRights.length > 1 ? filteredStudentnumbers : studentnumbers

  // fetch students that have transferred out of the programme and filter out these studentnumbers
-  if (!transferredStudents) {
-    const transferredOutStudents = (
+  if (!transferredOutStudents) {
+    const transfersOut = (
      await Transfer.findAll({
        attributes: ['studentnumber'],
        where: {
@@ -519,8 +562,63 @@ const studentnumbersWithAllStudyrightElements = async (
      })
    ).map(s => s.studentnumber)

-    const notTransferredStudents = studentnumberlist.filter(sn => !transferredOutStudents.includes(sn))
-    return notTransferredStudents
+    studentnumberlist = studentnumberlist.filter(sn => !transfersOut.includes(sn))
+  }
+
+  // fetch students that have transferred to the programme and filter out these studentnumbers
+  if (transferredToStudents) {
+    const transfersTo = (
+      await Transfer.findAll({
+        attributes: ['studentnumber'],
+        where: {
+          targetcode: {
+            [Op.in]: studyRights,
+          },
+          transferdate: {
+            [Op.gt]: startDate,
+          },
+          studentnumber: {
+            [Op.in]: studentnumberlist,
+          },
+        },
+        raw: true,
+      })
+    ).map(s => s.studentnumber)
+    studentnumberlist = studentnumberlist.filter(sn => !transfersTo.includes(sn))
+  }
+
+  // fetch students that have graduated from the programme and filter out these studentnumbers
+  if (graduatedStudents) {
+    const graduated = (
+      await Student.findAll({
+        attributes: ['studentnumber'],
+        include: [
+          {
+            model: Studyright,
+            include: [
+              {
+                model: StudyrightElement,
+                required: true,
+                where: {
+                  code: {
+                    [Op.in]: studyRights,
+                  },
+                },
+              },
+            ],
+            where: {
+              graduated: 1,
+            },
+          },
+        ],
+        where: {
+          studentnumber: {
+            [Op.in]: studentnumbers,
+          },
+        },
+      })
+    ).map(s => s.studentnumber)
+    studentnumberlist = studentnumberlist.filter(sn => !graduated.includes(sn))
  }

  return studentnumberlist
@@ -535,13 +633,11 @@ const parseQueryParams = query => {
    ? `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd[semesters.find(s => s === 'SPRING')]}`
    : `${year}-${semesterEnd[semesters.find(s => s === 'FALL')]}`
  const exchangeStudents = studentStatuses && studentStatuses.includes('EXCHANGE')
-  const cancelledStudents = studentStatuses && studentStatuses.includes('CANCELLED')
  const nondegreeStudents = studentStatuses && studentStatuses.includes('NONDEGREE')
  const transferredStudents = studentStatuses && studentStatuses.includes('TRANSFERRED')

  return {
    exchangeStudents,
-    cancelledStudents,
    nondegreeStudents,
    transferredStudents,
    // if someone passes something falsy like null as the studyright, remove it so it doesn't break
@@ -646,7 +742,7 @@ const getOptionsForStudents = async (students, code, level) => {
 }

 const formatStudentsForApi = async (
-  { students, credits, extents, semesters, elementdetails, courses },
+  { students, enrollments, credits, extents, semesters, elementdetails, courses },
  startDate,
  endDate,
  { studyRights },
@@ -667,6 +763,11 @@ const formatStudentsForApi = async (
    acc[e.student_studentnumber].push(e)
    return acc
  }, {})
+  enrollments = enrollments.reduce((acc, e) => {
+    acc[e.studentnumber] = acc[e.studentnumber] || []
+    acc[e.studentnumber].push(e)
+    return acc
+  }, {})
  const result = students.reduce(
    (stats, student) => {
      student.transfers.forEach(transfer => {
@@ -680,7 +781,15 @@ const formatStudentsForApi = async (
      if (student.studentnumber in optionData) student.option = optionData[student.studentnumber]
      else student.option = null
      stats.students.push(
-        formatStudentForPopulationStatistics(student, credits, startDate, endDate, startDateMoment, endDateMoment)
+        formatStudentForPopulationStatistics(
+          student,
+          enrollments,
+          credits,
+          startDate,
+          endDate,
+          startDateMoment,
+          endDateMoment
+        )
      )
      return stats
    },
@@ -743,22 +852,13 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
  if (
    formattedQueryParams.studentStatuses &&
    !formattedQueryParams.studentStatuses.every(
-      status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
+      status => status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
    )
  ) {
-    return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE or TRANSFERRED' }
+    return { error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' }
  }
-  const {
-    studyRights,
-    startDate,
-    months,
-    endDate,
-    exchangeStudents,
-    cancelledStudents,
-    nondegreeStudents,
-    transferredStudents,
-    tag,
-  } = parseQueryParams(formattedQueryParams)
+  const { studyRights, startDate, months, endDate, exchangeStudents, nondegreeStudents, transferredStudents, tag } =
+    parseQueryParams(formattedQueryParams)

  // db startdate is formatted to utc so need to change it when querying
  const formattedStartDate = new Date(moment.tz(startDate, 'Europe/Helsinki').format()).toUTCString()
@@ -770,7 +870,6 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
        formattedStartDate,
        endDate,
        exchangeStudents,
-        cancelledStudents,
        nondegreeStudents,
        transferredStudents
      )
@@ -784,48 +883,89 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
    optionData = await getOptionsForStudents(studentnumbers, code, 'BSC')
  }

-  const { students, credits, extents, semesters, elementdetails, courses } = await getStudentsIncludeCoursesBetween(
-    studentnumbers,
-    startDate,
-    dateMonthsFromNow(startDate, months),
-    studyRights,
-    tag
-  )
+  const { students, enrollments, credits, extents, semesters, elementdetails, courses } =
+    await getStudentsIncludeCoursesBetween(
+      studentnumbers,
+      startDate,
+      dateMonthsFromNow(startDate, months),
+      studyRights,
+      tag
+    )

  const formattedStudents = await formatStudentsForApi(
-    { students, credits, extents, semesters, elementdetails, courses },
+    { students, enrollments, credits, extents, semesters, elementdetails, courses },
    startDate,
    endDate,
    formattedQueryParams,
    optionData
  )
+
  return formattedStudents
 }

-const findCourses = async (studentnumbers, beforeDate) => {
-  const res = await Course.findAll({
-    attributes: ['code', 'name', 'coursetypecode', 'substitutions'],
-    include: [
-      {
-        required: true,
-        model: Credit,
-        attributes: ['grade', 'student_studentnumber', 'credittypecode', 'attainment_date', 'course_code'],
-        where: {
-          student_studentnumber: {
-            [Op.in]: studentnumbers,
-          },
-          attainment_date: {
-            [Op.lt]: beforeDate,
-          },
-        },
+const getSubstitutions = async codes => {
+  const courses = await Course.findAll({ where: { code: codes }, attributes: ['code', 'substitutions'], raw: true })
+  return [...new Set(courses.map(({ code, substitutions }) => [code, ...substitutions]).flat())]
+}
+
+const findCourses = async (studentnumbers, beforeDate, courses = [], studentCountLimit = 0) => {
+  const courseCodes = courses.length > 0 ? await getSubstitutions(courses) : ['DUMMY']
+
+  const res = await sequelize.query(
+    `
+      SELECT DISTINCT ON (course.id)
+        course.code,
+        course.name,
+        course.coursetypecode,
+        course.substitutions,
+        course.main_course_code,
+        course_types.name AS course_type,
+        credit.data AS credits,
+        enrollment.data AS enrollments
+      FROM course
+      INNER JOIN (
+        SELECT
+          course_code,
+          ARRAY_AGG(JSON_BUILD_OBJECT(
+            'grade', grade,
+            'student_studentnumber', student_studentnumber,
+            'attainment_date', attainment_date,
+            'credittypecode', credittypecode,
+            'course_code', course_code
+          )) AS data
+        FROM credit
+        WHERE student_studentnumber IN (:studentnumbers) AND attainment_date < :beforeDate
+        GROUP BY credit.course_code
+        HAVING COUNT(DISTINCT credit.student_studentnumber) >= :studentCountLimit
+      ) credit ON credit.course_code = course.code
+      INNER JOIN course_types ON course_types.coursetypecode = course.coursetypecode
+      LEFT JOIN (
+        SELECT
+          course_id,
+          ARRAY_AGG(JSON_BUILD_OBJECT(
+            'studentnumber', studentnumber,
+            'state', state,
+            'enrollment_date_time', enrollment_date_time
+          )) AS data
+        FROM enrollment
+        WHERE enrollment.studentnumber IN (:studentnumbers) AND enrollment.enrollment_date_time < :beforeDate
+        GROUP BY enrollment.course_id
+      ) enrollment ON enrollment.course_id = course.id 
+      WHERE :skipCourseCodeFilter OR course.code IN (:courseCodes)
+      -- GROUP BY 1, 2, 3, 4, 5, 6
+    `,
+    {
+      replacements: {
+        studentnumbers: studentnumbers.length > 0 ? studentnumbers : ['DUMMY'],
+        beforeDate,
+        studentCountLimit,
+        courseCodes,
+        skipCourseCodeFilter: courses.length === 0,
      },
-      {
-        attributes: ['coursetypecode', 'name'],
-        model: CourseType,
-        required: true,
-      },
-    ],
-  })
+      type: sequelize.QueryTypes.SELECT,
+    }
+  )
+
  return res
 }

@@ -841,37 +981,9 @@ const parseCreditInfo = credit => ({
  date: credit.attainment_date,
 })

-const findMainCourse = async code => {
-  const course = await Course.findOne({
-    where: { code: code },
-  })
-
-  let result = []
-
-  if (course.substitutions) {
-    const sortedCourses = sortMainCode([...course.substitutions, code])
-    result = await Course.findOne({
-      attributes: ['code', 'name'],
-      where: { code: sortedCourses[0] },
-    })
-  } else {
-    return course
-  }
-
-  return result
-}
-
 const bottlenecksOf = async (query, studentnumberlist) => {
  const isValidRequest = async (query, params) => {
-    const {
-      studyRights,
-      startDate,
-      endDate,
-      exchangeStudents,
-      cancelledStudents,
-      nondegreeStudents,
-      transferredStudents,
-    } = params
+    const { studyRights, startDate, endDate, exchangeStudents, nondegreeStudents, transferredStudents } = params

    if (!query.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
      return { error: 'Semester should be either SPRING OR FALL' }
@@ -879,10 +991,10 @@ const bottlenecksOf = async (query, studentnumberlist) => {
    if (
      query.studentStatuses &&
      !query.studentStatuses.every(
-        status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
+        status => status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
      )
    ) {
-      return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE or TRANSFERRED' }
+      return { error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' }
    }
    if (query.selectedStudents) {
      const allStudents = await studentnumbersWithAllStudyrightElements(
@@ -890,7 +1002,6 @@ const bottlenecksOf = async (query, studentnumberlist) => {
        startDate,
        endDate,
        exchangeStudents,
-        cancelledStudents,
        nondegreeStudents,
        transferredStudents,
        query.tag
@@ -904,19 +1015,10 @@ const bottlenecksOf = async (query, studentnumberlist) => {
    return null
  }

-  const getStudentsAndCourses = async (selectedStudents, studentnumberlist) => {
+  const getStudentsAndCourses = async (selectedStudents, studentnumberlist, courseCodes) => {
    if (!studentnumberlist) {
-      const {
-        months,
-        studyRights,
-        startDate,
-        endDate,
-        exchangeStudents,
-        cancelledStudents,
-        nondegreeStudents,
-        transferredStudents,
-        tag,
-      } = params
+      const { months, studyRights, startDate, endDate, exchangeStudents, nondegreeStudents, transferredStudents, tag } =
+        params
      const studentnumbers =
        selectedStudents ||
        (await studentnumbersWithAllStudyrightElements(
@@ -924,7 +1026,6 @@ const bottlenecksOf = async (query, studentnumberlist) => {
          startDate,
          endDate,
          exchangeStudents,
-          cancelledStudents,
          nondegreeStudents,
          transferredStudents,
          tag
@@ -935,20 +1036,28 @@ const bottlenecksOf = async (query, studentnumberlist) => {
        return numbers
      }, {})

-      const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months))
+      const courses = await findCourses(
+        studentnumbers,
+        dateMonthsFromNow(startDate, months),
+        courseCodes,
+        query.studentCountLimit
+      )
      return [allstudents, courses]
    } else {
+      const { months, startDate } = params
+      const beforeDate = months && startDate ? dateMonthsFromNow(startDate, months) : new Date()
+
      const allstudents = studentnumberlist.reduce((numbers, num) => {
        numbers[num] = true
        return numbers
      }, {})
-      const courses = await findCourses(studentnumberlist, new Date())
+      const courses = await findCourses(studentnumberlist, beforeDate, courseCodes, query.studentCountLimit)
      return [allstudents, courses]
    }
  }
  const params = parseQueryParams(query)
  const [[allstudents, courses], error] = await Promise.all([
-    getStudentsAndCourses(query.selectedStudents, studentnumberlist),
+    getStudentsAndCourses(query.selectedStudents, studentnumberlist, query.courses),
    isValidRequest(query, params),
  ])
  if (error) return error
@@ -962,17 +1071,19 @@ const bottlenecksOf = async (query, studentnumberlist) => {

  const startYear = parseInt(query.year, 10)
  const allstudentslength = Object.keys(allstudents).length
+  const coursesByCode = keyBy(courses, 'code')

  for (const course of courses) {
    let { course_type } = course
    let maincourse = course
-    const sortedMaincourse = await findMainCourse(course.code)
-    if (sortedMaincourse) maincourse = sortedMaincourse

-    /*  if (isOpenUniCourseCode) {
-      const nonOpen = unifyOpenUniversity(course.code)
-      const response = await findOneByCode(nonOpen)
- */
+    if (course.main_course_code && course.main_course_code !== course.code) {
+      let newmain = coursesByCode[course.main_course_code]
+
+      if (newmain) {
+        maincourse = newmain
+      }
+    }

    if (!stats[maincourse.code]) {
      stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudentslength)
@@ -989,6 +1100,14 @@ const bottlenecksOf = async (query, studentnumberlist) => {
      const semester = getPassingSemester(startYear, date)
      coursestats.markCredit(studentnumber, grade, passingGrade, failingGrade, improvedGrade, semester)
    })
+
+    if (course.enrollments) {
+      course.enrollments.forEach(({ studentnumber, state, enrollment_date_time }) => {
+        const semester = getPassingSemester(startYear, enrollment_date_time)
+        coursestats.markEnrollment(studentnumber, state, semester)
+      })
+    }
+
    stats[maincourse.code] = coursestats
  }

@@ -1,3 +1,4 @@
+const moment = require('moment')
 const sequelize = require('sequelize')
 const { Semester } = require('../models')
 const Op = sequelize.Op
@@ -28,8 +29,8 @@ const getSemestersAndYears = async () => {
        acc.years[yearcode] = {
          yearcode,
          yearname,
-          startdate: Math.min(acc.years[yearcode].startdate, startdate),
-          enddate: Math.max(acc.years[yearcode].enddate, enddate),
+          startdate: moment.min(moment(acc.years[yearcode].startdate), moment(startdate)),
+          enddate: moment.max(moment(acc.years[yearcode].enddate), moment(enddate)),
        }
      }
      return acc
@@ -26,7 +26,7 @@ const populationCourseStatsMerger = multiyearstats => {
        stats.coursestatistics[index].stats.retryPassed += c.stats.retryPassed

        stats.coursestatistics[index].stats.perStudent =
-          (stats.coursestatistics[index].stats.attempts / stats.coursestatistics[index].stats.students) * 100
+          stats.coursestatistics[index].stats.attempts / stats.coursestatistics[index].stats.students

        stats.coursestatistics[index].stats.percentage =
          (stats.coursestatistics[index].stats.passed / stats.coursestatistics[index].stats.students) * 100
@@ -12,6 +12,7 @@ const {
  ElementDetail,
  SemesterEnrollment,
  Semester,
+  Studyplan,
 } = require('../models')
 const { TagStudent, Tag } = require('../models/models_kone')
 const Op = Sequelize.Op
@@ -57,6 +58,10 @@ const byId = async id => {
        {
          model: SemesterEnrollment,
        },
+        {
+          model: Studyplan,
+          attributes: ['included_courses', 'programme_code', 'completed_credits'],
+        },
      ],
    }),
    TagStudent.findAll({
@@ -99,33 +104,79 @@ const byId = async id => {
  return student
 }

-const findByCourseAndSemesters = async (coursecodes, from, to, separate) =>
-  (
+const getUnifyStatus = unifyCourses => {
+  switch (unifyCourses) {
+    case 'unifyStats':
+      return [true, false]
+
+    case 'openStats':
+      return [true]
+
+    case 'reqularStats':
+      return [false]
+
+    default:
+      return [true, false]
+  }
+}
+
+const findByCourseAndSemesters = async (coursecodes, from, to, separate, unifyCourses = 'unifyStats') => {
+  const { startdate } = await Semester.findOne({
+    where: {
+      [separate ? 'semestercode' : 'yearcode']: from,
+    },
+    order: ['semestercode'],
+    limit: 1,
+    raw: true,
+  })
+  const { enddate } = await Semester.findOne({
+    where: {
+      [separate ? 'semestercode' : 'yearcode']: to,
+    },
+    order: [['semestercode', 'DESC']],
+    limit: 1,
+    raw: true,
+  })
+  return (
    await sequelize.query(
      `
  SELECT
    studentnumber
-  FROM student
-  INNER JOIN credit ON
-    student.studentnumber=credit.student_studentnumber
+  FROM
+    student s
  WHERE
-    course_code IN (:coursecodes) AND
-    attainment_date
-  BETWEEN
-    (select startdate FROM semesters where ${
-      separate ? 'semestercode' : 'yearcode'
-    }=:minYearCode ORDER BY semestercode LIMIT 1) AND
-    (select enddate FROM semesters where ${
-      separate ? 'semestercode' : 'yearcode'
-    }=:maxYearCode ORDER BY semestercode DESC LIMIT 1);
+    EXISTS (
+      SELECT
+        1
+      FROM
+        credit c
+      WHERE
+        c.student_studentnumber = s.studentnumber
+        AND c.course_code IN (:coursecodes)
+        AND c.is_open IN (:isOpen)
+        AND c.attainment_date BETWEEN '${startdate.toISOString()}' AND '${enddate.toISOString()}'
+    )
+    OR EXISTS (
+      SELECT
+        1
+      FROM
+        enrollment e
+      WHERE
+        e.studentnumber = s.studentnumber
+        AND e.course_code IN (:coursecodes)
+        AND e.enrollment_date_time BETWEEN '${startdate.toISOString()}' AND '${enddate.toISOString()}'
+        AND e.enrollment_date_time >= '2021-05-31'
+        AND e.state IN ('ENROLLED', 'CONFIRMED')
+    );
 `,
      {
-        replacements: { coursecodes, minYearCode: from, maxYearCode: to },
+        replacements: { coursecodes, minYearCode: from, maxYearCode: to, isOpen: getUnifyStatus(unifyCourses) },
        type: sequelize.QueryTypes.SELECT,
        raw: true,
      }
    )
  ).map(st => st.studentnumber)
+}

 const findByTag = async tag => {
  return (
@@ -140,7 +191,7 @@ const findByTag = async tag => {
  ).map(st => st.studentnumber)
 }

-const formatStudent = ({
+const formatStudent = async ({
  firstnames,
  lastname,
  studentnumber,
@@ -153,11 +204,13 @@ const formatStudent = ({
  studyrights,
  semester_enrollments,
  transfers,
+  studyplans,
  updatedAt,
  createdAt,
  tags,
+  sis_person_id,
 }) => {
-  const toCourse = ({ id, grade, credits, credittypecode, attainment_date, course, isStudyModule }) => {
+  const toCourse = ({ id, grade, credits, credittypecode, is_open, attainment_date, course, isStudyModule }) => {
    try {
      course = course.get()
    } catch (e) {
@@ -183,18 +236,21 @@ const formatStudent = ({
      credits,
      credittypecode,
      isStudyModuleCredit: isStudyModule,
+      isOpenCourse: is_open,
    }
  }

  studyrights = studyrights || []
  semester_enrollments = semester_enrollments || []
+  studyplans = studyplans || []
  const semesterenrollments = semester_enrollments.map(
-    ({ semestercode, enrollmenttype, name, yearname, startYear }) => ({
+    ({ semestercode, enrollmenttype, name, yearname, startYear, statutory_absence: statutoryAbsence }) => ({
      yearname,
      name,
      startYear,
      semestercode,
      enrollmenttype,
+      statutoryAbsence,
    })
  )

@@ -211,6 +267,13 @@ const formatStudent = ({
    .map(toCourse)
    .filter(c => c.course.name !== 'missing')

+  const hopsCourses = studyplans.map(sp => sp.included_courses).flat()
+
+  const allCourses = await sequelize.query(
+    'SELECT DISTINCT ON(course_code) course_code, credits FROM credit WHERE course_code IN (:hopsCourses)',
+    { replacements: { hopsCourses: hopsCourses.concat('id-to-fill-array') } }
+  )
+
  return {
    firstnames,
    lastname,
@@ -225,14 +288,17 @@ const formatStudent = ({
    email,
    semesterenrollments,
    updatedAt: updatedAt || createdAt,
+    studyplans,
+    allCourses: allCourses[0],
    tags,
+    sis_person_id: sis_person_id,
  }
 }

 const withId = async id => {
  try {
    const result = await byId(id)
-    return formatStudent(result)
+    return await formatStudent(result)
  } catch (e) {
    logger.error({ message: 'error when fetching single student', meta: e })
    return {
@@ -285,7 +351,7 @@ const bySearchTerm = async searchterm => {
      [Op.or]: [nameLike(terms), studentnumberLike(terms)],
    },
  })
-  return matches.map(formatStudent)
+  return await Promise.all(matches.map(async s => await formatStudent(s)))
 }

 const bySearchTermAndStudentNumbers = async (searchterm, studentnumbers) => {
@@ -300,7 +366,7 @@ const bySearchTermAndStudentNumbers = async (searchterm, studentnumbers) => {
      },
    },
  })
-  return matches.map(formatStudent)
+  return await Promise.all(matches.map(async s => await formatStudent(s)))
 }

 const filterStudentnumbersByAccessrights = async (studentnumbers, codes) =>
@@ -0,0 +1,149 @@
+const { indexOf } = require('lodash')
+const {
+  getCorrectStudentnumbers,
+  getStatsBasis,
+  defineYear,
+  getYearsArray,
+  getStartDate,
+  tableTitles,
+  alltimeStartDate,
+  alltimeEndDate,
+} = require('./studyprogrammeHelpers')
+const { graduatedStudyRights, startedStudyrights, transfersAway, transfersTo } = require('./studyprogramme')
+const { getYearStartAndEndDates } = require('../util/semester')
+
+const getStartedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+      const studyrights = await startedStudyrights(studyprogramme, since, studentnumbersOfTheYear)
+
+      studyrights.forEach(({ studystartdate }) => {
+        const startYear = defineYear(studystartdate, isAcademicYear)
+        graphStats[indexOf(years, startYear)] += 1
+        tableStats[startYear] += 1
+      })
+    })
+  )
+  return { graphStats, tableStats }
+}
+
+const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+    codes: [studyprogramme],
+    startDate: alltimeStartDate,
+    endDate: alltimeEndDate,
+    includeAllSpecials,
+  })
+
+  const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
+  studyrights.forEach(({ enddate }) => {
+    const graduationYear = defineYear(enddate, isAcademicYear)
+    graphStats[indexOf(years, graduationYear)] += 1
+    tableStats[graduationYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredAwayStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const studyrights = await transfersAway(studyprogramme, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredToStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const studyrights = await transfersTo(studyprogramme, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getBasicStatsForStudytrack = async ({ studyprogramme, settings }) => {
+  const { includeAllSpecials, isAcademicYear } = settings
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const started = await getStartedStats(queryParameters)
+  const graduated = await getGraduatedStats(queryParameters)
+  const transferredAway = await getTransferredAwayStats(queryParameters)
+  const transferredTo = await getTransferredToStats(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const titles = tableTitles['basics'][includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED']
+  const tableStats = reversedYears.map(year =>
+    includeAllSpecials
+      ? [
+          year,
+          started.tableStats[year],
+          graduated.tableStats[year],
+          transferredAway.tableStats[year],
+          transferredTo.tableStats[year],
+        ]
+      : [year, started.tableStats[year], graduated.tableStats[year]]
+  )
+
+  return {
+    id: studyprogramme,
+    years,
+    graphStats: includeAllSpecials
+      ? [
+          {
+            name: 'Started',
+            data: started.graphStats,
+          },
+          {
+            name: 'Graduated',
+            data: graduated.graphStats,
+          },
+          {
+            name: 'Transferred away',
+            data: transferredAway.graphStats,
+          },
+          {
+            name: 'Transferred To',
+            data: transferredTo.graphStats,
+          },
+        ]
+      : [
+          {
+            name: 'Started',
+            data: started.graphStats,
+          },
+          {
+            name: 'Graduated',
+            data: graduated.graphStats,
+          },
+        ],
+    tableStats,
+    titles,
+  }
+}
+
+module.exports = {
+  getBasicStatsForStudytrack,
+}
@@ -0,0 +1,209 @@
+const _ = require('lodash')
+
+const {
+  getStudentsForProgrammeCourses,
+  getCurrentStudyYearStartDate,
+  getOwnStudentsForProgrammeCourses,
+  getStudentsWithoutStudyrightForProgrammeCourses,
+  getOtherStudentsForProgrammeCourses,
+  getAllProgrammeCourses,
+} = require('./studyprogramme')
+const { mapToProviders } = require('../util/utils')
+
+const getCurrentYearStartDate = () => {
+  return new Date(new Date().getFullYear(), 0, 1)
+}
+
+const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+
+const getAllStudyprogrammeCourses = async studyprogramme => {
+  const providerCode = mapToProviders([studyprogramme])[0]
+  const normalCourses = await getAllProgrammeCourses(providerCode)
+  return normalCourses.reduce((acc, curr) => {
+    acc.push(curr.code)
+    if (curr.substitutions && curr.substitutions.includes('AY' + curr.code)) {
+      acc.push('AY' + curr.code)
+    }
+    return acc
+  }, [])
+}
+
+const makeYearlyPromises = (years, academicYear, type, programmeCourses, studyprogramme) => {
+  return years.map(
+    year =>
+      new Promise(async res => {
+        const from = academicYear == 'true' ? new Date(year, 7, 1, 0, 0, 0) : new Date(year, 0, 1, 0, 0, 0)
+        const to = academicYear == 'true' ? new Date(year + 1, 6, 31, 23, 59, 59) : new Date(year, 11, 31, 23, 59, 59)
+
+        let result = null
+
+        switch (type) {
+          case 'allStudents':
+            result = await getStudentsForProgrammeCourses(from, to, programmeCourses)
+            break
+          case 'ownStudents':
+            result = await getOwnStudentsForProgrammeCourses(from, to, programmeCourses, studyprogramme)
+            break
+          case 'withoutStudyright':
+            result = await getStudentsWithoutStudyrightForProgrammeCourses(from, to, programmeCourses)
+            break
+          case 'otherStudents':
+            result = await getOtherStudentsForProgrammeCourses(from, to, programmeCourses, studyprogramme)
+            break
+          default:
+            result = await getStudentsForProgrammeCourses(from, to, programmeCourses)
+        }
+
+        res(
+          result.map(c => {
+            c['year'] = year
+            return c
+          })
+        )
+      })
+  )
+}
+
+const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme, academicYear) => {
+  const startDate = academicYear == 'true' ? await getCurrentStudyYearStartDate(unixMillis) : getCurrentYearStartDate()
+  const startYear = startDate.getFullYear()
+  const yearRange = _.range(2017, startYear + 1)
+  const programmeCourses = await getAllStudyprogrammeCourses(studyprogramme)
+
+  const yearlyStudentByCoursePromises = makeYearlyPromises(yearRange, academicYear, 'allStudents', programmeCourses)
+  const yearlyProgrammeStudentsPromises = makeYearlyPromises(
+    yearRange,
+    academicYear,
+    'ownStudents',
+    programmeCourses,
+    studyprogramme
+  )
+  const yearlyStudentsWithoutStudyrightPromises = makeYearlyPromises(
+    yearRange,
+    academicYear,
+    'withoutStudyright',
+    programmeCourses
+  )
+  const yearlyOtherProgrammeStudentsPromises = makeYearlyPromises(
+    yearRange,
+    academicYear,
+    'otherStudents',
+    programmeCourses,
+    studyprogramme
+  )
+
+  const [
+    yearlyStudentByCourse,
+    yearlyProgrammeStudents,
+    yearlyStudentsWithoutStudyright,
+    yearlyOtherProgrammeStudents,
+  ] = await Promise.all([
+    Promise.all(yearlyStudentByCoursePromises),
+    Promise.all(yearlyProgrammeStudentsPromises),
+    Promise.all(yearlyStudentsWithoutStudyrightPromises),
+    Promise.all(yearlyOtherProgrammeStudentsPromises),
+  ])
+
+  let maxYear = 0
+  const allCourses = [
+    ...yearlyStudentByCourse.flat(),
+    ...yearlyProgrammeStudents.flat(),
+    ...yearlyStudentsWithoutStudyright.flat(),
+    ...yearlyOtherProgrammeStudents.flat(),
+  ].reduce((acc, curr) => {
+    if (curr.year > maxYear) maxYear = curr.year
+    if (!acc[curr.code]) {
+      acc[curr.code] = {
+        code: curr.code,
+        name: curr.name,
+        years: {},
+      }
+    }
+    if (!acc[curr.code]['years'][curr.year]) {
+      acc[curr.code]['years'][curr.year] = {
+        totalAllStudents: 0,
+        totalAllCredits: 0,
+        totalProgrammeStudents: 0,
+        totalProgrammeCredits: 0,
+        totalOtherProgrammeStudents: 0,
+        totalOtherProgrammeCredits: 0,
+        totalWithoutStudyrightStudents: 0,
+        totalWithoutStudyrightCredits: 0,
+      }
+    }
+    switch (curr.type) {
+      case 'total':
+        acc[curr.code]['years'][curr.year]['totalAllStudents'] += curr.totalAllStudents
+        acc[curr.code]['years'][curr.year]['totalAllCredits'] += curr.totalAllcredits
+        break
+      case 'ownProgramme':
+        acc[curr.code]['years'][curr.year]['totalProgrammeStudents'] += curr.totalProgrammeStudents
+        acc[curr.code]['years'][curr.year]['totalProgrammeCredits'] += curr.totalProgrammeCredits
+        break
+
+      case 'otherProgramme':
+        acc[curr.code]['years'][curr.year]['totalOtherProgrammeStudents'] += curr.totalOtherProgrammeStudents
+        acc[curr.code]['years'][curr.year]['totalOtherProgrammeCredits'] += curr.totalOtherProgrammeCredits
+        break
+
+      case 'noStudyright':
+        acc[curr.code]['years'][curr.year]['totalWithoutStudyrightStudents'] += curr.totalWithoutStudyrightStudents
+        acc[curr.code]['years'][curr.year]['totalWithoutStudyrightCredits'] += curr.totalWithoutStudyrightCredits
+        break
+    }
+
+    return acc
+  }, {})
+  const ayCourses = Object.keys(allCourses).filter(c => c.startsWith('AY'))
+  const properties = [
+    'totalAllStudents',
+    'totalAllCredits',
+    'totalProgrammeStudents',
+    'totalProgrammeCredits',
+    'totalOtherProgrammeStudents',
+    'totalOtherProgrammeCredits',
+    'totalWithoutStudyrightStudents',
+    'totalWithoutStudyrightCredits',
+  ]
+  ayCourses.forEach(ayCourse => {
+    const normCode = isOpenUniCourseCode(ayCourse)[1]
+
+    if (allCourses[normCode]) {
+      const mergedCourse = {}
+      mergedCourse['code'] = allCourses[normCode].code
+      mergedCourse['name'] = allCourses[normCode].name
+      mergedCourse['years'] = {}
+
+      yearRange
+        .filter(year => year <= maxYear)
+        .forEach(year => {
+          if (!allCourses[normCode]['years'][year]) {
+            mergedCourse['years'][year] = {
+              totalAllStudents: 0,
+              totalAllCredits: 0,
+              totalProgrammeStudents: 0,
+              totalProgrammeCredits: 0,
+              totalOtherProgrammeStudents: 0,
+              totalOtherProgrammeCredits: 0,
+              totalWithoutStudyrightStudents: 0,
+              totalWithoutStudyrightCredits: 0,
+            }
+          } else {
+            mergedCourse['years'][year] = { ...allCourses[normCode]['years'][year] }
+          }
+
+          if (allCourses[ayCourse]['years'][year]) {
+            properties.forEach(prop => {
+              mergedCourse['years'][year][prop] =
+                mergedCourse['years'][year][prop] + allCourses[ayCourse]['years'][year][prop]
+            })
+          }
+        })
+      allCourses[normCode] = mergedCourse
+      delete allCourses[ayCourse]
+    }
+  })
+  return Object.values(allCourses)
+}
+
+module.exports = { getStudyprogrammeCoursesForStudytrack, getAllStudyprogrammeCourses }
@@ -0,0 +1,167 @@
+const { indexOf } = require('lodash')
+
+const { mapToProviders } = require('../util/utils')
+const {
+  getStatsBasis,
+  defineYear,
+  isMajorStudentCredit,
+  isNonMajorCredit,
+  getYearsArray,
+  getStartDate,
+  isSpecialGroupCredit,
+  tableTitles,
+} = require('./studyprogrammeHelpers')
+const {
+  getStudyRights,
+  getCreditsForStudyProgramme,
+  getTransferredCredits,
+  allTransfers,
+  getCourseCodesForStudyProgramme,
+} = require('./studyprogramme')
+
+// Fetches all the credits for the studyprogramme and divides them into major-students and non-major students credits
+// Division is done on the basis that whether the student had a primary studyright to the programme on the attainment_date
+// If special groups are excluded, the transfer students are filtered away from the major-students as well
+const getRegularCreditStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const courses = await getCourseCodesForStudyProgramme(providercode)
+  const credits = await getCreditsForStudyProgramme(courses, since)
+  const students = [...new Set(credits.map(({ student_studentnumber }) => student_studentnumber))]
+
+  //let studyrights = await getProgrammesStudyrights(studyprogramme, since)
+  let studyrights = await getStudyRights(students, since)
+  const transfers = (await allTransfers(studyprogramme, since)).map(t => t.studyrightid)
+  if (!includeAllSpecials) {
+    studyrights = studyrights.filter(s => !transfers.includes(s.studyrightid))
+  }
+
+  let majors = getStatsBasis(years)
+  let nonMajors = getStatsBasis(years)
+  let nonDegree = getStatsBasis(years)
+
+  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
+    const studentStudyrights = studyrights.filter(studyright => studyright.studentnumber === student_studentnumber)
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+
+    if (!includeAllSpecials && isSpecialGroupCredit(studentStudyrights, attainment_date, transfers)) {
+      return
+    }
+
+    if (isMajorStudentCredit(studentStudyrights, attainment_date, studyprogramme)) {
+      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      majors.tableStats[attainmentYear] += credits || 0
+    } else if (isNonMajorCredit(studentStudyrights, attainment_date, studyprogramme)) {
+      nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      nonMajors.tableStats[attainmentYear] += credits || 0
+    } else {
+      nonDegree.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      nonDegree.tableStats[attainmentYear] += credits || 0
+    }
+  })
+
+  if (
+    majors.graphStats.every(year => year === 0) &&
+    nonMajors.graphStats.every(year => year === 0) &&
+    nonDegree.graphStats.every(year => year === 0)
+  ) {
+    return {
+      majors: { graphStats: [], tableStats: {} },
+      nonMajors: { graphStats: [], tableStats: {} },
+      nonDegree: { graphStats: [], tablestats: {} },
+    }
+  }
+
+  return { majors, nonMajors, nonDegree }
+}
+
+// Fetches all the credits with the type "transferred / hyväksiluettu" and divides them by year
+const getTransferredCreditStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const credits = await getTransferredCredits(providercode, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  credits.forEach(({ attainment_date, credits }) => {
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+    graphStats[indexOf(years, attainmentYear)] += credits || 0
+    tableStats[attainmentYear] += credits || 0
+  })
+
+  if (graphStats.every(year => year === 0)) {
+    return { graphStats: [], tableStats: {} }
+  }
+
+  return { graphStats, tableStats }
+}
+
+// Fetches all credits for the studytrack and combines them into the statistics for the table
+// and graph in the studyprogramme overview
+const getCreditStatsForStudytrack = async ({ studyprogramme, settings }) => {
+  const { isAcademicYear, includeAllSpecials } = settings
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const { majors, nonMajors, nonDegree } = await getRegularCreditStats(queryParameters)
+  const transferred = await getTransferredCreditStats(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const titles = tableTitles['credits'][includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED']
+  const dataFound = [majors, nonMajors, transferred].some(d => d.graphStats.length)
+
+  if (!dataFound) return null
+
+  const tableStats = reversedYears.map(year =>
+    includeAllSpecials
+      ? [
+          year,
+          majors.tableStats[year] +
+            nonMajors.tableStats[year] +
+            transferred.tableStats[year] +
+            nonDegree.tableStats[year],
+          majors.tableStats[year],
+          nonMajors.tableStats[year],
+          nonDegree.tableStats[year],
+          transferred.tableStats[year],
+        ]
+      : [
+          year,
+          majors.tableStats[year] +
+            nonMajors.tableStats[year] +
+            transferred.tableStats[year] +
+            nonDegree.tableStats[year],
+          majors.tableStats[year],
+          transferred.tableStats[year],
+        ]
+  )
+
+  const graphStats = [
+    {
+      name: 'Major students credits',
+      data: majors.graphStats,
+    },
+    {
+      name: 'Non-major students credits',
+      data: nonMajors.graphStats,
+    },
+    {
+      name: 'Non-degree credits',
+      data: nonDegree.graphStats,
+    },
+    {
+      name: 'Transferred credits',
+      data: transferred.graphStats,
+    },
+  ]
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    graphStats,
+    titles,
+  }
+}
+
+module.exports = {
+  getCreditStatsForStudytrack,
+}
@@ -0,0 +1,255 @@
+const moment = require('moment')
+const { indexOf, uniqBy } = require('lodash')
+
+const { mapToProviders } = require('../util/utils')
+const {
+  getCorrectStudentnumbers,
+  getStatsBasis,
+  defineYear,
+  getYearsObject,
+  getYearsArray,
+  getMedian,
+  getMean,
+  getStartDate,
+  getThesisType,
+  alltimeStartDate,
+  alltimeEndDate,
+} = require('./studyprogrammeHelpers')
+const {
+  graduatedStudyRights,
+  getThesisCredits,
+  followingStudyrights,
+  previousStudyrights,
+  allStudyrights,
+  allTransfers,
+} = require('./studyprogramme')
+const { getAllProgrammes } = require('./studyrights')
+
+const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  const studentnumbers = await getCorrectStudentnumbers({
+    codes: [studyprogramme],
+    startDate: alltimeStartDate,
+    endDate: alltimeEndDate,
+    includeAllSpecials,
+  })
+
+  const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbers)
+  studyrights.forEach(({ enddate }) => {
+    const graduationYear = defineYear(enddate, isAcademicYear)
+    graphStats[indexOf(years, graduationYear)] += 1
+    tableStats[graduationYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getThesisStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const thesisType = getThesisType(studyprogramme)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  const studentnumbers = await getCorrectStudentnumbers({
+    codes: [studyprogramme],
+    startDate: alltimeStartDate,
+    endDate: alltimeEndDate,
+    includeAllSpecials,
+  })
+
+  const credits = await getThesisCredits(providercode, since, thesisType, studentnumbers)
+
+  credits?.forEach(({ attainment_date }) => {
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+    graphStats[indexOf(years, attainmentYear)] += 1
+    tableStats[attainmentYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getGraduationTimeStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  let graduationAmounts = getYearsObject({ years })
+  let graduationTimes = getYearsObject({ years, emptyArrays: true })
+
+  const studentnumbers = await getCorrectStudentnumbers({
+    codes: [studyprogramme],
+    startDate: alltimeStartDate,
+    endDate: alltimeEndDate,
+    includeAllSpecials,
+  })
+
+  const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbers)
+
+  studyrights.forEach(({ enddate, studystartdate }) => {
+    const graduationYear = defineYear(enddate, isAcademicYear)
+    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+    graduationAmounts[graduationYear] += 1
+    graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
+  })
+  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
+  const comparisonValue = studyprogramme.includes('KH') ? 72 : 48
+
+  const medians = getYearsObject({ years, emptyArrays: true })
+  const means = getYearsObject({ years, emptyArrays: true })
+
+  // HighCharts graph require the data to have this format (ie. actual value, "empty value")
+  years.forEach(year => {
+    const median = getMedian(graduationTimes[year])
+    const mean = getMean(graduationTimes[year])
+    medians[year] = [
+      ['', median],
+      ['', comparisonValue - median],
+    ]
+    means[year] = [
+      ['', mean],
+      ['', comparisonValue - mean],
+    ]
+  })
+  return { medians, means, graduationAmounts }
+}
+
+// Creates (studyrightid, transfer)-map out of programmes transfers
+const getTransferStudyrightMap = async (studyprogramme, since) => {
+  const transfers = await allTransfers(studyprogramme, since)
+  const studyrightTransferMap = new Map()
+  transfers.forEach(t => studyrightTransferMap.set(t.studyrightid, t))
+  return studyrightTransferMap
+}
+
+const formatStats = (stats, years) => {
+  const tableStats = Object.values(stats)
+    .filter(p => years.map(year => p[year]).find(started => started !== 0)) // Filter out programmes with no-one started between the selected years
+    .map(p => [p.code, p.name['fi'], ...years.map(year => p[year])])
+
+  const graphStats = Object.values(stats)
+    .filter(p => years.map(year => p[year]).find(started => started !== 0)) // Filter out programmes with no-one started between the selected years
+    .map(p => ({ name: p.name['fi'], code: p.code, data: years.map(year => p[year]) }))
+
+  return { tableStats, graphStats }
+}
+
+const getProgrammesBeforeStarting = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const all = await allStudyrights(studyprogramme)
+  const allStudents = all.map(s => s.studentnumber)
+
+  // Get studyrights for bachelor studyprogrammes. This excludes studytracks.
+  const programmes = await getAllProgrammes()
+  let studyrightsBeforeMasters = await previousStudyrights(programmes, allStudents)
+  const studyprogrammeCodes = uniqBy(studyrightsBeforeMasters, 'code').map(s => ({ code: s.code, name: s.name }))
+  const stats = {}
+  studyprogrammeCodes.forEach(c => (stats[c.code] = { ...c, ...getYearsObject({ years }) }))
+
+  // Map transfers to the Master's programme
+  const transfers = await getTransferStudyrightMap(studyprogramme, since)
+
+  studyrightsBeforeMasters.forEach(({ studentnumber, code, givendate }) => {
+    const masterStudyright = all.find(s => s.studentnumber === studentnumber)
+    const transfer = transfers.get(masterStudyright.studyrightid)
+
+    // If special studyrights are excluded, transfers to the master's programme should be excluded
+    if (!includeAllSpecials && transfers.get(masterStudyright.studyrightid)) {
+      return
+    }
+
+    // If all studyrights are included, and transfer exists, define the start year in the programme, based on when the student transferred to it
+    // Otherwise, the year is defined based on when the student started in the chosen master's programme
+    const programmeStartDate = includeAllSpecials && transfer ? transfer.transferdate : masterStudyright?.studystartdate
+    const startYear = defineYear(programmeStartDate, isAcademicYear)
+
+    // The master studyright should have been given on the same date as the bachelor's
+    if (years.includes(startYear) && code && masterStudyright.givendate.getTime() === givendate.getTime()) {
+      stats[code][startYear] += 1
+    }
+  })
+
+  const graphAndTableStats = formatStats(stats, years)
+  return graphAndTableStats
+}
+
+const getProgrammesAfterGraduation = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const graduated = await graduatedStudyRights(studyprogramme, since)
+  const graduatedStudents = graduated.map(s => s.studentnumber)
+
+  // Get studyrights for masters studyprogramme. This excludes studytracks.
+  const programmes = await getAllProgrammes()
+  let studyrightsAfterThisProgramme = await followingStudyrights(since, programmes, graduatedStudents)
+  const studyprogrammeCodes = uniqBy(studyrightsAfterThisProgramme, 'code').map(s => ({ code: s.code, name: s.name }))
+  const stats = {}
+  studyprogrammeCodes.forEach(c => (stats[c.code] = { ...c, ...getYearsObject({ years }) }))
+
+  const transfers = await getTransferStudyrightMap(studyprogramme, since)
+
+  studyrightsAfterThisProgramme.forEach(({ studentnumber, studystartdate, code, givendate }) => {
+    const bachelorStudyright = graduated.find(s => s.studentnumber === studentnumber)
+
+    // If special studyrights are excluded, transfers to the bachelor programme should be excluded
+    if (!includeAllSpecials && transfers.get(bachelorStudyright.studyrightid)) {
+      return
+    }
+
+    // If all studyrights are included, and transfer exists, define the start year in the programme, based on when the student transferred to it
+    // Otherwise, the year is defined based on when the student started in the chosen master's programme
+    const startYear = defineYear(studystartdate, isAcademicYear)
+
+    // The master studyright should have been given on the same date as the bachelor's
+    if (years.includes(startYear) && code && bachelorStudyright.givendate.getTime() === givendate.getTime()) {
+      stats[code][startYear] += 1
+    }
+  })
+
+  const graphAndTableStats = formatStats(stats, years)
+  return graphAndTableStats
+}
+
+const getProgrammesBeforeOrAfter = async (studyprogramme, queryParameters) => {
+  if (studyprogramme.includes('KH')) return await getProgrammesAfterGraduation(queryParameters)
+  if (studyprogramme.includes('MH')) return await getProgrammesBeforeStarting(queryParameters)
+  return null
+}
+
+const getGraduationStatsForStudytrack = async ({ studyprogramme, settings }) => {
+  const { isAcademicYear, includeAllSpecials } = settings
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const thesis = await getThesisStats(queryParameters)
+
+  const graduated = await getGraduatedStats(queryParameters)
+  const graduationTimeStats = await getGraduationTimeStats(queryParameters)
+  const programmesBeforeOrAfter = await getProgrammesBeforeOrAfter(studyprogramme, queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+
+  const titles = ['', 'Graduated', 'Wrote thesis']
+  const programmesBeforeOrAfterTitles = ['Code', 'Programme', ...years]
+  const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    titles,
+    graphStats: [
+      {
+        name: 'Graduated students',
+        data: graduated.graphStats,
+      },
+      {
+        name: 'Wrote thesis',
+        data: thesis.graphStats,
+      },
+    ],
+    graduationMedianTime: graduationTimeStats.medians,
+    graduationMeanTime: graduationTimeStats.means,
+    graduationAmounts: graduationTimeStats.graduationAmounts,
+    programmesBeforeOrAfterTableStats: programmesBeforeOrAfter?.tableStats,
+    programmesBeforeOrAfterGraphStats: programmesBeforeOrAfter?.graphStats,
+    programmesBeforeOrAfterTitles,
+  }
+}
+
+module.exports = {
+  getGraduationStatsForStudytrack,
+}
@@ -1,16 +1,76 @@
 const { mean } = require('lodash')
+const { studentnumbersWithAllStudyrightElements } = require('./populations')

 // Helper functions
+
+const getCorrectStudentnumbers = async ({ codes, startDate, endDate, includeAllSpecials, includeGraduated = true }) => {
+  let studentnumbers = []
+  const exchangeStudents = includeAllSpecials
+  const nondegreeStudents = includeAllSpecials
+  const transferredOutStudents = includeAllSpecials
+  const transferredToStudents = !includeAllSpecials
+  const graduatedStudents = !includeGraduated
+
+  studentnumbers = await studentnumbersWithAllStudyrightElements(
+    codes,
+    startDate,
+    endDate,
+    exchangeStudents,
+    nondegreeStudents,
+    transferredOutStudents,
+    null,
+    transferredToStudents,
+    graduatedStudents
+  )
+
+  return studentnumbers
+}
+
+const defineStartDate = (studyright_elements, studystartdate) => {
+  if (studyright_elements?.length && studyright_elements[0].startdate > studystartdate) {
+    return studyright_elements[0].startdate
+  }
+  return studystartdate
+}
+
+const resolveStudyRightCode = studyright_elements => {
+  if (!studyright_elements) return null
+  const studyRightElement = studyright_elements
+    .filter(sre => sre.element_detail.type === 20)
+    .sort((a, b) => new Date(b.startdate) - new Date(a.startdate))[0]
+  if (studyRightElement) return studyRightElement.code
+  return null
+}
+
 const formatStudyright = studyright => {
-  const { studyrightid, studystartdate, enddate, graduated, prioritycode, extentcode, student } = studyright
-  return {
+  const {
    studyrightid,
    studystartdate,
    enddate,
+    givendate,
    graduated,
+    active,
+    prioritycode,
+    extentcode,
+    student,
+    studyright_elements,
+  } = studyright
+
+  return {
+    studyrightid,
+    studystartdate: defineStartDate(studyright_elements, studystartdate),
+    enddate,
+    givendate,
+    graduated,
+    active,
    prioritycode,
    extentcode,
    studentnumber: student.studentnumber,
+    code: resolveStudyRightCode(studyright_elements),
+    name:
+      studyright_elements?.length && studyright_elements[0].element_detail && studyright_elements[0].element_detail.name
+        ? studyright_elements[0].element_detail.name
+        : null,
  }
 }

@@ -24,8 +84,20 @@ const formatStudent = student => {
  }
 }

-const getYearsArray = (since, isAcademicYear) => {
+const formatTransfer = transfer => {
+  const { sourcecode, targetcode, transferdate, studyrightid } = transfer
+  return {
+    sourcecode,
+    targetcode,
+    transferdate,
+    studyrightid,
+  }
+}
+
+const getYearsArray = (since, isAcademicYear, yearsCombined) => {
  const years = []
+  const allYears = 'Total'
+  if (yearsCombined) years.push(allYears)
  for (let i = since; i <= new Date().getFullYear(); i++) {
    const year = isAcademicYear ? `${i} - ${i + 1}` : i
    years.push(year)
@@ -33,7 +105,7 @@ const getYearsArray = (since, isAcademicYear) => {
  return years
 }

-const getYearsObject = (years, emptyArrays = false) => {
+const getYearsObject = ({ years, emptyArrays = false }) => {
  let yearsObject = {}
  for (const year of years) {
    yearsObject = { ...yearsObject, [year]: emptyArrays ? [] : 0 }
@@ -41,19 +113,51 @@ const getYearsObject = (years, emptyArrays = false) => {
  return yearsObject
 }

+const getAcademicYearsObject = ({ years, emptyArrays = false }) => {
+  let yearsObject = {}
+  for (const year of years) {
+    yearsObject = { ...yearsObject, [`${year} - ${year + 1}`]: emptyArrays ? [] : 0 }
+  }
+  return yearsObject
+}
+
 const getStatsBasis = years => {
  return {
    graphStats: new Array(years.length).fill(0),
-    tableStats: getYearsObject(years),
+    tableStats: getYearsObject({ years }),
  }
 }

-const isMajorStudentCredit = (studyright, attainment_date) =>
-  studyright &&
-  (studyright.prioritycode === 1 || studyright.prioritycode === 30) && // Is studyright state = MAIN or state = GRADUATED
-  studyright.studystartdate <= attainment_date && // Has the credit been attained after studying in the programme started
-  studyright.enddate >= attainment_date && // Has the credit been attained before the studyright ended
-  (!studyright.canceldate || studyright.canceldate >= attainment_date) // If the studyright was cancelled, was the credit attained before it was cancelled
+const isMajorStudentCredit = (studyrights, attainment_date, code) =>
+  studyrights.some(studyright => {
+    if (!studyright) return false
+    if (studyright.code !== code) return false
+    if (!studyright.graduated) return new Date(attainment_date) >= new Date(studyright.studystartdate)
+    return (
+      new Date(attainment_date) >= new Date(studyright.studystartdate) &&
+      new Date(attainment_date) <= new Date(studyright.enddate)
+    )
+  })
+
+const isNonMajorCredit = (studyrights, attainment_date) =>
+  studyrights.some(studyright => {
+    if (!studyright) return false
+    if (!studyright.graduated) return new Date(attainment_date) >= new Date(studyright.studystartdate)
+    return (
+      new Date(attainment_date) >= new Date(studyright.studystartdate) &&
+      new Date(attainment_date) <= new Date(studyright.enddate)
+    )
+  })
+
+const isSpecialGroupCredit = (studyrights, attainment_date, transfers) =>
+  studyrights.some(studyright => {
+    if (!studyright) return true // If there is no studyright matching the credit, is not a major student credit
+    if (studyright.studystartdate > attainment_date) return true // Credits before the studyright started are not major student credits
+    if (studyright.enddate && attainment_date > studyright.enddate) return true // Credits after studyright are not major student credits
+    if ([7, 9, 16, 34, 33, 99, 14, 13].includes(studyright.extentcode)) return true // Excludes non-degree studyrights and exchange students
+    if (transfers.includes(studyright.studyrightid)) return true // Excludes both transfers in and out of the programme
+    return false
+  })

 const getMedian = values => {
  if (values.length === 0) return 0
@@ -79,10 +183,13 @@ const defineYear = (date, isAcademicYear) => {
 const getStartDate = (studyprogramme, isAcademicYear) => {
  if ((studyprogramme.includes('KH') || studyprogramme.includes('MH')) && isAcademicYear) return new Date('2017-08-01')
  if (studyprogramme.includes('KH') || studyprogramme.includes('MH')) return new Date('2017-01-01')
-  if (isAcademicYear) return new Date('2000-08-01')
-  return new Date('2000-01-01')
+  if (isAcademicYear) return new Date('2017-08-01')
+  return new Date('2017-01-01')
 }

+const alltimeStartDate = new Date('1900-01-01')
+const alltimeEndDate = new Date()
+
 // There are 9 course_unit_types
 // 1. urn:code:course-unit-type:regular
 // 2. urn:code:course-unit-type:bachelors-thesis
@@ -102,7 +209,7 @@ const getThesisType = studyprogramme => {
    return ['urn:code:course-unit-type:bachelors-thesis']
  if (/^(T)[0-9]{6}$/.test(studyprogramme))
    return ['urn:code:course-unit-type:doctors-thesis', 'urn:code:course-unit-type:licentiate-thesis']
-  return 'thesis-type-not-defined'
+  return ['urn:code:course-unit-type:doctors-thesis', 'urn:code:course-unit-type:licentiate-thesis']
 }

 const getPercentage = (value, total) => {
@@ -114,7 +221,7 @@ const getPercentage = (value, total) => {

 const getEmptyArray = length => new Array(length).fill(0)

-const getCreditGraphStats = years => ({
+const getBachelorCreditGraphStats = years => ({
  lte30: {
    name: 'Less than 30 credits',
    data: getEmptyArray(years.length),
@@ -135,27 +242,197 @@ const getCreditGraphStats = years => ({
    name: '120-149 credits',
    data: getEmptyArray(years.length),
  },
-  mte150: {
-    name: 'More than 150 credits',
+  lte180: {
+    name: '150-179 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte180: {
+    name: 'More than 180 credits',
    data: getEmptyArray(years.length),
  },
 })

-const creditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'mte150']
+const getMasterCreditGraphStats = years => ({
+  lte200: {
+    name: 'Less than 200 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte220: {
+    name: '200-219 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte240: {
+    name: '220-239 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte260: {
+    name: '240-259 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte280: {
+    name: '260-279 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte300: {
+    name: '280-299 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte300: {
+    name: 'More than 300 credits',
+    data: getEmptyArray(years.length),
+  },
+})
+
+const getDoctoralCreditGraphStats = years => ({
+  lte50: {
+    name: 'Less than 50 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte100: {
+    name: '50-99 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte150: {
+    name: '100-149 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte200: {
+    name: '150-199 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte250: {
+    name: '200-249 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte300: {
+    name: '250-299 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte300: {
+    name: 'More than 300 credits',
+    data: getEmptyArray(years.length),
+  },
+})
+
+const getCreditGraphStats = (studyprogramme, years) => {
+  if (studyprogramme.includes('KH')) return getBachelorCreditGraphStats(years)
+  if (studyprogramme.includes('MH')) return getMasterCreditGraphStats(years)
+  return getDoctoralCreditGraphStats(years)
+}
+
+const bachelorCreditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'lte180', 'mte180']
+const masterCreditThresholds = ['lte200', 'lte220', 'lte240', 'lte260', 'lte280', 'lte300', 'mte300']
+const doctoralCreditThresholds = ['lte50', 'lte100', 'lte150', 'lte200', 'lte250', 'lte300', 'mte300']
+const bachelorCreditAmounts = [30, 60, 90, 120, 150, 180, 180]
+const masterCreditAmounts = [200, 220, 240, 260, 280, 300, 300]
+const doctoralCreditAmounts = [50, 100, 150, 200, 250, 300, 300]
+
+const getCreditThresholds = studyprogramme => {
+  if (studyprogramme.includes('KH')) {
+    return { creditThresholdKeys: bachelorCreditThresholds, creditThresholdAmounts: bachelorCreditAmounts }
+  }
+  if (studyprogramme.includes('MH')) {
+    return { creditThresholdKeys: masterCreditThresholds, creditThresholdAmounts: masterCreditAmounts }
+  }
+  return { creditThresholdKeys: doctoralCreditThresholds, creditThresholdAmounts: doctoralCreditAmounts }
+}
+
+const tableTitles = {
+  basics: {
+    SPECIAL_EXCLUDED: ['', 'Started studying', 'Graduated'],
+    SPECIAL_INCLUDED: ['', 'Started studying', 'Graduated', 'Transferred away', 'Transferred to'],
+  },
+  credits: {
+    SPECIAL_EXCLUDED: ['', 'Total', 'Major students credits', 'Transferred credits'],
+    SPECIAL_INCLUDED: [
+      '',
+      'Total',
+      'Major students credits',
+      'Non major students credits',
+      'Non degree students credits',
+      'Transferred credits',
+    ],
+  },
+  creditProgress: {
+    bachelor: [
+      '',
+      'All',
+      '< 30 credits',
+      '30-59 credits',
+      '60-89 credits',
+      '90-119 credits',
+      '120-149 credits',
+      '150-179 credits',
+      '> 180 credits',
+    ],
+    master: [
+      '',
+      'All',
+      '< 200 credits',
+      '200-219 credits',
+      '220-239 credits',
+      '240-259 credits',
+      '260-279 credits',
+      '280-299 credits',
+      '> 300 credits',
+    ],
+    doctoral: [
+      '',
+      'All',
+      '< 50 credits',
+      '50-99 credits',
+      '100-149 credits',
+      '150-199 credits',
+      '200-249 credits',
+      '250-299 credits',
+      '> 300 credits',
+    ],
+  },
+  studytracks: [
+    '',
+    'All',
+    'Started studying',
+    'Currently enrolled',
+    'Absent',
+    'Inactive',
+    'Graduated',
+    'Men',
+    'Women',
+    'Finnish',
+  ],
+}
+
+const getCreditProgressTableTitles = studyprogramme => {
+  if (studyprogramme.includes('KH')) return tableTitles.creditProgress.bachelor
+  if (studyprogramme.includes('MH')) return tableTitles.creditProgress.master
+  return tableTitles.creditProgress.doctoral
+}

 module.exports = {
+  getCorrectStudentnumbers,
  formatStudyright,
  formatStudent,
+  formatTransfer,
  getYearsArray,
  getYearsObject,
+  getAcademicYearsObject,
  getStatsBasis,
  isMajorStudentCredit,
+  isSpecialGroupCredit,
  getMedian,
  getMean,
  defineYear,
  getStartDate,
+  alltimeStartDate,
+  alltimeEndDate,
  getThesisType,
  getPercentage,
+  getBachelorCreditGraphStats,
+  getMasterCreditGraphStats,
+  getDoctoralCreditGraphStats,
  getCreditGraphStats,
-  creditThresholds,
+  getCreditThresholds,
+  tableTitles,
+  getCreditProgressTableTitles,
+  isNonMajorCredit,
 }
@@ -1,315 +0,0 @@
-const moment = require('moment')
-const { indexOf } = require('lodash')
-
-const { mapToProviders } = require('../util/utils')
-const {
-  getStatsBasis,
-  defineYear,
-  isMajorStudentCredit,
-  getYearsObject,
-  getYearsArray,
-  getMedian,
-  getMean,
-  getStartDate,
-  getThesisType,
-} = require('./studyprogrammeHelpers')
-const {
-  graduatedStudyRights,
-  startedStudyrights,
-  cancelledStudyRights,
-  transfersAway,
-  transfersTo,
-  getProgrammesStudents,
-  getCreditsForStudyProgramme,
-  getTransferredCredits,
-  getThesisCredits,
-} = require('./newStudyprogramme')
-
-// const getEnrolledStats = async (studytrack, since, years, isAcademicYear) => {
-//   const students = await enrolledStudyrights(studytrack)
-//   const { graphStats, tableStats } = getStatsBasis(years)
-
-//   students.forEach(({ startdate }) => {
-//     if (startdate.getMonth() === 7) {
-//       const enrolledYear = defineYear(startdate, isAcademicYear)
-
-//       graphStats[indexOf(years, enrolledYear)] += 1
-//       tableStats[enrolledYear] += 1
-//     }
-//   })
-
-//   return { graphStats, tableStats }
-// }
-
-const getGraduatedStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await graduatedStudyRights(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ enddate }) => {
-    const graduationYear = defineYear(enddate, isAcademicYear)
-    graphStats[indexOf(years, graduationYear)] += 1
-    tableStats[graduationYear] += 1
-  })
-  return { graphStats, tableStats }
-}
-
-const getStartedStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await startedStudyrights(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ studystartdate }) => {
-    const startYear = defineYear(studystartdate, isAcademicYear)
-    graphStats[indexOf(years, startYear)] += 1
-    tableStats[startYear] += 1
-  })
-  return { graphStats, tableStats }
-}
-
-const getCancelledStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await cancelledStudyRights(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ canceldate }) => {
-    const cancelYear = defineYear(canceldate, isAcademicYear)
-    graphStats[indexOf(years, cancelYear)] += 1
-    tableStats[cancelYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getTransferredAwayStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await transfersAway(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ transferdate }) => {
-    const transferYear = defineYear(transferdate, isAcademicYear)
-    graphStats[indexOf(years, transferYear)] += 1
-    tableStats[transferYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getTransferredToStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await transfersTo(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ transferdate }) => {
-    const transferYear = defineYear(transferdate, isAcademicYear)
-    graphStats[indexOf(years, transferYear)] += 1
-    tableStats[transferYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getRegularCreditStats = async (studytrack, since, years, isAcademicYear) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const studyrights = await getProgrammesStudents(studytrack)
-  const credits = await getCreditsForStudyProgramme(providercode, since)
-
-  let majors = getStatsBasis(years)
-  let nonMajors = getStatsBasis(years)
-
-  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
-    const studyright = studyrights.find(studyright => studyright.studentnumber == student_studentnumber)
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-
-    if (isMajorStudentCredit(studyright, attainment_date)) {
-      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
-      majors.tableStats[attainmentYear] += credits || 0
-    } else {
-      nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
-      nonMajors.tableStats[attainmentYear] += credits || 0
-    }
-  })
-
-  return { majors, nonMajors }
-}
-
-const getTransferredCreditStats = async (studytrack, since, years, isAcademicYear) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const credits = await getTransferredCredits(providercode, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  credits.forEach(({ attainment_date, credits }) => {
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-    graphStats[indexOf(years, attainmentYear)] += credits || 0
-    tableStats[attainmentYear] += credits || 0
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getThesisStats = async (studytrack, since, years, isAcademicYear) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const thesisType = getThesisType(studytrack)
-  const credits = await getThesisCredits(providercode, since, thesisType)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  credits.forEach(({ attainment_date }) => {
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-    graphStats[indexOf(years, attainmentYear)] += 1
-    tableStats[attainmentYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getGraduationTimeStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await graduatedStudyRights(studytrack, since)
-  let graduationAmounts = getYearsObject(years)
-  let graduationTimes = getYearsObject(years, true)
-
-  studyrights.forEach(({ enddate, studystartdate }) => {
-    const graduationYear = defineYear(enddate, isAcademicYear)
-    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
-    graduationAmounts[graduationYear] += 1
-    graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
-  })
-
-  const medians = getYearsObject(years, true)
-  const means = getYearsObject(years, true)
-
-  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
-  const comparisonValue = studytrack.includes('KH') ? 72 : 48
-
-  // HighCharts graph require the data to have this format (ie. actual value, "empty value")
-  years.forEach(year => {
-    const median = getMedian(graduationTimes[year])
-    const mean = getMean(graduationTimes[year])
-    medians[year] = [
-      ['', median],
-      ['', comparisonValue - median],
-    ]
-    means[year] = [
-      ['', mean],
-      ['', comparisonValue - mean],
-    ]
-  })
-  return { medians, means, graduationAmounts }
-}
-
-const getBasicStatsForStudytrack = async ({ studyprogramme, yearType }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-  const started = await getStartedStats(studyprogramme, since, years, isAcademicYear)
-  const graduated = await getGraduatedStats(studyprogramme, since, years, isAcademicYear)
-  const cancelled = await getCancelledStats(studyprogramme, since, years, isAcademicYear)
-  const transferredAway = await getTransferredAwayStats(studyprogramme, since, years, isAcademicYear)
-  const transferredTo = await getTransferredToStats(studyprogramme, since, years, isAcademicYear)
-
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const tableStats = reversedYears.map(year => [
-    year,
-    started.tableStats[year],
-    graduated.tableStats[year],
-    cancelled.tableStats[year],
-    transferredAway.tableStats[year],
-    transferredTo.tableStats[year],
-  ])
-
-  return {
-    id: studyprogramme,
-    years,
-    graphStats: [
-      {
-        name: 'Started',
-        data: started.graphStats,
-      },
-      {
-        name: 'Graduated',
-        data: graduated.graphStats,
-      },
-      {
-        name: 'Cancelled',
-        data: cancelled.graphStats,
-      },
-      {
-        name: 'Transferred away',
-        data: transferredAway.graphStats,
-      },
-      {
-        name: 'Transferred To',
-        data: transferredTo.graphStats,
-      },
-    ],
-    tableStats,
-  }
-}
-
-const getCreditStatsForStudytrack = async ({ studyprogramme, yearType }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-  const { majors, nonMajors } = await getRegularCreditStats(studyprogramme, since, years, isAcademicYear)
-  const transferred = await getTransferredCreditStats(studyprogramme, since, years, isAcademicYear)
-
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const tableStats = reversedYears.map(year => [
-    year,
-    majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
-    majors.tableStats[year],
-    nonMajors.tableStats[year],
-    transferred.tableStats[year],
-  ])
-
-  return {
-    id: studyprogramme,
-    years,
-    tableStats,
-    graphStats: [
-      {
-        name: 'Major students credits',
-        data: majors.graphStats,
-      },
-      {
-        name: 'Non-major students credits',
-        data: nonMajors.graphStats,
-      },
-      {
-        name: 'Transferred credits',
-        data: transferred.graphStats,
-      },
-    ],
-  }
-}
-
-const getGraduationStatsForStudytrack = async ({ studyprogramme, yearType }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-  const thesis = await getThesisStats(studyprogramme, since, years, isAcademicYear)
-  const graduated = await getGraduatedStats(studyprogramme, since, years, isAcademicYear)
-  const graduationTimeStats = await getGraduationTimeStats(studyprogramme, since, years, isAcademicYear)
-
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
-
-  return {
-    id: studyprogramme,
-    years,
-    tableStats,
-    graphStats: [
-      {
-        name: 'Graduated students',
-        data: graduated.graphStats,
-      },
-      {
-        name: 'Wrote thesis',
-        data: thesis.graphStats,
-      },
-    ],
-    graduationMedianTime: graduationTimeStats.medians,
-    graduationMeanTime: graduationTimeStats.means,
-    graduationAmounts: graduationTimeStats.graduationAmounts,
-  }
-}
-
-module.exports = {
-  getBasicStatsForStudytrack,
-  getCreditStatsForStudytrack,
-  getGraduationStatsForStudytrack,
-}
@@ -1,8 +1,6 @@
-const {
-  getBasicStatsForStudytrack,
-  getCreditStatsForStudytrack,
-  getGraduationStatsForStudytrack,
-} = require('../services/studyprogrammeStats')
+const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics')
+const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
+const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
 const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
 const {
  setBasicStats,
@@ -10,53 +8,104 @@ const {
  setGraduationStats,
  setStudytrackStats,
 } = require('../services/analyticsService')
+const logger = require('../util/logger')

 const updateBasicView = async code => {
-  const academic = 'ACADEMIC_YEAR'
-  const calendar = 'CALENDAR_YEAR'
+  const specialCalendar = {
+    yearType: 'CALENDAR_YEAR',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const specialAcademic = {
+    yearType: 'ACADEMIC_YEAR',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const specialExcludedCalendar = {
+    yearType: 'CALENDAR_YEAR',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }

-  const basicStatsAcademic = await getBasicStatsForStudytrack({
-    studyprogramme: code,
-    yearType: academic,
-  })
-  await setBasicStats(basicStatsAcademic, academic)
+  const specialExcludedAcademic = {
+    yearType: 'ACADEMIC_YEAR',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }

-  const basicStatsCalendar = await getBasicStatsForStudytrack({
-    studyprogramme: code,
-    yearType: calendar,
-  })
-  await setBasicStats(basicStatsCalendar, calendar)
+  const options = [specialCalendar, specialAcademic, specialExcludedCalendar, specialExcludedAcademic]

-  const creditStatsAcademic = await getCreditStatsForStudytrack({
-    studyprogramme: code,
-    yearType: academic,
-  })
-  await setCreditStats(creditStatsAcademic, academic)
+  for (const option of options) {
+    try {
+      const basicStats = await getBasicStatsForStudytrack({
+        studyprogramme: code,
+        settings: {
+          isAcademicYear: option.yearType === 'ACADEMIC_YEAR',
+          includeAllSpecials: option.specialGroups === 'SPECIAL_INCLUDED',
+        },
+      })
+      await setBasicStats(basicStats, option.yearType, option.specialGroups)

-  const creditStatsCalendar = await getCreditStatsForStudytrack({
-    studyprogramme: code,
-    yearType: calendar,
-  })
-  await setCreditStats(creditStatsCalendar, calendar)
+      const creditStats = await getCreditStatsForStudytrack({
+        studyprogramme: code,
+        settings: {
+          isAcademicYear: option.yearType === 'ACADEMIC_YEAR',
+          includeAllSpecials: option.specialGroups === 'SPECIAL_INCLUDED',
+        },
+      })
+      await setCreditStats(creditStats, option.yearType, option.specialGroups)

-  const graduationStatsAcademic = await getGraduationStatsForStudytrack({
-    studyprogramme: code,
-    yearType: academic,
-  })
-  await setGraduationStats(graduationStatsAcademic, academic)
-
-  const graduationStatsCalendar = await getGraduationStatsForStudytrack({
-    studyprogramme: code,
-    yearType: calendar,
-  })
-  await setGraduationStats(graduationStatsCalendar, calendar)
-
-  return 'OK'
+      const graduationStats = await getGraduationStatsForStudytrack({
+        studyprogramme: code,
+        settings: {
+          isAcademicYear: option.yearType === 'ACADEMIC_YEAR',
+          includeAllSpecials: option.specialGroups === 'SPECIAL_INCLUDED',
+        },
+      })
+      await setGraduationStats(graduationStats, option.yearType, option.specialGroups)
+    } catch (e) {
+      logger.error(e)
+    }
+    return 'OK'
+  }
 }

 const updateStudytrackView = async code => {
-  const result = await getStudytrackStatsForStudyprogramme({ studyprogramme: code })
-  await setStudytrackStats(result)
+  const specialGraduated = {
+    graduated: 'GRADUATED_INCLUDED',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const specialExcludedGraduated = {
+    graduated: 'GRADUATED_INCLUDED',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
+  const specialGraduatedExcluded = {
+    graduated: 'GRADUATED_EXCLUDED',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const specialExcludedGraduatedExcluded = {
+    graduated: 'GRADUATED_EXCLUDED',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
+
+  const options = [
+    specialGraduated,
+    specialExcludedGraduated,
+    specialGraduatedExcluded,
+    specialExcludedGraduatedExcluded,
+  ]
+
+  for (const option of options) {
+    try {
+      const stats = await getStudytrackStatsForStudyprogramme({
+        studyprogramme: code,
+        settings: {
+          specialGroups: option.specialGroups === 'SPECIAL_INCLUDED',
+          graduated: option.graduated === 'GRADUATED_INCLUDED',
+        },
+      })
+      await setStudytrackStats(stats, option.graduated, option.specialGroups)
+    } catch (e) {
+      logger.error(e)
+    }
+  }
+
  return 'OK'
 }

@@ -1,58 +1,131 @@
 const { indexOf } = require('lodash')
+const moment = require('moment')

 const { getAssociations } = require('./studyrights')
-const { studentnumbersWithAllStudyrightElements } = require('./populations')
 const {
+  getMedian,
+  getMean,
  getStartDate,
  getYearsArray,
  getPercentage,
-  getCreditGraphStats,
  getYearsObject,
-  creditThresholds,
+  getCorrectStudentnumbers,
+  getCreditThresholds,
+  getCreditGraphStats,
+  tableTitles,
+  getCreditProgressTableTitles,
 } = require('./studyprogrammeHelpers')
-const { studytrackStudents, allStudyrights, startedStudyrights, graduatedStudyRights } = require('./newStudyprogramme')
+const {
+  studytrackStudents,
+  allStudyrights,
+  startedStudyrights,
+  inactiveStudyrights,
+  enrolledStudents,
+  absentStudents,
+  graduatedStudyRights,
+} = require('./studyprogramme')
 const { getAcademicYearDates } = require('../util/semester')

-const getStudentData = students => {
-  const data = { female: 0, male: 0, finnish: 0, lte30: 0, lte60: 0, lte90: 0, lte120: 0, lte150: 0, mte150: 0 }
+const getStudentData = (students, thresholdKeys, thresholdAmounts) => {
+  let data = { female: 0, male: 0, finnish: 0 }
+  thresholdKeys.forEach(t => (data[t] = 0))
+
  students.forEach(({ gender_code, home_country_en, creditcount }) => {
    data.male += gender_code === '1' ? 1 : 0
    data.female += gender_code === '2' ? 1 : 0
    data.finnish += home_country_en === 'Finland' ? 1 : 0
-    data.lte30 += creditcount < 30 ? 1 : 0
-    data.lte60 += creditcount >= 30 && creditcount < 60 ? 1 : 0
-    data.lte90 += creditcount >= 60 && creditcount < 90 ? 1 : 0
-    data.lte120 += creditcount >= 90 && creditcount < 120 ? 1 : 0
-    data.lte150 += creditcount >= 120 && creditcount < 150 ? 1 : 0
-    data.mte150 += creditcount >= 150 ? 1 : 0
+    data[thresholdKeys[0]] += creditcount < thresholdAmounts[0] ? 1 : 0
+    data[thresholdKeys[1]] += creditcount >= thresholdAmounts[0] && creditcount < thresholdAmounts[1] ? 1 : 0
+    data[thresholdKeys[2]] += creditcount >= thresholdAmounts[1] && creditcount < thresholdAmounts[2] ? 1 : 0
+    data[thresholdKeys[3]] += creditcount >= thresholdAmounts[2] && creditcount < thresholdAmounts[3] ? 1 : 0
+    data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
+    data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
+    data[thresholdKeys[6]] += creditcount >= thresholdAmounts[6] ? 1 : 0
  })
  return data
 }

+const getGraduationTimeStats = async ({
+  year,
+  graduated,
+  studyprogramme,
+  track,
+  graduationMeanTime,
+  graduationMedianTime,
+  graduationAmounts,
+}) => {
+  // Count how long each student took to graduate
+  let graduationTimes = []
+  graduated.forEach(({ enddate, studystartdate }) => {
+    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+    graduationAmounts[track][year] += 1
+    graduationTimes = [...graduationTimes, timeToGraduation]
+  })
+
+  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
+  const comparisonValue = studyprogramme.includes('KH') ? 72 : 48
+
+  // HighCharts graph requires the data to have this format (ie. actual value, "empty value")
+  const median = getMedian(graduationTimes)
+  const mean = getMean(graduationTimes)
+  graduationMedianTime[track][year] = [
+    ['', median],
+    ['', comparisonValue - median],
+  ]
+  graduationMeanTime[track][year] = [
+    ['', mean],
+    ['', comparisonValue - mean],
+  ]
+}
+
 // Goes through the programme and all its studytracks for the said year and adds the wanted stats to the data objects
-const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studytrackNames, year, years, data }) => {
-  const { mainStatsByYear, mainStatsByTrack, creditGraphStats, creditTableStats, emptyTracks } = data
-  const { startDate, endDate } = getAcademicYearDates(year)
+const getStudytrackDataForTheYear = async ({
+  studyprogramme,
+  since,
+  settings,
+  studytracks,
+  studytrackNames,
+  year,
+  years,
+  data,
+  creditThresholdKeys,
+  creditThresholdAmounts,
+}) => {
+  const { specialGroups: includeAllSpecials, graduated: includeGraduated } = settings
+  const { startDate, endDate } = getAcademicYearDates(year, since)
+
+  const {
+    mainStatsByYear,
+    mainStatsByTrack,
+    creditGraphStats,
+    creditTableStats,
+    graduationMeanTime,
+    graduationMedianTime,
+    graduationAmounts,
+    totalAmounts,
+    emptyTracks,
+  } = data

  await Promise.all(
    studytracks.map(async track => {
      const codes = studyprogramme === track ? [studyprogramme] : [studyprogramme, track]
-      const studentnumbers = await studentnumbersWithAllStudyrightElements(
+
+      const studentnumbers = await getCorrectStudentnumbers({
        codes,
        startDate,
        endDate,
-        true,
-        true,
-        true,
-        true
-      )
+        includeAllSpecials,
+        includeGraduated,
+      })

-      // All the stats are counted for the students who actually started studying
+      // Get all the studyrights and students for the calculations
      const all = await allStudyrights(track, studentnumbers)
-      const allStudentnumbers = [...new Set(all.map(s => s.studentnumber))]
-      const students = await studytrackStudents(allStudentnumbers)
-      const studentData = getStudentData(students)
+      const students = await studytrackStudents(studentnumbers)
+      const studentData = getStudentData(students, creditThresholdKeys, creditThresholdAmounts)
      const started = await startedStudyrights(track, startDate, studentnumbers)
+      const enrolled = await enrolledStudents(track, startDate, studentnumbers)
+      const absent = await absentStudents(track, studentnumbers)
+      const inactive = await inactiveStudyrights(track, studentnumbers)
      const graduated = await graduatedStudyRights(track, startDate, studentnumbers)

      // If the track has no stats for that year, it should be removed from the table and dropdown options
@@ -62,34 +135,31 @@ const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studyt
      }

      // Count stats for creditgraph for the year per track
-      creditThresholds.forEach(
+      creditThresholdKeys.forEach(
        threshold => (creditGraphStats[track][threshold].data[indexOf(years, year)] = studentData[threshold])
      )

      // Count stats for the credit progress table for the year per track
      creditTableStats[track] = [
        ...creditTableStats[track],
-        [
-          `${year} - ${year + 1}`,
-          all.length,
-          studentData.lte30,
-          studentData.lte60,
-          studentData.lte90,
-          studentData.lte120,
-          studentData.lte150,
-          studentData.mte150,
-        ],
+        [year, all.length, ...creditThresholdKeys.map(threshold => studentData[threshold])],
      ]

      // Count stats for the main studytrack table grouped by tracks
      mainStatsByTrack[track] = [
        ...mainStatsByTrack[track],
        [
-          `${year} - ${year + 1}`,
+          year,
          all.length,
          getPercentage(all.length, all.length),
          started.length,
          getPercentage(started.length, all.length),
+          enrolled.length,
+          getPercentage(enrolled.length, all.length),
+          absent.length,
+          getPercentage(absent.length, all.length),
+          inactive.length,
+          getPercentage(inactive.length, all.length),
          graduated.length,
          getPercentage(graduated.length, all.length),
          studentData.male,
@@ -105,11 +175,17 @@ const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studyt
      mainStatsByYear[year] = [
        ...mainStatsByYear[year],
        [
-          studytrackNames[track]?.name['fi'] || `${year} - ${year + 1}`,
+          studytrackNames[track]?.name['fi'] ? `${studytrackNames[track]?.name['fi']}, ${track}` : year,
          all.length,
          getPercentage(all.length, all.length),
          started.length,
          getPercentage(started.length, all.length),
+          enrolled.length,
+          getPercentage(enrolled.length, all.length),
+          absent.length,
+          getPercentage(absent.length, all.length),
+          inactive.length,
+          getPercentage(inactive.length, all.length),
          graduated.length,
          getPercentage(graduated.length, all.length),
          studentData.male,
@@ -120,6 +196,18 @@ const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studyt
          getPercentage(studentData.finnish, all.length),
        ],
      ]
+
+      // Count stats for the graduation time charts grouped by year
+      totalAmounts[track][year] = await all.length
+      await getGraduationTimeStats({
+        year,
+        graduated,
+        studyprogramme,
+        track,
+        graduationMedianTime,
+        graduationMeanTime,
+        graduationAmounts,
+      })
    })
  )
 }
@@ -138,27 +226,46 @@ const getStudytrackOptions = (studyprogramme, studytrackNames, studytracks, empt
 }

 // Creates empty objects for each statistic type, which are then updated with the studytrack data
-const getEmptyStatsObjects = (years, studytracks) => {
-  const mainStatsByYear = getYearsObject(years, true)
+const getEmptyStatsObjects = (years, studytracks, studyprogramme) => {
+  const mainStatsByYear = getYearsObject({ years, emptyArrays: true })
  const mainStatsByTrack = {}
  const creditGraphStats = {}
  const creditTableStats = {}
+  const graduationMedianTime = {}
+  const graduationMeanTime = {}
+  const graduationAmounts = {}
+  const totalAmounts = {}
  const emptyTracks = new Map()

  studytracks.forEach(async track => {
    mainStatsByTrack[track] = []
-    creditGraphStats[track] = await getCreditGraphStats(years)
+    creditGraphStats[track] = getCreditGraphStats(studyprogramme, years)
    creditTableStats[track] = []
+    graduationMedianTime[track] = getYearsObject({ years, emptyArrays: true })
+    graduationMeanTime[track] = getYearsObject({ years, emptyArrays: true })
+    graduationAmounts[track] = getYearsObject({ years })
+    totalAmounts[track] = getYearsObject({ years })
  })

-  return { mainStatsByYear, mainStatsByTrack, creditGraphStats, creditTableStats, emptyTracks }
+  return {
+    mainStatsByYear,
+    mainStatsByTrack,
+    creditGraphStats,
+    creditTableStats,
+    graduationMedianTime,
+    graduationMeanTime,
+    graduationAmounts,
+    totalAmounts,
+    emptyTracks,
+  }
 }

 // Combines all the data for the Populations and Studytracks -view
-const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {
+const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, settings }) => {
  const isAcademicYear = true
+  const includeYearsCombined = true
  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear())
+  const years = getYearsArray(since.getFullYear(), isAcademicYear, includeYearsCombined)

  const associations = await getAssociations()
  const studytracks = associations.programmes[studyprogramme]
@@ -167,17 +274,22 @@ const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {

  const studytrackNames = associations.studyTracks

-  const data = getEmptyStatsObjects(years, studytracks)
+  const data = getEmptyStatsObjects(years, studytracks, studyprogramme)
+  const { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(studyprogramme)

  await Promise.all(
    years.map(async year => {
      return await getStudytrackDataForTheYear({
        studyprogramme,
+        since,
+        settings,
        studytracks,
        studytrackNames,
        year,
        years,
        data,
+        creditThresholdKeys,
+        creditThresholdAmounts,
      })
    })
  )
@@ -186,12 +298,12 @@ const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {

  return {
    id: studyprogramme,
-    years: getYearsArray(since.getFullYear(), isAcademicYear),
-    mainStatsByTrack: data.mainStatsByTrack,
-    mainStatsByYear: data.mainStatsByYear,
-    creditTableStats: data.creditTableStats,
-    creditGraphStats: data.creditGraphStats,
+    years: getYearsArray(since.getFullYear(), isAcademicYear, includeYearsCombined),
+    ...data,
    studytrackOptions,
+    includeGraduated: settings.graduated,
+    populationTitles: tableTitles['studytracks'],
+    creditTableTitles: getCreditProgressTableTitles(studyprogramme),
  }
 }

@@ -1,29 +0,0 @@
-const { ThesisCourse } = require('../models/models_kone')
-
-const createThesisCourse = (programme, course, type) =>
-  ThesisCourse.create({
-    programmeCode: programme,
-    courseCode: course,
-    thesisType: type,
-  })
-
-const deleteThesisCourse = (programme, course) =>
-  ThesisCourse.destroy({
-    where: {
-      programmeCode: programme,
-      courseCode: course,
-    },
-  })
-
-const findProgrammeTheses = programme =>
-  ThesisCourse.findAll({
-    where: {
-      programmeCode: programme,
-    },
-  })
-
-module.exports = {
-  createThesisCourse,
-  deleteThesisCourse,
-  findProgrammeTheses,
-}
@@ -107,7 +107,9 @@ const findTopTeachers = async yearcode => {
    .filter(isRegularCourse)
    .map(credit => {
      const { credits, course, credittypecode } = credit
-      const teachers = credit.teachers.map(({ id, name }) => ({ id, name }))
+      const teachers = credit.teachers
+        .filter(({ id }) => !id.includes('hy-hlo-org')) // Remove faculties from the leaderboards
+        .map(({ id, name }) => ({ id, name }))
      const passed = Credit.passed(credit) || Credit.improved(credit)
      const failed = Credit.failed(credit)
      const transferred = credittypecode === 9
@@ -2,15 +2,28 @@ const _ = require('lodash')
 const { getRedisCDS, saveToRedis, getTargetStudentCounts } = require('./shared')
 const { getAssociations } = require('../studyrights')

-const REDIS_KEY_PROTOC = 'PROTOC_DATA_V2'
+const REDIS_KEY_PROTOC = 'PROTOC_DATA_V3.3'
 const REDIS_KEY_PROTOC_PROGRAMME = 'PROTOC_PROGRAMME_DATA_V2'

+const tryParseInt = (number, defaultValue) => {
+  try {
+    return parseInt(number, 10)
+  } catch (e) {
+    return defaultValue
+  }
+}
+
 const calculateProtoC = async query => {
  const associations = await getAssociations()

+  const startYear = tryParseInt(query.startYear, null)
+  const endYear = tryParseInt(query.endYear, null)
+
  const data = await getTargetStudentCounts({
    includeOldAttainments: query.include_old_attainments === 'true',
    excludeNonEnrolled: query.exclude_non_enrolled === 'true',
+    startYear,
+    endYear,
  })

  const programmeData = data.filter(d => d.programmeType !== 30)
@@ -97,10 +110,16 @@ const calculateProtoCProgramme = async query => {
  const codes = associations.programmes[query.code]
    ? [...associations.programmes[query.code].studytracks, query.code]
    : []
+
+  const startYear = tryParseInt(query.startYear, null)
+  const endYear = tryParseInt(query.endYear, null)
+
  const data = await getTargetStudentCounts({
-    codes: codes,
+    codes,
    includeOldAttainments: query.include_old_attainments === 'true',
    excludeNonEnrolled: query.exclude_non_enrolled === 'true',
+    startYear,
+    endYear,
  })

  const programmeData = data.find(d => d.programmeType === 20)
@@ -157,10 +176,10 @@ const calculateProtoCProgramme = async query => {
 }

 const getProtoC = async (query, doRefresh = false) => {
-  const { include_old_attainments, exclude_non_enrolled } = query
+  const { include_old_attainments, exclude_non_enrolled, startYear, endYear } = query

  // redis keys for different queries
-  const KEY = `${REDIS_KEY_PROTOC}_OLD_${include_old_attainments.toUpperCase()}_ENR_${exclude_non_enrolled.toUpperCase()}`
+  const KEY = `${REDIS_KEY_PROTOC}_OLD_${include_old_attainments.toUpperCase()}_ENR_${exclude_non_enrolled.toUpperCase()}_FROM_${startYear}_TO_${endYear}`
  const protoC = await getRedisCDS(KEY)
  if (!protoC || doRefresh) {
    const data = await calculateProtoC(query)
@@ -172,8 +191,8 @@ const getProtoC = async (query, doRefresh = false) => {

 // used for studytrack view
 const getProtoCProgramme = async (query, doRefresh = false) => {
-  const { include_old_attainments, exclude_non_enrolled, code } = query
-  const KEY = `${REDIS_KEY_PROTOC_PROGRAMME}_CODE_${code}_OLD_${include_old_attainments.toUpperCase()}_ENR_${exclude_non_enrolled.toUpperCase()}`
+  const { include_old_attainments, exclude_non_enrolled, code, startYear, endYear } = query
+  const KEY = `${REDIS_KEY_PROTOC_PROGRAMME}_CODE_${code}_OLD_${include_old_attainments.toUpperCase()}_ENR_${exclude_non_enrolled.toUpperCase()}_FROM_${startYear}_TO_${endYear}`
  const protoCProgramme = await getRedisCDS(KEY)

  if (!protoCProgramme || doRefresh) {
@@ -16,7 +16,16 @@ const saveToRedis = async (data, REDIS_KEY, expire = false) => {
  }
 }

-const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNonEnrolled }) => {
+const getTargetStudentCounts = async ({
+  codes,
+  includeOldAttainments,
+  excludeNonEnrolled,
+  startYear = 2017,
+  endYear = 2020,
+}) => {
+  // only pick studyrights during which the student has enrolled at least once, whether it's
+  // a present or non-present enrollment
+
  return await sequelize.query(
    `
    SELECT
@@ -25,39 +34,42 @@ const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNon
        ss.programme_code "programmeCode",
        ss.programme_name "programmeName",
        ss.programme_type "programmeType",
-        COUNT(ss.studentnumber) "programmeTotalStudents",
-        SUM(
-             public.is_in_target(
-                 CURRENT_TIMESTAMP,
-                 ss.studystartdate,
-                 public.next_date_occurrence(ss.studystartdate), -- e.g if studystartdate = 2017-07-31 and it's now 2020-02-02, this returns 2020-07-31. However, if it's now 2020-11-01 (we've passed the date already), it returns 2021-07-31.
-                 ss.credits,
-                 CASE
-                     -- HAX: instead of trying to guess which category the student is in if
-                     --      the studyright is currently cancelled, just make the target HUGE
-                     --      so we can subtract these from the non-goal students
-                     WHEN ss.currently_cancelled = 1 THEN 999999999
-                     -- credit target: 60 credits per year since starting
-                     ELSE 60 * ceil(EXTRACT(EPOCH FROM (now() - ss.studystartdate) / 365) / 86400)
-                 END
-             )
+        0 AS "students3y",
+        COUNT(ss.studentnumber) FILTER (
+          WHERE
+            public.is_in_target(
+                CURRENT_TIMESTAMP,
+                ss.studystartdate,
+                public.next_date_occurrence(ss.studystartdate), -- e.g if studystartdate = 2017-07-31 and it's now 2020-02-02, this returns 2020-07-31. However, if it's now 2020-11-01 (we've passed the date already), it returns 2021-07-31.
+                ss.credits,
+                CASE
+                    -- HAX: instead of trying to guess which category the student is in if
+                    --      the studyright is currently cancelled, just make the target HUGE
+                    --      so we can subtract these from the non-goal students
+                    WHEN ss.currently_cancelled = 1 THEN 999999999
+                    -- credit target: 60 credits per year since starting
+                    ELSE 60 * ceil(EXTRACT(EPOCH FROM (now() - ss.studystartdate) / 365) / 86400)
+                END
+            ) = 1
        ) "students3y",
-        SUM(
-             public.is_in_target(
-                 CURRENT_TIMESTAMP,
-                 ss.studystartdate,
-                 public.next_date_occurrence(ss.studystartdate),
-                 ss.credits,
-                 CASE
-                     -- HAX: instead of trying to guess which category the student is in if
-                     --      the studyright is currently cancelled, just make the target HUGE
-                     --      so we can subtract these from the non-goal students
-                     WHEN ss.currently_cancelled = 1 THEN 999999999
-                     -- credit target: 45 credits per year since starting
-                     ELSE 45 * ceil(EXTRACT(EPOCH FROM (now() - ss.studystartdate) / 365) / 86400)
-                 END
-             )
+        COUNT(ss.studentnumber) FILTER (
+          WHERE
+            public.is_in_target(
+                CURRENT_TIMESTAMP,
+                ss.studystartdate,
+                public.next_date_occurrence(ss.studystartdate),
+                ss.credits,
+                CASE
+                    -- HAX: instead of trying to guess which category the student is in if
+                    --      the studyright is currently cancelled, just make the target HUGE
+                    --      so we can subtract these from the non-goal students
+                    WHEN ss.currently_cancelled = 1 THEN 999999999
+                    -- credit target: 45 credits per year since starting
+                    ELSE 45 * ceil(EXTRACT(EPOCH FROM (now() - ss.studystartdate) / 365) / 86400)
+                END
+            ) = 1
        ) "students4y",
+        COUNT(ss.studentnumber) "programmeTotalStudents",
        SUM(ss.currently_cancelled) "currentlyCancelled"
    FROM (
        SELECT
@@ -68,92 +80,16 @@ const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNon
            programme_type,
            studystartdate,
            studentnumber,
-            SUM(credits) credits,
+            SUM(credits) FILTER (WHERE NOT credits.is_old OR :include_old_attainments) credits,
            CASE
                WHEN SUM(currently_cancelled) > 0 THEN 1
                ELSE 0
            END currently_cancelled
-        FROM (
-            SELECT
-                org.code org_code,
-                org.name->>'fi' org_name,
-                element_details.code programme_code,
-                element_details.name->>'fi' programme_name,
-                element_details.type programme_type,
-                studyright.studystartdate studystartdate,
-                studyright.student_studentnumber studentnumber,
-                credit.credits credits,
-                CASE
-                    WHEN studyright.canceldate IS NOT NULL AND studyright.graduated != 1 THEN 1
-                    ELSE 0
-                END currently_cancelled
-            FROM
-                organization org
-                ${
-                  excludeNonEnrolled
-                    ? `
-                -- only pick studyrights during which the student has enrolled at least once, whether it's
-                -- a present or non-present enrollment
-                INNER JOIN (
-                  SELECT DISTINCT -- remove duplicate join rows from se with DISTINCT
-                      studyright.*
-                  FROM
-                      studyright
-                      INNER JOIN (
-                          SELECT
-                              studentnumber,
-                              startdate
-                          FROM
-                              semester_enrollments
-                              INNER JOIN semesters
-                                  ON semester_enrollments.semestercode = semesters.semestercode
-                      ) se
-                          ON se.studentnumber = studyright.student_studentnumber
-                  WHERE
-                      se.startdate IS NOT NULL
-                      AND se.startdate >= studyright.studystartdate
-                ) studyright
-                    ON org.code = studyright.faculty_code`
-                    : `
-                INNER JOIN studyright
-                      ON org.code = studyright.faculty_code`
-                }
-                
-                INNER JOIN (
-                    -- HACK: fix updater writing duplicates where enddate has changed
-                    SELECT DISTINCT ON (studyrightid, startdate, code, studentnumber)
-                        *
-                    FROM studyright_elements
-                ) s_elements
-                    ON studyright.studyrightid = s_elements.studyrightid
-                INNER JOIN element_details
-                    ON s_elements.code = element_details.code
-                LEFT JOIN transfers
-                    ON studyright.studyrightid = transfers.studyrightid
-                LEFT JOIN LATERAL (
-                    SELECT
-                        student_studentnumber,
-                        attainment_date,
-                        credits
-                    FROM credit
-                    WHERE credit.credittypecode IN (4, 9) -- Completed or Transferred
-                        AND (credit."isStudyModule" = false OR credit."isStudyModule" IS NULL)
-                        ${
-                          includeOldAttainments
-                            ? ''
-                            : "AND credit.attainment_date >= studyright.studystartdate -- only include credits attained during studyright's time"
-                        }
-                ) credit
-                    ON credit.student_studentnumber = studyright.student_studentnumber
-            WHERE
-                studyright.extentcode = 1 -- Bachelor's
-                AND org.code NOT IN ('01', 'H02955')
-                AND studyright.prioritycode IN (1, 30) -- Primary or Graduated
-                AND studyright.studystartdate IN ('2017-08-01 00:00:00+00', '2018-08-01 00:00:00+00', '2019-08-01 00:00:00+00')
-                AND element_details.type IN (20,30) -- programme
-                ${!!codes && codes.length > 0 ? 'AND element_details.code IN (:codes)' : ''}
-                AND transfers.studyrightid IS NULL -- Not transferred within faculty
-        ) s
+        FROM organization_yearly_credits credits
+        WHERE
+          (NOT :exclude_non_enrolled OR credits.enrollment_exists) AND
+          (:ignore_codes OR credits.programme_code IN (:codes)) AND
+          credits.startyear BETWEEN :start_year AND :end_year
        GROUP BY (1,2), (3,4) ,5 , 6, 7
    ) ss
    GROUP BY (1, 2), (3, 4), 5;
@@ -161,7 +97,12 @@ const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNon
    {
      type: sequelize.QueryTypes.SELECT,
      replacements: {
-        codes: codes,
+        codes: !codes || codes.length === 0 ? ['DUMMY'] : codes,
+        ignore_codes: !codes || codes.length === 0,
+        exclude_non_enrolled: excludeNonEnrolled,
+        start_year: startYear,
+        end_year: endYear,
+        include_old_attainments: includeOldAttainments,
      },
    }
  )
@@ -200,11 +200,14 @@ const getTotalOpenUniCreditsOfCoursesBetween = async (a, b, alias = 'sum', alias
    INNER JOIN organization o ON o.id = cp.organizationcode
    AND cr.attainment_date BETWEEN :a AND :b
    AND (cr."isStudyModule" = false OR cr."isStudyModule" IS NULL)
-    AND cr.credittypecode IN (4, 9) AND cp.organizationcode = 'hy-org-48645785'
+    AND cr.credittypecode IN (4, 9) 
+    AND cr.is_open = true
    AND cr.student_studentnumber NOT IN
-      (SELECT student_studentnumber FROM studyright
-      WHERE studyright.student_studentnumber = cr.student_studentnumber
-      AND cr.attainment_date BETWEEN studyright.startdate AND studyright.enddate)
+        (
+        SELECT student_studentnumber FROM studyright_elements
+        WHERE studyright_elements.studentnumber = cr.student_studentnumber
+        AND cr.attainment_date BETWEEN studyright_elements.startdate AND studyright_elements.enddate
+      )
    GROUP BY co.id, o.code -- HAVING SUM(cr.credits) > 0
    `,
    {
@@ -284,6 +287,7 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
    from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
    to: new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS),
  })
+
  const yearlyAccCreditsPromises = makeYearlyCreditsPromises(
    startYear,
    yearRange,
@@ -342,7 +346,7 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
    if (!regexresult) return code
    return regexresult[1]
  }
-
+  // console.log('yearly open uni acc credits: ', yearlyOpenUniAccCredits)
  const fromAyToNormalCourses = [
    ...new Set(_.flatten(yearlyOpenUniAccCredits).map(c => unifyOpenUniversity(c.code))),
    ...new Set(_.flatten(yearlyOpenUniTotalCredits).map(c => unifyOpenUniversity(c.code))),
@@ -1,4 +1,5 @@
 const _ = require('lodash')
+const Sentry = require('@sentry/node')
 const {
  dbConnections: { sequelize },
 } = require('../../database/connection')
@@ -93,10 +94,15 @@ const calculateStatusGraduated = async (unixMillis, showByYear) => {
      // init faculty
      if (!acc[curr.faculty_code]) {
        const org = orgs.find(o => o.code === curr.faculty_code)
-        acc[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        if (org) {
+          acc[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        } else {
+          Sentry.captureException(new Error(`Failed to find org for faculty code: ${curr.faculty_code}`))
+        }
      }
-      if (!acc[curr.faculty_code]['yearly'][data.year])
+      if (!acc[curr.faculty_code]['yearly'][data.year]) {
        acc[curr.faculty_code]['yearly'][data.year] = { acc: 0, total: 0 }
+      }
      // init programme
      if (!acc[curr.faculty_code]['drill'][curr.code]) {
        const element = elements.find(e => e.code === curr.code)
@@ -128,7 +134,11 @@ const calculateStatusGraduated = async (unixMillis, showByYear) => {
      // if no accumulated init faculty
      if (!mankeled[curr.faculty_code]) {
        const org = orgs.find(o => o.code === curr.faculty_code)
-        mankeled[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        if (org) {
+          mankeled[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        } else {
+          Sentry.captureException(new Error(`Failed to find org for faculty code: ${curr.faculty_code}`))
+        }
      }
      // if no accumulated init programme
      if (!mankeled[curr.faculty_code]['drill'][curr.code]) {
@@ -29,16 +29,43 @@ const getPassingSemester = (startYear, date) => {
  return yearCount < 6 ? `${yearCount}-${semester}` : 'LATER'
 }

-const getAcademicYearDates = year => {
+const getAcademicYearDates = (year, since) => {
+  if (year === 'Total') {
+    return {
+      startDate: `${since.getFullYear()}-${semesterStart['FALL']}`,
+      endDate: `${moment(new Date()).add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`,
+    }
+  }
+  const startYear = typeof year === 'number' ? year : Number(year.slice(0, 4))
+  const endYear =
+    typeof year === 'number' ? moment(year, 'YYYY').add(1, 'years').format('YYYY') : Number(year.slice(7, 11))
  return {
-    startDate: `${year}-${semesterStart['FALL']}`,
-    endDate: `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`,
+    startDate: `${startYear}-${semesterStart['FALL']}`,
+    endDate: `${endYear}-${semesterEnd['SPRING']}`,
+  }
+}
+
+const getYearStartAndEndDates = (year, isAcademicYear) => {
+  if (isAcademicYear) {
+    const firstYear = year.slice(0, 4)
+    return {
+      startDate: new Date(`${firstYear}-${semesterStart['FALL']}`).toISOString(),
+      endDate: new Date(
+        `${moment(firstYear, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`
+      ).toISOString(),
+    }
+  }
+
+  return {
+    startDate: new Date(`${year}-01-01`).toISOString(),
+    endDate: new Date(`${year}-12-31`).toISOString(),
  }
 }

 module.exports = {
  getPassingSemester,
  getAcademicYearDates,
+  getYearStartAndEndDates,
  semesterEnd,
  semesterStart,
 }
@@ -1,5 +1,5 @@
-const mapToProviders = elementDetails =>
-  elementDetails.map(r => {
+const mapToProviders = elementDetails => {
+  return elementDetails.map(r => {
    const isNumber = str => !Number.isNaN(Number(str))
    if (r.includes('_')) {
      const [left, right] = r.split('_')
@@ -8,8 +8,13 @@ const mapToProviders = elementDetails =>
      const providercode = `${prefix}0-${suffix}`
      return providercode
    }
+    if (/^(T)[0-9]{6}$/.test(r)) {
+      const numbers = r.substring(1)
+      return `7${numbers}`
+    }
    return r
  })
+}

 // sort substitutions so that main code is first
 const newLetterBasedCode = /^[A-Za-z]/ // new letter based codes come first
@@ -1908,26 +1908,19 @@
      }
    },
    "@popperjs/core": {
-      "version": "2.9.2",
-      "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.9.2.tgz",
-      "integrity": "sha512-VZMYa7+fXHdwIq1TDhSXoVmSPEGM/aa+6Aiq3nVVJ9bXr24zScr+NlKFKC3iPljA7ho/GAZr+d2jOf5GIRC30Q=="
+      "version": "2.11.2",
+      "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.2.tgz",
+      "integrity": "sha512-92FRmppjjqz29VMJ2dn+xdyXZBrMlE42AV6Kq6BwjWV7CNUW1hs2FtxSNLQE+gJhaZ6AAmYuO9y8dshhcBl7vA=="
    },
    "@reduxjs/toolkit": {
-      "version": "1.6.2",
-      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.6.2.tgz",
-      "integrity": "sha512-HbfI/hOVrAcMGAYsMWxw3UJyIoAS9JTdwddsjlr5w3S50tXhWb+EMyhIw+IAvCVCLETkzdjgH91RjDSYZekVBA==",
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.8.0.tgz",
+      "integrity": "sha512-cdfHWfcvLyhBUDicoFwG1u32JqvwKDxLxDd7zSmSoFw/RhYLOygIRtmaMjPRUUHmVmmAGAvquLLsKKU/677kSQ==",
      "requires": {
-        "immer": "^9.0.6",
-        "redux": "^4.1.0",
-        "redux-thunk": "^2.3.0",
-        "reselect": "^4.0.0"
-      },
-      "dependencies": {
-        "immer": {
-          "version": "9.0.6",
-          "resolved": "https://registry.npmjs.org/immer/-/immer-9.0.6.tgz",
-          "integrity": "sha512-G95ivKpy+EvVAnAab4fVa4YGYn24J1SpEktnJX7JJ45Bd7xqME/SCplFzYFmTbrkwZbQ4xJK1xMTUYBkN6pWsQ=="
-        }
+        "immer": "^9.0.7",
+        "redux": "^4.1.2",
+        "redux-thunk": "^2.4.1",
+        "reselect": "^4.1.5"
      }
    },
    "@rollup/plugin-node-resolve": {
@@ -1978,13 +1971,13 @@
      }
    },
    "@sentry/browser": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-6.9.0.tgz",
-      "integrity": "sha512-4JnEPcwoNs6JqeEd4wscBq+hxpotEJ0DJ4eOIsaNZIMyqEHXBHTXCk/gfrSsiZFrkHM4PgvUHOxaC0HcZ92oBA==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-6.18.2.tgz",
+      "integrity": "sha512-EsqKSNboi2gOiMuEwQranLucxrARi00y2vgUnaPXcqTKTlVlHDetoWHvq8/r29idA1JHGka5tDrwrmWccWIkrg==",
      "requires": {
-        "@sentry/core": "6.9.0",
-        "@sentry/types": "6.9.0",
-        "@sentry/utils": "6.9.0",
+        "@sentry/core": "6.18.2",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
        "tslib": "^1.9.3"
      },
      "dependencies": {
@@ -1996,14 +1989,14 @@
      }
    },
    "@sentry/core": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/core/-/core-6.9.0.tgz",
-      "integrity": "sha512-oFX2qQcMLujCeIuCQGlhpTUIOXiU5n6V2lqDnvMXUV8gKpplBPalwdlR9bgbSi+VO8u7LjHR1IKM0RAPWgNHWw==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/core/-/core-6.18.2.tgz",
+      "integrity": "sha512-r5ad/gq5S/JHc9sd5CUhZQT9ojQ+f+thk/AoGeGawX/8HURZYAgIqD565d6FK0VsZEDkdRMl58z1Qon20h3y1g==",
      "requires": {
-        "@sentry/hub": "6.9.0",
-        "@sentry/minimal": "6.9.0",
-        "@sentry/types": "6.9.0",
-        "@sentry/utils": "6.9.0",
+        "@sentry/hub": "6.18.2",
+        "@sentry/minimal": "6.18.2",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
        "tslib": "^1.9.3"
      },
      "dependencies": {
@@ -2015,12 +2008,12 @@
      }
    },
    "@sentry/hub": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.9.0.tgz",
-      "integrity": "sha512-5mors7ojbo7G85ZmoVPQBgFBMONAJwyZfV0LNLy14GenoaVNuxTPyvAQiJb1FYq+x6YZ3CvqGX6r74KRKQU87w==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.18.2.tgz",
+      "integrity": "sha512-d0AugekMkbnN12b4EXMjseJxtLPc9S20DGobCPUb4oAQT6S2oDQEj1jwP6PQ5vtgyy+GMYWxBMgqAQ4pjVYISQ==",
      "requires": {
-        "@sentry/types": "6.9.0",
-        "@sentry/utils": "6.9.0",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
        "tslib": "^1.9.3"
      },
      "dependencies": {
@@ -2032,12 +2025,12 @@
      }
    },
    "@sentry/minimal": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.9.0.tgz",
-      "integrity": "sha512-GBZ6wG2Rc1wInYEl2BZTZc/t57O1Da876ifLsSPpEQAEnGWbqZWb8RLjZskH09ZIL/K4XCIDDi5ySzN8kFUWJw==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.18.2.tgz",
+      "integrity": "sha512-n7KYuo34W2LxE+3dnZ47of7XHuORINCnXq66XH72eoj67tf0XeWbIhEJrYGmoLRyRfoCYYrBLWiDl/uTjLzrzQ==",
      "requires": {
-        "@sentry/hub": "6.9.0",
-        "@sentry/types": "6.9.0",
+        "@sentry/hub": "6.18.2",
+        "@sentry/types": "6.18.2",
        "tslib": "^1.9.3"
      },
      "dependencies": {
@@ -2049,51 +2042,17 @@
      }
    },
    "@sentry/tracing": {
-      "version": "6.11.0",
-      "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.11.0.tgz",
-      "integrity": "sha512-9VA1/SY++WeoMQI4K6n/sYgIdRtCu9NLWqmGqu/5kbOtESYFgAt1DqSyqGCr00ZjQiC2s7tkDkTNZb38K6KytQ==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.18.2.tgz",
+      "integrity": "sha512-hg6NLqrqJ5sUPTyWEQ2RqdnhQVnyLtx8II0IyWxQLDWD8UCe3Mu6G7mroDtakPWcP+lWz6OnKfMEfuhMcxR8fw==",
      "requires": {
-        "@sentry/hub": "6.11.0",
-        "@sentry/minimal": "6.11.0",
-        "@sentry/types": "6.11.0",
-        "@sentry/utils": "6.11.0",
+        "@sentry/hub": "6.18.2",
+        "@sentry/minimal": "6.18.2",
+        "@sentry/types": "6.18.2",
+        "@sentry/utils": "6.18.2",
        "tslib": "^1.9.3"
      },
      "dependencies": {
-        "@sentry/hub": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.11.0.tgz",
-          "integrity": "sha512-pT9hf+ZJfVFpoZopoC+yJmFNclr4NPqPcl2cgguqCHb69DklD1NxgBNWK8D6X05qjnNFDF991U6t1mxP9HrGuw==",
-          "requires": {
-            "@sentry/types": "6.11.0",
-            "@sentry/utils": "6.11.0",
-            "tslib": "^1.9.3"
-          }
-        },
-        "@sentry/minimal": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.11.0.tgz",
-          "integrity": "sha512-XkZ7qrdlGp4IM/gjGxf1Q575yIbl5RvPbg+WFeekpo16Ufvzx37Mr8c2xsZaWosISVyE6eyFpooORjUlzy8EDw==",
-          "requires": {
-            "@sentry/hub": "6.11.0",
-            "@sentry/types": "6.11.0",
-            "tslib": "^1.9.3"
-          }
-        },
-        "@sentry/types": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.11.0.tgz",
-          "integrity": "sha512-gm5H9eZhL6bsIy/h3T+/Fzzz2vINhHhqd92CjHle3w7uXdTdFV98i2pDpErBGNTSNzbntqOMifYEB5ENtZAvcg=="
-        },
-        "@sentry/utils": {
-          "version": "6.11.0",
-          "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.11.0.tgz",
-          "integrity": "sha512-IOvyFHcnbRQxa++jO+ZUzRvFHEJ1cZjrBIQaNVc0IYF0twUOB5PTP6joTcix38ldaLeapaPZ9LGfudbvYvxkdg==",
-          "requires": {
-            "@sentry/types": "6.11.0",
-            "tslib": "^1.9.3"
-          }
-        },
        "tslib": {
          "version": "1.14.1",
          "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
@@ -2102,16 +2061,16 @@
      }
    },
    "@sentry/types": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.9.0.tgz",
-      "integrity": "sha512-v52HJqLoLapEnqS2NdVtUXPvT+aezQgNXQkp8hiQ3RUdTm5cffwBVG7wlbpE6OsOOIZxd6p1zKylFkwCypiIIA=="
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.18.2.tgz",
+      "integrity": "sha512-WzpJf/Q5aORTzrSwer/As1NlO90dBAQpaHV2ikDDKqOyMWEgjKb5/4gh59p9gH8JMMnLetP1AvQel0fOj5UnUw=="
    },
    "@sentry/utils": {
-      "version": "6.9.0",
-      "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.9.0.tgz",
-      "integrity": "sha512-PimDr6KAi4cCp5hQZ8Az2/pDcdfhTu7WAU30Dd9MZwknpHSTmD4G6QvkdrB5er6kMMnNQOC7rMo6w/Do3m6X3w==",
+      "version": "6.18.2",
+      "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.18.2.tgz",
+      "integrity": "sha512-EC619jesknyu4xpwud5WC/5odYLz6JUy7OSFy5405PpdGeh/m8XUvuJAx4zDx0Iz/Mlk0S1Md+ZcQwqkv39dkw==",
      "requires": {
-        "@sentry/types": "6.9.0",
+        "@sentry/types": "6.18.2",
        "tslib": "^1.9.3"
      },
      "dependencies": {
@@ -2330,9 +2289,9 @@
      }
    },
    "@types/hast": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.2.tgz",
-      "integrity": "sha512-Op5W7jYgZI7AWKY5wQ0/QNMzQM7dGQPyW1rXKNiymVCy5iTfdPuGu4HhYNOM2sIv8gUfIuIdcYlXmAepwaowow==",
+      "version": "2.3.4",
+      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.4.tgz",
+      "integrity": "sha512-wLEm0QvaoawEDoTRwzTXp4b4jpwiJDvR5KMnFnVodm3scufTlBOWRD6N1OBf9TZMhjlNsSfcO5V+7AF4+Vy+9g==",
      "requires": {
        "@types/unist": "*"
      }
@@ -2378,9 +2337,9 @@
      "integrity": "sha512-YSBPTLTVm2e2OoQIDYx8HaeWJ5tTToLH67kXR7zYNGupXMEHa2++G8k+DczX2cFVgalypqtyZIcU19AFcmOpmg=="
    },
    "@types/mdast": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.4.tgz",
-      "integrity": "sha512-gIdhbLDFlspL53xzol2hVzrXAbzt71erJHoOwQZWssjaiouOotf03lNtMmFm9VfFkvnLWccSVjUAZGQ5Kqw+jA==",
+      "version": "3.0.10",
+      "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.10.tgz",
+      "integrity": "sha512-W864tg/Osz1+9f4lrGTZpCSO5/z4608eUp19tbozkq2HJK6i3z1kT0H9tlADXuYIb1YYOBByU4Jsqkk75q48qA==",
      "requires": {
        "@types/unist": "*"
      }
@@ -2485,9 +2444,9 @@
      }
    },
    "@types/unist": {
-      "version": "2.0.5",
-      "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.5.tgz",
-      "integrity": "sha512-wnra4Vw9dopnuybR6HBywJ/URYpYrKLoepBTEtgfJup8Ahoi2zJECPP2cwiXp7btTvOT2CULv87aQRA4eZSP6g=="
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.6.tgz",
+      "integrity": "sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ=="
    },
    "@types/webpack": {
      "version": "4.41.30",
@@ -3160,9 +3119,9 @@
      "integrity": "sha512-99FZt8qS/xukgxU/8daV8WT7wAakqBzt6lF3XCweO6pwcf50/NgxxBj6ZC7/ejR+F4PWeFpkb9lAMH3y2quBXA=="
    },
    "axios": {
-      "version": "0.21.2",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.2.tgz",
-      "integrity": "sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg==",
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
      "requires": {
        "follow-redirects": "^1.14.0"
      }
@@ -4087,13 +4046,35 @@
      "integrity": "sha512-/4YgnZS8y1UXXmC02xD5rRrBEu6T5ub+mQHLNRj0fzTRbgdBYhsNo2V5EqwgqrExjxsjtF/OpAKAMkKsxbD5XQ=="
    },
    "cfb": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.0.tgz",
-      "integrity": "sha512-sXMvHsKCICVR3Naq+J556K+ExBo9n50iKl6LGarlnvuA2035uMlGA/qVrc0wQtow5P1vJEw9UyrKLCbtIKz+TQ==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.1.tgz",
+      "integrity": "sha512-wT2ScPAFGSVy7CY+aauMezZBnNrfnaLSrxHUHdea+Td/86vrk6ZquggV+ssBR88zNs0OnBkL2+lf9q0K+zVGzQ==",
      "requires": {
-        "adler-32": "~1.2.0",
+        "adler-32": "~1.3.0",
        "crc-32": "~1.2.0",
-        "printj": "~1.1.2"
+        "printj": "~1.3.0"
+      },
+      "dependencies": {
+        "adler-32": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.0.tgz",
+          "integrity": "sha512-f5nltvjl+PRUh6YNfUstRaXwJxtfnKEWhAWWlmKvh+Y3J2+98a0KKVYDEhz6NdKGqswLhjNGznxfSsZGOvOd9g==",
+          "requires": {
+            "printj": "~1.2.2"
+          },
+          "dependencies": {
+            "printj": {
+              "version": "1.2.3",
+              "resolved": "https://registry.npmjs.org/printj/-/printj-1.2.3.tgz",
+              "integrity": "sha512-sanczS6xOJOg7IKDvi4sGOUOe7c1tsEzjwlLFH/zgwx/uyImVM9/rgBkc8AfiQa/Vg54nRd8mkm9yI7WV/O+WA=="
+            }
+          }
+        },
+        "printj": {
+          "version": "1.3.1",
+          "resolved": "https://registry.npmjs.org/printj/-/printj-1.3.1.tgz",
+          "integrity": "sha512-GA3TdL8szPK4AQ2YnOe/b+Y1jUFwmmGMMK/qbY7VcE3Z7FU8JstbKiKRzO6CIiAKPhTO8m01NoQ0V5f3jc4OGg=="
+        }
      }
    },
    "chalk": {
@@ -4152,6 +4133,11 @@
      "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
      "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="
    },
+    "chroma-js": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/chroma-js/-/chroma-js-2.4.2.tgz",
+      "integrity": "sha512-U9eDw6+wt7V8z5NncY2jJfZa+hUH8XEj8FQHgFJTrUFnJfXYf4Ml4adI2vXZOjqRDpFWtYVWypDfZwnJ+HIR4A=="
+    },
    "chrome-trace-event": {
      "version": "1.0.3",
      "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz",
@@ -4299,20 +4285,9 @@
      }
    },
    "codepage": {
-      "version": "1.14.0",
-      "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.14.0.tgz",
-      "integrity": "sha1-jL4lSBMjVZ19MHVxsP/5HnodL5k=",
-      "requires": {
-        "commander": "~2.14.1",
-        "exit-on-epipe": "~1.0.1"
-      },
-      "dependencies": {
-        "commander": {
-          "version": "2.14.1",
-          "resolved": "https://registry.npmjs.org/commander/-/commander-2.14.1.tgz",
-          "integrity": "sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw=="
-        }
-      }
+      "version": "1.15.0",
+      "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz",
+      "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA=="
    },
    "collect-v8-coverage": {
      "version": "1.0.1",
@@ -4617,12 +4592,19 @@
      }
    },
    "crc-32": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.0.tgz",
-      "integrity": "sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.1.tgz",
+      "integrity": "sha512-Dn/xm/1vFFgs3nfrpEVScHoIslO9NZRITWGz/1E/St6u4xw99vfZzVkW0OSnzx2h9egej9xwMCEut6sqwokM/w==",
      "requires": {
        "exit-on-epipe": "~1.0.1",
-        "printj": "~1.1.0"
+        "printj": "~1.3.1"
+      },
+      "dependencies": {
+        "printj": {
+          "version": "1.3.1",
+          "resolved": "https://registry.npmjs.org/printj/-/printj-1.3.1.tgz",
+          "integrity": "sha512-GA3TdL8szPK4AQ2YnOe/b+Y1jUFwmmGMMK/qbY7VcE3Z7FU8JstbKiKRzO6CIiAKPhTO8m01NoQ0V5f3jc4OGg=="
+        }
      }
    },
    "create-ecdh": {
@@ -6657,11 +6639,6 @@
        "bser": "2.1.1"
      }
    },
-    "fflate": {
-      "version": "0.3.11",
-      "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.3.11.tgz",
-      "integrity": "sha512-Rr5QlUeGN1mbOHlaqcSYMKVpPbgLy0AWT/W0EHxA6NGI12yO1jpoui2zBBvU2G824ltM6Ut8BFgfHSBGfkmS0A=="
-    },
    "figgy-pudding": {
      "version": "3.5.2",
      "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz",
@@ -6837,9 +6814,9 @@
      }
    },
    "follow-redirects": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz",
-      "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg=="
+      "version": "1.14.8",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz",
+      "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA=="
    },
    "for-in": {
      "version": "1.0.2",
@@ -7355,9 +7332,9 @@
      "integrity": "sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ=="
    },
    "highcharts": {
-      "version": "9.1.2",
-      "resolved": "https://registry.npmjs.org/highcharts/-/highcharts-9.1.2.tgz",
-      "integrity": "sha512-RwCnJnxAUCGG4R/WDG+QP/4VJNtOpkQTkXcuf58DjDvm9ZILplSMdZvrbMRC9Y9ecuJQCjjZsiifvyS5E2IBvQ=="
+      "version": "9.3.3",
+      "resolved": "https://registry.npmjs.org/highcharts/-/highcharts-9.3.3.tgz",
+      "integrity": "sha512-QeOvm6cifeZYYdTLm4IxZsXcOE9c4xqfs0z0OJJ0z7hhA9WG0rmcVAyuIp5HBl/znjA/ayYHmpYjBYD/9PG4Fg=="
    },
    "highcharts-custom-events": {
      "version": "3.0.10",
@@ -7741,9 +7718,9 @@
      "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg=="
    },
    "immer": {
-      "version": "9.0.7",
-      "resolved": "https://registry.npmjs.org/immer/-/immer-9.0.7.tgz",
-      "integrity": "sha512-KGllzpbamZDvOIxnmJ0jI840g7Oikx58lBPWV0hUh7dtAyZpFqqrBZdKka5GlTwMTZ1Tjc/bKKW4VSFAt6BqMA=="
+      "version": "9.0.12",
+      "resolved": "https://registry.npmjs.org/immer/-/immer-9.0.12.tgz",
+      "integrity": "sha512-lk7UNmSbAukB5B6dh9fnh5D0bJTOFKxVg2cyJWTYrWRfhLrLMBquONcUs3aFq507hNoIZEDDh8lb8UtOizSMhA=="
    },
    "import-cwd": {
      "version": "2.1.0",
@@ -10412,9 +10389,9 @@
      "optional": true
    },
    "nanoid": {
-      "version": "3.1.23",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.23.tgz",
-      "integrity": "sha512-FiB0kzdP0FFVGDKlRLEQ1BgDzU87dy5NnzjeW9YZNt+/c3+q82EQDUwniSAUxp/F0gFNI1ZhKU1FqYsMuqZVnw=="
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.2.0.tgz",
+      "integrity": "sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA=="
    },
    "nanomatch": {
      "version": "1.2.13",
@@ -12351,13 +12328,13 @@
      }
    },
    "prop-types": {
-      "version": "15.7.2",
-      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz",
-      "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==",
+      "version": "15.8.1",
+      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
+      "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
      "requires": {
        "loose-envify": "^1.4.0",
        "object-assign": "^4.1.1",
-        "react-is": "^16.8.1"
+        "react-is": "^16.13.1"
      },
      "dependencies": {
        "react-is": {
@@ -12455,9 +12432,9 @@
      "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc="
    },
    "qs": {
-      "version": "6.10.1",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.1.tgz",
-      "integrity": "sha512-M528Hph6wsSVOBiYUnGf+K/7w0hNshs/duGsNXPUCLH5XAqjEtiPGwNONLV0tBH8NoGb0mvD5JubnUTrujKDTg==",
+      "version": "6.10.3",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz",
+      "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==",
      "requires": {
        "side-channel": "^1.0.4"
      }
@@ -12565,9 +12542,9 @@
      }
    },
    "react-datetime": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/react-datetime/-/react-datetime-3.0.4.tgz",
-      "integrity": "sha512-v6MVwCve+DRaLN2f22LTO5TlrPpkUXumPkp1zfrbhaFtSYGl2grZ2JtwJfLxRj/T4ACyePAV4srCR6cMSiQ/Iw==",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/react-datetime/-/react-datetime-3.1.1.tgz",
+      "integrity": "sha512-gHCTjAniCcMb6jdXpz+MpVe/uCeaHNDOofg+l41nLlJI3uBLBMV40CQbGB2TCTUpCzGT1mCs4vQzKGMjXO/WWQ==",
      "requires": {
        "prop-types": "^15.5.7"
      }
@@ -12734,15 +12711,15 @@
      "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w=="
    },
    "react-markdown": {
-      "version": "6.0.2",
-      "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-6.0.2.tgz",
-      "integrity": "sha512-Et2AjXAsbmPP1nLQQRqmVgcqzfwcz8uQJ8VAdADs8Nk/aaUA0YeU9RDLuCtD+GwajCnm/+Iiu2KPmXzmD/M3vA==",
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-6.0.3.tgz",
+      "integrity": "sha512-kQbpWiMoBHnj9myLlmZG9T1JdoT/OEyHK7hqM6CqFT14MAkgWiWBUYijLyBmxbntaN6dCDicPcUhWhci1QYodg==",
      "requires": {
        "@types/hast": "^2.0.0",
        "@types/unist": "^2.0.3",
        "comma-separated-tokens": "^1.0.0",
        "prop-types": "^15.7.2",
-        "property-information": "^5.0.0",
+        "property-information": "^5.3.0",
        "react-is": "^17.0.0",
        "remark-parse": "^9.0.0",
        "remark-rehype": "^8.0.0",
@@ -12772,6 +12749,11 @@
        }
      }
    },
+    "react-range": {
+      "version": "1.8.12",
+      "resolved": "https://registry.npmjs.org/react-range/-/react-range-1.8.12.tgz",
+      "integrity": "sha512-vY2eCNDvKlXtEpl8xx5AQVIBeE4ZZ46dLVL+82f4FcS9XrPqz4+XagPHmq3IPqYuVQ2H4b8UArXzvVAsd/B03Q=="
+    },
    "react-redux": {
      "version": "7.2.6",
      "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-7.2.6.tgz",
@@ -12800,12 +12782,17 @@
      "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.8.3.tgz",
      "integrity": "sha512-X8jZHc7nCMjaCqoU+V2I0cOhNW+QMBwSUkeXnTi8IPe6zaRWfn60ZzvFDZqWPfmSJfjub7dDW1SP0jaHWLu/hg=="
    },
+    "react-reverse-portal": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/react-reverse-portal/-/react-reverse-portal-2.1.0.tgz",
+      "integrity": "sha512-ck1igNI4tUFozJ89X7qqFtWy1wSL7WEd/5iZbjBjyf+6vjZ3leg4JJxdVPbFt9+zwMLcQSPsIkw9xGu9E+VqCQ=="
+    },
    "react-router": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/react-router/-/react-router-5.2.0.tgz",
-      "integrity": "sha512-smz1DUuFHRKdcJC0jobGo8cVbhO3x50tCL4icacOlcwDOEQPq4TMqwx3sY1TP+DvtTgz4nm3thuo7A+BK2U0Dw==",
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/react-router/-/react-router-5.2.1.tgz",
+      "integrity": "sha512-lIboRiOtDLFdg1VTemMwud9vRVuOCZmUIT/7lUoZiSpPODiiH1UQlfXy+vPLC/7IWdFYnhRwAyNqA/+I7wnvKQ==",
      "requires": {
-        "@babel/runtime": "^7.1.2",
+        "@babel/runtime": "^7.12.13",
        "history": "^4.9.0",
        "hoist-non-react-statics": "^3.1.0",
        "loose-envify": "^1.3.1",
@@ -12817,6 +12804,14 @@
        "tiny-warning": "^1.0.0"
      },
      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.17.2",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.2.tgz",
+          "integrity": "sha512-hzeyJyMA1YGdJTuWU0e/j4wKXrU4OMFvY2MSlaI9B7VQb0r5cxTE3EAIS2Q7Tn2RIcDkRvTA/v2JsAEhxe99uw==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        },
        "isarray": {
          "version": "0.0.1",
          "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
@@ -12838,17 +12833,27 @@
      }
    },
    "react-router-dom": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-5.2.0.tgz",
-      "integrity": "sha512-gxAmfylo2QUjcwxI63RhQ5G85Qqt4voZpUXSEqCwykV0baaOTQDR1f0PmY8AELqIyVc0NEZUj0Gov5lNGcXgsA==",
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-5.3.0.tgz",
+      "integrity": "sha512-ObVBLjUZsphUUMVycibxgMdh5jJ1e3o+KpAZBVeHcNQZ4W+uUGGWsokurzlF4YOldQYRQL4y6yFRWM4m3svmuQ==",
      "requires": {
-        "@babel/runtime": "^7.1.2",
+        "@babel/runtime": "^7.12.13",
        "history": "^4.9.0",
        "loose-envify": "^1.3.1",
        "prop-types": "^15.6.2",
-        "react-router": "5.2.0",
+        "react-router": "5.2.1",
        "tiny-invariant": "^1.0.2",
        "tiny-warning": "^1.0.0"
+      },
+      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.17.2",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.2.tgz",
+          "integrity": "sha512-hzeyJyMA1YGdJTuWU0e/j4wKXrU4OMFvY2MSlaI9B7VQb0r5cxTE3EAIS2Q7Tn2RIcDkRvTA/v2JsAEhxe99uw==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        }
      }
    },
    "react-scripts": {
@@ -13086,9 +13091,9 @@
      }
    },
    "redux-thunk": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.4.0.tgz",
-      "integrity": "sha512-/y6ZKQNU/0u8Bm7ROLq9Pt/7lU93cT0IucYMrubo89ENjxPa7i8pqLKu6V4X7/TvYovQ6x01unTeyeZ9lgXiTA=="
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.4.1.tgz",
+      "integrity": "sha512-OOYGNY5Jy2TWvTL1KgAlVy6dcx3siPJ1wTq741EPyUKfn6W6nChdICjZwCd0p8AZBs5kWpZlbkXW2nE/zjUa+Q=="
    },
    "regenerate": {
      "version": "1.4.2",
@@ -13261,9 +13266,9 @@
      "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
    },
    "reselect": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/reselect/-/reselect-4.1.2.tgz",
-      "integrity": "sha512-wg60ebcPOtxcptIUfrr7Jt3h4BR86cCW3R7y4qt65lnNb4yz4QgrXcbSioVsIOYguyz42+XTHIyJ5TEruzkFgQ=="
+      "version": "4.1.5",
+      "resolved": "https://registry.npmjs.org/reselect/-/reselect-4.1.5.tgz",
+      "integrity": "sha512-uVdlz8J7OO+ASpBYoz1Zypgx0KasCY20H+N8JD13oUMtPvSHQuscrHop4KbXrbsBcdB9Ds7lVK7eRkBIfO43vQ=="
    },
    "resolve": {
      "version": "1.18.1",
@@ -13837,9 +13842,9 @@
      }
    },
    "semantic-ui-react": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/semantic-ui-react/-/semantic-ui-react-2.0.3.tgz",
-      "integrity": "sha512-a0hGN6XXw64sRSKwWqMCKSI/AGLohxNeWuErS39eswvBbUnLjBij8ZoEdiqDiz/PuWpwYIRjgmQVrut+7h3b2g==",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/semantic-ui-react/-/semantic-ui-react-2.1.2.tgz",
+      "integrity": "sha512-NG7qMUi2Ab2gPWzOnGyBO9Ao2tjf9MZjjMMPbmg8uEUcEFm+z1QPqCKhW+eYMiZekH8L9azw9M+hzoTIB1lZgQ==",
      "requires": {
        "@babel/runtime": "^7.10.5",
        "@fluentui/react-component-event-listener": "~0.51.6",
@@ -13848,8 +13853,8 @@
        "@semantic-ui-react/event-stack": "^3.1.2",
        "clsx": "^1.1.1",
        "keyboard-key": "^1.1.0",
-        "lodash": "^4.17.19",
-        "lodash-es": "^4.17.15",
+        "lodash": "^4.17.21",
+        "lodash-es": "^4.17.21",
        "prop-types": "^15.7.2",
        "react-is": "^16.8.6 || ^17.0.0",
        "react-popper": "^2.2.4",
@@ -15343,9 +15348,9 @@
      }
    },
    "unfuck-spa-shibboleth-session": {
-      "version": "3.1.3",
-      "resolved": "https://registry.npmjs.org/unfuck-spa-shibboleth-session/-/unfuck-spa-shibboleth-session-3.1.3.tgz",
-      "integrity": "sha512-ZchOl94XvUz0eTLfZtx1iuqoKcNFN7WePB9Qz+6vPAkHO+F+H1zBatJo0lLgEdZGm1wqCl7gArrRuilcnsUp/A==",
+      "version": "3.1.4",
+      "resolved": "https://registry.npmjs.org/unfuck-spa-shibboleth-session/-/unfuck-spa-shibboleth-session-3.1.4.tgz",
+      "integrity": "sha512-eh20O3yTDRlRYmV8DQhELvnwFS78I8mdMUjetdvvIpO/tGSO5mzDooCrF9ssOUxGtCA6VtlcoVBs3FjFSm1Wng==",
      "requires": {
        "axios": "^0.21.1"
      }
@@ -15375,9 +15380,9 @@
      "integrity": "sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg=="
    },
    "unified": {
-      "version": "9.2.1",
-      "resolved": "https://registry.npmjs.org/unified/-/unified-9.2.1.tgz",
-      "integrity": "sha512-juWjuI8Z4xFg8pJbnEZ41b5xjGUWGHqXALmBZ3FC3WX0PIx1CZBIIJ6mXbYMcf6Yw4Fi0rFUTA1cdz/BglbOhA==",
+      "version": "9.2.2",
+      "resolved": "https://registry.npmjs.org/unified/-/unified-9.2.2.tgz",
+      "integrity": "sha512-Sg7j110mtefBD+qunSLO1lqOEKdrwBFBrR6Qd8f4uwkhWNlbkaqwHse6e7QvD3AP/MNoJdEDLaf8OxYyoWgorQ==",
      "requires": {
        "bail": "^1.0.0",
        "extend": "^3.0.0",
@@ -15616,9 +15621,9 @@
      }
    },
    "url-parse": {
-      "version": "1.5.3",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz",
-      "integrity": "sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ==",
+      "version": "1.5.10",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
+      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
      "requires": {
        "querystringify": "^2.1.1",
        "requires-port": "^1.0.0"
@@ -15629,6 +15634,11 @@
      "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz",
      "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ=="
    },
+    "use-context-selector": {
+      "version": "1.3.9",
+      "resolved": "https://registry.npmjs.org/use-context-selector/-/use-context-selector-1.3.9.tgz",
+      "integrity": "sha512-YgzRyeFjoJXwFn2qLVAuIbV6EQ8DOuzu3SS/eiCxyAyvBhcn02jYSz8c5v22QQU3LW6Ez/Iyo62kKvS7Kdqt3A=="
+    },
    "util": {
      "version": "0.11.1",
      "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz",
@@ -17189,27 +17199,17 @@
      "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg=="
    },
    "xlsx": {
-      "version": "0.17.0",
-      "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.17.0.tgz",
-      "integrity": "sha512-bZ36FSACiAyjoldey1+7it50PMlDp1pcAJrZKcVZHzKd8BC/z6TQ/QAN8onuqcepifqSznR6uKnjPhaGt6ig9A==",
+      "version": "0.17.5",
+      "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.17.5.tgz",
+      "integrity": "sha512-lXNU0TuYsvElzvtI6O7WIVb9Zar1XYw7Xb3VAx2wn8N/n0whBYrCnHMxtFyIiUU1Wjf09WzmLALDfBO5PqTb1g==",
      "requires": {
        "adler-32": "~1.2.0",
        "cfb": "^1.1.4",
-        "codepage": "~1.14.0",
-        "commander": "~2.17.1",
+        "codepage": "~1.15.0",
        "crc-32": "~1.2.0",
-        "exit-on-epipe": "~1.0.1",
-        "fflate": "^0.3.8",
        "ssf": "~0.11.2",
        "wmf": "~1.0.1",
        "word": "~0.3.0"
-      },
-      "dependencies": {
-        "commander": {
-          "version": "2.17.1",
-          "resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz",
-          "integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg=="
-        }
      }
    },
    "xml-name-validator": {
@@ -6,37 +6,41 @@
    "build": "DISABLE_ESLINT_PLUGIN=true react-scripts build"
  },
  "dependencies": {
-    "@reduxjs/toolkit": "^1.6.2",
-    "@sentry/browser": "^6.9.0",
-    "@sentry/tracing": "^6.11.0",
-    "axios": "^0.21.2",
+    "@reduxjs/toolkit": "^1.8.0",
+    "@sentry/browser": "^6.18.2",
+    "@sentry/tracing": "^6.18.2",
+    "axios": "^0.21.4",
+    "chroma-js": "^2.4.2",
    "formik": "^2.2.9",
-    "highcharts": "^9.1.2",
+    "highcharts": "^9.3.3",
    "highcharts-custom-events": "^3.0.10",
    "highcharts-exporting": "^0.1.7",
    "history": "^4.10.1",
-    "immer": "^9.0.7",
+    "immer": "^9.0.12",
    "lodash": "^4.17.21",
    "moment": "^2.29.1",
    "path-to-regexp": "^0.1.7",
    "piwik-react-router": "^0.12.1",
-    "prop-types": "^15.7.2",
-    "qs": "^6.10.1",
+    "prop-types": "^15.8.1",
+    "qs": "^6.10.3",
    "query-string": "^6.14.1",
    "react": "^16.14.0",
-    "react-datetime": "^3.0.4",
+    "react-datetime": "^3.1.1",
    "react-dom": "^16.14.0",
    "react-highcharts": "^16.1.0",
-    "react-markdown": "^6.0.2",
+    "react-markdown": "^6.0.3",
+    "react-range": "^1.8.12",
    "react-redux": "^7.2.6",
-    "react-router-dom": "^5.2.0",
+    "react-reverse-portal": "^2.1.0",
+    "react-router-dom": "^5.3.0",
    "react-scripts": "4.0.3",
    "react-scroll-to-component": "^1.0.2",
    "semantic-ui-css": "^2.4.1",
-    "semantic-ui-react": "^2.0.3",
-    "unfuck-spa-shibboleth-session": "^3.1.3",
+    "semantic-ui-react": "^2.1.2",
+    "unfuck-spa-shibboleth-session": "^3.1.4",
+    "use-context-selector": "^1.3.9",
    "uuid": "^3.4.0",
-    "xlsx": "^0.17.0"
+    "xlsx": "^0.17.5"
  },
  "proxy": "http://backend:8080",
  "browserslist": {
@@ -107,7 +107,7 @@ export const handleRequest = store => next => async action => {
 // All tags used for invalidating cache must be defined here
 export const RTKApi = createApi({
  reducerPath: 'api',
-  tagTypes: ['StudyGuidanceGroups'],
+  tagTypes: ['StudyGuidanceGroups', 'Semester'],
  baseQuery: fetchBaseQuery({
    baseUrl: apiBasePath,
    prepareHeaders: headers => {
@@ -6,9 +6,9 @@ export default {
  
  **4 vuoden tahdissa** olevat opiskelijat 45op × ((tämä päivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet,
  
-  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu).
+  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - passiivinen).

-  **peruutettu**, jos opinto-oikeus merkattu perutuksi
+  **passiivinen**, jos opinto-oikeus on päättynyt tai opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle, eikä opiskelija ole vielä valmistunut. 
    
  **Include only at least once enrolled students** suodattaa pois opiskelijat jotka eivät ole ollenkaan ilmoittautunut. 
  Vakiona opiskelijat jotka eivät ole ilmoittautunut millekkään lukuvuodelle (läsnä tai poissa olevaksi)
@@ -24,9 +24,9 @@ export default {
  
  **4 vuoden tahdissa** olevat opiskelijat 45op × ((tämä päivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet,
  
-  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu).
+  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - passiivinen).

-  **peruutettu**, jos opinto-oikeus merkattu perutuksi
+  **passiivinen**, jos opinto-oikeus on päättynyt tai opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle, eikä opiskelija ole vielä valmistunut. 
  
  **Include only at least once enrolled students** suodattaa pois opiskelijat,
  jotka eivät ole ollenkaan ilmoittautunut. Vakiona opiskelijat, jotka eivät ole ilmoittautunut millekkään lukuvuodelle (läsnä tai poissa olevaksi) on sisällytetty laskuihin.
@@ -41,11 +41,11 @@ export default {
  
  **4 vuoden tahdissa** olevat opiskelijat 45op × ((tarkastuspäivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet,

-  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu). 
+  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - passiivinen). 
  
  **Tarkastuspäivät** ovat 31.7, 30.11 ja 1.4 joka vuodelle.

-  **peruutettu**, jos opinto-oikeus merkattu perutuksi
+  **passiivinen**, jos opinto-oikeus on päättynyt tai opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle, eikä opiskelija ole vielä valmistunut. 

  **Include attainments attained before the studyright start** laskee mukaan opiskelijan ennen opinto-oikeuden alkua saadut opintopisteet.
  Vakiona opintopisteet ennen opinto-oikeutta suodatetaan pois.
@@ -1,7 +1,7 @@
 export default {
-  0: {
-    Attempts: `
-      Table - Attempts
+  Tables: {
+    ATTEMPTS: `
+      **Table - Attempts**

      Taulukko näyttää suorituskerrat lukuvuositasolla.
      Samalla opiskelijalla voi olla useampia suorituskertoja.
@@ -13,8 +13,8 @@ export default {
      Arvosanajakaumaa katsoessasi voit muuttaa luvut prosenteiksi, klikkaamalla painiketta "Relative".
      Mahdollinen sarake "Other passed" sisältää sekä arvosanan "Hyv." saaneet että kurssin hyväksilukeneet.
    `,
-    Students: `
-      Table - Students
+    STUDENT: `
+      **Table - Students**

      Taulukko näyttää yksittäiset opiskelijat, jotka ovat kurssia suorittaneet.
      Jokainen opiskelija on jaoteltu luokkiin: "läpäissyt ensiyrittämällä", "läpäissyt lopulta" tai "ei koskaan läpäissyt kurssia".
@@ -26,17 +26,17 @@ export default {
      Total-rivillä näkyy niiden opiskelijoiden kokonaismäärä, jotka ovat kurssista joskus arvosanan saaneet.
    `,
  },
-  1: {
-    Attempts: `
-      Pass rate chart - Attempts
+  PassRate: {
+    ATTEMPTS: `
+      **Pass rate chart - Attempts**

      Graafi näyttää suorituskerrat lukuvuositasolla jaoteltuina hyväksyttyihin ja hylättyihin suorituksiin.
      Hyväksytyt suoritukset sisältävät arvosanat 1-5 sekä arvosanan Hyv.
      Hylätyt suoritukset sisältävät arvosanat Hyl. ja 0, sekä suoritukset, joille opettaja on antanut merkinnän
      "Luop" (Luopunut suorituksesta) tai "Eisa" (Ei saapunut tenttiin).
    `,
-    Students: `
-      Pass rate chart - Students
+    STUDENT: `
+      **Pass rate chart - Students**

      Graafi näyttää yksittäiset opiskelijat, jotka ovat kurssia suorittaneet.
      Jokainen opiskelija on jaoteltu luokkiin: "läpäissyt ensiyrittämällä", "läpäissyt lopulta" tai "ei koskaan läpäissyt kurssia".
@@ -46,18 +46,9 @@ export default {
      myöhemmin keväällä 2019, tulisi merkityksi lukuvuodelle 2017-2018 pylvääseen "passed eventually".        
    `,
  },
-  2: {
-    Attempts: `
-      Grade distribution
-  
-      Graafi näyttää suorituskertojen arvosanajakauman eri lukuvuosille. Mahdollinen kategoria
-      "Other passed" sisältää sekä arvosanan "Hyv." että kurssin hyväksiluvut.
-    `,
-    Students: `
-      Grade distribution
+  GradeDistribution: `
+    **Grade distribution**

-      Graafi näyttää suorituskertojen arvosanajakauman eri lukuvuosille. Mahdollinen kategoria
-      "Other passed" sisältää sekä arvosanan "Hyv." että kurssin hyväksiluvut.
-    `,
-  },
+    Graafi näyttää suorituskertojen arvosanajakauman eri lukuvuosille. Mahdollinen kategoria
+  `,
 }
@@ -4,12 +4,23 @@ export default {
    short:
      'Suodata opiskelijoita sen mukaan ovatko he ilmoittautuneet läsnä- tai poissaoleviksi kyseiselle lukukaudelle',
  },
+  studyrightStatus: {
+    label: 'Rajaa opiskelijoita opinto-oikeuden statuksen mukaan',
+    short:
+      'Opinto-oikeus on aktiivinen, mikäli opiskelija on ilmoittautunut läsnä- tai poissaolevaksi tälle lukukaudelle ja opinto-oikeus ei ole vielä päättynyt. \n Opinto-oikeus on passiivinen, jos opiskelija ei ole ilmoittautunut läsnä- tai poissaolevaksi tälle lukukaudelle tai opinto-oikeus on vanhentunut. Valmistuneet opiskelijat on suodatettu pois molemmista kategorioista.',
+  },
  transferred: {
    label: 'Include and exclude students from this program',
-    short: 'Mahdollisuus suodattaa opiskelijat, jotka ovat siirtyneet tutkinto-ohjelmaan',
+    short:
+      'Mahdollisuus suodattaa opiskelijat, jotka ovat siirtyneet nykyiseen 2017 alkaneeseen tutkinto-ohjelmaan vanhasta tutkinto-ohjelmasta',
  },
  courseCredits: {
    label: null,
    short: 'Rajaa kurssisuorituksia tietyn ajanjakson mukaan. Tämä rajaus ei muuta opiskelijoiden määrää',
  },
+  cherrypicker: {
+    label: null,
+    short:
+      'Rajaa yksittäisiä opiskelijoita pois tilastoista (blocklist) tai näytä vain määritellyt opiskelijat (allowlist). Vain täsmälliset opiskelijanumerot toimivat.',
+  },
 }
@@ -1,5 +1,4 @@
 import PopulationStatistics from './populationStatistics'
-import PopulationOverview from './populationOverview'
 import CoolDataScience from './coolDataScience'
 import CourseStatistics from './courseStatistics'
 import CoursePopulation from './coursePopulation'
@@ -7,7 +6,6 @@ import Studyprogramme from './studyprogramme'

 export default {
  PopulationStatistics,
-  PopulationOverview,
  CoolDataScience,
  CourseStatistics,
  CoursePopulation,
@@ -1,24 +0,0 @@
-export default {
-  PopulationProgress: `Tämä koontisivu sisältää koulutusohjelman kaikkien opiskelijoiden perustiedot lukuvuosittain. Luvut perustuvat opinto-oikeuksiin kyseisessä ohjelmassa.
-
-  Population progress:
-  Vuosittain ohjelmassa aloittaneet, ohjelman keskeyttäneet / peruneet sekä ohjelmaan siirtyneet opiskelijat. 
-  Huomattavaa on, että sama opiskelija voi olla kahdessa luokassa, jos hän on esimerkiksi aloittanut opinnot jo, mutta keskeyttänyt ne lukuvuoden aikana.
-  Tarkemmat tiedot saat esille klikkaamalla nuolta kunkin lukuvuoden kohdalla.`,
-  YearlyProductivity: `
-  Yearly productivity:
-  Koulutusohjelman kunakin kalenterivuonna tuottamat opintopisteet, valmistuneiden määrän, ohjelman pääaineopiskelijoiden
-  opintopisteet sekä muiden kuin pääaineopiskelijoiden opintopisteet. Mukana on myös kuhunkin koulutusohjelmaan hyväksiluetut opintopisteet.
-  
-  Vanhojen koulutusohjelmien lakkauttaminen ja opiskelijoiden siirtäminen uusiin ohjelmiin voi aiheuttaa tavallista suuremmat opintopistemäärät lukuvuodelle 2020-2021.`,
-  Year: `Lukuvuosi`,
-  Students: `Opiskelijat, jotka ovat koulutusohjelmassa aloittaneet, opinto-oikeutensa peruneet sekä koulutusohjelmaan tai siitä pois siirtyneet.`,
-  Started: `Koulutusohjelmassa kunakin lukuvuonna aloittaneet opiskelijat. EI sisällä ohjelmaan siirtyneitä opiskelijoita.`,
-  Cancelled: `Koulutusohjelmaan oikeuttavan opinto-oikeutensa kunakin lukuvuonna peruttaneet`,
-  Graduated: `Tiettynä lukuvuonna aloittaneista opiskelijoista on valmistunut`,
-  Transferred: `Koulutusohjelmaan kyseisenä lukuvuonna siirtyneet opiskelijat. 
-  
-  Vanhojen koulutusohjelmien lakkauttaminen aiheuttaa tavallista suuremmat siirtoluvut lukuvuodelle 2020-2021.`,
-  Credits: `Kunakin vuonna aloittaneista opiskelijoista ne, jotka ovat ylittäneet tietyn opintopisterajapyykin`,
-  Thesis: `Loppututkielman kirjoittaneiden määrä`,
-}
@@ -1,7 +1,7 @@
 export default {
  Advanced: `**Semesters:**  Jos haluaa tarkastella vain syyslukukaudella tai kevätlukukaudella ilmoittautuneita, haluttu ajanjakso valitaan tästä.

-  **Include:** Koulutusohjelmapopulaatiosta on oletusarvoisesti suodatettu pois vaihto-opiskelijat, tutkinnonsuoritusoikeudesta luopuneet ja erillisopinto-oikeudella opiskelevat. Näitä ryhmiä voi sisällyttää mukaan lisäämällä ruksin haluamaansa kohtaan.`,
+  **Include:** Koulutusohjelmapopulaatiosta on oletusarvoisesti suodatettu pois vaihto-opiskelijat ja erillisopinto-oikeudella opiskelevat. Näitä ryhmiä voi sisällyttää mukaan lisäämällä ruksin haluamaansa kohtaan.`,
  QueryCard: `**Valitun koulutusohjelmavuosikurssin perustiedot.**
  
  **Sample size** luku sisältää läsnä- ja poissaolevaksi ilmoittautuneet opiskelijat, muttei "excludes"-ryhmiä, jotka voi määritellä "advanced settings" -kohdasta.
@@ -41,6 +41,12 @@ export default {
  + **Min ja Max** kertovat pienimmän ja suurimman opintopistemäärän, jotka jollain opiskelijoilla kyseisessä populaatiossa on.

  Mikäli valintatapatieto kyseiselle populaatiolle on saatavissa, näytetään myös samat statistiikat per valintatapa.  
+
+  **Distribution Development**
+
+  Pylväsdiagrammi kertoo opiskelijoiden jakauman opintopistehaarukoihin kalenterivuosittain, lukuvuosittain tai lukukausittain eriteltynä. Valmistuneita opiskelijoita ei sisällytetä valmistumisen jälkeisiin pylväisiin.
+
+  **Cumulative**-valinnan ollessa käytössä kunkin opiskelijan kaikki aiemmat opintopisteet lasketaan mukaan opintopistemäärään. Muutoin vain pylvästä vastaavalla aikavälillä kirjatut opintopisteet otetaan huomioon.
  `,
  CoursesOfPopulation: `**Courses of population**

@@ -60,6 +66,7 @@ export default {
  **Taulukot**
  
  - **pass/fail** - oletusnäkymä, joka sisältää kurssisuoritukset, hylätyt sekä yritykset. Taulukon tiedot voi järjestää sarakkeita (n, after retry, percentage, many times, per student, passed ja attempted) klikkaamalla.
+  - **pass/fail w/ enrollments** - näkymä, joka sisältää suoritustiedot ilmoittautumisten kanssa. Huom.! ilmoittautumistiedot ovat saatavilla vain Sisun käyttöönoton jälkeiseltä ajalta.
  - **grades** - arvosanajakauma.
  - **when passed** - kurssisuorituksen ajankohta. Huom.! *Passed*-sarakkeen kokonaissumma ei välttämättä täsmää puolivuosittaisten lukumäärien summan kanssa johtuen tuplakirjauksista Oodissa.
  `,
@@ -1,38 +1,68 @@
 export default {
+  YearToggle: `
+  Academic year: Näyttää tilastot akateemisille vuosille jaoteltuna, esimerkiksi 1.8.2019-31.7.2020
+  Calendar year: Näyttää tilastot kalenterivuosille jaoteltuna, esimerkiksi 1.1.2019-31.12.2019 
+  `,
+  StudentToggle: `
+  All studyrights: Valinnassa ovat mukana vaihto-opiskelijat, erillisoikeudella opiskelevat ja ohjelmaan siirtyneet opiskelijat sekä ohjelmasta pois siirtyneet opiskelijat.
+  
+  Special studyrights excluded: Valinnasta ja kaikista luvuista on poissuljettu kaikki edellä mainitut erikoisryhmät.
+  `,
+  GraduatedToggle: `
+  Graduated included: Valinnassa ovat mukana ohjelmasta jo valmistuneet opiskelijat.
+  
+  Graduated excluded: Valinnasta on poistettu ohjelmasta jo valmistuneet opiskelijat.
+  `,
  StudentsOfTheStudyprogramme: `
-  **Started**: Sisältää kyseisenä vuonna aloitetut opinto-oikeudet riippumatta siitä, minä vuonna kyseinen opiskeluoikeus on myönnetty. Opiskelija on myös voinut keskeyttää opintonsa tai siirtyä pois ohjelmasta.\n
+  **All**: Sisältää kaikki kyseisenä vuonna alkaneet opinto-oikeudet riippumatta siitä, onko opintoja aloitettu tai onko niitä aloitettu ko. vuonna. Mukana myös ohjelmaan siirtyneet opiskelijat.
+
+  **Started studying**: Sisältää kyseisenä vuonna aloittaneet opiskelijat (aloitetut opinto-oikeudet) riippumatta siitä, minä vuonna kyseinen opiskeluoikeus on myönnetty. Opiskelija on myös voinut myöhemmin keskeyttää opintonsa tai siirtyä pois ohjelmasta.\n
  **Graduated**: Sisältää kyseisenä vuonna valmistuneet opiskelijat riippumatta siitä, minä vuonna opiskeluoikeus ohjelmaan on myönnetty. Jaottelu on tehty opinto-oikeuden päättymispäivän (=valmistumispäivän) perusteella.
-  **Cancelled**: Sisältää kyseisenä vuonna perutut opiskeluoikeudet riippumatta siitä, onko kyseinen opiskelija ikinä edes alkanut opiskelemaan ohjelmassa tai opiskelun aloitusvuodesta.
+    
+  **Transferred away**: Sisältää kyseisenä vuonna pois ohjelmasta siirtyneet opiskelijat (opinto-oikeudet). Sama opiskelija voi esiintyä tilastoissa useana vuonna, mikäli hän on siirtynyt ohjelmaan ja siitä pois useasti.
  
-  **Transferred away**: Sisältää kyseisenä vuonna pois ohjelmasta siirtyneet opinto-oikeudet. Sama opiskelija voi esiintyä tilastoissa useana vuonna. Kyseiset opiskelijat ovat saattaneet aloittaa ohjelmassa tai eivät koskaan ole aloittaneet ohjelmassa. 
-  
-  **Transferred to**: Sisältää kyseisenä vuonna ohjelmaan siirtyneet opinto-oikeudet.
+  **Transferred to**: Sisältää kyseisenä vuonna ohjelmaan siirtyneet opiskelijat (opinto-oikeudet).

  HUOM! Ohjelmaan siirtyneiden määrä on erityisen suuri vuonna 2020, jolloin vanhoista koulutusohjelmista luovuttiin ja opiskelijat siirtyivät uusiin ohjelmiin.
  `,
  CreditsProducedByTheStudyprogramme: `
-  Sisältää opintopisteet suoritusvuosittain (Sisussa näkyvä kurssin suorituspäivä)
+  Sisältää opintopisteet suoritusvuosittain. Suoritukset on jaoteltu Sisussa näkyvän kurssin suorituspäivän mukaan.

-  **Major students' credits**: Sisältää ne opintopisteet, joiden suorittajalla on suoritushetkellä ollut ensisijainen opinto-oikeus kyseiseen koulutusohjelmaan.
+  **Major students' credits**: Sisältää ne opintopisteet, joiden suorittajalla on suoritushetkellä ollut ensisijainen opinto-oikeus kyseiseen koulutusohjelmaan. Sisältää myös ohjelmaan siirtyneet ensisijaiset opiskelijat.

-  **Non-major students' credits**: Sisältää koulutusohjelman tuottamat opintopisteet, joiden suorittajalla EI ole ollut suoritushetkellä ensisijaista opinto-oikeutta kyseiseen koulutusohjelmaan.
+  **Non-major students' credits**: Sisältää koulutusohjelman tuottamat opintopisteet, joiden suorittajalla EI ole ollut suoritushetkellä ensisijaista opinto-oikeutta kyseiseen koulutusohjelmaan. Sisältää
+  esimerkiksi vaihto-opiskelijat ja erillis-opinto-oikeudella opiskelevat. Sisältää myös opintopisteet, jotka opiskelija on suorittanut koulutusohjelmaan ennen opintojen aloittamista koulutusohjelmassa tai opintojen päätyttyä.
  
-  **Transferred credits**: Sisältää opintopisteet, jotka on suoritettu kyseisenä vuonna, ja hyväksiluettu tähän koulutusohjelmaan.
+  **Transferred credits**: Sisältää opintopisteet, jotka on suoritettu kyseisenä vuonna, ja hyväksiluettu tähän koulutusohjelmaan. Mukana ovat kaikenlaisilla opinto-oikeuksilla hyväksiluetut opintopisteet.
  `,
  GraduatedAndThesisWritersOfTheProgramme: `
-  **Wrote thesis**: Sisältää kyseisenä vuonna kandidaatintutkielman hyväksytysti suorittaneet opiskelijat. Mukana ei ole mahdollisia arvosanan korotuksia.
+  **Wrote thesis**: Sisältää kyseisenä vuonna kandidaatintutkielman tai pro gradu-tutkielman hyväksytysti suorittaneet opiskelijat. Mukana ei ole mahdollisia arvosanan korotuksia.

  **Graduated**: Sisältää kyseisenä vuonna valmistuneet opiskelijat riippumatta siitä, minä vuonna opiskeluoikeus ohjelmaan on myönnetty. Jaottelu on tehty opinto-oikeuden päättymispäivän (=valmistumispäivän) perusteella.
  `,
  AverageGraduationTimes: `
-  **Median time**: Näyttää kyseisenä vuonna valmistuneiden opintojen keston **mediaanin** kuukausina.
+  **Median time**: Näyttää kyseisenä vuonna valmistuneiden opiskelijoiden opintojen keston **mediaanin** kuukausina. Luku sisältää mahdolliset poissaolokuukaudet.

-  **Mean time**: Näyttää kyseisenä vuonna valmistuneiden opintojen keston **keskiarvon** kuukausina.
+  **Mean time**: Näyttää kyseisenä vuonna valmistuneiden opiskelijoiden opintojen keston **keskiarvon** kuukausina. Luku sisältää mahdolliset poissaolokuukaudet.
  
-  Mukana ovat **vain valmistuneet opiskelijat**. Kandiohjelmissa tavoiteaika on 36 kk, maisteriohjelmissa 24 kk`,
+  n = **valmistuneet opiskelijat / kaikki ohjelmassa kyseisenä vuonna alkaneet opinto-oikeudet**. Kandiohjelmissa tavoiteaika on 36 kk, maisteriohjelmissa 24 kk`,
+  ProgrammesBeforeOrAfter: `
+  **Mikäli valittuna on kandiohjelma:** Sisältää maisteriohjelmat, joissa tämän kandiohjelman opiskelijat ovat aloittaneet valmistuttuaan kandeiksi. Vuosijaottelu on tehty maisteriohjelmassa aloittamisen perusteella.
+
+  Mukana ovat **vain valmistuneet opiskelijat ja vain maisteriohjelmat.**
+
+  **Mikäli valittuna on maisteriohjelma:** Sisältää kandiohjelmat, joissa tämän ohjelman opiskelijat ovat opiskelleet ennen tuloaan maisteriohjelmaan. Vuosijaottelu on tehty maisteriohjelmassa aloittamisen perusteella.
+  `,
  StudytrackOverview: `  
  Kuvaa koulutusohjelman kyseiseen ohjelmaan opinto-oikeudella varustettujen opiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
-  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, tutkinnonsuoritusoikeudesta luopuneet ja erillisopinto-oikeudella opiskelevat.
+  
+  HUOM!
+  **All** Sisältää kaikki opiskelijat, joiden opinto-oikeudella on aloituspäivä kyseisenä vuonna, riippumatta siitä, onko opiskelija aloittanut opiskelemaan ko. vuonna.
+  **Started studying** Sisältää kaikki opiskelijat, jotka ovat aloittaneet opiskelemaan koulutusohjelmassa kyseisenä lukuvuonna.
+  **Inactive** Sisältää kaikki opinto-oikeudet, jotka ovat alkaneet kyseisenä vuonna, mutta jotka ovat sittemmin vanhentuneet ilman, että opiskelija on valmistunut ohjelmasta. Sisältää myös opinto-oikeudet,
+  joissa opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle.
+  
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, ohjelmaan ja siitä pois siirtyneet opiskelijat sekä erillisopinto-oikeudella opiskelevat.
  Taulukko näyttää myös näistä opiskelijoista miesten, naisten ja suomalaisten osuudet sekä valmistuneiden määrän. 

  Yläosan valikosta on mahdollista valita tarkasteluun yhden opintosuunnan opiskelijat. Sekä opiskelijoiden yleistiedot että edistyminen opintopisteittäin
@@ -40,8 +70,14 @@ export default {
  `,
  StudytrackProgress: `
  Kuvaa koulutusohjelman kyseiseen ohjelmaan opinto-oikeudella varustettujen opiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
-  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, tutkinnonsuoritusoikeudesta luopuneet ja erillisopinto-oikeudella opiskelevat.   
+  
+  HUOM! Mukana ovat myös muuhun kuin valittuun koulutusohjelmaan tai opintosuuntaan tehdyt opintopisteet.
+
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, erillisopinto-oikeudella opiskelevat ja ohjelmaan siirtyneet.   

  Mikäli yläosan valikosta valittuna on jokin tietty opintosuunta, myös nämä graafit ja taulukot kuvaavat kyseisen opintosuunnan opiskelijoita.    
  `,
+  NonDegree: `
+  Opiskelijat, joilla ei ole tutkinto-oikeutta Helsingin yliopistossa.
+  `,
 }
@@ -271,6 +271,55 @@ export const useLanguage = () => {
  return language
 }

+export const useDebounce = (value, timeout, onChange) => {
+  const [innerValue, setInnerValue] = useState(value)
+  const [dirty, setDirty] = useState(false)
+
+  const timeoutRef = useRef(null)
+
+  useEffect(() => {
+    if (!isEqual(value, innerValue)) {
+      setInnerValue(value)
+      setDirty(false)
+
+      if (timeoutRef.current) {
+        clearTimeout(timeoutRef.current)
+        timeoutRef.current = null
+      }
+    }
+  }, [value])
+
+  const setValue = useCallback(
+    value => {
+      setInnerValue(value)
+      setDirty(true)
+
+      if (timeoutRef.current) {
+        clearTimeout(timeoutRef.current)
+        timeoutRef.current = null
+      }
+
+      timeoutRef.current = setTimeout(() => {
+        onChange(value)
+        setDirty(false)
+      }, timeout)
+    },
+    [innerValue, setInnerValue, timeoutRef, onChange, timeout]
+  )
+
+  const flush = useCallback(() => {
+    if (timeoutRef.current) {
+      clearTimeout(timeoutRef.current)
+      timeoutRef.current = null
+    }
+
+    onChange(innerValue)
+    setDirty(false)
+  }, [timeoutRef, onChange, innerValue])
+
+  return [innerValue, setValue, flush, dirty]
+}
+
 // From: https://www.joshwcomeau.com/snippets/react-hooks/use-toggle/
 export const useToggle = (initialValue = false) => {
  const [value, setValue] = useState(initialValue)
@@ -101,6 +101,52 @@ export const getTextIn = (texts, language) => {
  return null
 }

+export const getTextInWithOpen = (texts, language, isOpenCourse) => {
+  const prefixes = ['Avoin', 'Öppna', 'Open']
+  if (texts) {
+    const lanText = texts[language] || texts.fi || texts.en || texts.sv || Object.values(texts)[0]
+    const splitText = lanText.split(':')
+    if (prefixes.some(word => splitText[0].startsWith(word))) {
+      splitText.shift()
+    }
+    const newText = [...splitText].join()
+
+    if (isOpenCourse) {
+      switch (language) {
+        case 'fi':
+          return `${prefixes[0]} yo: ${newText}`
+
+        case 'sv':
+          return `${prefixes[1]} uni: ${newText}`
+
+        case 'en':
+          return `${prefixes[2]} uni: ${newText}`
+
+        default:
+          return `${prefixes[0]} yo: ${newText}`
+      }
+    }
+    return newText
+  }
+  return null
+}
+
+export const getUnifyTextIn = unifyCourses => {
+  switch (unifyCourses) {
+    case 'reqularStats':
+      return '(normal)'
+
+    case 'openStats':
+      return '(open)'
+
+    case 'unifyStats':
+      return '(open and normal)'
+
+    default:
+      return ''
+  }
+}
+
 export const cancelablePromise = promise => {
  let hasCanceled = false

@@ -160,7 +206,12 @@ export const getNewestProgramme = (studyrights, studentNumber, studentToTargetCo
        elementDetails[srE.code] &&
        elementDetails[srE.code].type === 20 &&
        (studentToTargetCourseDateMap && studentNumber
-          ? moment(studentToTargetCourseDateMap[studentNumber]).isBetween(srE.startdate, srE.enddate, 'day', '[]')
+          ? moment(studentToTargetCourseDateMap[studentNumber]).isBetween(
+              srE.startdate,
+              srE.enddate || moment(),
+              'day',
+              '[]'
+            )
          : true)
    )
    if (studyrightElements.length > 0) {
@@ -210,12 +261,100 @@ export const getHighestGradeOfCourseBetweenRange = (courses, lowerBound, upperBo
  return maxBy(grades, grade => grade.value)
 }

+export const getHighestGradeOrEnrollmentOfCourseBetweenRange = (courses, enrollments, lowerBound, upperBound) => {
+  const grade = getHighestGradeOfCourseBetweenRange(courses, lowerBound, upperBound)
+  if (!grade) return enrollments.length ? { grade: 'No grade' } : undefined
+  return grade
+}
+
+const getEarliestStudyRightElement = studyright => {
+  if (!studyright) return null
+  return (
+    studyright.studyright_elements
+      // eslint-disable-next-line camelcase
+      .filter(({ element_detail }) => element_detail.type === 20)
+      .sort((a, b) => new Date(a.startdate) - new Date(b.startdate))[0]
+  )
+}
+
+export const getTargetCreditsForProgramme = code => {
+  if (code === 'MH30_001') return 360
+  if (code === 'MH30_003') return 330
+  if (code === 'MH30_004') return 150
+  if (code === 'MH90_001') return 180
+  if (code.includes('MH')) return 120
+  return 180
+}
+
+const getMonthsForDegree = code => getTargetCreditsForProgramme(code) / (60 / 12)
+
+/**
+ * Get start and end dates for study right element. For bachelor a 3 year target is used
+ * and for masters a 2 year target. Any absences within the study right element extends the end date.
+ * If any absence is in the start of study right element the start date is postponed.
+ */
+export const getStudyRightElementTargetDates = (studyRight, absences = []) => {
+  const studyRightElement = getEarliestStudyRightElement(studyRight)
+  if (!studyRightElement) return []
+  const { code, startdate: sreStartDate } = studyRightElement
+  const months = getMonthsForDegree(code)
+  const end =
+    code.includes('KH') || code.includes('ba') || ['MH30_001', 'MH30_003'].includes(code)
+      ? moment(sreStartDate).add(months, 'months').set('month', 7).endOf('month')
+      : moment(sreStartDate).add(months, 'months')
+
+  if (!absences) return [new Date(sreStartDate), new Date(end)]
+  const absencesWithinStudyRightElement = absences.filter(
+    ({ startdate, enddate }) => startdate >= new Date(sreStartDate).getTime() && enddate <= new Date(end).getTime()
+  )
+
+  if (!absencesWithinStudyRightElement.length) return [new Date(sreStartDate), new Date(end)]
+
+  const absenceInStartOfStudyRight = absencesWithinStudyRightElement.find(
+    ({ startdate }) => new Date(sreStartDate).getTime() === startdate
+  )
+  const absentMonthsDuringStudy = Math.round(
+    absencesWithinStudyRightElement
+      .filter(({ startdate, enddate }) => {
+        if (!absenceInStartOfStudyRight) return true
+        return startdate !== absenceInStartOfStudyRight.startdate && enddate !== absenceInStartOfStudyRight.enddate
+      })
+      .filter(({ startdate }) => startdate < new Date(end).getTime())
+      .reduce((acc, absent) => {
+        const { startdate, enddate } = absent
+        const diff = moment(startdate).diff(moment(enddate), 'days') / 30
+        return acc + Math.abs(diff)
+      }, 0)
+  )
+  const absentMonthsBeforeStudy = absenceInStartOfStudyRight
+    ? Math.round(
+        Math.abs(
+          moment(absenceInStartOfStudyRight.startdate).diff(moment(absenceInStartOfStudyRight.enddate), 'days') / 30
+        )
+      )
+    : 0
+  return [
+    new Date(moment(sreStartDate).add(absentMonthsBeforeStudy, 'months')),
+    new Date(end.add(absentMonthsDuringStudy + absentMonthsBeforeStudy, 'months')),
+  ]
+}
+
 export const validateInputLength = (input, minLength) => input && input.trim().length >= minLength

 export const splitByEmptySpace = str => str.replace(/\s\s+/g, ' ').split(' ')

 export const isNewHYStudyProgramme = code => !!(code && code.match(/^[A-Z]*[0-9]*_[0-9]*$/))

+export const resolveStudyPlan = (studyPlans, studyRight) => {
+  if (!studyRight) return null
+  const { code } =
+    studyRight.studyright_elements
+      .filter(e => e.element_detail.type === 20)
+      .sort((a, b) => new Date(b.startdate) - new Date(a.startdate))[0] || {}
+  if (!code) return null
+  return studyPlans.find(p => p.programme_code === code)
+}
+
 // These are the new Bachelor's programmes in Matlu, that have BH possibility
 export const bachelorHonoursProgrammes = [
  'KH50_001',
@@ -3,20 +3,26 @@ import React, { useState, useEffect, useCallback, useMemo } from 'react'
 import { connect } from 'react-redux'
 import Highcharts from 'highcharts'
 import ReactHighcharts from 'react-highcharts'
-import { Segment, Loader, Dimmer, Checkbox, Button, Message, Icon } from 'semantic-ui-react'
+import { Menu, Segment, Loader, Dimmer, Checkbox, Button, Message, Icon, Dropdown } from 'semantic-ui-react'
 import _ from 'lodash'
 import HighchartsCustomEvents from 'highcharts-custom-events'

 import TSA from '../../common/tsa'
 import InfoToolTips from '../../common/InfoToolTips'
-import { getProtoC, getProtoCProgramme } from '../../redux/coolDataScience'
+import { useGetProtoCQuery, getProtoC, getProtoCProgramme, useGetProtoCProgrammeQuery } from '../../redux/coolDataScience'
 import ReactMarkdown from 'react-markdown'
+import moment from 'moment'
+import RangeSelector from 'components/RangeSelector'

 HighchartsCustomEvents(Highcharts)

 const ANALYTICS_CATEGORY = 'Trends'
 const sendAnalytics = (action, name, value) => TSA.Matomo.sendEvent(ANALYTICS_CATEGORY, action, name, value)

+const upToYear = moment().isBefore({ day: 1, month: 7 })
+  ? moment().year() - 2
+  : moment().year() - 1
+
 const defaultConfig = (pointer = true) => {
  return {
    chart: {
@@ -55,6 +61,9 @@ const defaultConfig = (pointer = true) => {
      reversed: false
    },
    plotOptions: {
+      series: {
+        animation: false
+      },
      area: {
        cursor: pointer ? 'pointer' : undefined,
        fillOpacity: 0.5,
@@ -70,9 +79,8 @@ const defaultConfig = (pointer = true) => {
            function round(x) {
              return Math.round(x * 100) / 100
            }
-            return `${point.index + 1}, ${point.category}, ${point.y}, ${round(point.percentage)}%, ${
-              point.series.name
-            }`
+            return `${point.index + 1}, ${point.category}, ${point.y}, ${round(point.percentage)}%, ${point.series.name
+              }`
          }
        }
      }
@@ -80,7 +88,7 @@ const defaultConfig = (pointer = true) => {
  }
 }

-const makeClickableChartConfig = (sortedData, onPointClicked, org) => {
+const makeClickableChartConfig = (sortedData, onPointClicked, org, [startYear, endYear]) => {
  const addPointClickHandler = serie => {
    serie.point = {
      events: {
@@ -124,7 +132,7 @@ const makeClickableChartConfig = (sortedData, onPointClicked, org) => {
  const series = [
    {
      color: '#7f8c8d',
-      name: 'tällä hetkellä peruutettu',
+      name: 'tällä hetkellä passiivinen',
      data: sortedData.map(data => ({
        y: data.currentlyCancelled,
        // pass % of total as z so we can display it in the tooltip
@@ -266,16 +274,15 @@ const makeClickableChartConfig = (sortedData, onPointClicked, org) => {
    },
    yAxis: {
      title: {
-        text: ` ${
-          org ? `2017-2019 aloittaneet uudet kandiopiskelijat<br/>${org.name}` : '% tiedekunnan opiskelijoista'
-        }`
+        text: ` ${org ? `${startYear}-${endYear} aloittaneet uudet kandiopiskelijat<br/>${org.name}` : '% tiedekunnan opiskelijoista'
+          }`
      }
    },
    series
  })
 }

-const makeNonClickableChartConfig = programme => {
+const makeNonClickableChartConfig = (programme, [startYear, endYear]) => {
  const addMouseOverHandler = serie => {
    serie.point = {
      events: {
@@ -311,7 +318,7 @@ const makeNonClickableChartConfig = programme => {
  const series = [
    {
      color: '#7f8c8d',
-      name: 'tällä hetkellä peruutettu',
+      name: 'tällä hetkellä passiivinen',
      data: programme.studytracks.map(p => ({
        y: p.currentlyCancelled,
        // pass % of total as z so we can display it in the tooltip
@@ -347,7 +354,7 @@ const makeNonClickableChartConfig = programme => {

  return Highcharts.merge(defaultConfig(false), {
    title: {
-      text: `2017-2019 aloittaneet uudet kandiopiskelijat<br/>${programme.name}`
+      text: `${startYear}-${endYear} aloittaneet uudet kandiopiskelijat<br/>${programme.name}`
    },
    xAxis: {
      categories: programme.studytracks.map(data => data.name),
@@ -450,24 +457,29 @@ const sorters = {
  'ei tahdissa': (a, b) =>
    (countNotInTarget(a) + a.currentlyCancelled) / a.totalStudents -
    (countNotInTarget(b) + b.currentlyCancelled) / b.totalStudents,
-  peruutettu: (a, b) => a.currentlyCancelled / a.totalStudents - b.currentlyCancelled / b.totalStudents
+  passiivinen: (a, b) => a.currentlyCancelled / a.totalStudents - b.currentlyCancelled / b.totalStudents
 }

-const OrgChart = React.memo(({ orgs, onOrgClicked }) => {
-  return <ReactHighcharts highcharts={Highcharts} config={makeClickableChartConfig(orgs, onOrgClicked)} />
-})
-
-const ProgrammeChart = React.memo(({ programmes, onProgrammeClicked, org }) => {
+const OrgChart = React.memo(({ orgs, onOrgClicked, yearRange }) => {
  return (
-    <ReactHighcharts highcharts={Highcharts} config={makeClickableChartConfig(programmes, onProgrammeClicked, org)} />
+    <ReactHighcharts
+      highcharts={Highcharts}
+      config={makeClickableChartConfig(orgs, onOrgClicked, undefined, yearRange)}
+    />
  )
 })

-const StudytrackChart = React.memo(({ programme }) => {
-  return <ReactHighcharts highcharts={Highcharts} config={makeNonClickableChartConfig(programme)} />
+const ProgrammeChart = React.memo(({ programmes, onProgrammeClicked, org, yearRange }) => {
+  return (
+    <ReactHighcharts highcharts={Highcharts} config={makeClickableChartConfig(programmes, onProgrammeClicked, org, yearRange)} />
+  )
 })

-const ProgrammeDrilldown = ({ org, sorter, sortDir, onProgrammeClicked }) => {
+const StudytrackChart = React.memo(({ programme, yearRange }) => {
+  return <ReactHighcharts highcharts={Highcharts} config={makeNonClickableChartConfig(programme, yearRange)} />
+})
+
+const ProgrammeDrilldown = ({ org, sorter, sortDir, onProgrammeClicked, yearRange }) => {
  const orgSortedProgrammes = useMemo(() => {
    return { ...org, programmes: [...org.programmes].sort((a, b) => sorters[sorter](a, b) * sortDir) }
  }, [org, sorter, sortDir])
@@ -476,12 +488,13 @@ const ProgrammeDrilldown = ({ org, sorter, sortDir, onProgrammeClicked }) => {
    <ProgrammeChart
      programmes={orgSortedProgrammes.programmes}
      onProgrammeClicked={onProgrammeClicked}
+      yearRange={yearRange}
      org={orgSortedProgrammes}
    />
  )
 }

-const StudytrackDrilldown = ({ programme, sorter, sortDir }) => {
+const StudytrackDrilldown = ({ programme, sorter, sortDir, yearRange }) => {
  if (!programme.studytracks || programme.studytracks.length < 1)
    return (
      <Segment>
@@ -491,49 +504,54 @@ const StudytrackDrilldown = ({ programme, sorter, sortDir }) => {
  const programmeSortedStudytracks = useMemo(() => {
    return { ...programme, studytracks: [...programme.studytracks].sort((a, b) => sorters[sorter](a, b) * sortDir) }
  }, [programme, sorter, sortDir])
-  return <StudytrackChart programme={programmeSortedStudytracks} />
+  return <StudytrackChart programme={programmeSortedStudytracks} yearRange={yearRange} />
 }

 const ProtoC = ({
  programme,
-  getProtoCDispatch,
-  getProtoCProgrammeDispatch,
-  protoC,
-  protoCProgramme,
-  loadingProtoC,
-  loadingProtoCProgramme
 }) => {
+  const [firstLoad, setFirstLoad] = useState(true)
  const [sorter, setSorter] = useState('3v tahti')
  const [sortDir, setSortDir] = useState(1)
-  const [drilldownOrg, setDrilldownOrg] = useState(null)
-  const [drilldownProgramme, setDrilldownProgramme] = useState(null)
+  const [drilldownOrgCode, setDrilldownOrgCode] = useState(null)
+  const [drilldownProgrammeCode, setDrilldownProgrammeCode] = useState(null)

  const [includeOldAttainments, setIncludeOldAttainments] = useState(false)
  const [excludeNonEnrolled, setExcludeNonEnrolled] = useState(false)

-  useEffect(() => {
-    const load = async () => {
-      if (!!programme) {
-        getProtoCProgrammeDispatch({
-          includeOldAttainments: includeOldAttainments.toString(),
-          excludeNonEnrolled: excludeNonEnrolled.toString(),
-          code: programme
-        })
-      } else {
-        getProtoCDispatch({
-          includeOldAttainments: includeOldAttainments.toString(),
-          excludeNonEnrolled: excludeNonEnrolled.toString()
-        })
-      }
-    }
-    load()
-  }, [includeOldAttainments, excludeNonEnrolled])
+  const [yearRange, setYearRange] = useState([2017, upToYear])
+  const [startYear, endYear] = yearRange

-  useEffect(() => {
-    if (!!programme) {
-      setDrilldownProgramme(protoCProgramme)
-    }
-  }, [protoCProgramme])
+  const { data: protoC, isLoading: loadingProtoC, isFetching: fetchingProtoC } = useGetProtoCQuery({
+    excludeNonEnrolled,
+    includeOldAttainments,
+    startYear,
+    endYear,
+  })
+
+  const { date: protoCProgramme, isLoading: loadingProtoCProgramme, isFetching: fetchingProtoCProgramme } = useGetProtoCProgrammeQuery({
+    includeOldAttainments: includeOldAttainments.toString(),
+    excludeNonEnrolled: excludeNonEnrolled.toString(),
+    code: programme,
+    startYear,
+    endYear,
+  })
+
+  const drilldownOrg = useMemo(
+    () => protoC
+      ? Object.values(protoC).find(org => org.code === drilldownOrgCode)
+      : null,
+    [drilldownOrgCode, protoC],
+  )
+
+  const drilldownProgramme = useMemo(
+    () => protoC
+      ? Object.values(protoC)
+        .find(org => org.code === drilldownOrgCode)
+        ?.programmes?.find?.(prog => prog.code === drilldownProgrammeCode)
+      : null,
+    [drilldownOrgCode, drilldownProgrammeCode, protoC],
+  )

  const handleOldAttainmentToggled = useCallback(() => {
    setIncludeOldAttainments(previous => !previous)
@@ -546,13 +564,13 @@ const ProtoC = ({
  }, [])

  const handleOrgClicked = useCallback(org => {
-    setDrilldownOrg(org)
-    setDrilldownProgramme(null)
+    setDrilldownOrgCode(org.code)
+    setDrilldownProgrammeCode(null)
    sendAnalytics('C Org drilldown clicked', 'ProtoC')
  }, [])

  const handleProgrammeClicked = useCallback(programme => {
-    setDrilldownProgramme(programme)
+    setDrilldownProgrammeCode(programme.code)
    sendAnalytics('C Programme drilldown clicked', 'ProtoC')
  }, [])

@@ -577,14 +595,39 @@ const ProtoC = ({
      return a > b ? 1 : -1
    })

+  const RangeButtons = () => (
+    <div align="center" style={{ marginTop: '15px' }}>
+      <Menu compact>
+        <Menu.Item>
+          Started between
+        </Menu.Item>
+        <Dropdown
+          className="link item"
+          value={startYear}
+          onChange={(_, { value }) => setYearRange([value, endYear])}
+          options={_.range(2017, upToYear + 1).map((year) => ({ key: year, value: year, text: year, disabled: year > endYear }))}
+        />
+        <Menu.Item>
+          and
+        </Menu.Item>
+        <Dropdown
+          className="link item"
+          value={endYear}
+          onChange={(_, { value }) => setYearRange([startYear, value])}
+          options={_.range(2017, upToYear + 1).map((year) => ({ key: year, value: year, text: year, disabled: year < startYear }))}
+        />
+      </Menu>
+    </div>
+  )
+
  const SorterButtons = () => (
    <div align="center" style={{ marginTop: '15px' }}>
-      <Button.Group>
-        <Button style={{ cursor: 'default' }} active color="black">
+      <Menu compact>
+        <Menu.Item style={{ cursor: 'default' }} active color="black">
          Sort by:
-        </Button>
+        </Menu.Item>
        {sorterNames.map(sorterName => (
-          <Button
+          <Menu.Item
            basic={sorter !== sorterName}
            color={sorter === sorterName ? 'blue' : 'black'}
            key={sorterName}
@@ -595,7 +638,7 @@ const ProtoC = ({
            content={sorterName}
          />
        ))}
-      </Button.Group>
+      </Menu>
    </div>
  )
  // legend and checkboxes, custom legend because highcharts is pita
@@ -606,7 +649,7 @@ const ProtoC = ({
          <Icon style={{ marginLeft: '10px', color: '#6ab04c' }} name="circle" size="small" /> 3v tahdissa
          <Icon style={{ marginLeft: '10px', color: '#f9ca24' }} name="circle" size="small" /> 4v tahdissa
          <Icon style={{ marginLeft: '10px', color: '#ff7979' }} name="circle" size="small" /> ei tahdissa
-          <Icon style={{ marginLeft: '10px', color: '#7f8c8d' }} name="circle" size="small" /> tällä hetkellä peruutettu
+          <Icon style={{ marginLeft: '10px', color: '#7f8c8d' }} name="circle" size="small" /> tällä hetkellä passiivinen
        </span>
      </div>
      <div align="center">
@@ -637,9 +680,9 @@ const ProtoC = ({
      <Segment>
        <SorterButtons />
        <Segment placeholder={loadingProtoC || loadingProtoCProgramme} vertical>
-          <Dimmer inverted active={loadingProtoC || loadingProtoCProgramme} />
-          <Loader active={loadingProtoC || loadingProtoCProgramme} />
-          <StudytrackDrilldown programme={drilldownProgramme} sorter={sorter} sortDir={sortDir} />
+          <Dimmer inverted active={fetchingProtoC || fetchingProtoCProgramme} />
+          <Loader active={fetchingProtoC || fetchingProtoCProgramme} />
+          <StudytrackDrilldown yearRange={yearRange} programme={drilldownProgramme} sorter={sorter} sortDir={sortDir} />
        </Segment>
        <RenderBelowGraph />
      </Segment>
@@ -649,26 +692,37 @@ const ProtoC = ({
  return (
    <Segment>
      <div align="center">
-        <h2>Kandiohjelmat: Suhteellinen tavoiteaikaerittely, 2017-2019 aloittaneet</h2>
+        <h2>Kandiohjelmat: Suhteellinen tavoiteaikaerittely, {startYear}-{endYear} aloittaneet</h2>
      </div>
      <DrilldownMessage />
-      <SorterButtons />
+      <div style={{ display: 'flex', gapX: 15, gapY: 0, justifyContent: 'center', flexFlow: 'wrap' }}>
+        <RangeButtons />
+        <SorterButtons />
+      </div>
      <Segment placeholder={loadingProtoC || loadingProtoCProgramme} vertical>
-        <Dimmer inverted active={loadingProtoC || loadingProtoCProgramme} />
-        <Loader active={loadingProtoC || loadingProtoCProgramme} />
-        {!(loadingProtoC || loadingProtoCProgramme) && protoC && (
-          <OrgChart orgs={sortedOrgs} onOrgClicked={handleOrgClicked} />
-        )}
-        {!(loadingProtoC || loadingProtoCProgramme) && protoC && drilldownOrg && (
+        <Dimmer inverted active={fetchingProtoC || fetchingProtoCProgramme} />
+        <Loader active={fetchingProtoC || fetchingProtoCProgramme} />
+        <OrgChart
+          orgs={sortedOrgs}
+          onOrgClicked={handleOrgClicked}
+          yearRange={yearRange}
+        />
+        {drilldownOrg && (
          <ProgrammeDrilldown
            org={drilldownOrg}
            sorter={sorter}
            sortDir={sortDir}
+            yearRange={yearRange}
            onProgrammeClicked={handleProgrammeClicked}
          />
        )}
        {!(loadingProtoC || loadingProtoCProgramme) && protoC && drilldownProgramme && (
-          <StudytrackDrilldown programme={drilldownProgramme} sorter={sorter} sortDir={sortDir} />
+          <StudytrackDrilldown
+            programme={drilldownProgramme}
+            sorter={sorter}
+            sortDir={sortDir}
+            yearRange={yearRange}
+          />
        )}
      </Segment>
      <RenderBelowGraph />
@@ -679,14 +733,4 @@ const ProtoC = ({
  )
 }

-const mapStateToProps = ({ coolDataScience }) => ({
-  protoC: coolDataScience.data.protoC || {},
-  protoCProgramme: coolDataScience.data.protoCProgramme || {},
-  loadingProtoC: coolDataScience.pending.protoC,
-  loadingProtoCProgramme: coolDataScience.pending.protoCProgramme
-})
-
-export default connect(
-  mapStateToProps,
-  { getProtoCDispatch: getProtoC, getProtoCProgrammeDispatch: getProtoCProgramme }
-)(ProtoC)
+export default ProtoC
@@ -1,21 +1,23 @@
-import React, { useState, useEffect, useCallback, useMemo } from 'react'
-import PropTypes, { func, bool, shape, string } from 'prop-types'
-import { connect } from 'react-redux'
+import React, { useState, useCallback, useMemo } from 'react'
 import Highcharts from 'highcharts'
 import ReactHighcharts from 'react-highcharts'
-import { Segment, Loader, Dimmer, Checkbox, Button, Message } from 'semantic-ui-react'
+import _ from 'lodash'
+import { Dropdown, Menu, Segment, Loader, Dimmer, Checkbox, Message } from 'semantic-ui-react'
 import ReactMarkdown from 'react-markdown'
 import HighchartsCustomEvents from 'highcharts-custom-events'

+import moment from 'moment'
 import TSA from '../../common/tsa'
 import InfoToolTips from '../../common/InfoToolTips'
-import { getProtoC, getProtoCProgramme } from '../../redux/coolDataScience'
+import { useGetProtoCQuery, useGetProtoCProgrammeQuery } from '../../redux/coolDataScience'

 HighchartsCustomEvents(Highcharts)

 const ANALYTICS_CATEGORY = 'Trends'
 const sendAnalytics = (action, name, value) => TSA.Matomo.sendEvent(ANALYTICS_CATEGORY, action, name, value)

+const upToYear = moment().isBefore({ day: 1, month: 7 }) ? moment().year() - 2 : moment().year() - 1
+
 const defaultConfig = () => {
  return {
    chart: {
@@ -111,7 +113,7 @@ const makeConfig = (data, sorter, type = 'column', clickHandler) => {
  const orgSeries = [
    {
      color: '#7f8c8d',
-      name: 'tällä hetkellä peruutettu',
+      name: 'tällä hetkellä passiivinen',
      data: data.map(entry => ({
        y: entry.currentlyCancelled,
        // pass % of total as z so we can display it in the tooltip
@@ -168,7 +170,7 @@ const makeConfig = (data, sorter, type = 'column', clickHandler) => {
            const { chart } = this
            chart.myLabel.destroy()
            const clickedLabel = data.find(entry => entry.name === this.value)
-            if (clickedLabel) clickHandler(clickedLabel.programmes || clickedLabel.studytracks)
+            if (clickedLabel) clickHandler(clickedLabel)
            sendAnalytics('Drilldown clicked', 'ProtoC2')
          },
          mouseover() {
@@ -248,7 +250,7 @@ const makeConfig = (data, sorter, type = 'column', clickHandler) => {
                // clicked on top-level, drill down
                sendAnalytics('Org drilldown clicked', 'ProtoC2')
                const datapoint = data.find(entry => entry.code === point.custom.code)
-                clickHandler(datapoint.programmes || datapoint.studytracks)
+                clickHandler(datapoint)
              }
            },
          },
@@ -264,7 +266,7 @@ const sorters = {
  '4v tahti': (a, b) => a.students4y - b.students4y,
  '3v tahti': (a, b) => a.students3y - b.students3y,
  'ei tahdissa': (a, b) => countNotInTarget(a) + a.currentlyCancelled - (countNotInTarget(b) + b.currentlyCancelled),
-  peruutettu: (a, b) => a.currentlyCancelled - b.currentlyCancelled,
+  passiivinen: (a, b) => a.currentlyCancelled - b.currentlyCancelled,
 }

 const ClickableChart = React.memo(({ data, sorter, isSideways, clickHandler }) => {
@@ -290,78 +292,60 @@ NonClickableChart.defaultProps = {
  isSideways: false,
 }

-NonClickableChart.propTypes = {
-  data: PropTypes.arrayOf(
-    PropTypes.shape({
-      name: PropTypes.string,
-      code: PropTypes.string,
-      students3y: PropTypes.number,
-      students4y: PropTypes.number,
-      totalStudents: PropTypes.number,
-      currentlyCancelled: PropTypes.number,
-    })
-  ).isRequired,
-  sorter: PropTypes.func.isRequired,
-  isSideways: PropTypes.bool,
-}
-
 ClickableChart.defaultProps = {
  isSideways: false,
 }

-ClickableChart.propTypes = {
-  data: PropTypes.arrayOf(
-    PropTypes.shape({
-      name: PropTypes.string,
-      code: PropTypes.string,
-      students3y: PropTypes.number,
-      students4y: PropTypes.number,
-      totalStudents: PropTypes.number,
-      currentlyCancelled: PropTypes.number,
-    })
-  ).isRequired,
-  sorter: PropTypes.func.isRequired,
-  clickHandler: func.isRequired,
-  isSideways: PropTypes.bool,
-}
-
-const ProtoC = ({
-  getProtoCDispatch,
-  getProtoCProgrammeDispatch,
-  data,
-  isLoading,
-  programme = 'KH50_005',
-  protoCProgrammeData,
-  protoCProgrammeLoading,
-}) => {
+const ProtoC = ({ programme = 'KH50_005' }) => {
  const [sorter, setSorter] = useState('3v tahti')
  const [sortDir, setSortDir] = useState(1)
-  const [drilldownOrg, setDrilldownOrg] = useState(null)
-  const [drilldownProgramme, setDrilldownProgramme] = useState(null)
+  const [drilldownOrgCode, setDrilldownOrgCode] = useState(null)
+  const [drilldownProgrammeCode, setDrilldownProgrammeCode] = useState(null)

  const [includeOldAttainments, setIncludeOldAttainments] = useState(false)
  const [excludeNonEnrolled, setExcludeNonEnrolled] = useState(false)

-  useEffect(() => {
-    if (programme) {
-      getProtoCProgrammeDispatch({
-        includeOldAttainments: includeOldAttainments.toString(),
-        excludeNonEnrolled: excludeNonEnrolled.toString(),
-        code: programme,
-      })
-    } else {
-      getProtoCDispatch({
-        includeOldAttainments: includeOldAttainments.toString(),
-        excludeNonEnrolled: excludeNonEnrolled.toString(),
-      })
-    }
-  }, [includeOldAttainments, excludeNonEnrolled])
+  const [yearRange, setYearRange] = useState([2017, upToYear])
+  const [startYear, endYear] = yearRange

-  useEffect(() => {
-    if (programme) {
-      setDrilldownProgramme(protoCProgrammeData.studytracks)
+  const { data, isLoading, isFetching } = useGetProtoCQuery({
+    excludeNonEnrolled,
+    includeOldAttainments,
+    startYear,
+    endYear,
+  })
+
+  const {
+    data: protoCProgrammeData,
+    isLoading: protoCProgrammeLoading,
+    isFetching: protoCProgrammeFetching,
+  } = useGetProtoCProgrammeQuery(
+    {
+      includeOldAttainments: includeOldAttainments.toString(),
+      excludeNonEnrolled: excludeNonEnrolled.toString(),
+      code: programme,
+      startYear,
+      endYear,
+    },
+    {
+      skip: !programme,
    }
-  }, [protoCProgrammeData])
+  )
+
+  const drilldownOrg = useMemo(
+    () => (data ? Object.values(data).find(org => org.code === drilldownOrgCode) : null),
+    [drilldownOrgCode, data]
+  )
+
+  const drilldownProgramme = useMemo(
+    () =>
+      data
+        ? Object.values(data)
+            .find(org => org.code === drilldownOrgCode)
+            ?.programmes?.find?.(prog => prog.code === drilldownProgrammeCode)
+        : null,
+    [drilldownOrgCode, drilldownProgrammeCode, data]
+  )

  const handleOldAttainmentToggled = useCallback(() => {
    setIncludeOldAttainments(previous => !previous)
@@ -380,19 +364,21 @@ const ProtoC = ({
  }, [data, currentSorter])

  const sortedProgrammes = useMemo(() => {
-    return (drilldownOrg || []).sort(currentSorter)
+    return [...(drilldownOrg?.programmes || [])].sort(currentSorter)
  }, [drilldownOrg, currentSorter])

  const sortedStudytracks = useMemo(() => {
-    return (drilldownProgramme || []).sort(currentSorter)
-  }, [drilldownProgramme, currentSorter])
+    const studytracks = programme ? protoCProgrammeData?.studytracks : drilldownProgramme?.studytracks
+
+    return [...(studytracks || [])].sort(currentSorter)
+  }, [programme, drilldownProgramme, protoCProgrammeData, currentSorter])

  const drilldownOrgClick = useCallback(org => {
-    setDrilldownOrg(org)
+    setDrilldownOrgCode(org.code)
  })

  const drilldownProgrammeClick = useCallback(programme => {
-    setDrilldownProgramme(programme)
+    setDrilldownProgrammeCode(programme.code)
  })

  const { CoolDataScience } = InfoToolTips
@@ -423,15 +409,46 @@ const ProtoC = ({
    return (
      <Segment>
        <div align="center">
-          <h2>Prototyyppi: Tavoiteaikaerittely, 2017-2019 aloittaneet</h2>
+          <h2>
+            Prototyyppi: Tavoiteaikaerittely, {startYear}-{endYear} aloittaneet
+          </h2>
        </div>
-        <div align="center" style={{ marginTop: '10px' }}>
-          <Button.Group>
-            <Button style={{ cursor: 'default' }} active color="black">
+        <div
+          align="center"
+          style={{ marginTop: '10px', display: 'flex', gap: '10px 20px', flexFlow: 'wrap', justifyContent: 'center' }}
+        >
+          <Menu compact>
+            <Menu.Item>Started between</Menu.Item>
+            <Dropdown
+              className="link item"
+              value={startYear}
+              onChange={(_, { value }) => setYearRange([value, endYear])}
+              options={_.range(2017, upToYear + 1).map(year => ({
+                key: year,
+                value: year,
+                text: year,
+                disabled: year > endYear,
+              }))}
+            />
+            <Menu.Item>and</Menu.Item>
+            <Dropdown
+              className="link item"
+              value={endYear}
+              onChange={(_, { value }) => setYearRange([startYear, value])}
+              options={_.range(2017, upToYear + 1).map(year => ({
+                key: year,
+                value: year,
+                text: year,
+                disabled: year < startYear,
+              }))}
+            />
+          </Menu>
+          <Menu compact>
+            <Menu.Item style={{ cursor: 'default' }} active color="black">
              Sort by:
-            </Button>
+            </Menu.Item>
            {sorterNames.map(sorterName => (
-              <Button
+              <Menu.Item
                basic={sorter !== sorterName}
                color={sorter === sorterName ? 'blue' : 'black'}
                key={sorterName}
@@ -442,11 +459,11 @@ const ProtoC = ({
                content={sorterName}
              />
            ))}
-          </Button.Group>
+          </Menu>
        </div>
        <Segment placeholder={isLoading} vertical>
-          <Dimmer inverted active={isLoading} />
-          <Loader active={isLoading} />
+          <Dimmer inverted active />
+          <Loader active />
          {!protoCProgrammeLoading && protoCProgrammeData && (
            <>
              <NonClickableChart data={drilldownProgramme} sorter={currentSorter} isSideways />
@@ -476,19 +493,54 @@ const ProtoC = ({
    )
  }

+  const [studytracksLoading, studytracksFetching] = !programme
+    ? [isLoading, isFetching]
+    : [protoCProgrammeLoading, protoCProgrammeFetching]
+
  return (
    <Segment>
      <div align="center">
-        <h2>Kandiohjelmat: Tavoiteaikaerittely, 2017-2019 aloittaneet</h2>
+        <h2>
+          Kandiohjelmat: Tavoiteaikaerittely, {startYear}-{endYear} aloittaneet
+        </h2>
      </div>
      <DrilldownMessage />
-      <div align="center" style={{ marginTop: '10px' }}>
-        <Button.Group>
-          <Button style={{ cursor: 'default' }} active color="black">
+      <div
+        align="center"
+        style={{ marginTop: '10px', display: 'flex', gap: '10px 20px', flexFlow: 'wrap', justifyContent: 'center' }}
+      >
+        <Menu compact>
+          <Menu.Item>Started between</Menu.Item>
+          <Dropdown
+            className="link item"
+            value={startYear}
+            onChange={(_, { value }) => setYearRange([value, endYear])}
+            options={_.range(2017, upToYear + 1).map(year => ({
+              key: year,
+              value: year,
+              text: year,
+              disabled: year > endYear,
+            }))}
+          />
+          <Menu.Item>and</Menu.Item>
+          <Dropdown
+            className="link item"
+            value={endYear}
+            onChange={(_, { value }) => setYearRange([startYear, value])}
+            options={_.range(2017, upToYear + 1).map(year => ({
+              key: year,
+              value: year,
+              text: year,
+              disabled: year < startYear,
+            }))}
+          />
+        </Menu>
+        <Menu compact>
+          <Menu.Item style={{ cursor: 'default' }} active color="black">
            Sort by:
-          </Button>
+          </Menu.Item>
          {sorterNames.map(sorterName => (
-            <Button
+            <Menu.Item
              basic={sorter !== sorterName}
              color={sorter === sorterName ? 'blue' : 'black'}
              key={sorterName}
@@ -499,11 +551,11 @@ const ProtoC = ({
              content={sorterName}
            />
          ))}
-        </Button.Group>
+        </Menu>
      </div>
      <Segment placeholder={isLoading} vertical>
-        <Dimmer inverted active={isLoading} />
-        <Loader active={isLoading} />
+        <Dimmer inverted active={isFetching} />
+        <Loader active={isFetching} />
        {!isLoading && data && (
          <>
            <ClickableChart data={sortedOrgs} sorter={currentSorter} isSideways clickHandler={drilldownOrgClick} />
@@ -515,9 +567,17 @@ const ProtoC = ({
                clickHandler={drilldownProgrammeClick}
              />
            )}
-            {drilldownProgramme && <NonClickableChart data={sortedStudytracks} sorter={currentSorter} isSideways />}
          </>
        )}
+      </Segment>
+      {drilldownProgramme && (
+        <Segment placeholder={studytracksLoading} vertical>
+          <Dimmer inverted active={studytracksFetching} />
+          <Loader active={studytracksFetching} />
+          <NonClickableChart data={sortedStudytracks} sorter={currentSorter} isSideways />
+        </Segment>
+      )}
+      <Segment>
        <div align="center">
          <Checkbox
            label="Include only at least once enrolled students"
@@ -546,24 +606,4 @@ ProtoC.defaultProps = {
  programme: '',
 }

-ProtoC.propTypes = {
-  isLoading: bool.isRequired,
-  data: shape({}).isRequired,
-  getProtoCDispatch: func.isRequired,
-  getProtoCProgrammeDispatch: func.isRequired,
-  protoCProgrammeLoading: bool.isRequired,
-  protoCProgrammeData: shape({}).isRequired,
-  programme: string,
-}
-
-const mapStateToProps = ({ coolDataScience }) => ({
-  data: coolDataScience.data.protoC,
-  protoCProgrammeData: coolDataScience.data.protoCProgramme,
-  isLoading: coolDataScience.pending.protoC,
-  protoCProgrammeLoading: coolDataScience.pending.protoCProgramme,
-})
-
-export default connect(mapStateToProps, {
-  getProtoCDispatch: getProtoC,
-  getProtoCProgrammeDispatch: getProtoCProgramme,
-})(ProtoC)
+export default ProtoC
@@ -22,7 +22,7 @@ const settingDefinitions = _.map(
    {
      key: 'showByYear',
      type: 'checkbox',
-      defaultValue: false,
+      defaultValue: true,
    },
    {
      key: 'showYearlyValues',
@@ -59,6 +59,7 @@ const StatusContainer = ({
  max1,
  showYearlyValues,
  showRelativeValues,
+  showCountingFrom,
  showByYear,
  showStudentCounts,
  clickable,
@@ -84,11 +85,31 @@ const StatusContainer = ({
  let yearlyValues = null

  if (showYearlyValues) {
-    yearlyValues = _.orderBy(Object.entries(stats.yearly), ([y]) => y, ['desc']).map(([year, yearStats]) => ({
-      label: showByYear ? year : `${year}-${`${Number(year) + 1}`.slice(-2)}`,
-      accumulated: showStudentCounts && stats.type === 'course' ? yearStats.accStudents : yearStats.acc,
-      total: showStudentCounts && stats.type === 'course' ? yearStats.totalStudents : yearStats.total,
-    }))
+    yearlyValues = _.chain(stats.yearly)
+      .toPairs()
+      .map(([year, stats]) => ({
+        dateRange: {
+          start: showByYear ? moment([year]).startOf('year') : moment([year]).month(7).date(1),
+          end: showByYear
+            ? moment([year]).startOf('year')
+            : moment([parseInt(year, 10) + 1])
+                .month(6)
+                .date(31),
+        },
+        stats,
+      }))
+      .filter(({ dateRange: { start } }) => moment(showCountingFrom).isSameOrAfter(start))
+      .orderBy(({ dateRange: { start } }) => start, ['desc'])
+      .map(({ dateRange, stats }) => ({
+        label: _.chain([dateRange.start, dateRange.end])
+          .map(date => date.year())
+          .uniq()
+          .join('-')
+          .value(),
+        accumulated: showStudentCounts && stats.type === 'course' ? stats.accStudents : stats.acc,
+        total: showStudentCounts && stats.type === 'course' ? stats.totalStudents : stats.total,
+      }))
+      .value()
  }

  return (
@@ -108,7 +129,7 @@ const StatusContainer = ({
 }

 const StatusContent = ({ data, settings }) => {
-  const { showByYear, showRelativeValues, showYearlyValues, showStudentCounts } = settings
+  const { showByYear, showRelativeValues, showYearlyValues, showStudentCounts, showCountingFrom } = settings

  return (
    <DrillStack
@@ -124,6 +145,7 @@ const StatusContent = ({ data, settings }) => {
          showRelativeValues={showRelativeValues}
          showYearlyValues={showYearlyValues}
          showStudentCounts={showStudentCounts}
+          showCountingFrom={showCountingFrom}
          min1={-2 * medianDiff}
          max1={2 * medianDiff}
          showByYear={showByYear}
@@ -3,6 +3,7 @@ import { connect } from 'react-redux'
 import { Progress, Table } from 'semantic-ui-react'
 import { intersection, orderBy } from 'lodash'
 import { shape, bool, arrayOf, string, number } from 'prop-types'
+import moment from 'moment'
 import { getHighestGradeOfCourseBetweenRange } from '../../../common'
 import ExternalGradeFilterToggle from './ExternalGradeFilterToggle'

@@ -10,13 +11,21 @@ const CoursePopulationCreditDist = ({ singleCourseStats, pending, selectedStuden
  const [courseGrades, setCourseGrades] = useState([])

  useEffect(() => {
-    if (samples && singleCourseStats.alternatives) {
+    if (samples && singleCourseStats?.unifyStats?.alternatives) {
      const filteredGradeArray = []
      const grades = {}

      samples.forEach(student => {
        const courses = student.courses.filter(c => codes.includes(c.course_code))
+        const hasEnrollment =
+          student.enrollments?.some(
+            e => codes.includes(e.course_code) && moment(e.enrollment_date_time).isBetween(moment(from), moment(to))
+          ) ?? false
        const highestGrade = getHighestGradeOfCourseBetweenRange(courses, from, to)
+        if (!highestGrade && hasEnrollment) {
+          if (!grades['No grade']) grades['No grade'] = []
+          grades['No grade'].push(student.studentNumber)
+        }

        if (highestGrade) {
          if (!grades[highestGrade.grade]) {
@@ -10,7 +10,7 @@ import { getCoursePopulation } from '../../redux/populations'
 import { getSingleCourseStats } from '../../redux/singleCourseStats'
 import { useGetStudentListCourseStatisticsQuery } from '../../redux/populationCourses'
 import { getFaculties } from '../../redux/faculties'
-import { getSemesters } from '../../redux/semesters'
+import { useGetSemestersQuery } from '../../redux/semesters'
 import { getElementDetails } from '../../redux/elementdetails'
 import PopulationStudents from '../PopulationStudents'
 import CoursePopulationGradeDist from './CoursePopulationGradeDist'
@@ -19,7 +19,7 @@ import CoursePopulationCreditGainTable from './CoursePopulationCreditGainTable'
 import CustomPopulationProgrammeDist from '../CustomPopulation/CustomPopulationProgrammeDist'
 import CustomPopulationCourses from '../CustomPopulation/CustomPopulationCourses'
 import ProgressBar from '../ProgressBar'
-import { getStudentToTargetCourseDateMap, getTextIn } from '../../common'
+import { getStudentToTargetCourseDateMap, getTextIn, getUnifyTextIn } from '../../common'
 import { useProgress, useTitle } from '../../common/hooks'
 import infotooltips from '../../common/InfoToolTips'
 import {
@@ -31,11 +31,18 @@ import {
  startYearAtUniFilter,
  programmeFilter,
  studyTrackFilter,
+  cherrypickFilter,
 } from '../FilterView/filters'
 import FilterView from '../FilterView'
 import useLanguage from '../LanguagePicker/useLanguage'
 import { queryParamsFromUrl } from '../../common/query'

+const NO_PROGRAMME = {
+  code: '00000',
+  name: { en: 'No programme', fi: 'Ei ohjelmaa' },
+  startdate: '',
+}
+
 const CoursePopulation = ({
  getCoursePopulationDispatch,
  getSingleCourseStatsDispatch,
@@ -44,12 +51,10 @@ const CoursePopulation = ({
  pending,
  history,
  courseData,
-  getSemestersDispatch,
-  semesters,
  getFacultiesDispatch,
+  unifyCourses,
 }) => {
  const { language } = useLanguage()
-
  const [codes, setCodes] = useState([])
  const [headerYears, setYears] = useState('')
  const [dateFrom, setDateFrom] = useState(null)
@@ -75,9 +80,10 @@ const CoursePopulation = ({
    studentNumbers: studentData.students ? studentData.students.map(student => student.studentNumber) : [],
  })

+  const { data: semesters = {} } = useGetSemestersQuery()
+
  useEffect(() => {
    getElementDetails()
-    getSemestersDispatch()
  }, [])

  useEffect(() => {
@@ -100,9 +106,9 @@ const CoursePopulation = ({

  useEffect(() => {
    if (semesters.years && semesters.semesters) {
-      const { coursecodes, from, to, years, years2, separate } = queryParamsFromUrl(history.location)
+      const { coursecodes, from, to, years, years2, separate, unifyCourses } = queryParamsFromUrl(history.location)
      const parsedCourseCodes = JSON.parse(coursecodes)
-      getCoursePopulationDispatch({ coursecodes, from, to, onProgress, separate })
+      getCoursePopulationDispatch({ coursecodes, from, to, onProgress, separate, unifyCourses })
      getSingleCourseStatsDispatch({
        fromYear: from,
        toYear: to,
@@ -124,7 +130,8 @@ const CoursePopulation = ({
    }
  }, [semesters])

-  const header = courseData ? `${getTextIn(courseData.name, language)} ${headerYears}` : null
+  const avoin = getUnifyTextIn(unifyCourses)
+  const header = courseData ? `${getTextIn(courseData.name, language)} ${headerYears} ${avoin}` : null

  const subHeader = codes.join(', ')

@@ -215,6 +222,7 @@ const CoursePopulation = ({
            <CustomPopulationProgrammeDist
              studentToTargetCourseDateMap={studentToTargetCourseDateMap}
              samples={filtered}
+              coursecode={codes}
              selectedStudents={filtered.map(s => s.studentNumber)}
            />
          </div>
@@ -284,6 +292,8 @@ const CoursePopulation = ({
              coursePopulation
              language={language}
              coursecode={codes}
+              from={dateFrom}
+              to={dateTo}
            />
          </div>
        ),
@@ -301,7 +311,7 @@ const CoursePopulation = ({
      .max()
      .value()

-    return moment(date).isBetween(sre.startdate, sre.enddate, 'day', '[]')
+    return sre.code === NO_PROGRAMME.code || moment(date).isBetween(sre.startdate, sre.enddate, 'day', '[]')
  }

  return (
@@ -309,6 +319,7 @@ const CoursePopulation = ({
      name="CoursePopulation"
      filters={[
        genderFilter,
+        cherrypickFilter,
        ageFilter,
        courseFilter({ courses: courseStatistics?.coursestatistics }),
        creditsEarnedFilter,
@@ -364,12 +375,12 @@ const CustomPopulationCoursesWrapper = props => {
  return <CustomPopulationCourses {...props} courses={courseStatistics} />
 }

-const mapStateToProps = ({ singleCourseStats, populations, semesters }) => {
+const mapStateToProps = ({ singleCourseStats, populations, courseSearch }) => {
  return {
    studentData: populations.data,
    pending: populations.pending,
-    courseData: singleCourseStats.stats || {},
-    semesters: semesters.data,
+    courseData: singleCourseStats.stats?.[courseSearch.openOrReqular],
+    unifyCourses: courseSearch.openOrReqular,
    elementDetails: populations?.data?.elementdetails?.data,
  }
 }
@@ -378,7 +389,6 @@ export default withRouter(
  connect(mapStateToProps, {
    getCoursePopulationDispatch: getCoursePopulation,
    getSingleCourseStatsDispatch: getSingleCourseStats,
-    getSemestersDispatch: getSemesters,
    getFacultiesDispatch: getFaculties,
    getElementDetails,
  })(CoursePopulation)
@@ -84,11 +84,13 @@ const CourseTable = ({ course, courseInstance, language, selectedYear }) => {

 const FacultyLevelStatistics = () => {
  const { language: activeLanguage } = useLanguage()
+  const openOrReqular = useSelector(state => state.courseSearch.openOrReqular)
  const { courseStats, language } = useSelector(({ courseStats }) => ({
    courseStats: courseStats.data,
    language: activeLanguage,
  }))
-  const yearcodes = uniq(flatten(Object.values(courseStats).map(c => Object.keys(c.facultyStats))))
+
+  const yearcodes = uniq(flatten(Object.values(courseStats).map(c => Object.keys(c[openOrReqular].facultyStats))))
    .sort()
    .reverse()

@@ -103,13 +105,13 @@ const FacultyLevelStatistics = () => {
  const yearsCourseStats = Object.values(courseStats)
    .map(course => ({
      course,
-      courseInstance: course.facultyStats[selectedYear],
+      courseInstance: course[openOrReqular].facultyStats[selectedYear],
    }))
    .sort((a, b) => (a.courseInstance === undefined) - (b.courseInstance === undefined))

  const courseTables = yearsCourseStats.map(({ course, courseInstance }) => (
    <CourseTable
-      course={course}
+      course={course[openOrReqular]}
      courseInstance={courseInstance}
      language={language}
      key={course.coursecode}
@@ -0,0 +1,31 @@
+import React from 'react'
+import ReactMarkdown from 'react-markdown'
+
+import { Popup, Button, Icon } from 'semantic-ui-react'
+
+import infotooltips from '../../../common/InfoToolTips'
+
+export default ({ tab, viewMode }) => {
+  let content = infotooltips.CourseStatistics[tab]
+
+  if (viewMode) {
+    content = content[viewMode]
+  }
+
+  content = content.replace(/\n +/g, '\n')
+
+  return (
+    <Popup
+      on="click"
+      wide="very"
+      trigger={
+        <Button icon labelPosition="left">
+          <Icon name="question circle outline" />
+          Help
+        </Button>
+      }
+    >
+      <ReactMarkdown>{content}</ReactMarkdown>
+    </Popup>
+  )
+}
@@ -1,22 +1,15 @@
 import React from 'react'
 import qs from 'query-string'
+import _, { uniq } from 'lodash'
 import { Link } from 'react-router-dom'
-import { arrayOf, number, oneOfType, shape, string, bool } from 'prop-types'
 import { connect } from 'react-redux'
 import { Header, Icon, Item } from 'semantic-ui-react'
-import { uniq } from 'lodash'
+
+import { row } from 'components/SortableTable'
 import SortableTable from '../../../../SortableTable'
-import { defineCellColor, getGradeSpread, getThesisGradeSpread, isThesisGrades, THESIS_GRADE_KEYS } from '../util'
+import { getGradeSpread, getThesisGradeSpread, isThesisGrades, resolveGrades, getSortableColumn } from '../util'

-const getSortableColumn = (key, title, getRowVal, getRowContent) => ({
-  key,
-  title,
-  getRowVal,
-  getRowContent,
-  getCellProps: s => defineCellColor(s),
-})
-
-const getTableData = (stats, notThesisGrades, isRelative) =>
+const getTableData = (stats, useThesisGrades, isRelative) =>
  stats.map(stat => {
    const {
      name,
@@ -27,61 +20,52 @@ const getTableData = (stats, notThesisGrades, isRelative) =>
    } = stat

    const attempts = Object.values(grades).reduce((cur, acc) => acc + cur, 0)
-    const gradeSpread = notThesisGrades
-      ? getGradeSpread([grades], isRelative)
-      : getThesisGradeSpread([grades], isRelative)
+    const gradeSpread = useThesisGrades
+      ? getThesisGradeSpread([grades], isRelative)
+      : getGradeSpread([grades], isRelative)

-    return {
+    const mapped = {
      name,
      code,
      coursecode,
      passed: stat.attempts.categories.passed,
      failed: stat.attempts.categories.failed,
+      enrollmentsByState: stat.attempts.enrollmentsByState,
+      totalEnrollments: stat.attempts.totalEnrollments,
      passRate: stat.attempts.passRate,
      attempts,
      rowObfuscated,
-      ...gradeSpread,
+      ..._.mapValues(gradeSpread, x => x[0]),
    }
+
+    if (mapped.name === 'Total') {
+      return row(mapped, { ignoreFilters: true })
+    }
+
+    return mapped
  })

-const includesHTOrTT = stats =>
-  stats.some(({ attempts }) => ['HT', 'TT'].some(grade => Object.keys(attempts.grades).includes(grade)))
-
-const getGradeColumns = (notThesisGrades, addHTAndTT) => {
-  if (!notThesisGrades) return THESIS_GRADE_KEYS.map(k => getSortableColumn(k, k, s => (s.rowObfuscated ? 'NA' : s[k])))
-  const columns = [
-    getSortableColumn('0', '0', s => (s.rowObfuscated ? 'NA' : s['0'])),
-    getSortableColumn('1', '1', s => (s.rowObfuscated ? 'NA' : s['1'])),
-    getSortableColumn('2', '2', s => (s.rowObfuscated ? 'NA' : s['2'])),
-    getSortableColumn('3', '3', s => (s.rowObfuscated ? 'NA' : s['3'])),
-    getSortableColumn('4', '4', s => (s.rowObfuscated ? 'NA' : s['4'])),
-    getSortableColumn('5', '5', s => (s.rowObfuscated ? 'NA' : s['5'])),
-    getSortableColumn('OTHER_PASSED', 'Other passed', s => (s.rowObfuscated ? 'NA' : s['Hyv.'])),
-  ]
-  if (addHTAndTT)
-    columns.splice(
-      6,
-      0,
-      getSortableColumn('HT', 'HT', s => (s.rowObfuscated ? 'NA' : s.HT)),
-      getSortableColumn('TT', 'TT', s => (s.rowObfuscated ? 'NA' : s.TT))
-    )
-  return columns
-}
+const getGradeColumns = grades =>
+  grades.map(({ key, title }) =>
+    getSortableColumn({
+      key,
+      title,
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s[key] || 0),
+    })
+  )

 const AttemptsTable = ({
-  stats,
-  name,
+  data: { stats, name },
+  settings: { showGrades, separate },
  alternatives,
-  separate,
  isRelative,
  userHasAccessToAllStats,
  headerVisible = false,
-  showGrades,
 }) => {
  const {
    attempts: { grades },
  } = stats[0]
-  const notThesisGrades = !isThesisGrades(grades)
+  const useThesisGrades = isThesisGrades(grades)

  const showPopulation = (yearcode, years) => {
    const queryObject = {
@@ -96,46 +80,99 @@ const AttemptsTable = ({
  }

  const timeColumn = {
-    ...getSortableColumn(
-      'TIME',
-      'Time',
-      s => s.code,
-      s => (
-        <div>
-          {s.name}
-          {s.name === 'Total' && !userHasAccessToAllStats && <strong>*</strong>}
-          {s.name !== 'Total' && userHasAccessToAllStats && (
+    key: 'TIME-PARENT',
+    merge: true,
+    mergeHeader: true,
+    children: [
+      getSortableColumn({
+        key: 'TIME',
+        title: 'Time',
+        filterType: 'range',
+        cellProps: {},
+        getRowVal: s => s.code + (2011 - 62),
+        getRowExportVal: s => s.name,
+        getRowContent: s => (
+          <div>
+            {s.name}
+            {s.name === 'Total' && !userHasAccessToAllStats && <strong>*</strong>}
+          </div>
+        ),
+      }),
+      {
+        key: 'TIME-ICON',
+        export: false,
+        getRowContent: s =>
+          s.name !== 'Total' &&
+          userHasAccessToAllStats && (
            <Item as={Link} to={showPopulation(s.code, s.name, s)}>
              <Icon name="level up alternate" />
            </Item>
-          )}
-        </div>
-      )
-    ),
+          ),
+      },
+    ],
  }

  let columns = [
    timeColumn,
-    getSortableColumn('ATTEMPTS', 'Total attempts', s => (s.rowObfuscated ? '5 or less students' : s.attempts)),
-    getSortableColumn('PASSED', 'Passed', s => (s.rowObfuscated ? 'NA' : s.passed)),
-    getSortableColumn('FAILED', 'Failed', s => (s.rowObfuscated ? 'NA' : s.failed)),
-    getSortableColumn(
-      'PASSRATE',
-      'Pass rate',
-      s => (s.rowObfuscated ? 'NA' : s.passRate),
-      s => (s.rowObfuscated ? 'NA' : `${Number(s.passRate || 0).toFixed(2)} %`)
-    ),
+    getSortableColumn({
+      key: 'ATTEMPTS',
+      title: 'Total attempts',
+      getRowVal: s => (s.rowObfuscated ? '5 or less students' : s.attempts),
+    }),
+    getSortableColumn({
+      key: 'PASSED',
+      title: 'Passed',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.passed),
+    }),
+    getSortableColumn({
+      key: 'FAILED',
+      title: 'Failed',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.failed),
+    }),
+    getSortableColumn({
+      key: 'PASSRATE',
+      title: 'Pass rate',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.passRate),
+      getRowContent: s => (s.rowObfuscated ? 'NA' : `${Number(s.passRate || 0).toFixed(2)} %`),
+    }),
+    getSortableColumn({
+      key: 'TOTAL_ENROLLMENTS',
+      title: 'Total enrollments',
+      helpText: 'All enrollments, including all rejected and aborted states.',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.totalEnrollments),
+    }),
+    getSortableColumn({
+      key: 'ENROLLMENTS_ENROLLED',
+      title: 'Enrolled',
+      helpText: 'All enrollments with enrolled or confirmed state.',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.enrollmentsByState.ENROLLED),
+    }),
+    getSortableColumn({
+      key: 'ENROLLMENTS_REJECTED',
+      title: 'Rejected',
+      helpText: 'All enrollments with rejected state.',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.enrollmentsByState.REJECTED),
+    }),
+    getSortableColumn({
+      key: 'ENROLLMENTS_ABORTED',
+      title: 'Aborted',
+      helpText: 'All enrollments with aborted by student or teacher state.',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.enrollmentsByState.ABORTED),
+    }),
  ]

-  if (showGrades) {
+  if (showGrades)
    columns = [
      timeColumn,
-      getSortableColumn('ATTEMPTS', 'Total attempts', s => (s.rowObfuscated ? '5 or less students' : s.attempts)),
-      ...getGradeColumns(notThesisGrades, includesHTOrTT(stats)),
+      getSortableColumn({
+        key: 'ATTEMPTS',
+        title: 'Total attempts',
+        getRowVal: s => (s.rowObfuscated ? '5 or less students' : s.attempts),
+      }),
+      ...getGradeColumns(resolveGrades(stats)),
    ]
-  }

-  const data = getTableData(stats, notThesisGrades, isRelative)
+  const data = getTableData(stats, useThesisGrades, isRelative)

  return (
    <div>
@@ -145,9 +182,8 @@ const AttemptsTable = ({
        </Header>
      )}
      <SortableTable
-        defaultdescending
-        getRowKey={s => s.code}
-        tableProps={{ celled: true, structured: true }}
+        title={`Yearly attempt statistics for group ${name}`}
+        defaultSort={['TIME', 'desc']}
        columns={columns}
        data={data}
      />
@@ -158,19 +194,4 @@ const AttemptsTable = ({
  )
 }

-AttemptsTable.propTypes = {
-  stats: arrayOf(shape({})).isRequired,
-  name: oneOfType([number, string]).isRequired,
-  alternatives: arrayOf(string).isRequired,
-  separate: bool,
-  isRelative: bool.isRequired,
-  userHasAccessToAllStats: bool.isRequired,
-  headerVisible: bool.isRequired,
-  showGrades: bool.isRequired,
-}
-
-AttemptsTable.defaultProps = {
-  separate: false,
-}
-
 export default connect(null)(AttemptsTable)
@@ -1,16 +1,27 @@
-import React from 'react'
+import React, { useMemo } from 'react'
 import qs from 'query-string'
 import { Link } from 'react-router-dom'
 import { Header, Icon, Item } from 'semantic-ui-react'
 import { connect } from 'react-redux'
 import { uniq } from 'lodash'
-import { shape, string, number, oneOfType, arrayOf, bool } from 'prop-types'
+import { string, object, arrayOf, bool } from 'prop-types'
+import { row } from 'components/SortableTable'
 import SortableTable from '../../../../SortableTable'
-import { defineCellColor } from '../util'
+import { defineCellColor, resolveGrades, getSortableColumn } from '../util'

 const formatPercentage = p => `${(p * 100).toFixed(2)} %`

-const StudentTable = ({ stats, name, alternatives, separate, userHasAccessToAllStats, headerVisible = false }) => {
+const getGradeColumns = grades =>
+  grades.map(({ key, title }) =>
+    getSortableColumn({
+      key,
+      title,
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.grades[key] || 0),
+      onlyInGradeView: true,
+    })
+  )
+
+const getColumns = (stats, showDetails, showGrades, userHasAccessToAllStats, alternatives, separate, unifyCourses) => {
  const showPopulation = (yearcode, years) => {
    const queryObject = {
      from: yearcode,
@@ -18,11 +29,187 @@ const StudentTable = ({ stats, name, alternatives, separate, userHasAccessToAllS
      coursecodes: JSON.stringify(uniq(alternatives)),
      years,
      separate,
+      unifyCourses,
    }
    const searchString = qs.stringify(queryObject)
    return `/coursepopulation?${searchString}`
  }

+  const columns = [
+    {
+      key: 'TIME-PARENT',
+      merge: true,
+      mergeHeader: true,
+      children: [
+        {
+          key: 'TIME',
+          title: 'Time',
+          filterType: 'range',
+          getRowVal: s => s.code + (2011 - 62),
+          getRowExportVal: s => s.name,
+          getRowContent: s => (
+            <div style={{ whiteSpace: 'nowrap' }}>
+              {s.name}
+              {s.name === 'Total' && !userHasAccessToAllStats && <strong>*</strong>}
+            </div>
+          ),
+        },
+        {
+          key: 'TIME-ICON',
+          export: false,
+          getRowContent: s => (
+            <>
+              {s.name !== 'Total' && userHasAccessToAllStats && (
+                <Item as={Link} to={showPopulation(s.code, s.name, s)}>
+                  <Icon name="level up alternate" />
+                </Item>
+              )}
+            </>
+          ),
+        },
+      ],
+    },
+    {
+      key: 'TOTAL',
+      title: 'Total Students',
+      helpText: 'Total count of students, including enrolled students with no grade.',
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
+      filterType: 'range',
+      getRowVal: s => (s.rowObfuscated ? 5 : s.students.withEnrollments.total),
+      getRowContent: s => (s.rowObfuscated ? '5 or less students' : s.students.withEnrollments.total),
+      getCellProps: s => defineCellColor(s),
+    },
+    {
+      key: 'TOTAL_PASSED',
+      title: 'Passed',
+      filterType: 'range',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.totalPassed || 0),
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
+      hideWhenGradesVisible: true,
+    },
+    {
+      key: 'TOTAL_FAILED',
+      title: 'Failed',
+      filterType: 'range',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.totalFailed || 0),
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
+      hideWhenGradesVisible: true,
+    },
+    ...getGradeColumns(resolveGrades(stats)),
+    {
+      key: 'ENROLLMENTS_MISSING_GRADE',
+      title: 'Enrolled no grade',
+      filterType: 'range',
+      helpText: 'Total count of students with a valid enrollment and no passing or failing grade.',
+      getRowVal: s => (s.rowObfuscated ? 5 : s.students.enrolledStudentsWithNoGrade),
+      getRowContent: s => (s.rowObfuscated ? '5 or less students' : s.students.enrolledStudentsWithNoGrade),
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
+    },
+    {
+      key: 'PASS_RATE',
+      title: 'Pass-%',
+      getRowVal: s => (s.rowObfuscated ? 0 : s.students.withEnrollments.passRate * 100),
+      getRowContent: s =>
+        s.rowObfuscated ? '5 or less students' : formatPercentage(s.students.withEnrollments.passRate),
+      filterType: 'range',
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
+    },
+    {
+      key: 'PASS_FIRST',
+      title: 'On First Attempt',
+      filterType: 'range',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedFirst || 0),
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
+      onlyInDetailedView: true,
+    },
+    {
+      key: 'PASS_EVENTUALLY',
+      title: 'Eventually',
+      filterType: 'range',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedEventually || 0),
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
+      onlyInDetailedView: true,
+    },
+    {
+      key: 'FAIL_RATE',
+      title: 'Fail-%',
+      filterType: 'range',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : (s.students.failRate || 0) * 100),
+      getRowContent: s => (s.rowObfuscated ? 'NA' : formatPercentage(s.students.failRate || 0)),
+      cellProps: s => ({
+        style: {
+          textAlign: 'right',
+          color: s.rowObfuscated ? 'gray' : 'inherit',
+        },
+      }),
+      onlyInDetailedView: true,
+    },
+  ]
+
+  return columns.filter(column => {
+    if (showDetails && column.onlyInDetailedView) return true
+    if (showGrades && column.onlyInGradeView) return true
+    if (showGrades && column.hideWhenGradesVisible) return false
+    return !column.onlyInDetailedView && !column.onlyInGradeView
+  })
+}
+
+const StudentTable = ({
+  data: { name, stats },
+  settings: { showDetails, separate, showGrades },
+  alternatives,
+  unifyCourses,
+  userHasAccessToAllStats,
+  headerVisible = false,
+}) => {
+  const columns = useMemo(
+    () => getColumns(stats, showDetails, showGrades, userHasAccessToAllStats, alternatives, separate, unifyCourses),
+    [stats, showDetails, showGrades, userHasAccessToAllStats, alternatives, separate, unifyCourses]
+  )
+
+  const data = stats.map(stats => {
+    if (stats.name === 'Total') {
+      return row(stats, { ignoreFilters: true })
+    }
+
+    return stats
+  })
+
  return (
    <div>
      {headerVisible && (
@@ -31,91 +218,13 @@ const StudentTable = ({ stats, name, alternatives, separate, userHasAccessToAllS
        </Header>
      )}
      <SortableTable
+        title={`Student statistics for group ${name}`}
+        defaultSort={['TIME', 'desc']}
        defaultdescending
        getRowKey={s => s.code}
-        tableProps={{ celled: true, structured: true }}
-        columns={[
-          {
-            key: 'TIME',
-            title: 'Time',
-            getRowVal: s => s.code,
-            getRowContent: s => (
-              <div>
-                {s.name}
-                {s.name === 'Total' && !userHasAccessToAllStats && <strong>*</strong>}
-                {s.name !== 'Total' && userHasAccessToAllStats && (
-                  <Item as={Link} to={showPopulation(s.code, s.name, s)}>
-                    <Icon name="level up alternate" />
-                  </Item>
-                )}
-              </div>
-            ),
-            getCellProps: s => defineCellColor(s),
-            headerProps: { rowSpan: 2, width: 3 },
-          },
-          {
-            key: 'TOTAL',
-            title: 'All students',
-            getRowVal: s => (s.rowObfuscated ? '5 or less students' : s.students.total),
-            getCellProps: s => defineCellColor(s),
-            headerProps: { rowSpan: 2, width: 3 },
-          },
-          {
-            key: 'PASSED',
-            title: 'Passed',
-            parent: true,
-            headerProps: { colSpan: 3, width: 5 },
-          },
-          {
-            key: 'PASS_FIRST',
-            title: 'first try',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedFirst || 0),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 2 },
-            child: true,
-          },
-          {
-            key: 'PASS_EVENTUALLY',
-            title: 'passed eventually',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedEventually || 0),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 2 },
-            child: true,
-          },
-          {
-            key: 'PASS_RATE',
-            title: 'percentage',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.passRate),
-            getRowContent: s => (s.rowObfuscated ? 'NA' : formatPercentage(s.students.passRate || 0)),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 1 },
-            child: true,
-          },
-          {
-            key: 'FAIL',
-            title: 'Failed',
-            parent: true,
-            headerProps: { colSpan: 3, width: 5 },
-          },
-          {
-            key: 'NEVER PASSED',
-            title: 'never passed',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.neverPassed || 0),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 2 },
-            child: true,
-          },
-          {
-            key: 'FAIL_RATE',
-            title: 'percentage',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.failRate),
-            getRowContent: s => (s.rowObfuscated ? 'NA' : formatPercentage(s.students.failRate || 0)),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 1 },
-            child: true,
-          },
-        ]}
-        data={stats}
+        tableProps={{ celled: true, fixed: true }}
+        columns={columns}
+        data={data}
      />
      {!userHasAccessToAllStats && (
        <span className="totalsDisclaimer">* Years with 5 students or less are NOT included in the total</span>
@@ -125,12 +234,10 @@ const StudentTable = ({ stats, name, alternatives, separate, userHasAccessToAllS
 }

 StudentTable.propTypes = {
-  stats: arrayOf(shape({})).isRequired,
-  name: oneOfType([number, string]).isRequired,
+  data: object.isRequired,
  alternatives: arrayOf(string).isRequired,
  separate: bool,
  userHasAccessToAllStats: bool.isRequired,
-  headerVisible: bool.isRequired,
 }

 StudentTable.defaultProps = {
@@ -1,12 +1,11 @@
 /* eslint-disable quotes */
 import React from 'react'
-import { Grid } from 'semantic-ui-react'
-import { bool } from 'prop-types'
+import { Menu, Radio } from 'semantic-ui-react'
+import HelpButton from '../HelpButton'
 import { gradeGraphOptions } from '../../../../constants'
 import {
  getDataObject,
  getMaxValueOfSeries,
-  dataSeriesType,
  getGradeSpread,
  getThesisGradeSpread,
  isThesisSeries,
@@ -71,61 +70,47 @@ const getGradeSeries = series => {
      }
 }

-const Distribution = ({ primary, comparison, isRelative, userHasAccessToAllStats }) => {
-  const primaryStats = primary.stats.filter(stat => stat.name !== 'Total' || isRelative)
-
-  const statYears = primaryStats.map(year => year.name)
-  const comparisonStats = comparison ? comparison.stats.filter(stat => stat.name !== 'Total' || isRelative) : []
-  const primaryGrades = primaryStats.flatMap(s => s.attempts.grades)
-  const comparisonGrades = comparisonStats.flatMap(s => s.attempts.grades)
-
-  const gradeGraphSeries = getGradeSeries(primaryGrades)
-  const comparisonGraphSeries = getGradeSeries(comparisonGrades)
-
-  const maxGradeValue = isRelative ? 1 : getMaxValueOfSeries(gradeGraphSeries.absolute)
-
-  const primaryDistributionOptions = comparison
-    ? gradeGraphOptions(statYears, maxGradeValue, 'Primary Grades')
-    : gradeGraphOptions(statYears, maxGradeValue, 'Grades')
-  const comparisonDistributionOptions = gradeGraphOptions(statYears, maxGradeValue, 'Comparison Grades')
+export const DistributionSettings = ({ value, onChange }) => {
+  const { isRelative } = value

  return (
-    <>
-      <Grid.Row>
-        <Grid.Column>
-          <StackedBarChart
-            options={primaryDistributionOptions}
-            series={isRelative ? gradeGraphSeries.relative : gradeGraphSeries.absolute}
-          />
-        </Grid.Column>
-      </Grid.Row>
-      {comparison && (
-        <Grid.Row>
-          <Grid.Column>
-            <StackedBarChart
-              options={comparisonDistributionOptions}
-              series={isRelative ? comparisonGraphSeries.relative : comparisonGraphSeries.absolute}
-            />
-          </Grid.Column>
-        </Grid.Row>
-      )}
-      {!userHasAccessToAllStats && (
-        <span className="totalsDisclaimer">* Years with 5 students or less are shown as 0 in the chart</span>
-      )}
-    </>
+    <Menu secondary style={{ marginBottom: 0 }}>
+      <Menu.Item>
+        <Radio
+          toggle
+          label="Show relative"
+          checked={isRelative}
+          onChange={() => onChange({ ...value, isRelative: !isRelative })}
+        />
+        <Menu.Item>
+          <HelpButton tab="GradeDistribution" />
+        </Menu.Item>
+      </Menu.Item>
+    </Menu>
  )
 }

-Distribution.propTypes = {
-  primary: dataSeriesType.isRequired,
-  comparison: dataSeriesType,
-  isRelative: bool,
-  userHasAccessToAllStats: bool.isRequired,
-}
+export const Distribution = ({ data, settings: { isRelative }, userHasAccessToAllStats }) => {
+  const stats = data.stats.filter(stat => stat.name !== 'Total' || isRelative)

-Distribution.defaultProps = {
-  comparison: undefined,
-  isRelative: false,
-}
+  const statYears = stats.map(year => year.name)
+  const grades = stats.flatMap(s => s.attempts.grades)

-export default Distribution
+  const gradeGraphSeries = getGradeSeries(grades)
+
+  const maxGradeValue = isRelative ? 1 : getMaxValueOfSeries(gradeGraphSeries.absolute)
+
+  const primaryDistributionOptions = gradeGraphOptions(statYears, maxGradeValue, 'Grades')
+
+  return (
+    <div>
+      <StackedBarChart
+        options={primaryDistributionOptions}
+        series={isRelative ? gradeGraphSeries.relative : gradeGraphSeries.absolute}
+      />
+      {!userHasAccessToAllStats && (
+        <span className="totalsDisclaimer">* Years with 5 students or less are shown as 0 in the chart</span>
+      )}
+    </div>
+  )
+}
@@ -1,17 +1,10 @@
 /* eslint-disable quotes */
 import React from 'react'
-import { Grid } from 'semantic-ui-react'
-import { bool } from 'prop-types'
+import { Menu, Radio } from 'semantic-ui-react'
 import StackedBarChart from '../../../StackedBarChart'
 import { passRateAttemptGraphOptions, passRateStudGraphOptions } from '../../../../constants'
-import {
-  viewModeNames,
-  getDataObject,
-  getMaxValueOfSeries,
-  dataSeriesType,
-  viewModeType,
-  absoluteToRelative,
-} from './util'
+import HelpButton from '../HelpButton'
+import { viewModeNames, getDataObject, getMaxValueOfSeries, absoluteToRelative } from './util'

 const getPassRateAttemptSeriesFromStats = stats => {
  const all = []
@@ -70,62 +63,62 @@ const getPassRateStudSeriesFromStats = stats => {
  }
 }

-const PassRate = ({ primary, comparison, viewMode, isRelative = false, userHasAccessToAllStats }) => {
-  const isAttemptsMode = viewMode === viewModeNames.ATTEMPTS
-
-  const primaryStats = primary.stats.filter(stat => stat.name !== 'Total')
-  const statYears = primaryStats.map(year => year.name)
-  const comparisonStats = comparison ? comparison.stats : []
-  const passGraphSerieFn = isAttemptsMode ? getPassRateAttemptSeriesFromStats : getPassRateStudSeriesFromStats
-
-  const passGraphSerie = passGraphSerieFn(primaryStats)
-  const comparisonGraphSerie = passGraphSerieFn(comparisonStats)
-
-  const maxPassRateVal = isRelative ? 1 : getMaxValueOfSeries(passGraphSerie.absolute)
-  const graphOptionsFn = isAttemptsMode ? passRateAttemptGraphOptions : passRateStudGraphOptions
-  const primaryGraphOptions = comparison
-    ? graphOptionsFn(statYears, maxPassRateVal, 'Primary pass rate chart', isRelative)
-    : graphOptionsFn(statYears, maxPassRateVal, 'Pass rate chart', isRelative)
-  const comparisonGraphOptions = graphOptionsFn(statYears, maxPassRateVal, 'Comparison pass rate chart', isRelative)
+export const PassRateSettings = ({ value, onChange }) => {
+  const { viewMode, isRelative } = value

  return (
-    <>
-      <Grid.Row>
-        <Grid.Column>
-          <StackedBarChart
-            options={primaryGraphOptions}
-            series={isRelative ? passGraphSerie.relative : passGraphSerie.absolute}
-          />
-        </Grid.Column>
-      </Grid.Row>
-      {comparison && (
-        <Grid.Row>
-          <Grid.Column>
-            <StackedBarChart
-              options={comparisonGraphOptions}
-              series={isRelative ? comparisonGraphSerie.relative : comparisonGraphSerie.absolute}
-            />
-          </Grid.Column>
-        </Grid.Row>
-      )}
-      {!userHasAccessToAllStats && (
-        <span className="totalsDisclaimer">* Years with 5 students or less are shown as 0 in the chart</span>
-      )}
-    </>
+    <Menu secondary style={{ marginBottom: 0 }}>
+      {Object.entries(viewModeNames).map(([key, name]) => (
+        <Menu.Item
+          active={viewMode === key}
+          name={name}
+          onClick={() =>
+            onChange({
+              ...value,
+              viewMode: key,
+            })
+          }
+        />
+      ))}
+      <Menu.Item>
+        <Radio
+          toggle
+          label="Show relative"
+          checked={isRelative}
+          onChange={() => onChange({ ...value, isRelative: !isRelative })}
+        />
+      </Menu.Item>
+      <Menu.Item>
+        <HelpButton tab="PassRate" viewMode={viewMode} />
+      </Menu.Item>
+    </Menu>
  )
 }

-PassRate.propTypes = {
-  primary: dataSeriesType.isRequired,
-  comparison: dataSeriesType,
-  viewMode: viewModeType.isRequired,
-  isRelative: bool,
-  userHasAccessToAllStats: bool.isRequired,
-}
+export const PassRate = ({ data, settings: { viewMode, isRelative }, userHasAccessToAllStats }) => {
+  const isAttemptsMode = viewMode === 'ATTEMPTS'

-PassRate.defaultProps = {
-  comparison: undefined,
-  isRelative: false,
-}
+  const stats = data.stats.filter(stat => stat.name !== 'Total')
+  const statYears = stats.map(year => year.name)
+  // const comparisonStats = comparison ? comparison.stats : []
+  const passGraphSerieFn = isAttemptsMode ? getPassRateAttemptSeriesFromStats : getPassRateStudSeriesFromStats

-export default PassRate
+  const passGraphSerie = passGraphSerieFn(stats)
+  // const comparisonGraphSerie = passGraphSerieFn(comparisonStats)
+
+  const maxPassRateVal = isRelative ? 1 : getMaxValueOfSeries(passGraphSerie.absolute)
+  const graphOptionsFn = isAttemptsMode ? passRateAttemptGraphOptions : passRateStudGraphOptions
+  const primaryGraphOptions = graphOptionsFn(statYears, maxPassRateVal, 'Pass rate chart', isRelative)
+
+  return (
+    <div>
+      <StackedBarChart
+        options={primaryGraphOptions}
+        series={isRelative ? passGraphSerie.relative : passGraphSerie.absolute}
+      />
+      {!userHasAccessToAllStats && (
+        <span className="totalsDisclaimer">* Years with 5 students or less are shown as 0 in the chart</span>
+      )}
+    </div>
+  )
+}
@@ -1,85 +1,75 @@
-import React, { Fragment } from 'react'
-import { connect } from 'react-redux'
-import { Grid } from 'semantic-ui-react'
-import { string, arrayOf, bool } from 'prop-types'
-import { dataSeriesType, viewModeNames, viewModeType } from './util'
+import React from 'react'
+import { useSelector } from 'react-redux'
+import { Menu, Radio } from 'semantic-ui-react'
+import { viewModeNames } from './util'
+import HelpButton from '../HelpButton'
 import StudentTable from './Tables/student'
 import AttemptsTable from './Tables/attempts'
+import selectors from '../../../../selectors/courseStats'
+import UnifyRadioButtons from '../UnifyRadioButtons'

-const Tables = ({
-  primary,
-  comparison,
-  viewMode,
-  alternatives,
-  separate,
-  isRelative,
-  showGrades,
-  userHasAccessToAllStats,
-}) => {
-  const getViewMode = statistics => {
-    const { name, stats } = statistics
-    const headerVisible = !!comparison
-
-    switch (viewMode) {
-      case viewModeNames.ATTEMPTS:
-        return (
-          <AttemptsTable
-            separate={separate}
-            name={name}
-            stats={stats}
-            alternatives={alternatives}
-            isRelative={isRelative}
-            userHasAccessToAllStats={userHasAccessToAllStats}
-            headerVisible={headerVisible}
-            showGrades={showGrades}
-          />
-        )
-      case viewModeNames.STUDENT:
-        return (
-          <StudentTable
-            separate={separate}
-            name={name}
-            stats={stats}
-            alternatives={alternatives}
-            userHasAccessToAllStats={userHasAccessToAllStats}
-            headerVisible={headerVisible}
-          />
-        )
-      default:
-        return null
-    }
-  }
+export const TablesSettings = ({ value, onChange, availableStats, onSeparateChange }) => {
+  const { viewMode, showDetails, showGrades, separate } = value

  return (
-    <>
-      <Grid.Column id="PrimaryDataTable">{getViewMode(primary)}</Grid.Column>
-      {comparison && <Grid.Column id="ComparisonDataTable">{getViewMode(comparison)}</Grid.Column>}
-    </>
+    <div>
+      <Menu style={{ flexWrap: 'wrap' }} secondary>
+        {Object.entries(viewModeNames).map(([key, name]) => (
+          <Menu.Item
+            key={key}
+            active={viewMode === key}
+            data-cy={`viewMode-${name}`}
+            name={name}
+            onClick={() =>
+              onChange({
+                ...value,
+                viewMode: key,
+              })
+            }
+          />
+        ))}
+        <Menu.Item>
+          <Radio
+            toggle
+            label="Show details"
+            disabled={viewMode !== 'STUDENT'}
+            checked={showDetails}
+            onChange={() => onChange({ ...value, showDetails: !showDetails })}
+          />
+        </Menu.Item>
+        <Menu.Item>
+          <Radio
+            toggle
+            label="Show grades"
+            data-cy="gradeToggle"
+            checked={showGrades}
+            onChange={() => onChange({ ...value, showGrades: !showGrades })}
+          />
+        </Menu.Item>
+        <Menu.Item>
+          <Radio
+            toggle
+            label="Separate by semesters"
+            data-cy="separateToggle"
+            checked={separate}
+            onChange={() => onSeparateChange(!separate)}
+          />
+        </Menu.Item>
+
+        <Menu.Item>
+          <HelpButton tab="Tables" viewMode={viewMode} />
+        </Menu.Item>
+      </Menu>
+      <UnifyRadioButtons availableStats={availableStats} />
+    </div>
  )
 }

-Tables.propTypes = {
-  primary: dataSeriesType.isRequired,
-  comparison: dataSeriesType,
-  viewMode: viewModeType.isRequired,
-  alternatives: arrayOf(string).isRequired,
-  separate: bool,
-  isRelative: bool.isRequired,
-  showGrades: bool.isRequired,
-  userHasAccessToAllStats: bool.isRequired,
+export const Tables = props => {
+  const alternatives = useSelector(selectors.getCourseAlternatives)
+  const viewModes = { ATTEMPTS: AttemptsTable, STUDENT: StudentTable }
+  const Content = viewModes[props.settings.viewMode]
+  const openOrReqular = useSelector(state => state.courseSearch.openOrReqular)
+
+  return <Content {...props} alternatives={alternatives} unifyCourses={openOrReqular} />
 }
-
-Tables.defaultProps = {
-  comparison: null,
-  separate: false,
-}
-
-const mapStateToProps = state => {
-  const { selectedCourse } = state.singleCourseStats
-
-  return {
-    alternatives: state.courseStats.data[selectedCourse].alternatives,
-  }
-}
-
-export default connect(mapStateToProps)(Tables)
@@ -1,8 +1,27 @@
 import { arrayOf, number, oneOfType, shape, string, oneOf } from 'prop-types'
+import { flatten } from 'lodash'
+
+const gradesMap = {
+  0: 0,
+  1: 10,
+  2: 20,
+  3: 30,
+  4: 40,
+  5: 50,
+  'Hyv.': 60,
+  I: 0,
+  A: 1,
+  LUB: 2,
+  NSLA: 3,
+  CL: 4,
+  MCLA: 5,
+  ECLA: 6,
+  L: 7,
+}

 export const viewModeNames = {
-  ATTEMPTS: 'Attempts',
  STUDENT: 'Students',
+  ATTEMPTS: 'Attempts',
 }

 export const getDataObject = (name, data, stack) => ({ name, data, stack })
@@ -23,12 +42,55 @@ export const viewModeType = oneOf(Object.values(viewModeNames))

 export const THESIS_GRADE_KEYS = ['I', 'A', 'NSLA', 'LUB', 'CL', 'MCLA', 'ECLA', 'L']

+export const sortGrades = (a, b) => gradesMap[a] - gradesMap[b]
+
 export const isThesisGrades = grades => Object.keys(grades).some(k => THESIS_GRADE_KEYS.includes(k))

 export const isThesisSeries = series => series && series.some(s => isThesisGrades(s))

 export const absoluteToRelative = all => (p, i) => p / all[i] || 0

+export const resolveGrades = stats => {
+  const failedGrades = ['eisa', 'hyl.', 'hyl', '0', 'luop']
+  const otherPassedGrades = ['hyv.', 'hyv']
+
+  const allGrades = [
+    '0',
+    ...flatten(
+      stats.map(({ students }) =>
+        [...Object.keys(students.grades)].map(grade => {
+          const parsedGrade = Number(grade) ? Math.round(Number(grade)).toString() : grade
+          if (failedGrades.includes(parsedGrade.toLowerCase())) return '0'
+          if (parsedGrade === 'LA') return 'LUB' // merge LA and LUB grades
+          return parsedGrade
+        })
+      )
+    ),
+  ]
+
+  // If any of grades 1-5 is present, make sure that full the grade scale is present
+  if (allGrades.filter(grade => ['1', '2', '3', '4', '5'].includes(grade)).length)
+    allGrades.push(...['1', '2', '3', '4', '5'])
+  const grades = [...new Set(allGrades)]
+
+  return grades.sort(sortGrades).map(grade => {
+    if (grade === '0') return { key: grade, title: 'Failed' }
+    if (otherPassedGrades.includes(grade.toLowerCase())) return { key: grade, title: 'Other passed' }
+    return { key: grade, title: grade.charAt(0).toUpperCase() + grade.slice(1) }
+  })
+}
+
+export const getSortableColumn = opts => ({
+  filterType: 'range',
+  cellProps: s => ({
+    style: {
+      textAlign: 'right',
+      color: s.rowObfuscated ? 'gray' : 'inherit',
+    },
+  }),
+  ...opts,
+})
+
 export const getThesisGradeSpread = (series, isRelative) => {
  const thesisGradeAccumulator = {
    L: [],
@@ -39,13 +101,20 @@ export const getThesisGradeSpread = (series, isRelative) => {
    NSLA: [],
    A: [],
    I: [],
+    0: [],
+    1: [],
+    2: [],
+    3: [],
+    4: [],
+    5: [],
  }
  const newSeries = series.reduce(
    (acc, cur, i) => {
      const currentEntries = Object.entries(cur)

      currentEntries.forEach(([k, v]) => {
-        acc[k].push(v)
+        const merged = k === 'LA' ? 'LUB' : k
+        acc[merged].push(v)
      })

      Object.entries(acc).forEach(([k, v]) => {
@@ -94,7 +163,8 @@ export const getGradeSpread = (series, isRelative) => {
        if (failedKeys.includes(k.toLowerCase())) {
          failed += v
        } else {
-          acc[k].push(v)
+          const parsedGrade = Number(k) ? Math.round(Number(k)) : k
+          acc[parsedGrade].push(v)
        }
      })
      acc[0].push(failed)
@@ -0,0 +1,60 @@
+import React from 'react'
+import { useSelector, useDispatch } from 'react-redux'
+import { Menu, Radio, Form } from 'semantic-ui-react'
+import { toggleOpenAndReqularCourses } from '../../../redux/coursesearch'
+
+const UnifyRadioButtons = ({ availableStats }) => {
+  const dispatch = useDispatch()
+
+  const openOrReqular = useSelector(state => state.courseSearch.openOrReqular)
+  const toggleUnifyRadioValue = (event, { value }) => {
+    dispatch(toggleOpenAndReqularCourses(value))
+  }
+
+  return (
+    <Menu secondary style={{ marginLeft: '0.5rem' }}>
+      <Form>
+        <Form.Group>
+          <Form.Field>
+            <b>Provider organization:</b>
+          </Form.Field>
+          <Form.Field>
+            <Radio
+              label={availableStats.university ? 'university' : 'university (no data)'}
+              name="radioGroup"
+              value="reqularStats"
+              checked={openOrReqular === 'reqularStats'}
+              disabled={!availableStats.university}
+              onChange={toggleUnifyRadioValue}
+              data-cy="unify_radio_reqular"
+            />
+          </Form.Field>
+          <Form.Field>
+            <Radio
+              label={availableStats.open ? 'open' : 'open (no data)'}
+              name="radioGroup"
+              value="openStats"
+              checked={openOrReqular === 'openStats'}
+              onChange={toggleUnifyRadioValue}
+              disabled={!availableStats.open}
+              data-cy="unify_radio_open"
+            />
+          </Form.Field>
+          <Form.Field>
+            <Radio
+              label={availableStats.unify ? 'both' : 'both (no data)'}
+              name="radioGroup"
+              value="unifyStats"
+              checked={openOrReqular === 'unifyStats'}
+              onChange={toggleUnifyRadioValue}
+              disabled={!availableStats.unify}
+              data-cy="unify_radio_unify"
+            />
+          </Form.Field>
+        </Form.Group>
+      </Form>
+    </Menu>
+  )
+}
+
+export default UnifyRadioButtons
@@ -1,166 +1,150 @@
-import React, { useEffect, useState } from 'react'
-import { Tab, Grid, Radio, Menu } from 'semantic-ui-react'
+import React, { useState } from 'react'
+import { Tab, Segment, Menu, Icon } from 'semantic-ui-react'
 import { withRouter } from 'react-router-dom'
-import { shape, bool } from 'prop-types'
-import { dataSeriesType, viewModeNames } from './Panes/util'
-import PassRate from './Panes/passRate'
-import Distribution from './Panes/distribution'
-import Tables from './Panes/tables'
-import InfoBox from '../../Info/InfoBox'
-import { useTabs } from '../../../common/hooks'
+import qs from 'query-string'
+import { useDispatch, useSelector } from 'react-redux'
+import { getCourseStats } from 'redux/coursestats'
+import { PassRate, PassRateSettings } from './Panes/passRate'
+import { Distribution, DistributionSettings } from './Panes/distribution'
+import { Tables, TablesSettings } from './Panes/tables'
+import { useProgress, useTabs } from '../../../common/hooks'

 import './resultTabs.css'
 import TSA from '../../../common/tsa'
-import infotooltips from '../../../common/InfoToolTips'

 const ANALYTICS_CATEGORY = 'Course Statistics'
 const sendAnalytics = (action, name, value) => TSA.Matomo.sendEvent(ANALYTICS_CATEGORY, action, name, value)

-const paneViewIndex = {
-  TABLE: 0,
-  PASS_RAGE: 1,
-  GRADE_DISTRIBUTION: 2,
-}
+const PaneContent = ({
+  component: Component,
+  settings: SettingsComponent,
+  initialSettings,
+  datasets,
+  availableStats,
+  updateQuery,
+  ...rest
+}) => {
+  const [settings, setSettings] = useState(initialSettings)
+  const [splitDirection, setSplitDirection] = useState('row')

-const ResultTabs = props => {
-  const [tab, setTab] = useTabs('cs_tab', 0, props.history)
-  const [viewMode, setViewMode] = useState(viewModeNames.ATTEMPTS)
-  const [selectedView, setSelectedView] = useState(true)
-  const [isRelative, setIsRelative] = useState(false)
-  const [showGrades, setShowGrades] = useState(false)
-
-  const handleModeChange = newViewMode => {
-    sendAnalytics(`Current view mode '${newViewMode}'`, 'Course statistics')
-    setViewMode(newViewMode)
+  const toggleSeparate = separate => {
+    setSettings({ ...settings, separate })
+    updateQuery(separate)
  }

-  useEffect(() => {
-    const newViewMode = selectedView ? viewModeNames.ATTEMPTS : viewModeNames.STUDENT
-    handleModeChange(newViewMode)
-  }, [selectedView])
+  return (
+    <Tab.Pane>
+      <Segment basic>
+        <div style={{ display: 'flex', marginBottom: '2em' }}>
+          <SettingsComponent
+            value={settings}
+            onChange={setSettings}
+            onSeparateChange={toggleSeparate}
+            availableStats={availableStats}
+          />
+          <div style={{ flexGrow: 1 }} />
+          {datasets.filter(i => i).length > 1 && (
+            <div style={{ display: 'flex', alignItems: 'center', gap: '1em' }}>
+              <label>Split direction: </label>
+              <Menu style={{ margin: 0 }}>
+                <Menu.Item active={splitDirection === 'row'} onClick={() => setSplitDirection('row')}>
+                  <Icon name="arrows alternate horizontal" />
+                </Menu.Item>
+                <Menu.Item active={splitDirection === 'column'} onClick={() => setSplitDirection('column')}>
+                  <Icon name="arrows alternate vertical" />
+                </Menu.Item>
+              </Menu>
+            </div>
+          )}
+        </div>
+        <div style={{ display: 'flex', flexDirection: splitDirection, gap: '2em' }}>
+          {datasets
+            .filter(i => i)
+            .map(data => (
+              <div style={{ flexGrow: 1, flexBasis: 1, width: '100%' }}>
+                <h3>{data.name}</h3>
+                <Component data={data} settings={settings} {...rest} />
+              </div>
+            ))}
+        </div>
+      </Segment>
+    </Tab.Pane>
+  )
+}
+
+const ResultTabs = ({ primary, comparison, history, separate, availableStats, location }) => {
+  const [tab, setTab] = useTabs('cs_tab', 0, history)
+  const { userHasAccessToAllStats } = primary
+  const courseStats = useSelector(({ courseStats }) => courseStats)
+  const { pending: loading } = courseStats
+  const { onProgress } = useProgress(loading)
+  const dispatch = useDispatch()

  const handleTabChange = (...params) => {
-    const resetViewMode = params[1].activeIndex === paneViewIndex.TABLE && viewMode === viewModeNames.ATTEMPTS
    const { activeIndex } = params[1]
    const currentTab = params[1].panes[activeIndex]
    sendAnalytics(`Current tab '${currentTab.menuItem.content}'`, 'Course statistics')
    setTab(...params)
-    setViewMode(resetViewMode ? viewMode : viewModeNames.ATTEMPTS)
  }

-  const getRadioButton = (firstLabel, secondLabel, value, setValue) => (
-    <div className="toggleContainer">
-      <label className="toggleLabel">{firstLabel}</label>
-      <Radio toggle data-cy="gradeToggle" checked={value} onChange={() => setValue(!value)} />
-      <label className="toggleLabel">{secondLabel}</label>
-    </div>
-  )
-
-  const renderViewModeSelector = () => {
-    const isTogglePane = tab !== 0
-    const getButtonMenu = () => (
-      <Menu secondary>
-        {Object.values(viewModeNames).map(name => (
-          <Menu.Item key={name} name={name} active={viewMode === name} onClick={() => handleModeChange(name)} />
-        ))}
-        {viewMode === 'Attempts' && getRadioButton('Totals', 'Grade distribution', showGrades, setShowGrades)}
-        {viewMode === 'Attempts' && showGrades && getRadioButton('Absolute', 'Relative', isRelative, setIsRelative)}
-      </Menu>
-    )
-
-    const getToggle = () => {
-      return (
-        <div className="chartToggleContainer">
-          {tab === 1 && getRadioButton('Student', 'Attempts', selectedView, setSelectedView)}
-          {(tab === 2 || props.comparison) && getRadioButton('Absolute', 'Relative', isRelative, setIsRelative)}
-        </div>
-      )
+  const updateSeparate = separate => {
+    const { courseCodes, ...params } = qs.parse(location.search)
+    const query = {
+      ...params,
+      courseCodes: JSON.parse(courseCodes),
+      separate,
    }
-
-    // Remove "false" and activate infoboxes when the texts for them are ready
-    return (
-      <div className="modeSelectorContainer">
-        {isTogglePane ? getToggle() : getButtonMenu()}
-        <InfoBox content={infotooltips.CourseStatistics[tab][viewMode]} />
-      </div>
-    )
+    dispatch(getCourseStats(query, onProgress))
+    const queryToString = { ...query, courseCodes: JSON.stringify(query.courseCodes) }
+    history.replace({ search: qs.stringify(queryToString) })
  }

-  const getPanes = () => {
-    const { primary, comparison, separate } = props
-    const { userHasAccessToAllStats } = primary
+  const paneTypes = [
+    {
+      label: 'Tables',
+      icon: 'table',
+      initialSettings: { showDetails: false, viewMode: 'STUDENT', separate },
+      settings: TablesSettings,
+      component: Tables,
+    },
+    {
+      label: 'Pass rate chart',
+      icon: 'balance',
+      initialSettings: { viewMode: 'STUDENT', separate },
+      settings: PassRateSettings,
+      component: PassRate,
+    },
+    {
+      label: 'Grade distribution chart',
+      icon: 'chart bar',
+      initialSettings: { isRelative: false, viewMode: 'STUDENT' },
+      settings: DistributionSettings,
+      component: Distribution,
+    },
+  ]

-    const paneMenuItems = [
-      {
-        menuItem: { key: 'Table', icon: 'table', content: 'Table' },
-        renderFn: () => (
-          <Tables
-            separate={separate}
-            comparison={comparison}
-            primary={primary}
-            viewMode={viewMode}
-            isRelative={isRelative}
-            showGrades={showGrades}
-            userHasAccessToAllStats={userHasAccessToAllStats}
-          />
-        ),
-      },
-      {
-        menuItem: { key: 'pass', icon: 'balance', content: 'Pass rate chart' },
-        renderFn: () => (
-          <PassRate
-            comparison={comparison}
-            primary={primary}
-            viewMode={viewMode}
-            isRelative={isRelative && !!comparison}
-            userHasAccessToAllStats={userHasAccessToAllStats}
-          />
-        ),
-      },
-      {
-        menuItem: { key: 'grade', icon: 'chart bar', content: 'Grade distribution chart' },
-        renderFn: () => (
-          <Distribution
-            comparison={comparison}
-            primary={primary}
-            isRelative={isRelative}
-            userHasAccessToAllStats={userHasAccessToAllStats}
-          />
-        ),
-      },
-    ]
-
-    return paneMenuItems.map(p => {
-      const { menuItem, renderFn } = p
-      return {
-        menuItem,
-        render: () => (
-          <Grid padded="vertically" columns="equal">
-            <Grid.Row className="modeSelectorRow">{renderViewModeSelector()}</Grid.Row>
-            {renderFn()}
-          </Grid>
-        ),
-      }
+  const panes = paneTypes.map(
+    ({ icon, label, initialSettings, component: Component, settings: SettingsComponent }) => ({
+      menuItem: { icon, content: label },
+      render: () => (
+        <PaneContent
+          component={Component}
+          settings={SettingsComponent}
+          updateQuery={updateSeparate}
+          userHasAccessToAllStats={userHasAccessToAllStats}
+          initialSettings={initialSettings}
+          datasets={[primary, comparison]}
+          availableStats={availableStats}
+        />
+      ),
    })
-  }
+  )

  return (
-    <div>
-      <Tab id="CourseStatPanes" panes={getPanes()} onTabChange={handleTabChange} activeIndex={tab} />
-    </div>
+    <Segment loading={loading} basic>
+      <Tab id="CourseStatPanes" panes={panes} onTabChange={handleTabChange} activeIndex={tab} />
+    </Segment>
  )
 }

-ResultTabs.propTypes = {
-  primary: dataSeriesType.isRequired,
-  comparison: dataSeriesType,
-  history: shape({}).isRequired,
-  separate: bool,
-}
-
-ResultTabs.defaultProps = {
-  comparison: undefined,
-  separate: false,
-}
-
 export default withRouter(ResultTabs)
@@ -4,32 +4,34 @@ import { Form } from 'semantic-ui-react'

 const YearFilter = ({ years, fromYear, toYear, handleChange }) => (
  <>
-    <Form.Group inline>
-      <Form.Dropdown
-        label="From:"
-        name="fromYear"
-        options={toYear ? years.filter(({ value }) => value <= toYear) : years}
-        selection
-        inline
-        placeholder="Select academic year"
-        onChange={handleChange}
-        value={fromYear}
-        selectOnBlur={false}
-        selectOnNavigation={false}
-      />
-      <Form.Dropdown
-        label="To:"
-        name="toYear"
-        options={fromYear ? years.filter(({ value }) => value >= fromYear) : years}
-        inline
-        selection
-        placeholder="Select academic year"
-        onChange={handleChange}
-        value={toYear}
-        selectOnBlur={false}
-        selectOnNavigation={false}
-      />
-    </Form.Group>
+    <Form>
+      <Form.Group inline>
+        <Form.Dropdown
+          label="From:"
+          name="fromYear"
+          options={toYear ? years.filter(({ value }) => value <= toYear) : years}
+          selection
+          inline
+          placeholder="Select academic year"
+          onChange={handleChange}
+          value={fromYear}
+          selectOnBlur={false}
+          selectOnNavigation={false}
+        />
+        <Form.Dropdown
+          label="To:"
+          name="toYear"
+          options={fromYear ? years.filter(({ value }) => value >= fromYear) : years}
+          inline
+          selection
+          placeholder="Select academic year"
+          onChange={handleChange}
+          value={toYear}
+          selectOnBlur={false}
+          selectOnNavigation={false}
+        />
+      </Form.Group>
+    </Form>
  </>
 )

@@ -1,11 +1,11 @@
-import React, { useState, useEffect, Fragment } from 'react'
+import React, { useState, useEffect } from 'react'
 import { Segment, Header, Form } from 'semantic-ui-react'
-import { connect } from 'react-redux'
+import { useSelector, useDispatch } from 'react-redux'
 import { withRouter } from 'react-router-dom'
 import qs from 'query-string'
 import { sortBy } from 'lodash'
-import { func, arrayOf, shape, bool } from 'prop-types'
-import { clearCourses, findCoursesV2, toggleUnifyOpenUniCourses } from '../../../redux/coursesearch'
+import { func, shape } from 'prop-types'
+import { clearCourses, findCoursesV2 } from '../../../redux/coursesearch'
 import { getCourseStats, clearCourseStats } from '../../../redux/coursestats'
 import { getCourseSearchResults } from '../../../selectors/courses'
 import { useSearchHistory, usePrevious } from '../../../common/hooks'
@@ -15,7 +15,6 @@ import AutoSubmitSearchInput from '../../AutoSubmitSearchInput'
 import CourseTable from '../CourseTable'
 import SearchHistory from '../../SearchHistory'
 import useLanguage from '../../LanguagePicker/useLanguage'
-import filterCourseSearchResults from './searchFormUtils'

 const sendAnalytics = (action, name, value) => TSA.Matomo.sendEvent('Course statistics search', action, name, value)

@@ -55,13 +54,18 @@ const useTSASearchResultsHook = (coursesLoading, courseName, courseCode, matchin

 const SearchForm = props => {
  const { language } = useLanguage()
+  const dispatch = useDispatch()
+  const isLoading = useSelector(state => state.courseStats.pending)
+  const coursesLoading = useSelector(state => state.courseSearch.pending)
+  const matchingCourses = useSelector(getCourseSearchResults)
  const [state, setState] = useState({
    ...INITIAL,
  })
  const [searchHistory, addItemToSearchHistory, updateItemInSearchHistory] = useSearchHistory('courseSearch', 6)

  const { courseName, courseCode, selectedCourses, separate } = state
-  const { coursesLoading, isLoading, matchingCourses, unifyOpenUniCourses } = props
+
+  // const { coursesLoading, isLoading, matchingCourses /* unifyOpenUniCourses */ } = props

  const parseQueryFromUrl = () => {
    const { location } = props
@@ -71,7 +75,7 @@ const SearchForm = props => {
      ...rest,
      courseCodes: JSON.parse(courseCodes),
      separate: JSON.parse(separate),
-      unifyOpenUniCourses: JSON.parse(unifyOpenUniCourses || false),
+      // unifyOpenUniCourses: JSON.parse(unifyOpenUniCourses || false),
    }
    return query
  }
@@ -79,14 +83,14 @@ const SearchForm = props => {
  const fetchStatisticsFromUrlParams = () => {
    const query = parseQueryFromUrl()
    setState({ ...state, ...query })
-    props.getCourseStats(query, props.onProgress)
+    dispatch(getCourseStats(query, props.onProgress))
  }

  useEffect(() => {
    const { location } = props
    if (!location.search) {
-      props.clearCourses()
-      props.clearCourseStats()
+      dispatch(clearCourses())
+      dispatch(clearCourseStats())
    }
  }, [])

@@ -143,7 +147,6 @@ const SearchForm = props => {
    const params = {
      courseCodes: codes,
      separate,
-      unifyOpenUniCourses: props.unifyOpenUniCourses,
    }
    const searchHistoryText = codes.map(code => `${getTextIn(selectedCourses[code].name, language)} ${code}`)
    addItemToSearchHistory({
@@ -158,28 +161,14 @@ const SearchForm = props => {
    const isValidCode = validateInputLength(courseCode, 2)

    if (isValidName || isValidCode) {
-      return props.findCoursesV2({ name: courseName, code: courseCode })
+      return dispatch(findCoursesV2({ name: courseName, code: courseCode }))
    }
    if (courseName.length < 5 && courseCode.length < 2) {
-      props.clearCourses()
+      dispatch(clearCourses())
    }
    return Promise.resolve()
  }

-  const onToggleSeparateStatistics = () => {
-    const newValue = !state.separate
-    setState({ ...state, separate: newValue })
-    sendAnalytics('Toggle Separate stats for spring & fall', `${newValue}`)
-  }
-
-  const onToggleUnifyOpenUniCoursesCheckbox = () => {
-    const newValue = !unifyOpenUniCourses
-    sendAnalytics('Toggle Unify open university courses', `${newValue}`)
-
-    setState({ ...state, selectedCourses: {} })
-    props.toggleUnifyOpenUniCourses()
-  }
-
  const courses = matchingCourses.filter(c => !selectedCourses[c.code])

  const disabled = isLoading || Object.keys(selectedCourses).length === 0
@@ -235,16 +224,6 @@ const SearchForm = props => {
                  data-cy="course-code-input"
                />
              </Form.Field>
-              <Form.Field width={4} style={{ display: 'flex', flexDirection: 'column' }}>
-                <label>Unify open university courses:</label>
-                <span style={{ flex: 1, display: 'flex', alignItems: 'center' }}>
-                  <Form.Checkbox
-                    name="unifyOpenUniCourses"
-                    onChange={onToggleUnifyOpenUniCoursesCheckbox}
-                    checked={unifyOpenUniCourses}
-                  />
-                </span>
-              </Form.Field>
            </Form.Group>
            <div style={!noSelectedCourses ? searchBoxStyle : null}>
              <CourseTable
@@ -256,26 +235,18 @@ const SearchForm = props => {
                selectedTable
              />
              {!noSelectedCourses && (
-                <>
-                  <Form.Checkbox
-                    label="Separate statistics for Spring and Fall semesters"
-                    name="separate"
-                    onChange={onToggleSeparateStatistics}
-                    checked={separate}
-                  />
-                  <Form.Button
-                    type="button"
-                    disabled={disabled}
-                    fluid
-                    size="huge"
-                    primary
-                    basic
-                    positive
-                    content="Fetch statistics"
-                    onClick={onSubmitFormClick}
-                    data-cy="fetch-stats-button"
-                  />
-                </>
+                <Form.Button
+                  type="button"
+                  disabled={disabled}
+                  fluid
+                  size="huge"
+                  primary
+                  basic
+                  positive
+                  content="Fetch statistics"
+                  onClick={onSubmitFormClick}
+                  data-cy="fetch-stats-button"
+                />
              )}
            </div>
            <CourseTable
@@ -314,38 +285,9 @@ SearchForm.defaultProps = {
 }

 SearchForm.propTypes = {
-  findCoursesV2: func.isRequired,
-  getCourseStats: func.isRequired,
-  clearCourses: func.isRequired,
-  clearCourseStats: func.isRequired,
-  matchingCourses: arrayOf(shape({})).isRequired,
-  isLoading: bool.isRequired,
-  coursesLoading: bool.isRequired,
  history: shape({}).isRequired,
  location: shape({}).isRequired,
-  unifyOpenUniCourses: bool.isRequired,
-  toggleUnifyOpenUniCourses: func.isRequired,
  onProgress: func,
 }

-const mapStateToProps = state => {
-  const { courses } = getCourseSearchResults(state)
-  const { pending: courseStatsPending } = state.courseStats
-  const { unifyOpenUniCourses } = state.courseSearch
-  return {
-    matchingCourses: filterCourseSearchResults(courses, unifyOpenUniCourses),
-    isLoading: courseStatsPending,
-    coursesLoading: state.courseSearch.pending,
-    unifyOpenUniCourses,
-  }
-}
-
-export default withRouter(
-  connect(mapStateToProps, {
-    getCourseStats,
-    clearCourses,
-    findCoursesV2,
-    clearCourseStats,
-    toggleUnifyOpenUniCourses,
-  })(SearchForm)
-)
+export default withRouter(SearchForm)
@@ -1,9 +1,9 @@
-import { orderBy } from 'lodash'
+// import { orderBy } from 'lodash'

 // handle special case where the course's actual code starts with "A" so it is mistakenly taken as Open Uni course
-const isAvoin = course => !!course.code.match(/^[A][0-9]|^AY/)
+// const isAvoin = course => !!course.code.match(/^[A][0-9]|^AY/)

-const sortAlternatives = alternatives => {
+/* const sortAlternatives = alternatives => {
  const result = orderBy(
    alternatives,
    [
@@ -20,17 +20,18 @@ const sortAlternatives = alternatives => {
  )

  return result
-}
+} */

 // If special case of Open Uni course that starts with A let's just sort them length, that
 // should be good enough in this case.
-const getAlternatives = course => sortAlternatives(course.alternatives)

-const filterCourseSearchResults = (courses, unifyOpenUniCourses) => {
+/* const getAlternatives = course => sortAlternatives(course.alternatives)
+
+const filterCourseSearchResults = courses => {
  const mergedCourses = {}
  const sortedCourses = sortAlternatives(courses)
  sortedCourses.forEach(course => {
-    const groupId = isAvoin(course) && !unifyOpenUniCourses ? course.code : course.subsId
+    const groupId = course.subsId

    if (!(course.max_attainment_date && course.min_attainment_date)) {
      return
@@ -64,6 +65,4 @@ const filterCourseSearchResults = (courses, unifyOpenUniCourses) => {
  }))

  return result
-}
-
-export default filterCourseSearchResults
+} */
@@ -1,3 +1,9 @@
+const parseGrade = grade => {
+  const parsedGrade = Number(grade) ? Math.round(Number(grade)).toString() : grade
+  if (parsedGrade === 'LA') return 'LUB' // merge LA and LUB grades
+  return parsedGrade
+}
+
 const countTotalStats = (formattedStats, userHasAccessToAllStats) => {
  const totals = formattedStats.reduce(
    (acc, curr) => {
@@ -8,18 +14,26 @@ const countTotalStats = (formattedStats, userHasAccessToAllStats) => {
      let { passed } = acc.attempts.categories
      let { failed } = acc.attempts.categories
      const cgrades = acc.attempts.grades
+      const bestEffortGrades = acc.students.grades

      Object.keys(curr.attempts.grades).forEach(grade => {
-        if (!cgrades[grade]) cgrades[grade] = 0
-        cgrades[grade] += curr.attempts.grades[grade]
+        const parsedGrade = parseGrade(grade)
+        if (!cgrades[parsedGrade]) cgrades[parsedGrade] = 0
+        cgrades[parsedGrade] += curr.attempts.grades[grade]

-        if (['EISA', 'Hyl.', 'HYL', '0', 'LUOP'].includes(grade)) {
+        if (['eisa', 'hyl.', 'hyl', '0', 'luop'].includes(parsedGrade.toLowerCase())) {
          failed += curr.attempts.grades[grade]
        } else {
          passed += curr.attempts.grades[grade]
        }
      })

+      Object.keys(curr.students.grades).forEach(grade => {
+        const parsedGrade = parseGrade(grade)
+        if (!bestEffortGrades[parsedGrade]) bestEffortGrades[parsedGrade] = 0
+        bestEffortGrades[parsedGrade] += curr.students.grades[grade]
+      })
+
      const { passedFirst, passedEventually, neverPassed } = curr.students.categories

      const newPassedFirst = passedFirst
@@ -34,16 +48,42 @@ const countTotalStats = (formattedStats, userHasAccessToAllStats) => {
        ? acc.students.categories.neverPassed + neverPassed
        : acc.students.categories.neverPassed

+      Object.keys(curr.students.enrollmentsByState).forEach(k => {
+        if (acc.students.enrollmentsByState[k] === undefined) acc.students.enrollmentsByState[k] = 0
+        acc.students.enrollmentsByState[k] += curr.students.enrollmentsByState[k] || 0
+      })
+
+      Object.keys(curr.attempts.enrollmentsByState).forEach(k => {
+        if (acc.attempts.enrollmentsByState[k] === undefined) acc.attempts.enrollmentsByState[k] = 0
+        acc.attempts.enrollmentsByState[k] += curr.attempts.enrollmentsByState[k] || 0
+      })
+
      return {
        ...acc,
        coursecode: curr.coursecode,
-        attempts: { categories: { passed, failed }, grades: cgrades },
+        attempts: {
+          categories: { passed, failed },
+          grades: cgrades,
+          enrollmentsByState: { ...acc.attempts.enrollmentsByState },
+          totalEnrollments: acc.students.totalEnrollments + (curr.students.totalEnrollments || 0),
+        },
        students: {
+          totalEnrollments: acc.students.totalEnrollments + (curr.students.totalEnrollments || 0),
+          totalPassed: acc.students.totalPassed + curr.students.totalPassed,
+          totalFailed: acc.students.totalFailed + curr.students.totalFailed,
+          enrolledStudentsWithNoGrade:
+            acc.students.enrolledStudentsWithNoGrade + (curr.students.enrolledStudentsWithNoGrade || 0),
          categories: {
            passedFirst: newPassedFirst,
            passedEventually: newPassedEventually,
            neverPassed: newNeverPassed,
          },
+          grades: bestEffortGrades,
+          enrollmentsByState: { ...acc.students.enrollmentsByState },
+          withEnrollments: {
+            total: acc.students.withEnrollments.total + curr.students.withEnrollments.total,
+            totalFailed: acc.students.withEnrollments.totalFailed + curr.students.withEnrollments.totalFailed,
+          },
        },
      }
    },
@@ -59,6 +99,12 @@ const countTotalStats = (formattedStats, userHasAccessToAllStats) => {
        },
        passRate: 0,
        grades: {},
+        totalEnrollments: 0,
+        enrollmentsByState: {
+          ENROLLED: 0,
+          REJECTED: 0,
+          ABORTED: 0,
+        },
      },
      students: {
        categories: {
@@ -66,9 +112,23 @@ const countTotalStats = (formattedStats, userHasAccessToAllStats) => {
          passedEventually: 0,
          neverPassed: 0,
        },
+        grades: {},
        passRate: 0,
        failRate: 0,
        total: 0,
+        totalPassed: 0,
+        totalFailed: 0,
+        totalEnrollments: 0,
+        enrolledStudentsWithNoGrade: 0,
+        enrollmentsByState: {
+          ENROLLED: 0,
+          REJECTED: 0,
+          ABORTED: 0,
+        },
+        withEnrollments: {
+          total: 0,
+          totalFailed: 0,
+        },
      },
      studentnumbers: [],
    }
@@ -76,11 +136,16 @@ const countTotalStats = (formattedStats, userHasAccessToAllStats) => {

  // Count pass- and failrates also for "Total"-lines
  const { passedFirst = 0, passedEventually = 0, neverPassed = 0 } = totals.students.categories
+  const { enrolledStudentsWithNoGrade = 0 } = totals.students
  const total = passedFirst + passedEventually + neverPassed
  totals.students.total = total
  totals.students.passRate = (passedFirst + passedEventually) / total
  totals.students.failRate = neverPassed / total

+  totals.students.withEnrollments.passRate = (passedFirst + passedEventually) / (total + enrolledStudentsWithNoGrade)
+  totals.students.withEnrollments.failRate =
+    (neverPassed + enrolledStudentsWithNoGrade) / (total + enrolledStudentsWithNoGrade)
+
  const { failed, passed } = totals.attempts.categories
  totals.attempts.passRate = (100 * passed) / (passed + failed)

@@ -1,4 +1,4 @@
-import React, { useState, useEffect } from 'react'
+import React, { useState, useMemo, useEffect } from 'react'
 import { Segment, Header, Form, Grid, Button, Popup } from 'semantic-ui-react'
 import { shape, string, arrayOf, objectOf, oneOfType, number, func, bool } from 'prop-types'
 import { connect } from 'react-redux'
@@ -15,7 +15,7 @@ import ProgrammeDropdown from '../ProgrammeDropdown'
 import selectors, { ALL } from '../../../selectors/courseStats'
 import YearFilter from '../SearchForm/YearFilter'
 import { getTextIn } from '../../../common'
-import { getSemesters } from '../../../redux/semesters'
+import { useGetSemestersQuery } from '../../../redux/semesters'
 import TSA from '../../../common/tsa'
 import useLanguage from '../../LanguagePicker/useLanguage'
 import countTotalStats from './countTotalStats'
@@ -34,14 +34,12 @@ const countFilteredStudents = (stat, filter) =>

 const SingleCourseStats = ({
  stats,
+  availableStats,
  setSelectedCourse,
  clearSelectedCourse,
  history,
  location,
  stats: { coursecode },
-  getSemesters,
-  years,
-  semesters,
  programmes,
  maxYearsToCreatePopulationFrom,
  getMaxYearsToCreatePopulationFrom,
@@ -54,6 +52,27 @@ const SingleCourseStats = ({
  const [toYear, setToYear] = useState(0)
  const [separate, setSeparate] = useState(null)

+  const { data: semesterData } = useGetSemestersQuery()
+  const { semesters, years } = useMemo(() => {
+    const semesters = Object.values(semesterData?.semesters ?? [])
+      .map(({ semestercode, name, yearcode }) => ({
+        key: semestercode,
+        texts: Object.values(name),
+        value: yearcode,
+      }))
+      .reverse()
+
+    const years = Object.values(semesterData?.years ?? [])
+      .map(({ yearcode, yearname }) => ({
+        key: yearcode,
+        text: yearname,
+        value: yearcode,
+      }))
+      .reverse()
+
+    return { semesters, years }
+  }, [semesterData])
+
  const parseQueryFromUrl = () => {
    const { separate } = qs.parse(location.search)
    return {
@@ -62,7 +81,6 @@ const SingleCourseStats = ({
  }

  useEffect(() => {
-    if (years.length === 0 || semesters.length === 0) getSemesters()
    if (location.search) {
      const { separate } = parseQueryFromUrl()
      setSeparate(separate)
@@ -81,6 +99,13 @@ const SingleCourseStats = ({
    return () => clearSelectedCourse()
  }, [])

+  useEffect(() => {
+    if (location.search) {
+      const { separate } = parseQueryFromUrl()
+      setSeparate(separate)
+    }
+  }, [location.search])
+
  useEffect(() => {
    if (primary.every(c => !programmes.map(p => p.key).includes(c))) {
      setPrimary([ALL.value])
@@ -170,19 +195,76 @@ const SingleCourseStats = ({
    }
  }

-  const countStudentStats = (allstudents, filter) => {
+  const countStudentStats = (allstudents, enrolledNoGrade = 0, filter) => {
    const categories = countFilteredStudents(allstudents.categories, filter)
-
+    const grades = countFilteredStudents(allstudents.grades, filter)
    const { passedFirst = 0, passedEventually = 0, neverPassed = 0 } = categories
    const total = passedFirst + passedEventually + neverPassed
    const passRate = (passedFirst + passedEventually) / total
    const failRate = neverPassed / total
+    const withEnrollments = {
+      total: total + enrolledNoGrade,
+      passRate: (passedFirst + passedEventually) / (total + enrolledNoGrade),
+      failRate: (neverPassed + enrolledNoGrade) / (total + enrolledNoGrade),
+    }

    return {
+      totalPassed: passedFirst + passedEventually,
+      totalFailed: neverPassed,
      categories,
      passRate,
      failRate,
      total,
+      withEnrollments,
+      grades,
+    }
+  }
+
+  const countEnrollmentStates = filteredEnrollments => {
+    const combined = { CONFIRMED: 'ENROLLED', ABORTED_BY_TEACHER: 'ABORTED', ABORTED_BY_STUDENT: 'ABORTED' }
+    return filteredEnrollments.reduce(
+      (acc, enrollment) => {
+        const state = combined[enrollment.state] || enrollment.state
+        if (acc[state] === undefined) acc[state] = 0
+        acc[state] += 1
+        return acc
+      },
+      {
+        ENROLLED: 0,
+        REJECTED: 0,
+        ABORTED: 0,
+      }
+    )
+  }
+
+  const countAttemptEnrollmentStats = (filteredEnrollments, displayEnrollments) => {
+    const enrollmentsByState = countEnrollmentStates(filteredEnrollments)
+    const totalEnrollments = displayEnrollments ? filteredEnrollments.length : undefined
+    if (!displayEnrollments)
+      Object.keys(enrollmentsByState).forEach(k => {
+        enrollmentsByState[k] = undefined
+      })
+    return { enrollmentsByState, totalEnrollments }
+  }
+
+  const countStudentEnrollmentStats = (allAttempts, filteredEnrollments, displayEnrollments) => {
+    const enrolledStudentsWithNoGrade = filteredEnrollments.filter(({ studentnumber, state }) => {
+      if (state !== 'ENROLLED') return false
+      const hasFailed = allAttempts.categories.failed ? allAttempts.categories.failed.includes(studentnumber) : false
+      const hasPassed = allAttempts.categories.passed ? allAttempts.categories.passed.includes(studentnumber) : false
+      return !hasFailed && !hasPassed
+    })
+    const enrollmentsByState = countEnrollmentStates(filteredEnrollments)
+    if (!displayEnrollments) {
+      Object.keys(enrollmentsByState).forEach(k => {
+        enrollmentsByState[k] = undefined
+      })
+      return { enrolledStudentsWithNoGrade: undefined, enrollmentsByState, totalEnrollments: undefined }
+    }
+    return {
+      enrolledStudentsWithNoGrade: enrolledStudentsWithNoGrade.length,
+      enrollmentsByState,
+      totalEnrollments: filteredEnrollments.length,
    }
  }

@@ -191,20 +273,40 @@ const SingleCourseStats = ({
    const filter = belongsToAtLeastOneProgramme(progCodes)
    const formattedStats = statistics
      .filter(isStatInYearRange)
-      .map(({ code, name, students: allStudents, attempts: allAttempts, coursecode, obfuscated }) => {
-        const attempts = countAttemptStats(allAttempts, filter)
-        const students = countStudentStats(allStudents, filter)
-        const parsedName = separate ? getTextIn(name, language) : name
-        return {
+      .map(
+        ({
          code,
-          name: parsedName,
-          attempts,
-          students,
+          name,
+          students: allStudents,
+          attempts: allAttempts,
          coursecode,
-          rowObfuscated: obfuscated,
-          userHasAccessToAllStats,
+          obfuscated,
+          enrollments = [],
+          allEnrollments = [],
+          yearcode,
+        }) => {
+          const displayEnrollments = yearcode >= 72 // Display enrollments only for Sisu era
+          const filteredEnrollments = enrollments.filter(({ studentnumber }) => filter(studentnumber))
+          const filteredAllEnrollments = allEnrollments.filter(({ studentnumber }) => filter(studentnumber))
+
+          const attempts = countAttemptStats(allAttempts, filter)
+          const attemptsEnrollments = countAttemptEnrollmentStats(filteredAllEnrollments, displayEnrollments)
+          const studentsEnrollments = countStudentEnrollmentStats(allAttempts, filteredEnrollments, displayEnrollments)
+          const students = countStudentStats(allStudents, studentsEnrollments.enrolledStudentsWithNoGrade, filter)
+          const parsedName = separate ? getTextIn(name, language) : name
+
+          return {
+            name: parsedName,
+            students: { ...students, ...studentsEnrollments },
+            attempts: { ...attempts, ...attemptsEnrollments },
+            enrollments: filteredEnrollments,
+            code,
+            coursecode,
+            rowObfuscated: obfuscated,
+            userHasAccessToAllStats,
+          }
        }
-      })
+      )

    const totals = countTotalStats(formattedStats, userHasAccessToAllStats)

@@ -247,7 +349,6 @@ const SingleCourseStats = ({

  const filteredProgrammeStatistics = () => {
    const filter = p => validProgCode(p)
-
    const excludedProgrammes = getExcluded()

    const primaryProgrammes = primary
@@ -266,6 +367,7 @@ const SingleCourseStats = ({
          comparisonProgrammes.length === 1 ? getProgrammeName(comparisonProgrammes[0]) : 'Comparison'
        )
      : undefined
+
    return {
      primary: pstats || undefined,
      comparison: cstats || undefined,
@@ -324,6 +426,7 @@ const SingleCourseStats = ({
      return { ...e, students: [...students], size: students.size }
    })
    .filter(e => e.size > 0)
+
  if (stats.statistics.length < 1) return <Segment>No data for selected course</Segment>

  const options = filteredProgrammes
@@ -389,7 +492,12 @@ const SingleCourseStats = ({
          </Form>
        </Segment>
      )}
-      <ResultTabs separate={separate} primary={statistics.primary} comparison={statistics.comparison} />
+      <ResultTabs
+        separate={separate}
+        primary={statistics.primary}
+        comparison={statistics.comparison}
+        availableStats={availableStats}
+      />
    </div>
  )
 }
@@ -419,46 +527,27 @@ SingleCourseStats.propTypes = {
    coursecode: string,
  }).isRequired,
  programmes: arrayOf(shape({})).isRequired,
-  years: arrayOf(shape({})).isRequired,
-  semesters: arrayOf(shape({})).isRequired,
  location: shape({}).isRequired,
  setSelectedCourse: func.isRequired,
  clearSelectedCourse: func.isRequired,
-  getSemesters: func.isRequired,
  history: shape({
    push: func,
  }).isRequired,
-  getMaxYearsToCreatePopulationFrom: func.isRequired,
+  getMaxYearsToCreatePopulationFrom: Number.isRequired,
  maxYearsToCreatePopulationFrom: number.isRequired,
  userHasAccessToAllStats: bool.isRequired,
 }

 const mapStateToProps = state => {
-  const { semesters = [], years = [] } = state.semesters.data
  return {
    programmes: selectors.getAllStudyProgrammes(state),
-    years: Object.values(years)
-      .map(({ yearcode, yearname }) => ({
-        key: yearcode,
-        text: yearname,
-        value: yearcode,
-      }))
-      .reverse(),
-    semesters: Object.values(semesters)
-      .map(({ semestercode, name, yearcode }) => ({
-        key: semestercode,
-        texts: Object.values(name),
-        value: yearcode,
-      }))
-      .reverse(),
-    maxYearsToCreatePopulationFrom: state.singleCourseStats.maxYearsToCreatePopulationFrom,
+    maxYearsToCreatePopulationFrom: selectors.getMaxYearsToCreatePopulationFrom(state),
  }
 }

 const mapDispatchToProps = {
  setSelectedCourse,
  clearSelectedCourse,
-  getSemesters,
  getMaxYearsToCreatePopulationFrom,
 }

@@ -1,21 +1,30 @@
 import React, { useState, useEffect } from 'react'
-import { connect } from 'react-redux'
+import { useSelector } from 'react-redux'
 import { Segment, Label, Header, Divider, Form } from 'semantic-ui-react'
-import { shape, arrayOf, oneOfType, number, string, bool } from 'prop-types'
+import { oneOfType, number, string, bool } from 'prop-types'
 import SingleCourseStats from '../SingleCourseStats'
 import useLanguage from '../../LanguagePicker/useLanguage'
 import selectors from '../../../selectors/courseStats'
 import { getTextIn } from '../../../common'

-const SingleCourseTab = ({ selected, stats, courses, userHasAccessToAllStats }) => {
+const SingleCourseTab = ({ selected, userHasAccessToAllStats }) => {
  const [selection, setSelection] = useState(selected)
  const { language } = useLanguage()
+  const stats = useSelector(selectors.getCourseStats)
+  const availableStats = useSelector(selectors.getAvailableStats)
+  const courses = useSelector(selectors.getCourses).map(({ code, name }) => ({
+    key: code,
+    value: code,
+    text: <Header content={<>{name}</>} />,
+    content: name,
+  }))

  useEffect(() => {
    setSelection(selected)
  }, [selected])

  if (!stats[selection]) return null
+
  return (
    <div>
      <Segment>
@@ -44,26 +53,20 @@ const SingleCourseTab = ({ selected, stats, courses, userHasAccessToAllStats })
          </Label.Group>
        </Form>
      </Segment>
-      {selection && <SingleCourseStats stats={stats[selection]} userHasAccessToAllStats={userHasAccessToAllStats} />}
+      {selection && (
+        <SingleCourseStats
+          stats={stats[selection]}
+          userHasAccessToAllStats={userHasAccessToAllStats}
+          availableStats={availableStats[selected]}
+        />
+      )}
    </div>
  )
 }

 SingleCourseTab.propTypes = {
-  stats: shape({}).isRequired,
-  courses: arrayOf(shape({})).isRequired,
  selected: oneOfType([number, string]).isRequired,
  userHasAccessToAllStats: bool.isRequired,
 }

-const mapStateToProps = state => ({
-  stats: selectors.getCourseStats(state),
-  courses: selectors.getCourses(state).map(({ code, name }) => ({
-    key: code,
-    value: code,
-    text: <Header content={<>{name}</>} />,
-    content: name,
-  })),
-})
-
-export default connect(mapStateToProps, {})(SingleCourseTab)
+export default SingleCourseTab
@@ -3,6 +3,9 @@
  margin-bottom: 15px !important;
 }

-.graphOptions {
-  float: right;
+.graph-options {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  margin: 1rem 0;
 }
@@ -3,14 +3,14 @@ import { renderToString } from 'react-dom/server'
 import moment from 'moment'
 import { useHistory } from 'react-router-dom'
 import Highcharts from 'highcharts/highstock'
-import { Button } from 'semantic-ui-react'
+import { Button, Radio } from 'semantic-ui-react'
 import boostcanvas from 'highcharts/modules/boost-canvas'
 import _ from 'lodash/fp'
 import boost from 'highcharts/modules/boost'
 import ReactHighstock from 'react-highcharts/ReactHighstock'
 import './creditAccumulationGraphHC.css'
 import CreditGraphTooltip from '../CreditGraphTooltip'
-import { reformatDate, getTextIn } from '../../common'
+import { reformatDate, getTextIn, getStudyRightElementTargetDates } from '../../common'
 import useLanguage from '../LanguagePicker/useLanguage'
 import { DISPLAY_DATE_FORMAT, API_DATE_FORMAT } from '../../constants'

@@ -29,6 +29,8 @@ const createGraphOptions = ({
  onPointClicked,
  startDate,
  endDate,
+  graduations,
+  studyRightStartLine,
 }) => {
  const tooltip = {
    shared: false,
@@ -38,7 +40,7 @@ const createGraphOptions = ({
  if (singleStudent) {
    Object.assign(tooltip, {
      formatter() {
-        tooltipFormatter(this)
+        return tooltipFormatter(this)
      },
      useHTML: true,
      style: {
@@ -90,6 +92,27 @@ const createGraphOptions = ({
      ordinal: false,
      min: startDate,
      max: endDate,
+      plotLines: graduations
+        .map(value => ({
+          value,
+          color: '#a333c8',
+          width: 2,
+          dashStyle: 'dash',
+          label: {
+            text: `Graduation`,
+          },
+        }))
+        .concat(
+          studyRightStartLine.map(value => ({
+            value,
+            color: 'red',
+            width: 3,
+            dashStyle: 'dash',
+            label: {
+              text: `Population study start`,
+            },
+          }))
+        ),
    },
    series: seriesData,
    chart: {
@@ -102,8 +125,58 @@ const createGraphOptions = ({
 const sortCoursesByDate = courses =>
  [...courses].sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime())

+const filterCoursesByStudyPlan = (plan, courses) =>
+  // eslint-disable-next-line camelcase
+  !plan ? courses : courses.filter(({ course_code }) => plan.included_courses.includes(course_code))
+
 const filterCoursesByDate = (courses, date) => courses.filter(c => moment(c.date).isSameOrAfter(moment(date)))

+const filterCourses = (
+  student,
+  singleStudent,
+  byStudyPlanOfCode,
+  cutStudyPlanCredits,
+  startDate,
+  customStudyStartYear
+) => {
+  if (byStudyPlanOfCode && cutStudyPlanCredits)
+    return filterCoursesByDate(
+      filterCoursesByStudyPlan(
+        student.studyplans.find(p => p.programme_code === byStudyPlanOfCode),
+        student.courses
+      ),
+      customStudyStartYear || student.studyrightStart
+    )
+  if (byStudyPlanOfCode)
+    return filterCoursesByStudyPlan(
+      student.studyplans.find(p => p.programme_code === byStudyPlanOfCode),
+      student.courses
+    )
+  if (singleStudent) return student.courses
+  return filterCoursesByDate(student.courses, startDate)
+}
+
+const reduceCreditsToPoints = ({ credits, points, singleStudent }, course) => {
+  const gainedCredits = course.passed && !course.isStudyModuleCredit ? course.credits : 0
+
+  const point = {
+    x: new Date(course.date).getTime(),
+    y: credits + gainedCredits,
+  }
+
+  if (singleStudent) {
+    Object.assign(point, {
+      name: course.course_code,
+    })
+  }
+
+  return {
+    points: [...points, point],
+    credits: credits + gainedCredits,
+    singleStudent,
+  }
+}
+
 const getXAxisMonth = (date, startDate) =>
  Math.max(moment(date, API_DATE_FORMAT).diff(moment(startDate, API_DATE_FORMAT), 'days') / 30, 0)

@@ -137,29 +210,44 @@ const singleStudentTooltipFormatter = (point, student, language) => {
 const createGoalSeries = (starting, ending, absences) => {
  const lastMonth = Math.ceil(getXAxisMonth(moment(ending), moment(starting)))

+  const getColor = (i, type, statutoryAbsence) => {
+    if (i % 2 === 0) return '#96d7c3'
+    if (statutoryAbsence) return '#8e24aa'
+    if (type === 2) return '#ffb300'
+    if (type === -1) return '#e0e0e0'
+    return '#e53935'
+  }
+
  let totalAbsenceMonths = 0
+  let previousTargetCreditsBeforeAbsence = 0
  const absencePoints = absences
    .filter(a => a.startdate >= starting || (a.startdate <= starting && a.enddate >= starting))
-    .reduce((res, { startdate, enddate }) => {
+    .reduce((res, { startdate, enddate, enrollmenttype, statutoryAbsence }) => {
      const targetCreditsBeforeAbsence =
-        (Math.ceil(getXAxisMonth(moment(startdate), moment(starting))) - totalAbsenceMonths) * (55 / 12)
-      if (enddate < ending) {
-        res.push([startdate, targetCreditsBeforeAbsence])
-        res.push([enddate, targetCreditsBeforeAbsence])
-        totalAbsenceMonths += moment(enddate).diff(moment(startdate), 'months')
-      }
+        (Math.abs(moment(startdate).diff(moment(starting), 'months')) - totalAbsenceMonths) * (60 / 12)
+      const newCredits =
+        targetCreditsBeforeAbsence - previousTargetCreditsBeforeAbsence > 5 // Snap to previous credits when absent type changes
+          ? targetCreditsBeforeAbsence
+          : previousTargetCreditsBeforeAbsence
+      previousTargetCreditsBeforeAbsence = newCredits
+      res.push([startdate, newCredits, enrollmenttype, statutoryAbsence])
+      res.push([enddate, newCredits, enrollmenttype, statutoryAbsence])
+      totalAbsenceMonths += Math.abs(moment(enddate).diff(moment(startdate), 'months'))
      return res
    }, [])

  const zoneStart = absencePoints.length ? [absencePoints[0][0]] : []
-  const zones = absencePoints.reduce(
-    (acc, [start], i) => [...acc, { value: start, color: i % 2 === 0 ? '#96d7c3' : 'red' }],
-    zoneStart
+  const zones = absencePoints.reduce((acc, [start, , enrollmenttype, statutoryAbsence], i) => {
+    return [...acc, { value: start, color: getColor(i, enrollmenttype, statutoryAbsence) }]
+  }, zoneStart)
+
+  const lastCredits = (lastMonth - totalAbsenceMonths) * (60 / 12)
+  const endingCredits = absences.some(a => moment(ending).isBetween(moment(a.startdate), moment(a.enddate)))
+    ? absencePoints[absencePoints.length - 1][1]
+    : lastCredits
+  const data = [...[[starting, 0], ...absencePoints, [ending, endingCredits]].sort((a, b) => a[0] - b[0])].filter(
+    r => r[0] <= ending
  )
-
-  const lastCredits = (lastMonth - totalAbsenceMonths) * (55 / 12)
-  const data = [...[[starting, 0], ...absencePoints, [ending, lastCredits]].sort((a, b) => a[0] - b[0])]
-
  return {
    name: SINGLE_GRAPH_GOAL_SERIES_NAME,
    data,
@@ -173,37 +261,49 @@ const createGoalSeries = (starting, ending, absences) => {
  }
 }

-const createStudentCreditLines = (students, singleStudent) =>
-  students.map(student => {
-    const { started, studyrightStart } = student
+// eslint-disable-next-line camelcase
+const resolveStudyRightElement = ({ studyright_elements }) => {
+  // eslint-disable-next-line camelcase
+  if (!studyright_elements || !studyright_elements.length) return {}
+  return (
+    studyright_elements
+      .filter(e => e.element_detail.type === 20)
+      .sort((a, b) => new Date(b.startdate) - new Date(a.startdate))[0] || {}
+  )
+}

-    const startDate = students.length === 1 ? started : studyrightStart
+const createStudentCreditLines = (
+  students,
+  singleStudent,
+  selectedStartDate,
+  studyRightId,
+  studyPlanFilterIsActive,
+  cutStudyPlanCredits,
+  programmeCode,
+  customStudyStartYear
+) =>
+  students.map(student => {
+    const { studyrightStart } = student
+
+    const startDate = singleStudent ? selectedStartDate : studyrightStart
+    const { code } = resolveStudyRightElement(
+      student.studyrights.find(({ studyrightid }) => studyrightid === studyRightId) || {}
+    )
+    const studyPlanProgrammeCode = singleStudent ? code : studyPlanFilterIsActive && programmeCode

    const { points } = _.flow(
+      () =>
+        filterCourses(
+          student,
+          singleStudent,
+          studyPlanProgrammeCode,
+          cutStudyPlanCredits,
+          startDate,
+          customStudyStartYear
+        ),
+      courses => [...courses].filter(({ date }) => new Date(date) <= new Date()),
      sortCoursesByDate,
-      courses => filterCoursesByDate(courses, startDate),
-      _.reduce(
-        ({ credits, points }, course) => {
-          const gainedCredits = course.passed && !course.isStudyModuleCredit ? course.credits : 0
-
-          const point = {
-            x: new Date(course.date).getTime(),
-            y: credits + gainedCredits,
-          }
-
-          if (singleStudent) {
-            Object.assign(point, {
-              name: course.course_code,
-            })
-          }
-
-          return {
-            points: [...points, point],
-            credits: credits + gainedCredits,
-          }
-        },
-        { credits: 0, points: [] }
-      )
+      courses => courses.reduce(reduceCreditsToPoints, { credits: 0, points: [], singleStudent })
    )(student.courses)

    return {
@@ -217,32 +317,122 @@ const createStudentCreditLines = (students, singleStudent) =>
    }
  })

-const CreditAccumulationGraphHighCharts = ({ students, singleStudent, absences, startDate, endDate }) => {
+const filterGraduations = (student, selectedStudyRight) => {
+  const graduated = student.studyrights.filter(({ graduated }) => graduated)
+  // eslint-disable-next-line camelcase
+  if (!selectedStudyRight) return graduated.map(({ enddate }) => new Date(enddate).getTime())
+  const selectedGraduation = graduated.find(({ studyrightid }) => studyrightid === selectedStudyRight.studyrightid)
+  if (!selectedGraduation) return []
+  return [new Date(selectedGraduation.enddate).getTime()]
+}
+
+const CreditAccumulationGraphHighCharts = ({
+  students,
+  singleStudent,
+  absences,
+  startDate,
+  endDate,
+  studyRightId,
+  programmeCode,
+  customPopulation = false,
+  studyPlanFilterIsActive,
+  customStudyStartYear,
+}) => {
  const history = useHistory()
  const chartRef = useRef()
  const language = useLanguage()
-  const [graphHeight, setGraphHeight] = useState(600)

-  const seriesData = useMemo(() => createStudentCreditLines(students, singleStudent), [students, singleStudent])
+  const [graphHeight, setGraphHeight] = useState(700)
+  const [cutStudyPlanCredits, setCutStudyPlanCredits] = useState(false)
+  const selectedStudyRight =
+    singleStudent && studyRightId
+      ? students[0].studyrights.find(({ studyrightid }) => studyrightid === studyRightId)
+      : null
+
+  const seriesData = useMemo(
+    () =>
+      createStudentCreditLines(
+        students,
+        singleStudent,
+        startDate,
+        studyRightId,
+        studyPlanFilterIsActive,
+        cutStudyPlanCredits,
+        programmeCode,
+        customStudyStartYear
+      ),
+    [
+      students,
+      singleStudent,
+      startDate,
+      studyRightId,
+      studyPlanFilterIsActive,
+      programmeCode,
+      cutStudyPlanCredits,
+      customStudyStartYear,
+    ]
+  )

  if (singleStudent) {
-    const starting = _.chain(students[0].studyRights || students[0].courses)
-      .map(elem => new Date(elem.startdate || elem.date))
-      .sortBy()
-      .head()
-      .defaultTo(new Date())
-      .value()
-      .getTime()
-    const ending = endDate ? new Date(endDate).getTime() : new Date().getTime()
+    const startDate = selectedStudyRight
+      ? selectedStudyRight.studyright_elements
+          // eslint-disable-next-line camelcase
+          .filter(({ element_detail }) => element_detail.type === 20)
+          .sort((a, b) => new Date(a.startdate) - new Date(b.startdate))[0].startdate
+      : _.chain(students[0].studyrights || students[0].courses)
+          .map(elem => new Date(elem.startdate || elem.date))
+          .sortBy()
+          .head()
+          .defaultTo(new Date())
+          .value()
+          .getTime()
+    const [, studyRightTargetEnd] = getStudyRightElementTargetDates(selectedStudyRight, absences)
+    const ending = selectedStudyRight
+      ? new Date(studyRightTargetEnd).getTime()
+      : new Date(endDate || new Date()).getTime()
+    const starting = new Date(startDate).getTime()
+    const filteredAbsences = selectedStudyRight
+      ? absences.filter(({ startdate, enddate }) => startdate >= starting && enddate <= ending)
+      : absences

-    seriesData.push(createGoalSeries(starting, ending, absences))
+    seriesData.push(createGoalSeries(starting, ending, filteredAbsences))
  }
+  const getStudyRightStart = () => {
+    if (customStudyStartYear) return new Date(customStudyStartYear).getTime()
+    const studyRightStartFromStudent = new Date(students[0]?.studyrightStart ?? new Date(null))
+    if (studyRightStartFromStudent.getFullYear() < 2000)
+      return Math.min(..._.flatten(students.map(({ courses }) => courses.map(({ date }) => new Date(date).getTime()))))
+    return studyRightStartFromStudent.getTime()
+  }
+
+  const getGraphStart = () => {
+    if (startDate) return new Date(startDate).getTime()
+    if (customPopulation)
+      return Math.min(
+        ..._.flatten(
+          students.map(({ courses }) => {
+            return courses.map(({ date }) => new Date(date).getTime())
+          })
+        )
+      )
+    const studyRightStart = getStudyRightStart()
+    if (studyPlanFilterIsActive && cutStudyPlanCredits) return studyRightStart
+    if (studyPlanFilterIsActive)
+      return Math.min(..._.flatten(students.map(({ courses }) => courses.map(({ date }) => new Date(date).getTime()))))
+    return studyRightStart
+  }
+
+  const graduations = singleStudent ? filterGraduations(students[0], selectedStudyRight) : []
+  const studyRightStartLine =
+    !singleStudent && studyPlanFilterIsActive
+      ? [new Date(customStudyStartYear || students[0].studyrightStart).getTime()]
+      : []

  const options = createGraphOptions({
    singleStudent,
    seriesData,
    graphHeight,
-    startDate: startDate ? new Date(startDate).getTime() : undefined,
+    startDate: getGraphStart(),
    endDate,
    tooltipFormatter: point => singleStudent && singleStudentTooltipFormatter(point, students[0], language),
    onPointClicked: point => {
@@ -250,6 +440,8 @@ const CreditAccumulationGraphHighCharts = ({ students, singleStudent, absences,
        history.push(`/students/${point.series.name}`)
      }
    },
+    graduations,
+    studyRightStartLine,
  })

  const makeGraphSizeButton = (height, label) => (
@@ -258,10 +450,23 @@ const CreditAccumulationGraphHighCharts = ({ students, singleStudent, absences,

  return (
    <div className="graphContainer">
-      <div className="graphOptions">
-        {makeGraphSizeButton(400, 'Small')}
-        {makeGraphSizeButton(600, 'Medium')}
-        {makeGraphSizeButton(1000, 'Large')}
+      <div className="graph-options">
+        <div>
+          {!singleStudent && studyPlanFilterIsActive ? (
+            <Radio
+              checked={cutStudyPlanCredits}
+              onChange={() => setCutStudyPlanCredits(!cutStudyPlanCredits)}
+              label="Display credits from study right start"
+              toggle
+            />
+          ) : null}
+        </div>
+
+        <div>
+          {makeGraphSizeButton(400, 'Small')}
+          {makeGraphSizeButton(600, 'Medium')}
+          {makeGraphSizeButton(1000, 'Large')}
+        </div>
      </div>

      <ReactHighstock highcharts={Highcharts} ref={chartRef} constructorType="stockChart" config={options} />
@@ -1,6 +1,6 @@
 import React from 'react'
 import { Card, Icon } from 'semantic-ui-react'
-import { bool, arrayOf, object } from 'prop-types'
+import { arrayOf, object } from 'prop-types'

 import './creditGraphTooltip.css'

@@ -53,8 +53,7 @@ const getCardDescription = (credits, grade, passed, isStudyModuleCredit) => (
 )

 const CreditGraphTooltip = props => {
-  const { active } = props
-  if (active && props.payload && props.payload.length > 0) {
+  if (props.payload && props.payload.length > 0) {
    const { payload } = props
    const { name } = payload[0]
    const { title, credits, date, grade, passed, isStudyModuleCredit } = payload[0].payload
@@ -72,12 +71,10 @@ const CreditGraphTooltip = props => {
 }

 CreditGraphTooltip.defaultProps = {
-  active: false,
  payload: [],
 }

 CreditGraphTooltip.propTypes = {
-  active: bool,
  payload: arrayOf(object),
 }

@@ -14,6 +14,7 @@ const CustomPopulationProgrammeDist = ({
  selectedStudents,
  studentToTargetCourseDateMap,
  populationStatistics,
+  coursecode,
 }) => {
  const { language } = useLanguage()

@@ -25,12 +26,24 @@ const CustomPopulationProgrammeDist = ({
      const filteredSamples = samples.filter(student => selectedStudents.includes(student.studentNumber))

      filteredSamples.forEach(student => {
-        const programme = getNewestProgramme(
+        let programme = getNewestProgramme(
          student.studyrights,
          student.studentNumber,
          studentToTargetCourseDateMap,
          populationStatistics.elementdetails.data
        )
+        if (programme && programme.code === '00000' && coursecode) {
+          const filteredEnrollments = (student.enrollments || [])
+            // eslint-disable-next-line camelcase
+            .filter(({ course_code }) => coursecode.includes(course_code))
+            .sort((a, b) => new Date(b.enrollment_date_time) - new Date(a.enrollment_date_time))
+          programme = getNewestProgramme(
+            student.studyrights,
+            student.studentNumber,
+            { [student.studentNumber]: (filteredEnrollments[0] || {}).enrollment_date_time },
+            populationStatistics.elementdetails.data
+          )
+        }
        if (programme) {
          if (allProgrammes[programme.code]) {
            allProgrammes[programme.code].students.push({ studentnumber: student.studentNumber })
@@ -4,11 +4,11 @@ import { useSelector, useDispatch } from 'react-redux'
 import { Segment, Header, Accordion, Message } from 'semantic-ui-react'
 import _ from 'lodash'
 import scrollToComponent from 'react-scroll-to-component'
+import semestersApi from 'redux/semesters'
 import { useProgress, useTitle } from '../../common/hooks'
 import infotooltips from '../../common/InfoToolTips'
 import { getCustomPopulationSearches } from '../../redux/customPopulationSearch'
 import { useGetStudentListCourseStatisticsQuery } from '../../redux/populationCourses'
-import { getSemesters } from '../../redux/semesters'
 import CreditAccumulationGraphHighCharts from '../CreditAccumulationGraphHighCharts'
 import PopulationStudents from '../PopulationStudents'
 import CustomPopulationCourses from './CustomPopulationCourses'
@@ -34,10 +34,10 @@ import CustomPopulationSearch from './CustomPopulationSearch'
 import UnihowDataExport from './UnihowDataExport'

 const selectCustomPopulationData = createSelector(
-  state => state.semesters.data,
+  semestersApi.endpoints.getSemesters.select(),
  state => state.populations.data,
  (semesters, populations) => ({
-    allSemesters: semesters?.semesters ?? [],
+    allSemesters: semesters?.data?.semesters ?? [],
    custompop: populations?.students ?? [],
  })
 )
@@ -55,7 +55,6 @@ const CustomPopulation = () => {
  useTitle('Custom population')

  useEffect(() => {
-    dispatch(getSemesters())
    dispatch(getCustomPopulationSearches())
  }, [])

@@ -103,7 +102,7 @@ const CustomPopulationContent = ({ students, custompop }) => {
  const populations = useSelector(state => state.populations)
  const { customPopulationFlag } = populations

-  const { progress } = useProgress(populations.loading)
+  const { progress } = useProgress(populations.pending)

  const { data: courseStats } = useGetStudentListCourseStatisticsQuery({
    studentNumbers: students.map(s => s.studentNumber),
@@ -143,7 +142,7 @@ const CustomPopulationContent = ({ students, custompop }) => {
      content: {
        content: (
          <div ref={creditGainRef}>
-            <CreditAccumulationGraphHighCharts students={students} />
+            <CreditAccumulationGraphHighCharts students={students} customPopulation />
          </div>
        ),
      },
@@ -0,0 +1,30 @@
+import React from 'react'
+import { Form } from 'semantic-ui-react'
+import DateTimeSelector from 'components/DateTimeSelector'
+
+const DateRangeSelector = ({ value, onChange, ...rest }) => {
+  return (
+    <Form style={{}}>
+      <Form.Field>
+        <label>Beginning:</label>
+        <DateTimeSelector
+          value={value?.[0]}
+          before={value?.[1]}
+          onChange={date => onChange([date, value?.[1]])}
+          {...rest}
+        />
+      </Form.Field>
+      <Form.Field>
+        <label>Ending:</label>
+        <DateTimeSelector
+          value={value?.[1]}
+          after={value?.[0]}
+          onChange={date => onChange([value?.[0], date])}
+          {...rest}
+        />
+      </Form.Field>
+    </Form>
+  )
+}
+
+export default DateRangeSelector
@@ -0,0 +1,121 @@
+import React, { useRef } from 'react'
+import Datetime from 'react-datetime'
+import 'moment/locale/fi'
+import { Icon, Button } from 'semantic-ui-react'
+import moment from 'moment'
+import { useGetSemestersQuery } from 'redux/semesters'
+import { getTextIn } from '../../common'
+
+import './style.css'
+
+const semesterListStyles = {
+  maxHeight: '10em',
+  overflowY: 'auto',
+  padding: 0,
+  margin: '0 -4px -4px -4px',
+  borderTop: '1px solid #f9f9f9',
+}
+
+const DateTime = ({ value, onChange, before, after, showSemesters }) => {
+  const datetimeRef = useRef()
+  const semesterRequest = useGetSemestersQuery()
+  const allSemesters = semesterRequest.data?.semesters ?? []
+
+  return (
+    <Datetime
+      ref={datetimeRef}
+      value={value}
+      onChange={onChange}
+      timeFormat={false}
+      locale="fi"
+      closeOnSelect
+      isValidDate={date => (!before || date.isBefore(before)) && (!after || date.isAfter(after))}
+      renderView={(mode, renderDefault) => {
+        const createDateButton = (date, label) => (
+          <button
+            type="button"
+            onClick={() => {
+              datetimeRef.current.setViewDate(moment(date))
+              onChange(moment(date))
+            }}
+            style={{
+              height: '1.75em',
+              background: value && value.isSame(date, 'day') ? '#428bca' : undefined,
+              color: value && value.isSame(date, 'day') ? 'white' : 'inherit',
+            }}
+            className="date-picker-semester-button"
+          >
+            {label}
+          </button>
+        )
+
+        return (
+          <>
+            {renderDefault()}
+            {showSemesters && (
+              <>
+                <div
+                  style={{
+                    ...semesterListStyles,
+                    height: '1.75em',
+                    lineHeight: '1.75em',
+                    marginBottom: 0,
+                    paddingLeft: '0.3em',
+                    fontWeight: 'bold',
+                  }}
+                >
+                  Semesters:
+                </div>
+                <ul style={semesterListStyles}>
+                  {allSemesters &&
+                    Object.values(allSemesters)
+                      .filter(({ startdate, enddate }) => {
+                        const start = moment(startdate)
+                        const end = moment(enddate)
+
+                        return start.isBefore() && (!before || start.isBefore(before)) && (!after || end.isAfter(after))
+                      })
+                      .sort((a, b) => new Date(b.startdate) - new Date(a.startdate))
+                      .map(({ name, startdate, enddate }) => (
+                        <li style={{ padding: '0.2em 0.3em', display: 'flex', alignItems: 'center' }}>
+                          <span style={{ flexGrow: 1 }}>{getTextIn(name)}</span>
+                          {createDateButton(startdate, 'Start')}
+                          {createDateButton(moment(enddate).subtract(1, 'days'), 'End')}
+                        </li>
+                      ))}
+                </ul>
+              </>
+            )}
+          </>
+        )
+      }}
+      renderInput={(_, open) => (
+        <Button
+          icon={value !== null}
+          labelPosition={value !== null && 'right'}
+          onClick={open}
+          className="credit-date-filter-input"
+          size="mini"
+          style={{
+            whiteSpace: 'nowrap',
+            paddingRight: value ? '3.5em !important' : undefined,
+            paddingLeft: value ? '1em !important' : undefined,
+          }}
+        >
+          {value === null ? 'Select Date' : moment(value).format('DD.MM.YYYY')}
+          {value !== null && (
+            <Icon
+              name="x"
+              onClick={evt => {
+                evt.stopPropagation()
+                onChange(null)
+              }}
+            />
+          )}
+        </Button>
+      )}
+    />
+  )
+}
+
+export default DateTime
@@ -0,0 +1,3 @@
+.date-picker-semester-button:hover {
+  background-color: red;
+}
--- a/Show More
+++ b/Show More