Potential fix for problems with Courses of Population -components

Bump follow-redirects from 1.14.3 to 1.14.8 in /services/backend

.github/workflows/release-sis-updater-scheduler.yaml

@@ -1,54 +0,0 @@
-name: "Release sis-updater-scheduler"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-sis-updater-scheduler.yaml'
-      - 'updater/sis-updater-scheduler/**'
-
-jobs:
-  build_and_release:
-    name: "Build and release sis-updater-scheduler"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: docker/setup-buildx-action@v1
-    - name: "Login to DockerHub"
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: "Build and push"
-      uses: docker/build-push-action@v2
-      with:
-        cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
-        cache-to: type=inline
-        context: updater/sis-updater-scheduler/
-        push: true
-        tags: toska/sis-updater-scheduler:latest
-        target: production
-    - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-      with:
-        webhook-url: ${{ secrets.WEBHOOK_URL }}
-        message-type: deployment
-        softa-url: https://oodikone.cs.helsinki.fi
-        deployment-target: production
-        softa-to-deploy: sis-updater-scheduler
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: production
-          softa-to-deploy: sis-updater-scheduler

.github/workflows/release-sis-updater-worker.yaml

@@ -1,54 +0,0 @@
-name: "Release sis-updater-worker"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-sis-updater-worker.yaml'
-      - 'updater/sis-updater-worker/**'
-
-jobs:
-  build_and_release:
-    name: "Build and release sis-updater-worker"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: docker/setup-buildx-action@v1
-    - name: "Login to DockerHub"
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: "Build and push"
-      uses: docker/build-push-action@v2
-      with:
-        cache-from: type=registry,ref=toska/sis-updater-worker:latest
-        cache-to: type=inline
-        context: updater/sis-updater-worker/
-        push: true
-        tags: toska/sis-updater-worker:latest
-        target: production
-    - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-      with:
-        webhook-url: ${{ secrets.WEBHOOK_URL }}
-        message-type: deployment
-        softa-url: https://oodikone.cs.helsinki.fi
-        deployment-target: production
-        softa-to-deploy: sis-updater-worker
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: production
-          softa-to-deploy: sis-updater-worker

.github/workflows/test-and-release-updater.yaml

@@ -0,0 +1,138 @@
+name: 'Test and release updater'
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - '.github/workflows/test-and-release-updater.yaml'
+      - 'updater/sis-updater-worker/**'
+      - 'updater/sis-updater-scheduler/**'
+
+jobs:
+  build_worker:
+    name: 'Build sis-updater-worker'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Build the image'
+        uses: docker/build-push-action@v2
+        with:
+          context: updater/sis-updater-worker/
+          target: production
+          push: false
+          outputs: type=docker,dest=/tmp/worker.tar
+          tags: toska/sis-updater-worker:latest
+          cache-from: type=registry,ref=toska/sis-updater-worker:latest
+          cache-to: type=inline
+      - name: 'Store built image'
+        uses: actions/upload-artifact@v2
+        with:
+          path: /tmp/worker.tar
+
+  build_scheduler:
+    name: 'Build sis-updater-scheduler'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Build the image'
+        uses: docker/build-push-action@v2
+        with:
+          context: updater/sis-updater-scheduler/
+          target: production
+          push: false
+          outputs: type=docker,dest=/tmp/scheduler.tar
+          tags: toska/sis-updater-scheduler:latest
+          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
+          cache-to: type=inline
+      - name: 'Store built image'
+        uses: actions/upload-artifact@v2
+        with:
+          path: /tmp/scheduler.tar
+
+  test_updater:
+    name: 'Test updater'
+    runs-on: ubuntu-latest
+    needs:
+      - build_worker
+      - build_scheduler
+    steps:
+      - uses: actions/checkout@v2
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Download built images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load worker image'
+        run: docker load --input /tmp/artifact/worker.tar
+      - name: 'Load scheduler image'
+        run: docker load --input /tmp/artifact/scheduler.tar
+      - name: 'Start worker'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
+      - name: 'Run hourly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: hourly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run weekly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: weekly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run prepurge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: prepurge
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run purge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: purge
+          EXIT_AFTER_IMMEDIATES: yes
+
+  release_updater:
+    name: 'Release updater'
+    runs-on: ubuntu-latest
+    needs:
+      - test_updater
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Login to DockerHub'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: 'Build and push scheduler'
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
+          cache-to: type=inline
+          context: updater/sis-updater-scheduler/
+          push: true
+          tags: toska/sis-updater-scheduler:latest
+          target: production
+      - name: 'Build and push worker'
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=registry,ref=toska/sis-updater-worker:latest
+          cache-to: type=inline
+          context: updater/sis-updater-worker/
+          push: true
+          tags: toska/sis-updater-worker:latest
+          target: production
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone.cs.helsinki.fi
+          deployment-target: production
+          softa-to-deploy: sis-updater

.github/workflows/test-oodikone.yaml

@@ -9,6 +9,7 @@ on:
       - 'cypress/**'
       - 'cypress.json'
       - 'services/**'
+      - '!services/backend/src/rapodiff.js'
       - 'docker-compose.ci.yml'
       - 'docker-compose.test.yml'
       - 'nginx.conf'

anonyymioodi/README.md

@@ -2,11 +2,15 @@
 
 (name anonyymioodi for :culture: reasons)
 
-This repository contains a guide and scripts to create docker images from database dumps and/or push database images to Toska Hub docker registry. Images are mostly used by oodikone.
+This folder contains a guide and scripts to create anonymous database images for oodikone, mainly used for testing.
 
-## How to update database images
+## How to create / modify database images
 
-Updating images varies a bit depending on the db you're trying to update. Complete steps as defined below and run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the image you've created.
+Creating / modifying images varies a bit depending on the db. Generally, to modify images, you can either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose. 
+
+By default, previous db images from Toska Hub are used as a base. If needed, comment out Toska hub image and uncomment corresponding clean postgres image in docker-compose before running other steps.
+
+See more details on how to create / modify databases below.
 
 ### Sis-importer-db
 
@@ -15,11 +19,11 @@ The following steps are for building a clean sis-importer-db image from scratch.
 - See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
 - Move `sis-importer-db.sqz` to this folder
 - Run `./create_container_from_dump.sh sis-importer-db`
-- Enrich data according to what's in `enrich_data.sql` file
+- Enrich db with command in file `enrich_data.sql` either by piping command to container or by running command inside container.
 
 ### Sis-db:
 
-The following steps are for building a clean sis-importer-db image from scratch using the sis-updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+The following steps are for building a clean sis-db image from scratch using oodikone's updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
 
 - Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
 - Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
@@ -32,25 +36,21 @@ curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
 curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
 curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
 ```
-
-(It might be a good idea to follow sis-updater-worker logs while updater is running)
-
+(or run these from frontend. You might also want to follow sis-updater-workers logs to ensure things are running smoothly.)
 - Clean up any extra stuff you don't want to keep in sis-db.
-- (Bonus: check that cypress tests run okay)
+- Bonus: check that cypress tests run okay
 
 ### User-db / kone-db
 
-- Temporarily replace user-db/kone-db -service in oodikone's docker-compose.yml with the same service from this repo's docker-compose.yml
-- Ensure the newest user-db/kone-db image is pulled from the registry with `docker pull`
-- Start oodikone, let it run migrations
+- Temporarily replace the db service in oodikone's docker-compose.yml with the same db service from this folder's docker-compose.yml
+- Ensure the newest user-db/kone-db image is pulled from the registry with `docker-compose pull`
 - Modify db to wanted state. Tips:
-  - Easy way to create new users is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
+  - Run oodikone if you want to upgrade something from frontend (e.g. tags in kone-db, courses in sis-db etc.).
+  - Easy way to create new users to user-db is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
 
-By default, previous user-db/kone-db images are used as a base. If you want to build the image from a clean Postgres image, just comment out toska hub image and uncomment the clean Postgres image.
+## Publishing the image
 
-## How to modify database images
-
-If you need to clean up / modify / do whatever with dbs, either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose.
+After completing the steps above and making sure your local version has the correct data, run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the db image to Toska Hub.
 
 ## How to update Postgres versions
 

cypress/integration/Course_statistics.js

@@ -20,15 +20,13 @@ describe('Course Statistics tests', () => {
 
       cy.contains('.tabular.menu a', 'Table').click()
       cy.contains('All')
-      cy.contains('.modeSelectorRow a', 'Attempts').click()
-      cy.contains('.modeSelectorRow a', 'Students').click()
+      cy.cs('viewMode-Attempts').click()
+      cy.cs('viewMode-Students').click()
 
       cy.contains('.tabular.menu a', 'Pass rate chart').click()
-      cy.get('div.modeSelectorContainer').click()
       cy.contains('svg', 'Pass rate chart')
 
       cy.contains('.tabular.menu a', 'Grade distribution chart').click()
-      cy.get('div.modeSelectorContainer').click()
       cy.contains('svg', 'Grades')
 
       cy.contains('a', 'New query').click()
@@ -186,7 +184,7 @@ describe('Course Statistics tests', () => {
       cy.contains('td', '010674989').siblings().contains('1')
     })
 
-    it.skip('Language distribution is correct', () => {
+    it('Language distribution is correct', () => {
       cy.get("input[placeholder='Search by a course code']").type('TKT20003')
       cy.contains(/^TKT20003$/).click()
       cy.contains('Fetch statistics').should('be.enabled').click()
@@ -194,7 +192,7 @@ describe('Course Statistics tests', () => {
       cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
       cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
       cy.contains('Language distribution').click()
-      cy.contains('td', 'finnish').siblings().contains('122')
+      cy.contains('td', 'finnish').siblings().contains('107')
       cy.contains('td', 'english').siblings().contains('5')
     })
 
@@ -304,7 +302,7 @@ describe('Course Statistics tests', () => {
     })
   })
 
-  it('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
+  it.only('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
     cy.init('/coursestatistics', 'onlycoursestatistics')
     cy.get('[data-cy=navbar-courseStatistics]').click()
     cy.get('[data-cy=course-code-input]').type('TKT20003')
@@ -314,10 +312,14 @@ describe('Course Statistics tests', () => {
     cy.contains('Faculty statistics').should('not.exist')
     cy.contains('Show population').should('not.exist')
 
+    cy.cs('viewMode-Attempts').click()
+
     const attemptsTableContents = [
       // [time, passed, failed, passrate]
       ['Total*', 288, 213, 75, '73.96 %'],
+      // ['Total', 295, 216, 79, '73.22 %'],
       ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
+      //['2020-2021', 5, 2, 3, '40.00 %'],
       ['2019-2020', 164, 121, 43, '73.78 %'],
       ['2018-2019', 85, 60, 25, '70.59 %'],
       ['2017-2018', 39, 32, 7, '82.05 %'],

cypress/integration/Filters.js

@@ -22,11 +22,25 @@ const clearSingleDropdownSelection = dataCyAttribute => {
 // functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
 const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
   return (filterName, testStepFunctionToRun) => {
-    cy.get(`[data-cy="${filterName}-header"]`).click()
-    checkFilteringResult(amountWithoutFiltering)
-    testStepFunctionToRun()
-    checkFilteringResult(amountWithoutFiltering)
-    cy.get(`[data-cy="${filterName}-header"]`).click({ force: true })
+    const card = cy.cs(`${filterName}-filter-card`)
+
+    card.invoke('attr', 'data-open').then(open => {
+      console.log('open', open)
+
+      const getHeader = () => cy.cs(`${filterName}-header`)
+
+      if (open === 'false') {
+        getHeader().click({ force: true })
+      }
+
+      checkFilteringResult(amountWithoutFiltering)
+      testStepFunctionToRun()
+      checkFilteringResult(amountWithoutFiltering)
+
+      if (open === 'false') {
+        getHeader().click({ force: true })
+      }
+    })
   }
 }
 
@@ -42,33 +56,58 @@ describe('Population Statistics', () => {
 
   it('Transfer filter is on not transferred by default', () => {
     cy.contains('By default only students who have not transferred to this study programme are shown.')
-    const card = cy.cs('TransferredToProgramme-filter-card')
-    card.cs('TransferredToProgramme-header').click()
-    card.get('[data-cy="option-havenot"] input').should('be.checked')
-    card.cs('TransferredToProgramme-header').click()
+
+    runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
+      const card = cy.cs('TransferredToProgramme-filter-card')
+      card.get('[data-cy="option-havenot"] input').should('be.checked')
+    })
+  })
+
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(5)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
   })
 
   it('Graduation filter works', () => {
     runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
-      const card = cy.cs('GraduatedFromProgramme-filter-card')
+      const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
 
       const graduated = 11
-      card.cs('option-graduated-true').click()
+      getCard().cs('option-graduated-true').click()
       checkFilteringResult(graduated)
-      card.cs('option-graduated-false').click()
+      getCard().cs('option-graduated-false').click()
       checkFilteringResult(defaultAmountOfStudents - graduated)
-      card.get('[data-cy="GraduatedFromProgramme-filter-card"] [data-cy="option-all"]').click()
+      getCard().cs('option-all').click()
     })
   })
 
   it('Transfer filter works', () => {
     runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
       const transferred = 1
-      cy.cs('option-have').click()
+      cy.cs('TransferredToProgramme-filter-card').cs('option-have').click()
       checkFilteringResult(transferred)
-      cy.cs('option-all').click()
+      cy.cs('TransferredToProgramme-filter-card').cs('option-all').click()
       checkFilteringResult(defaultAmountOfStudents + transferred)
-      cy.cs('option-havenot').click()
+      cy.cs('TransferredToProgramme-filter-card').cs('option-havenot').click()
     })
   })
 
@@ -95,7 +134,7 @@ describe('Population Statistics', () => {
     runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(41)
+      checkFilteringResult(40)
       cy.cs('ageFilter-min').find('input').clear()
       cy.cs('ageFilter-max').find('input').clear()
     })
@@ -120,8 +159,8 @@ describe('Population Statistics', () => {
       cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.cs('startYearAtUni-dropdown').selectFromDropdown(14)
-      checkFilteringResult(121)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(7)
+      checkFilteringResult(141)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
@@ -159,14 +198,14 @@ describe('Population Statistics', () => {
   it('Filter combinations work', () => {
     runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
       runTestStepWithPreAndPostParts('Age', () => {
-        const card = cy.cs('GraduatedFromProgramme-filter-card')
-        card.cs('option-graduated-true').click()
+        const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
+        getCard().cs('option-graduated-true').click()
         cy.cs('ageFilter-min').type('20')
         cy.cs('ageFilter-max').type('30')
         checkFilteringResult(11)
         cy.cs('ageFilter-min').find('input').clear()
         cy.cs('ageFilter-max').find('input').clear()
-        card.cs('option-all').click()
+        getCard().cs('option-all').click()
       })
     })
   })
@@ -193,11 +232,35 @@ describe('Course Statistics', () => {
     })
   })
 
+  it('Programme defaults to "Attainment" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Attainment')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(116)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
   it('Age filter works', () => {
     runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(32)
+      checkFilteringResult(31)
       cy.cs('ageFilter-min').find('input').clear()
       cy.cs('ageFilter-max').find('input').clear()
     })
@@ -222,8 +285,8 @@ describe('Course Statistics', () => {
       cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.cs('startYearAtUni-dropdown').selectFromDropdown(12)
-      checkFilteringResult(29)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(3)
+      checkFilteringResult(84)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
@@ -268,6 +331,21 @@ describe('Custom Population Statistics', () => {
     })
   })
 
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(5)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
   it('Gender filter works', () => {
     runTestStepWithPreAndPostParts('Gender', () => {
       cy.cs('genderFilter-dropdown').selectFromDropdown(0)
@@ -290,7 +368,7 @@ describe('Custom Population Statistics', () => {
     })
   })
 
-  it('Courses filter works', () => {
+  it.skip('Courses filter works', () => {
     runTestStepWithPreAndPostParts('Courses', () => {
       const courses = ['MAT11001', 'TKT20004']
       cy.cs('courseFilter-course-dropdown')

cypress/integration/Population_statistics.js

@@ -80,7 +80,7 @@ describe('Population Statistics tests', () => {
 
       cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
       // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000) // a bit hacky way, wait until ui is ready
+      cy.wait(15000) // a bit hacky way, wait until ui is ready
       cy.get('[data-cy=expand-TKT1]').click()
       cy.get('[data-cy=coursestats-link-TKT10002]').click()
       cy.wait('@coursePage')

cypress/integration/Student_statistics.js

@@ -38,7 +38,7 @@ describe('Student Statistics tests', () => {
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.studentnumber)
     cy.contains(student.studentnumber).click()
-    cy.contains('Started: 01.08.2011')
+    cy.contains('Started: 01.08.2017')
     cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 01.08.2025')
     cy.contains(student.lastname).should('not.exist')
     cy.contains(student.firstnames).should('not.exist')

cypress/integration/Studyprogramme_overview.js

@@ -7,161 +7,341 @@ const deleteTag = name => {
   cy.contains(name).should('not.exist')
 }
 
-describe('Studyprogramme overview', () => {
-  // check is used by couple of tests
-  const checkProgressAndProductivity = (user = 'basic') => {
-    // Column amount depends on user. Note that with dev user, you'll see even more
-    // stuff than with admin or basic.
-    const CSBachPopulationProgress2018ForAdmin = [
-      '162',
-      '44 (27%)',
-      '118 (72%)',
-      '161 (99%)',
-      '161',
-      '1',
-      '11',
-      '11',
-      '152',
-      '125',
-      '78',
-      '42',
-      '16',
-    ]
-    const CSBachPopulationProgress2018ForBasic = [
-      '162',
-      '44 (27%)',
-      '118 (72%)',
-      '161 (99%)',
-      '161',
-      '1',
-      '11',
-      '11',
-      '152',
-      '125',
-      '78',
-      '42',
-      '16',
-    ] // cancelled and transferred from are missing
-
-    const population = user === 'basic' ? CSBachPopulationProgress2018ForBasic : CSBachPopulationProgress2018ForAdmin
-    cy.contains('Population progress')
-    cy.contains('Yearly productivity')
-
-    cy.contains('2018-2019')
-      .siblings()
-      .each((elem, index) => {
-        cy.wrap(elem).contains(population[index])
-      })
-    const populationproductivity2019 = ['7388.0', '17', '7388.00', '0.00', '349.00']
-    cy.get('table')
-      .eq(1)
-      .contains('2019')
-      .siblings()
-      .each((elem, index) => {
-        cy.wrap(elem).contains(populationproductivity2019[index])
-      })
-  }
-
-  describe('when opening programme page with admin user', () => {
-    beforeEach(() => {
-      cy.init('/study-programme/KH50_005', 'admin')
-    })
-
-    it('progress should not be recalculating when opened for the first time', () => {
-      cy.contains('Recalculating').should('not.exist')
-      cy.contains('Last updated').should('exist')
-    })
-
-    // Following vars are used to compare values before and after stats calculation
-    // Taken from https://docs.cypress.io/api/commands/should#Compare-text-values-of-two-elements
-    const normalizeText = s => s.replace(/\s/g, '').toLowerCase()
-    let originalProgressCalculatedText
-    let originalProductivityCalculatedText
-    const doRecalculation = () => {
-      // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
-      cy.wait(5000)
-      cy.get('.attached').contains('Admin').click()
-      cy.contains('recalculate productivity').click()
-      cy.contains('recalculate throughput').click()
-      cy.get('.attached').contains('Overview').click() // TODO: add data-cy tags for tabs
-      // Wait to "recalculating" to disappear, TODO: change this to cy.intercept
-      cy.wait(5000)
-      cy.reload()
+const getEmptyYears = isAcademicYear => {
+  const thisYear = new Date().getFullYear()
+  const years = []
+  for (let year = thisYear; year > 2021; year--) {
+    if (isAcademicYear) {
+      years.push(`${year} - ${year + 1}`)
+    } else {
+      years.push(year)
     }
+  }
+  return years
+}
 
-    it('renders progress and productivity tables with calculated status', () => {
-      doRecalculation()
-
-      // check contains also study programme options for admin
-      cy.contains('Study programme options')
-      cy.get('table').should('have.length', 3)
-
-      checkProgressAndProductivity('admin')
-
-      // Grab update dates to be compared later
-      cy.cs('throughputUpdateStatus')
-        .invoke('text')
-        .then(text => {
-          originalProgressCalculatedText = normalizeText(text)
-          expect(originalProgressCalculatedText).not.to.contain('recalculating')
-          expect(originalProgressCalculatedText).not.to.contain('refresh')
-        })
-
-      cy.cs('productivityUpdateStatus')
-        .invoke('text')
-        .then(text => {
-          originalProductivityCalculatedText = normalizeText(text)
-          expect(originalProductivityCalculatedText).not.to.contain('recalculating')
-          expect(originalProgressCalculatedText).not.to.contain('refresh')
-        })
+describe('Studyprogramme overview', () => {
+  /* Basic information overview -tests*/
+  describe('Basic information -view works for basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme')
     })
 
-    it('renders progress and productivity tables with calculated status after recalculating stats again', () => {
-      doRecalculation()
-      checkProgressAndProductivity('admin')
+    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
+    it('Basic information -tab loads', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('[data-cy=Section-StudentsOfTheStudyprogramme]')
+      cy.get('[data-cy=Section-CreditsProducedByTheStudyprogramme]')
+      cy.get('[data-cy=Section-GraduatedAndThesisWritersOfTheProgramme]')
+      cy.get('[data-cy=Section-ProgrammesAfterGraduation]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
+    })
 
-      // Check new calculation statuses are reported
-      const newProgressCalculatedTextElement = cy.cs('throughputUpdateStatus').invoke('text')
-      const newProductivityCalculatedTextElement = cy.cs('productivityUpdateStatus').invoke('text')
+    it('Basic information contains correct students', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Started, Graduated, Cancelled, Transferred Away, Transferred to]
+        ...years.map(year => [year, 0, 0, 0, 0, 0]),
+        [2021, 0, 0, 0, 1, 0],
+        [2020, 10, 36, 16, 3, 2],
+        [2019, 86, 17, 14, 0, 1],
+        [2018, 162, 1, 3, 0, 0],
+        [2017, 173, 0, 0, 0, 0],
+      ]
 
-      newProgressCalculatedTextElement.should(text => {
-        const newProgressCalculatedText = normalizeText(text)
-        expect(newProgressCalculatedText).not.to.contain('recalculating')
-        expect(newProgressCalculatedText).not.to.contain('refresh')
-        expect(newProgressCalculatedText).not.to.contain(originalProgressCalculatedText)
-      })
+      cy.checkTableStats(tableContents, 'StudentsOfTheStudyprogramme')
+    })
 
-      newProductivityCalculatedTextElement.should(text => {
-        const newProductivityCalculatedText = normalizeText(text)
-        expect(newProductivityCalculatedText).not.to.contain('recalculating')
-        expect(newProductivityCalculatedText).not.to.contain('refresh')
-        expect(newProductivityCalculatedText).not.to.contain(originalProgressCalculatedText)
-      })
+    it('Basic information contains correct credits', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Total, Major credits, Non-major credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        [2021, 0, 0, 0, 0],
+        [2020, 4928, 548, 4213, 167],
+        [2019, 7737, 1644, 5744, 349],
+        [2018, 6730, 1820, 4638, 272],
+        [2017, 2900, 788, 1724, 388],
+      ]
+
+      cy.checkTableStats(tableContents, 'CreditsProducedByTheStudyprogramme')
+    })
+
+    it('Basic information contains correct thesis writers and graduates', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Graduated, Wrote thesis]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 36, 35],
+        [2019, 17, 25],
+        [2018, 1, 5],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(tableContents, 'GraduatedAndThesisWritersOfTheProgramme')
+    })
+
+    it('Special studyrights can be excluded and basic data changes accordingly', () => {
+      cy.contains('Tietojenkäsittelytieteen maisteriohjelma').click()
+      cy.get('[data-cy=StudentToggle]').click()
+      const years = getEmptyYears()
+      const studentTableContents = [
+        // [Year, Started, Graduated]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 22, 0],
+        [2019, 12, 0],
+        [2018, 0, 0],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
+
+      const creditTableContents = [
+        // [Year, Total, Major students credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0]),
+        [2021, 0, 0, 0],
+        [2020, 117.5, 0, 7.5],
+        [2019, 67, 0, 0],
+        [2018, 0, 0, 0],
+        [2017, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+
+      const graduatedTableContents = [
+        // [Year, Graduated, Wrote thesis]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 0, 0],
+        [2019, 0, 0],
+        [2018, 0, 0],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(graduatedTableContents, 'GraduatedAndThesisWritersOfTheProgramme')
+    })
+
+    it('Year can be changed to academic year, and data changes accordingly', () => {
+      cy.contains('Tietojenkäsittelytieteen maisteriohjelma').click()
+      cy.get('[data-cy=YearToggle]').click()
+      const isAcademicYear = true
+      const years = getEmptyYears(isAcademicYear)
+      const studentTableContents = [
+        // [Year, Started, Graduated, Cancelled, Transferred away, Transferred to]
+        ...years.map(year => [year, 0, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0, 0],
+        ['2020 - 2021', 5, 0, 1, 0, 0],
+        ['2019 - 2020', 27, 0, 1, 0, 0],
+        ['2018 - 2019', 4, 0, 0, 0, 0],
+        ['2017 - 2018', 0, 0, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
+
+      const creditTableContents = [
+        // [Year, Total, Major students credits, Non-major students credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0],
+        ['2020 - 2021', 0, 0, 0, 0],
+        ['2019 - 2020', 338.5, 0, 331, 7.5],
+        ['2018 - 2019', 173, 0, 173, 0],
+        ['2017 - 2018', 42, 0, 42, 0],
+      ]
+
+      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+    })
+
+    it('Basic information graphs render', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+
+      cy.get('[data-cy=Graph-StudentsOfTheStudyprogramme')
+        .should('contain', 'Started')
+        .should('contain', 'Graduated')
+        .should('contain', 'Transferred away')
+
+      cy.get('[data-cy=Graph-CreditsProducedByTheStudyprogramme')
+        .should('contain', 'Major students credits')
+        .should('contain', 'Non-major students credits')
+        .should('contain', 'Transferred credits')
+        .should('contain', 788)
+        .should('contain', 1820)
+
+      cy.get('[data-cy=Graph-GraduatedAndThesisWritersOfTheProgramme')
+        .should('contain', 'Graduated students')
+        .should('contain', 'Wrote thesis')
+        .should('contain', 1)
+        .should('contain', 5)
+        .should('contain', 17)
+        .should('contain', 25)
+        .should('contain', 36)
+        .should('contain', 35)
+
+      cy.get('[data-cy=Graph-2019-AverageGraduationTimes]')
+        .should('contain', '2019')
+        .should('contain', 'n = 17')
+        .should('contain', '22 kk')
+
+      cy.get('[data-cy=Graph-ProgrammesAfterGraduation')
+        .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
+        .should('contain', 'Datatieteen maisteriohjelma')
+        .should('contain', 'Matematiikan ja tilastotieteen maisteriohjelma')
+        .should('contain', 'Kielellisen diversiteetin ja digitaalisten ihmistieteiden maisteriohjelma')
+        .should('contain', 1)
+        .should('contain', 4)
+        .should('contain', 13)
     })
   })
 
-  describe('when opening programme page with basic user', () => {
+  /* Studytrack overview -tests*/
+  describe('Basic information -view works for basic user', () => {
     beforeEach(() => {
-      cy.init('/study-programme/KH50_005')
+      cy.init('/study-programme')
     })
 
-    it('renders progress and productivity tables with previosly calculated values', () => {
-      // check doesn't contain study programme options for admin
-      cy.contains('Study programme options').should('not.exist')
-      cy.get('table').should('have.length', 2)
-      checkProgressAndProductivity()
-    })
-
-    it('can open Thesis page', () => {
+    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
+    it('Studytracks and student populations -tab loads', () => {
       cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.contains('Thesis Courses').click()
-      cy.contains('Add thesis course').click()
-      cy.contains('No results')
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      cy.get('[data-cy=Section-StudytrackOverview]')
+      cy.get('[data-cy=Section-StudytrackProgress]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
     })
 
-    it('can move to Population statistics page by clickin', () => {
-      cy.get('i.level.up.alternate.icon').eq(0).click()
-      cy.contains('Students (10)')
+    it('Students of the studyprogramme are shown correctly', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      const tableContents = [
+        // [Year, All, Started studying, Currently enrolled, Absent, Cancelled, Graduated, Men, Women, Finnish]
+        [
+          '2020 - 2021',
+          12,
+          '100.0 %',
+          10,
+          '83.3 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          10,
+          '83.3 %',
+          2,
+          '16.7 %',
+          12,
+          '100.0 %',
+        ],
+        [
+          '2019 - 2020',
+          86,
+          '100.0 %',
+          86,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          1,
+          '1.2 %',
+          1,
+          '1.2 %',
+          65,
+          '75.6 %',
+          21,
+          '24.4 %',
+          86,
+          '100.0 %',
+        ],
+        [
+          '2018 - 2019',
+          162,
+          '100.0 %',
+          161,
+          '99.4 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          11,
+          '6.8 %',
+          11,
+          '6.8 %',
+          118,
+          '72.8 %',
+          44,
+          '27.2 %',
+          161,
+          '99.4 %',
+        ],
+        [
+          '2017 - 2018',
+          171,
+          '100.0 %',
+          171,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          21,
+          '12.3 %',
+          42,
+          '24.6 %',
+          135,
+          '78.9 %',
+          36,
+          '21.1 %',
+          166,
+          '97.1 %',
+        ],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudytrackOverview')
+    })
+
+    it('Student progress data is shown correctly', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      const tableContents = [
+        ['2020 - 2021', 12, 5, 2, 2, 0, 2, 1],
+        ['2019 - 2020', 86, 13, 15, 32, 11, 2, 13],
+        ['2018 - 2019', 162, 7, 16, 35, 33, 27, 44],
+        ['2017 - 2018', 171, 10, 22, 18, 22, 24, 75],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudytrackProgress')
+    })
+
+    it('Studytrack overview graphs render', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+
+      cy.get('[data-cy=Graph-StudytrackProgress]')
+        .should('contain', 'Less than 30 credits')
+        .should('contain', '30-59 credits')
+        .should('contain', 'More than 150 credits')
+        .should('contain', '5.8%')
+        .should('contain', '4.3%')
+        .should('contain', '15.1%')
+
+      cy.get('[data-cy=Graph-2018-AverageGraduationTimes]')
+        .should('contain', '2018 - 2019')
+        .should('contain', 'n = 11 / 162')
+        .should('contain', '22 kk')
+    })
+  })
+
+  /* Tag-tests*/
+  describe('Basic information -view works for basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme')
     })
 
     it('can create and delete tags for population', () => {

cypress/support/commands.js

@@ -7,7 +7,7 @@ const adminUserHeaders = {
   uid: 'admin',
   displayname: 'Admin User',
   'shib-session-id': 'mock-cypress-session',
-  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users;grp-toska',
   edupersonaffiliation: 'member;employee;faculty',
   mail: 'grp-toska+mockadminuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-6666666',
@@ -62,9 +62,13 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
  * Shorthand for using "Cypress Selectors" (CS), i.e., `data-cy` attributes.
  */
 Cypress.Commands.add('cs', { prevSubject: 'optional' }, (subject, name) => {
-  const p = subject === undefined ? cy : cy.wrap(subject)
+  const selector = `[data-cy="${name}"]`
 
-  return p.get(`[data-cy='${name}']`)
+  if (subject === undefined) {
+    return cy.get(selector)
+  } else {
+    return cy.wrap(subject).find(selector)
+  }
 })
 
 /**
@@ -77,7 +81,12 @@ Cypress.Commands.add('selectFromDropdown', { prevSubject: true }, (s, index) =>
 
   indexes.forEach(i => {
     subject.click()
-    subject.children('.menu').children().eq(i).click({ force: true })
+
+    if (typeof i === 'number') {
+      subject.children('.menu').children().eq(i).click({ force: true })
+    } else {
+      subject.children('.menu').containing(i).eq(0).click({ force: true })
+    }
   })
 
   // Close multiple selection so it does not block elements underneath it.
@@ -95,3 +104,18 @@ Cypress.Commands.add('selectStudyProgramme', name => {
   cy.cs('select-study-programme').click().children().contains(name).click()
   cy.contains('See population').click()
 })
+
+Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {
+  cy.get(`[data-cy=Table-${tableName}]>tbody`).within(() => {
+    correctStats.forEach((values, trIndex) => {
+      cy.get('tr')
+        .eq(trIndex)
+        .within(() => {
+          values.forEach((value, tdIndex) => {
+            cy.get('td').eq(tdIndex).contains(value)
+          })
+        })
+    })
+    cy.get('tr').should('have.length', correctStats.length)
+  })
+})

docker-compose.ci.yml

@@ -103,9 +103,35 @@ services:
     profiles:
       - updater
 
+  sis-updater-scheduler:
+    <<: *defaults
+    build:
+      target: development
+      context: updater/sis-updater-scheduler
+    container_name: sis-updater-scheduler
+    depends_on:
+      - redis
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      REDIS_URI: //redis:6379
+      SECRET_TOKEN: dev
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
+    ports:
+      - 8082:8082
+    profiles:
+      - updater
+
   sis-updater-nats:
     <<: *defaults
-    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_channels 1000 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:

docker-compose.yml

@@ -1,3 +1,5 @@
+version: '2.1'
+
 x-defaults: &defaults
   restart: unless-stopped
 
@@ -97,7 +99,7 @@ services:
 
   sis-updater-nats:
     <<: *defaults
-    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_channels 1000 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:
@@ -122,6 +124,8 @@ services:
       SIS_IMPORTER_USER: postgres
       SIS_NATS_TOKEN: dev
       SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
     ports:
       - 8082:8082
     profiles:

package.json

@@ -15,7 +15,8 @@
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
     "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "rapodiff": "docker exec backend node /usr/src/app/src/rapodiff.js"
   },
   "repository": {
     "type": "git",

services/backend/Dockerfile

@@ -17,7 +17,7 @@ ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
 ARG SENTRY_RELEASE
 ENV SENTRY_RELEASE=$SENTRY_RELEASE
 COPY --chown=node:node ./package* /usr/src/app/
-RUN npm ci --production
+RUN npm ci --production && apk add --no-cache vim curl nano
 COPY --chown=node:node . /usr/src/app/
 USER node
 CMD ["npm", "start"]

services/backend/package-lock.json

@@ -1003,9 +1003,9 @@
       "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="
     },
     "follow-redirects": {
-      "version": "1.14.3",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.3.tgz",
-      "integrity": "sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw=="
+      "version": "1.14.8",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz",
+      "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA=="
     },
     "for-in": {
       "version": "1.0.2",

services/backend/package.json

@@ -4,11 +4,7 @@
   "scripts": {
     "start": "node --max_old_space_size=4096 index.js",
     "dev": "nodemon --max_old_space_size=4096 index.js",
-    "diff:students": "node /usr/src/app/src/sisoodi_diff/students/index.js",
-    "diff:populations": "node /usr/src/app/src/sisoodi_diff/populations.js",
-    "diff:courses": "node /usr/src/app/src/sisoodi_diff/courses/index.js",
-    "diff:transfers": "node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "node /usr/src/app/src/sisoodi_diff/trends.js"
+    "rapodiff": "node /usr/src/app/src/rapodiff.js"
   },
   "dependencies": {
     "@sentry/node": "^5.30.0",

services/backend/src/database/connection.js

@@ -16,7 +16,7 @@ class DbConnection extends EventEmitter {
       pool: {
         max: 25,
         min: 0,
-        acquire: 10000,
+        acquire: 30000,
         idle: 300000000,
       },
       logging: false,

services/backend/src/database/migrations_user/20220112_00_drop_hy_groups.js

@@ -0,0 +1,9 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('user_hy_group')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS user_hy_group_id_seq')
+    await queryInterface.dropTable('hy_groups')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS hy_groups_id_seq')
+  },
+  down: async () => {},
+}

services/backend/src/events.js

@@ -10,9 +10,8 @@ const {
 } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
-const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studyprogramme')
+const { updateBasicView, updateStudytrackView } = require('./services/studyprogrammeUpdates')
 const { findAndSaveTeachers } = require('./services/topteachers')
-const { setProductivity, setThroughput, patchProductivity, patchThroughput } = require('./services/analyticsService')
 const { isProduction } = require('./conf-backend')
 const { getCurrentSemester } = require('./services/semesters')
 const logger = require('./util/logger')
@@ -24,47 +23,21 @@ const refreshStudyrightAssociations = async () => {
   logger.info('Studyright associations refreshed!')
 }
 
-const refreshOverview = async () => {
-  logger.info('Refreshing throughput and productivity for programmes...')
+const refreshNewOverviews = async () => {
+  logger.info('Refreshing studyprogramme and studytrack overview statistics for all programmes')
   const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH') || code.includes('MH'))
   let ready = 0
   for (const code of codes) {
-    let programmeStatsSince = new Date('2017-07-31')
-    if (code.includes('MH') || code.includes('KH')) {
-      programmeStatsSince = new Date('2017-07-31')
-    } else {
-      programmeStatsSince = new Date('2000-07-31')
-    }
     try {
-      await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-      const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
-      await setThroughput(data)
+      await updateBasicView(code)
+      await updateStudytrackView(code)
+      ready += 1
     } catch (e) {
-      logger.error({ message: `failed to recalculate throughput for ${code}`, meta: e })
-      try {
-        await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-      } catch (e) {
-        logger.error({ message: 'failed to update throughtput status to error', meta: e })
-      }
+      logger.error({ message: `Failed to update overview stats for programme ${code}`, meta: e })
     }
-    try {
-      await patchProductivity({ [code]: { status: 'RECALCULATING' } })
-      const data = await productivityStatsForStudytrack(code, programmeStatsSince)
-      await setProductivity(data)
-    } catch (e) {
-      logger.error({ message: `failed to recalculate productivity for ${code}`, meta: e })
-      try {
-        await patchProductivity({
-          [code]: { status: 'RECALCULATION ERRORED' },
-        })
-      } catch (e) {
-        logger.error({ message: 'failed to update productivity status to error', meta: e })
-      }
-    }
-    ready += 1
     logger.info(`${ready}/${codes.length} programmes done`)
   }
-  logger.info('Throughput and productivity for programmes refreshed')
+  logger.info('Studyprogramme and studytrack overview stats refreshed!')
 }
 
 const refreshTeacherLeaderboard = async () => {
@@ -147,7 +120,7 @@ const refreshProtoCProgrammeToRedis = async () => {
 }
 
 const refreshStatistics = async () => {
-  const statfuncs = [refreshStudyrightAssociations, refreshOverview, refreshTeacherLeaderboard]
+  const statfuncs = [refreshStudyrightAssociations, refreshTeacherLeaderboard]
   logger.info('Refreshing statistics')
   for (const func of statfuncs) {
     await func()
@@ -166,9 +139,10 @@ const refreshTrends = async () => {
 
 const startCron = () => {
   if (isProduction) {
+    logger.info('Cronjob for refreshing stats started: runs at 3am every day.')
     // refresh 3am every day
     schedule('0 3 * * *', async () => {
-      for (const func of [refreshStatistics, refreshTrends]) {
+      for (const func of [refreshStatistics, refreshTrends, refreshNewOverviews]) {
         await func()
       }
     })

services/backend/src/middleware/accessLogger.js

@@ -1,5 +1,6 @@
 const morgan = require('morgan')
 const logger = require('../util/logger')
+const _ = require('lodash')
 
 // So this appears to be a hack to get neatly formatted stats like response-time etc. from morgan
 // without actually using morgan what it's used for (LOGGING REQUESTS!).
@@ -7,7 +8,8 @@ const logger = require('../util/logger')
 // so don't e.g. make it into an async function or it'll print [object Promise] on every request
 const accessLogger = morgan((tokens, req, res) => {
   const fields = ['method', 'url', 'status', 'response-time', 'remote-addr', 'remote-user', 'user-agent', 'referrer']
-  const meta = { ...req.decodedToken }
+  const { user } = req
+  const meta = { ...user }
   fields.forEach(field => (meta[field] = tokens[field](req, res)))
   meta['time'] = tokens['date'](req, res, 'iso')
   if (req.route) {
@@ -15,7 +17,7 @@ const accessLogger = morgan((tokens, req, res) => {
   }
 
   const message = [
-    req.decodedToken.name,
+    user.name,
     ':',
     tokens['method'](req, res),
     tokens['url'](req, res),
@@ -26,7 +28,8 @@ const accessLogger = morgan((tokens, req, res) => {
   ].join(' ')
 
   logger.info(message, {
-    ...meta,
+    // don't log student list which might be huge
+    ..._.omit(meta, ['studentsUserCanAccess']),
     // pass this as a custom field so we can filter by it in graylog
     isUsageStats: true,
   })

services/backend/src/middleware/auth.js

@@ -1,6 +1,8 @@
 const roles = requiredRoles => async (req, res, next) => {
-  if (req.decodedToken) {
-    const { roles } = req
+  if (req.user) {
+    const {
+      user: { roles },
+    } = req
 
     if (requiredRoles.every(r => roles.indexOf(r) >= 0) || roles.includes('admin')) {
       return next()

services/backend/src/middleware/currentUserMiddleware.js

@@ -1,6 +1,6 @@
 const Sentry = require('@sentry/node')
 const { ApplicationError } = require('../util/customErrors')
-const { getUser, getUserDataFor, getMockedUser } = require('../services/userService')
+const { getUser, getMockedUser } = require('../services/userService')
 const { requiredGroup } = require('../conf-backend')
 const _ = require('lodash')
 
@@ -8,47 +8,45 @@ const parseIamGroups = iamGroups => iamGroups?.split(';').filter(Boolean) ?? []
 
 const hasRequiredIamGroup = iamGroups => _.intersection(iamGroups, requiredGroup).length > 0
 
-const currentUserMiddleware = async (req, _, next) => {
-  const { uid: username, 'shib-session-id': sessionId, shib_logout_url: logoutUrl } = req.headers
+const currentUserMiddleware = async (req, _res, next) => {
+  const {
+    'shib-session-id': sessionId,
+    'x-show-as-user': showAsUser,
+    displayname: name,
+    hygroupcn,
+    hypersonsisuid: sisId,
+    mail: email,
+    shib_logout_url: logoutUrl,
+    uid: username,
+  } = req.headers
 
-  if (!sessionId) {
-    throw new ApplicationError('No session id found in request headers', 403, { logoutUrl })
+  if (!sessionId || !username) {
+    throw new ApplicationError('Not enough data in request headers', 403, { logoutUrl })
   }
 
-  if (!username) {
-    throw new ApplicationError('No username found in request headers', 403, { logoutUrl })
-  }
-  const { displayname: name, mail: email, hygroupcn: iamGroups, hypersonsisuid: sisId } = req.headers
-  const parsedIamGroups = parseIamGroups(iamGroups)
+  const iamGroups = parseIamGroups(hygroupcn)
 
-  req.decodedToken = await getUser({
+  if (!hasRequiredIamGroup(iamGroups)) {
+    throw new ApplicationError('User does not have required iam group', 403, { logoutUrl })
+  }
+
+  let user = await getUser({
     username,
     name,
     email,
-    iamGroups: parsedIamGroups,
+    iamGroups,
     sisId,
   })
 
-  if (!hasRequiredIamGroup(parsedIamGroups)) {
-    throw new ApplicationError('User is not enabled', 403, { logoutUrl })
+  if (showAsUser && user.isAdmin) {
+    user = await getMockedUser({ userToMock: showAsUser, mockedBy: username })
   }
 
-  const showAsUser = req.headers['x-show-as-user']
+  Sentry.setUser({ username: user.mockedBy ?? username })
 
-  if (showAsUser) {
-    const mockedUser = await getMockedUser(username, showAsUser)
-    if (mockedUser) {
-      req.decodedToken = mockedUser
-    }
-  }
+  req.user = user
   req.logoutUrl = logoutUrl
 
-  Sentry.setUser({ username: req.decodedToken.mockedBy || req.decodedToken.userId })
-
-  // TODO: remove this garbo
-  const userData = await getUserDataFor(req.decodedToken.userId)
-  Object.assign(req, userData)
-
   next()
 }
 

services/backend/src/models/credit.js

@@ -78,6 +78,9 @@ Credit.init(
     language: {
       type: STRING,
     },
+    is_open: {
+      type: BOOLEAN,
+    },
   },
   {
     underscored: false,

services/backend/src/models/models_user.js

@@ -83,23 +83,6 @@ const User = sequelizeUser.define(
     timestamps: false,
   }
 )
-const HyGroup = sequelizeUser.define('hy_group', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true,
-  },
-
-  code: { type: Sequelize.STRING },
-
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})
 
 User.hasMany(UserElementDetails, { as: 'programme' })
 UserElementDetails.belongsTo(User)
@@ -110,16 +93,12 @@ User.belongsToMany(AccessGroup, {
 })
 AccessGroup.belongsToMany(User, { through: 'user_accessgroup' })
 
-User.belongsToMany(HyGroup, { through: 'user_hy_group', as: 'hy_group' })
-HyGroup.belongsToMany(User, { through: 'user_hy_group' })
-
 User.hasMany(UserFaculties, { as: 'faculty', foreignKey: 'userId' })
 
 module.exports = {
   User,
   UserElementDetails,
   AccessGroup,
-  HyGroup,
   UserFaculties,
   sequelizeUser,
 }

services/backend/src/rapodiff.js

@@ -0,0 +1,64 @@
+/* eslint no-console: 0 */
+
+// script needs rapo certs, so you need to run this in production pannu
+//
+// possible workflows:
+// - go to oodikone pannu, then "docker exec -it backend sh", then run this script with "npm
+// run rapodiff". Modify with vim / nano until you're happy, then copy changes to local version and commit.
+// - or bind mount a file to backend container in pannu, overriding this file. Modify that file
+// inside or outside container, however you want. Keep file in pannu, or put it to pannu's git repo (oodikone-server-setup).
+
+const axios = require('axios').default
+const fs = require('fs')
+const https = require('https')
+const _ = require('lodash')
+
+const { optimizedStatisticsOf } = require('./services/populations')
+
+const { RAPO_CERT_PATH, RAPO_KEY_PATH, RAPO_API_KEY } = process.env
+
+const agent = new https.Agent({
+  cert: fs.readFileSync(RAPO_CERT_PATH, 'utf8'),
+  key: fs.readFileSync(RAPO_KEY_PATH, 'utf8'),
+})
+
+const api = axios.create({
+  baseURL: 'https://gw.api.helsinki.fi/secure/rapo/report/',
+  headers: { 'X-Api-Key': RAPO_API_KEY },
+  httpsAgent: agent,
+})
+
+const main = async () => {
+  const facultyCode = 'H50'
+  const programme = 'KH50_005'
+  const studyrightStartDate = '2020-08-01'
+  const presence = 'LÄSNÄOLEVA'
+
+  const months = 12 // some amount of months, dunno
+  const year = 2020
+
+  const oodikoneQuery = {
+    semesters: ['FALL', 'SPRING'],
+    months,
+    studyRights: { programme },
+    year,
+  }
+
+  const oodikoneStuff = await optimizedStatisticsOf(oodikoneQuery)
+  const oodikoneStudents = oodikoneStuff.students.map(s => s.studentNumber)
+
+  const rapoUrl = encodeURI(
+    `studyrights?faculty=${facultyCode}&presence=${presence}&education=${programme}&studyrightStartDate=${studyrightStartDate}`
+  )
+
+  const { data: rapoStuff } = await api.get(rapoUrl)
+  const rapoStudents = rapoStuff.map(s => s.opiskelijanumero)
+
+  const onlyInOodikone = _.difference(oodikoneStudents, rapoStudents)
+  const onlyInRapo = _.difference(rapoStudents, oodikoneStudents)
+
+  console.log('Only in Oodikone:', onlyInOodikone)
+  console.log('Only in Rapo:', onlyInRapo)
+}
+
+main().then(() => process.exit(0))

services/backend/src/routes/courses.js

@@ -16,7 +16,9 @@ router.get('/v2/coursesmulti', async (req, res) => {
 })
 
 router.get('/v3/courseyearlystats', async (req, res) => {
-  const { rights, roles } = req
+  const {
+    user: { rights, roles },
+  } = req
 
   const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
 
@@ -42,6 +44,4 @@ router.get('/v3/courseyearlystats', async (req, res) => {
   }
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/customPopulationSearch.js

@@ -8,7 +8,7 @@ const {
 
 router.get('/', async (req, res) => {
   const {
-    decodedToken: { id },
+    user: { id },
   } = req
   const customPopulationSearches = await getCustomPopulationSearchesByUser(id)
   res.json(customPopulationSearches)
@@ -16,7 +16,7 @@ router.get('/', async (req, res) => {
 
 router.post('/', async (req, res) => {
   const {
-    decodedToken: { id },
+    user: { id },
     body: { name, students },
   } = req
 
@@ -29,7 +29,7 @@ router.post('/', async (req, res) => {
 
 router.put('/:id', async (req, res) => {
   const {
-    decodedToken: { id: userId },
+    user: { id: userId },
     body: { students },
   } = req
   const { id } = req.params
@@ -46,7 +46,7 @@ router.put('/:id', async (req, res) => {
 
 router.delete('/:id', async (req, res) => {
   const {
-    decodedToken: { id: userId },
+    user: { id: userId },
   } = req
   const { id } = req.params
 

services/backend/src/routes/elementdetails.js

@@ -1,23 +1,7 @@
 const router = require('express').Router()
 const MandatoryCourses = require('../services/mandatoryCourses')
-const {
-  productivityStatsForStudytrack,
-  throughputStatsForStudytrack,
-  optionData,
-} = require('../services/studyprogramme')
 const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
-const {
-  getProductivity,
-  setProductivity,
-  getThroughput,
-  setThroughput,
-  patchProductivity,
-  patchThroughput,
-} = require('../services/analyticsService')
 const { getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
-const logger = require('../util/logger')
-
-const programmeStatsSince = new Date('2017-07-31')
 
 router.get('/elementdetails/all', async (req, res) => {
   const elementdetails = await getAllElementDetails()
@@ -29,22 +13,6 @@ router.get('/studyprogrammes', async (req, res) => {
   res.json(studyrights)
 })
 
-router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
-  const code = req.params.id
-
-  let level
-  if (code.includes('MH')) {
-    level = 'MSC'
-  } else if (code.includes('KH')) {
-    level = 'BSC'
-  } else {
-    return res.json([])
-  }
-
-  const data = await optionData(new Date('2017-07-31'), new Date(), code, level)
-  res.json(data)
-})
-
 router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
   if (req.params.id) {
     const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
@@ -54,109 +22,6 @@ router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
   }
 })
 
-router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    let data = null
-    try {
-      data = await getProductivity(code)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} productivity`)
-    }
-    if (!data) {
-      try {
-        const stats = await productivityStatsForStudytrack(code, programmeStatsSince)
-        data = await setProductivity(stats)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      }
-    }
-    return res.json(data)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-  res.status(200).end()
-
-  try {
-    if (code.includes('MH') || code.includes('KH')) {
-      const data = await productivityStatsForStudytrack(code, new Date('2017-07-31'))
-      await setProductivity(data)
-    } else {
-      const data = await productivityStatsForStudytrack(code, new Date('2000-07-31'))
-      await setProductivity(data)
-    }
-  } catch (e) {
-    try {
-      await patchProductivity({
-        [code]: { status: 'RECALCULATION ERRORED' },
-      })
-    } catch (e) {
-      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      return
-    }
-
-    logger.error(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-  }
-  logger.info(`Produdtiviy stats recalculation for studyprogramme ${code} done`)
-})
-
-router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    const thesisPromise = findProgrammeTheses(req.params.id)
-
-    let data = null
-    try {
-      data = await getThroughput(code)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} throughput`)
-    }
-    if (!data) {
-      try {
-        const result = await throughputStatsForStudytrack(req.params.id, programmeStatsSince.getFullYear())
-        data = await setThroughput(result)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      }
-    }
-    return res.json({ ...data, thesis: await thesisPromise })
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-  res.status(200).end()
-
-  try {
-    if (code.includes('MH') || code.includes('KH')) {
-      const data = await throughputStatsForStudytrack(code, 2017)
-      await setThroughput(data)
-    } else {
-      const data = await throughputStatsForStudytrack(code, 2000)
-      await setThroughput(data)
-    }
-  } catch (e) {
-    try {
-      await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-    } catch (e) {
-      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      return
-    }
-    logger.error(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
-  }
-  logger.info(`Throughput stats recalculation for studyprogramme ${code} done`)
-})
-
 router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {
   const { id } = req.params
   if (id) {
@@ -188,6 +53,4 @@ router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
   }
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/feedback.js

@@ -1,28 +1,19 @@
 const router = require('express').Router()
-const { getUserDataFor } = require('../services/userService')
 const { sendFeedbackToToska } = require('../services/mailservice')
 const logger = require('../util/logger')
 
-router.post('/email', async (req, res) => {
-  const { content } = req.body
+router.post('/email', async req => {
   const {
-    decodedToken: { userId },
+    body: { content },
+    user,
   } = req
 
-  const { email, full_name } = await getUserDataFor(userId)
-
-  const result = await sendFeedbackToToska({
+  await sendFeedbackToToska({
     feedbackContent: content,
-    userId,
-    userEmail: email,
-    userFullName: full_name,
+    user,
   })
-  if (result.error) {
-    return res.status(400).json(result).end()
-  }
-  logger.info(`${userId} succesfully sent some feedback mail to toska`)
 
-  res.status(200).json('success').end()
+  logger.info(`${user.userId} succesfully sent feedback to toska`)
 })
 
 module.exports = router

services/backend/src/routes/language.js

@@ -2,8 +2,10 @@ const router = require('express').Router()
 const userService = require('../services/userService')
 
 router.post('/language', async (req, res) => {
-  const { language } = req.body
-  const { userId } = req.decodedToken
+  const {
+    body: { language },
+    user: { userId },
+  } = req
   if (!['fi', 'sv', 'en'].includes(language)) {
     return res.status(400).json('invalid language')
   }

services/backend/src/routes/login.js

@@ -1,16 +1,19 @@
 const router = require('express').Router()
 const { ApplicationError } = require('../util/customErrors')
+const _ = require('lodash')
 
 router.get('/login', async (req, res) => {
-  const { decodedToken, logoutUrl } = req
+  const { user, logoutUrl } = req
 
-  if (!decodedToken) {
+  if (!user) {
     throw new ApplicationError('User not found', 404)
   }
 
   res.send({
-    token: decodedToken,
+    // don't send possibly huge list of students to frontend
+    user: _.omit(user, ['studentsUserCanAccess']),
     logoutUrl,
   })
 })
+
 module.exports = router

services/backend/src/routes/population.js

@@ -1,15 +1,14 @@
 const crypto = require('crypto')
 const Sentry = require('@sentry/node')
 const router = require('express').Router()
+const _ = require('lodash')
 const Population = require('../services/populations')
 const Student = require('../services/students')
 const StudyrightService = require('../services/studyrights')
-const UserService = require('../services/userService')
 const TagService = require('../services/tags')
 const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
-const { getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')
-const { logger } = require('../util/logger')
+const { ApplicationError } = require('../util/customErrors')
 
 const filterPersonalTags = (population, userId) => {
   return {
@@ -80,20 +79,13 @@ router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) =>
   if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
     return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
   }
-  let studentnumberlist
-  const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
+
   const {
-    decodedToken: { userId },
-    roles,
+    user: { isAdmin, studentsUserCanAccess },
   } = req
 
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = studentnumbers
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-  }
+  const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
+  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
 
   const result = await Population.bottlenecksOf(
     {
@@ -120,19 +112,12 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
     res.status(400).json({ error: 'The body should have a tag defined' })
     return
   }
-  let studentnumberlist
-  const studentnumbers = await Student.findByTag(tag)
   const {
-    decodedToken: { userId },
-    roles,
+    user: { isAdmin, studentsUserCanAccess },
   } = req
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = studentnumbers
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-  }
+  const studentnumbers = await Student.findByTag(tag)
+  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
+
   const result = await Population.bottlenecksOf(
     {
       year: 1900,
@@ -154,27 +139,17 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
 })
 
 router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
-  if (!req.body.studentnumberlist) {
-    res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
-    return
-  }
-  let studentnumberlist
   const {
-    decodedToken: { userId, sisPersonId },
-    roles,
+    user: { isAdmin, studentsUserCanAccess },
+    body: { studentnumberlist: studentnumbersInReq },
   } = req
 
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = req.body.studentnumberlist
-  } else if (req.body.usingStudyGuidanceGroups) {
-    const studentsUserCanAccess = await getAllStudentsUserHasInGroups(sisPersonId)
-    studentnumberlist = req.body.studentnumberlist.filter(s => studentsUserCanAccess.has(s))
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
+  if (!studentnumbersInReq) {
+    throw new ApplicationError('The body should have a studentnumberlist defined', 400)
   }
 
+  const studentnumberlist = isAdmin ? studentnumbersInReq : _.intersection(studentnumbersInReq, studentsUserCanAccess)
+
   const result = await Population.bottlenecksOf(
     {
       year: 1900,
@@ -195,7 +170,7 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
 
 router.get('/v3/populationstatistics', async (req, res) => {
   const { year, semesters, studyRights: studyRightsJSON } = req.query
-  const { decodedToken } = req
+  const { user } = req
 
   if (!year || !semesters || !studyRightsJSON) {
     res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
@@ -204,13 +179,13 @@ router.get('/v3/populationstatistics', async (req, res) => {
   let studyRights = null
   try {
     studyRights = JSON.parse(studyRightsJSON)
-    const { rights, roles } = req
+    const {
+      user: { rights, isAdmin },
+    } = req
 
-    if (!roles || !roles.includes('admin')) {
-      if (!rights.includes(studyRights.programme)) {
-        res.status(403).json([])
-        return
-      }
+    if (!isAdmin && !rights.includes(studyRights.programme)) {
+      res.status(403).json([])
+      return
     }
   } catch (e) {
     res.status(400).json({ error: 'The query had invalid studyRights' })
@@ -245,7 +220,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
       return
     }
 
-    res.json(filterPersonalTags(result, decodedToken.id))
+    res.json(filterPersonalTags(result, user.id))
   } else {
     const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
     if (result.error) {
@@ -254,32 +229,26 @@ router.get('/v3/populationstatistics', async (req, res) => {
       return
     }
 
-    res.json(filterPersonalTags(result, decodedToken.id))
+    res.json(filterPersonalTags(result, user.id))
   }
 })
 
 router.get('/v3/populationstatisticsbytag', async (req, res) => {
-  const { tag, studyRights: studyRightsJSON, months, year } = req.query
-  const { decodedToken } = req
+  const {
+    user: { id, isAdmin, studentsUserCanAccess },
+    query: { tag, studyRights: studyRightsJSON, months, year },
+  } = req
 
   if (!tag) return res.status(400).json({ error: 'The query should have a tag defined' })
   const foundTag = await TagService.findTagById(tag)
   if (!foundTag) return res.status(404).json({ error: 'Tag not found' })
 
   const semesters = ['FALL', 'SPRING']
-  let studentnumberlist
+
   const studentnumbers = await Student.findByTag(tag)
-  const {
-    decodedToken: { userId },
-    roles,
-  } = req
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = studentnumbers
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-  }
+
+  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
+
   const studyRights = JSON.parse(studyRightsJSON)
   const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
   const yearDifference = Number(year) - Number(newStartYear)
@@ -301,12 +270,14 @@ router.get('/v3/populationstatisticsbytag', async (req, res) => {
     return
   }
 
-  res.json(filterPersonalTags(result, decodedToken.id))
+  res.json(filterPersonalTags(result, id))
 })
 
 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
   const { coursecodes, from, to, separate: sep } = req.query
-  const { decodedToken, roles } = req
+  const {
+    user: { id, isAdmin, studentsUserCanAccess: allStudentsUserCanAccess },
+  } = req
   const separate = sep ? JSON.parse(sep) : false
 
   if (!coursecodes || !from || !to) {
@@ -335,14 +306,9 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
     studentnumbers
   )
 
-  let studentsUserCanAccess
-  if (roles && roles.includes('admin')) {
-    studentsUserCanAccess = new Set(studentnumbers)
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(decodedToken.userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentsUserCanAccess = new Set(await Student.filterStudentnumbersByAccessrights(studentnumbers, codes))
-  }
+  const studentsUserCanAccess = isAdmin
+    ? new Set(studentnumbers)
+    : new Set(_.intersection(studentnumbers, allStudentsUserCanAccess))
 
   const randomHash = crypto.randomBytes(12).toString('hex')
   const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
@@ -368,26 +334,16 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
     return
   }
 
-  res.json(filterPersonalTags(result, decodedToken.id))
+  res.json(filterPersonalTags(result, id))
 })
 
 router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
-  const { studentnumberlist, usingStudyGuidanceGroups } = req.body
-  const { roles, decodedToken } = req
+  const { studentnumberlist } = req.body
+  const {
+    user: { isAdmin, id, studentsUserCanAccess },
+  } = req
 
-  if (usingStudyGuidanceGroups) {
-    if (!decodedToken.sisPersonId) {
-      logger.error(
-        `User ${decodedToken.userId} tried to get person groups but personId was ${decodedToken.sisPersonId} in header`
-      )
-      return res.status(400).json({ error: 'Not possible to get groups without personId' })
-    }
-  }
-
-  const studentsUserCanAccess = usingStudyGuidanceGroups
-    ? await getAllStudentsUserHasInGroups(decodedToken.sisPersonId)
-    : await UserService.getStudentsUserCanAccess(studentnumberlist, roles, decodedToken.userId)
-  const filteredStudentNumbers = studentnumberlist.filter(s => studentsUserCanAccess.has(s))
+  const filteredStudentNumbers = isAdmin ? studentnumberlist : _.intersection(studentnumberlist, studentsUserCanAccess)
 
   const result = await Population.optimizedStatisticsOf(
     {
@@ -402,12 +358,14 @@ router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
     Sentry.captureException(new Error(result.error))
     return
   }
-  res.status(200).json(filterPersonalTags(result, decodedToken.id))
+  res.status(200).json(filterPersonalTags(result, id))
 })
 
 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
-  const { rights, roles } = req
-  if (roles && roles.includes('admin')) {
+  const {
+    user: { rights, roles },
+  } = req
+  if (roles?.includes('admin')) {
     const studyrights = await StudyrightService.getAssociations()
     res.json(studyrights)
   } else {
@@ -427,6 +385,4 @@ router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req
   return res.json(maxYearsToCreatePopulationFrom)
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/semesters.js

@@ -6,6 +6,4 @@ router.get('/semesters/codes', async (req, res) => {
   res.json(providers)
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/students.js

@@ -1,8 +1,6 @@
 const router = require('express').Router()
 const Student = require('../services/students')
-const userService = require('../services/userService')
-const Unit = require('../services/units')
-const { getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')
+const { ApplicationError } = require('../util/customErrors')
 
 const filterStudentTags = (student, userId) => {
   return {
@@ -13,8 +11,7 @@ const filterStudentTags = (student, userId) => {
 
 router.get('/students', async (req, res) => {
   const {
-    roles,
-    decodedToken: { userId, sisPersonId },
+    user: { isAdmin, studentsUserCanAccess },
     query: { searchTerm },
   } = req
 
@@ -26,68 +23,30 @@ router.get('/students', async (req, res) => {
       .slice(0, 2)
       .find(t => t.length > 3)
   ) {
-    return res.status(400).json({ error: 'at least one search term must be longer than 3 characters' })
+    throw new ApplicationError('at least one search term must be longer than 3 characters', 400)
   }
 
-  if (roles && roles.includes('admin')) {
-    let results = []
-    if (trimmedSearchTerm) {
-      results = await Student.bySearchTerm(trimmedSearchTerm)
-    }
-    return res.status(200).json(results).end()
-  } else {
-    const unitsUserCanAccess = await userService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    let extraStudents = []
-    if (roles.includes('studyGuidanceGroups')) {
-      const matchingOnlyByTerm = await Student.bySearchTerm(trimmedSearchTerm)
-      const studentsUserCanAccess = await getAllStudentsUserHasInGroups(sisPersonId)
-      extraStudents.push(...matchingOnlyByTerm.filter(s => studentsUserCanAccess.has(s.studentNumber)))
-    }
-    const matchingStudents = await Student.bySearchTermAndElements(trimmedSearchTerm, codes)
-    res.json([...matchingStudents, ...extraStudents])
+  let results = []
+  if (trimmedSearchTerm) {
+    results = isAdmin
+      ? await Student.bySearchTerm(trimmedSearchTerm)
+      : await Student.bySearchTermAndStudentNumbers(trimmedSearchTerm, studentsUserCanAccess)
   }
+  res.json(results)
 })
 
 router.get('/students/:id', async (req, res) => {
   const { id: studentId } = req.params
-  const { roles, decodedToken } = req
+  const {
+    user: { id, isAdmin, studentsUserCanAccess },
+  } = req
 
-  if (roles?.includes('admin')) {
-    const results = await Student.withId(studentId)
-    return results.error
-      ? res.status(400).json({ error: 'error finding student' }).end()
-      : res.status(200).json(filterStudentTags(results, decodedToken.id)).end()
+  if (!isAdmin && !studentsUserCanAccess.includes(studentId)) {
+    throw new ApplicationError('Error finding student', 400)
   }
 
-  const uid = req.decodedToken.userId
   const student = await Student.withId(studentId)
-  if (student.error) {
-    return res.status(400).json({ error: 'error finding student' }).end()
-  }
-  const units = await userService.getUnitsFromElementDetails(uid)
-
-  const rights = await Promise.all(
-    units.map(async unit => {
-      const jtn = await Unit.hasStudent(unit.id, student.studentNumber)
-      return jtn
-    })
-  )
-
-  if (roles?.includes('studyGuidanceGroups')) {
-    const studentsUserCanAccess = await getAllStudentsUserHasInGroups(decodedToken.sisPersonId)
-    if (studentsUserCanAccess.has(studentId)) {
-      return res.status(200).json(filterStudentTags(student, decodedToken.id)).end()
-    }
-  }
-
-  if (rights.some(right => right !== null)) {
-    res.status(200).json(filterStudentTags(student, decodedToken.id)).end()
-  } else {
-    res.status(400).json({ error: 'error finding student' }).end()
-  }
+  res.json(filterStudentTags(student, id))
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/studyGuidanceGroups.js

@@ -4,7 +4,7 @@ const { getAllGroupsAndStudents, changeGroupTags } = require('../services/studyG
 
 router.get('/', async (req, res) => {
   const {
-    decodedToken: { uid, sisPersonId },
+    user: { uid, sisPersonId },
   } = req
   if (!sisPersonId) {
     logger.error(`User ${uid} tried to get person groups but personId was ${sisPersonId} in header`)
@@ -15,7 +15,7 @@ router.get('/', async (req, res) => {
 
 router.put('/:id/tags', async (req, res) => {
   const {
-    decodedToken: { uid, sisPersonId },
+    user: { uid, sisPersonId },
     body: tags,
     params: { id: groupId },
   } = req

services/backend/src/routes/studyProgramme.js

@@ -1,11 +1,8 @@
 const router = require('express').Router()
-const {
-  getBasicStatsForStudytrack,
-  getCreditStatsForStudytrack,
-  getGraduationStatsForStudytrack,
-} = require('../services/studyprogrammeStats')
+const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics')
+const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
+const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
 const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
-
 const {
   getBasicStats,
   setBasicStats,
@@ -16,16 +13,18 @@ const {
   getStudytrackStats,
   setStudytrackStats,
 } = require('../services/analyticsService')
+const { updateBasicView, updateStudytrackView } = require('../services/studyprogrammeUpdates')
 const logger = require('../util/logger')
 
 router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
 
   if (code) {
     let data = null
     try {
-      data = await getBasicStats(code, yearType)
+      data = await getBasicStats(code, yearType, specialGroups)
     } catch (e) {
       logger.error(`Failed to get code ${code} basic stats: ${e}`)
     }
@@ -34,8 +33,9 @@ router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
         const result = await getBasicStatsForStudytrack({
           studyprogramme: req.params.id,
           yearType,
+          specialGroups,
         })
-        data = await setBasicStats(result, yearType)
+        data = await setBasicStats(result, yearType, specialGroups)
       } catch (e) {
         logger.error(`Failed to update code ${code} basic stats: ${e}`)
       }
@@ -49,11 +49,12 @@ router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
 router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
 
   if (code) {
     let data = null
     try {
-      data = await getCreditStats(code, yearType)
+      data = await getCreditStats(code, yearType, specialGroups)
     } catch (e) {
       logger.error(`Failed to get code ${code} credit stats`)
     }
@@ -62,8 +63,9 @@ router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
         const result = await getCreditStatsForStudytrack({
           studyprogramme: req.params.id,
           yearType,
+          specialGroups,
         })
-        data = await setCreditStats(result, yearType)
+        data = await setCreditStats(result, yearType, specialGroups)
       } catch (e) {
         logger.error(`Failed to update code ${code} credit stats`)
       }
@@ -77,11 +79,12 @@ router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
 router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
 
   if (code) {
     let data = null
     try {
-      data = await getGraduationStats(code, yearType)
+      data = await getGraduationStats(code, yearType, specialGroups)
     } catch (e) {
       logger.error(`Failed to get code ${code} graduation stats`)
     }
@@ -90,8 +93,9 @@ router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
         const result = await getGraduationStatsForStudytrack({
           studyprogramme: req.params.id,
           yearType,
+          specialGroups,
         })
-        data = await setGraduationStats(result, yearType)
+        data = await setGraduationStats(result, yearType, specialGroups)
       } catch (e) {
         logger.error(`Failed to update code ${code} graduation stats`)
       }
@@ -104,18 +108,20 @@ router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
 
 router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
   const code = req.params.id
+  const graduated = req.query?.graduated
+  const specialGroups = req.query?.special_groups
 
   if (code) {
     let data = null
     try {
-      data = await getStudytrackStats(code)
+      data = await getStudytrackStats(code, graduated, specialGroups)
     } catch (e) {
       logger.error(`Failed to get code ${code} studytrack stats: ${e}`)
     }
     if (!data) {
       try {
-        const result = await getStudytrackStatsForStudyprogramme({ studyprogramme: req.params.id })
-        data = await setStudytrackStats(result)
+        const result = await getStudytrackStatsForStudyprogramme({ studyprogramme: code, specialGroups, graduated })
+        data = await setStudytrackStats(result, graduated, specialGroups)
       } catch (e) {
         logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
       }
@@ -126,6 +132,34 @@ router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
   }
 })
 
-router.use('*', (req, res, next) => next())
+router.get('/v2/studyprogrammes/:id/update_basicview', async (req, res) => {
+  const code = req.params.id
+  if (code) {
+    let result = null
+    try {
+      result = await updateBasicView(code)
+    } catch (e) {
+      logger.error(`Failed to update code ${code} basic stats: ${e}`)
+    }
+    return res.json(result)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/update_studytrackview', async (req, res) => {
+  const code = req.params.id
+  if (code) {
+    let result = null
+    try {
+      result = await updateStudytrackView(code)
+    } catch (e) {
+      logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
+    }
+    return res.json(result)
+  } else {
+    res.status(422).end()
+  }
+})
 
 module.exports = router

services/backend/src/routes/tags.js

@@ -12,48 +12,50 @@ const filterRelevantStudentTags = (studentTags, userId) => {
   return studentTags.filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
 }
 
-router.get('/tags', async (req, res) => {
+router.get('/tags', async (_req, res) => {
   const tags = await Tags.findTags()
   res.status(200).json(tags)
 })
 
 router.get('/tags/:studytrack', async (req, res) => {
   const { studytrack } = req.params
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const tags = await Tags.findTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantTags(tags, decodedToken.id))
+  res.status(200).json(filterRelevantTags(tags, id))
 })
 
 router.post('/tags', async (req, res) => {
   const {
     tag: { studytrack, tagname, year, personal_user_id },
   } = req.body
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
   const tags = await Tags.findTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantTags(tags, decodedToken.id))
+  res.status(200).json(filterRelevantTags(tags, id))
 })
 
 router.delete('/tags', async (req, res) => {
   const { tag } = req.body
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (
-    !rights.includes(tag.studytrack) &&
-    !(roles && roles.includes('admin')) &&
-    !(tag.personal_user_id === decodedToken.id)
-  )
+  if (!rights.includes(tag.studytrack) && !roles?.includes('admin') && !(tag.personal_user_id === id))
     return res.status(403).end()
 
   await Tags.deleteTag(tag)
   const t = await Tags.findTagsByStudytrack(tag.studytrack)
-  res.status(200).json(filterRelevantTags(t, decodedToken.id))
+  res.status(200).json(filterRelevantTags(t, id))
 })
 
 router.get('/studenttags', async (req, res) => {
@@ -63,37 +65,43 @@ router.get('/studenttags', async (req, res) => {
 
 router.get('/studenttags/:studytrack', async (req, res) => {
   const { studytrack } = req.params
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.get('/studenttags/:studentnumber', async (req, res) => {
   const { studentnumber } = req.params
-  const { decodedToken } = req
+  const { user } = req
   const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, user.id))
 })
 
 router.post('/studenttags/:studentnumber', async (req, res) => {
   const { tag, studytrack } = req.body
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { roles, rights, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   await TagStudent.createStudentTag(tag)
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.post('/studenttags', async (req, res) => {
   const { tags, studytrack } = req.body
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const existingtags = await Tags.findTagsByStudytrack(studytrack)
   const existingTagids = existingtags.map(t => t.tag_id)
@@ -110,37 +118,39 @@ router.post('/studenttags', async (req, res) => {
 
   await TagStudent.createMultipleStudentTags(tags)
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.delete('/studenttags/delete_one', async (req, res) => {
   const { tag_id, studentnumber, studytrack } = req.body
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
   if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
 
   await TagStudent.deleteStudentTag(studentnumber, tag_id)
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.delete('/studenttags/delete_many', async (req, res) => {
   const { tagId, studentnumbers, studytrack } = req.body
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
   if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
 
   await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/teachers.js

@@ -42,7 +42,9 @@ router.get('/top/categories', async (req, res) => {
 })
 
 router.get('/stats', async (req, res) => {
-  const { rights, roles } = req
+  const {
+    user: { rights, roles },
+  } = req
 
   const { providers, semesterStart, semesterEnd } = req.query
   if (!providers || !semesterStart) {
@@ -78,6 +80,4 @@ router.get('/:id', async (req, res) => {
   res.json(result)
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/tsaAnalytics.js

@@ -9,8 +9,12 @@ router.post('/event', (req, res) => {
     return res.status(400).json({ error: 'group, name and value are required' })
   }
 
+  const {
+    user: { userId },
+  } = req
+
   // don't await here because frontend doesn't care if it succeeds
-  tsaService.sendTsaEvent(req.decodedToken.userId, { group, name, label, value }).catch(e => {
+  tsaService.sendTsaEvent(userId, { group, name, label, value }).catch(e => {
     Sentry.withScope(scope => {
       // if error was caused by a 400, dig out the JSON response for Sentry
       if (e.isAxiosError && e.response && e.response.data) {

services/backend/src/services/analyticsService.js

@@ -1,106 +1,23 @@
 const { redisClient } = require('../services/redis')
 const moment = require('moment')
 
-const createRedisKeyForProductivity = id => `PRODUCTIVITY_${id}`
-const createRedisKeyForThroughput = id => `THROUGHPUT_${id}`
-const createRedisKeyForBasicStats = (id, yearType) => `BASIC_STATS_${id}_${yearType}`
-const createRedisKeyForCreditStats = (id, yearType) => `CREDIT_STATS_${id}_${yearType}`
-const createRedisKeyForGraduationStats = (id, yearType) => `GRADUATION_STATS_${id}_${yearType}`
-const createRedisKeyForStudytrackStats = id => `STUDYTRACK_STATS_${id}`
+const createRedisKeyForBasicStats = (id, yearType, specialGroups) => `BASIC_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForCreditStats = (id, yearType, specialGroups) => `CREDIT_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForGraduationStats = (id, yearType, specialGroups) =>
+  `GRADUATION_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForStudytrackStats = (id, graduated, specialGroups) =>
+  `STUDYTRACK_STATS_${id}_${graduated}_${specialGroups}`
 
-const getProductivity = async id => {
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return {
-    [id]: JSON.parse(dataFromRedis),
-  }
-}
-
-const setProductivity = async data => {
-  const { id } = data
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: dataToRedis,
-  }
-}
-
-const patchProductivity = async data => {
-  const { id } = data
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataFromRedis = JSON.parse(await redisClient.getAsync(redisKey))
-  const patchedData = {
-    ...dataFromRedis,
-    ...data,
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(patchedData))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: patchedData,
-  }
-}
-
-const getThroughput = async id => {
-  const redisKey = createRedisKeyForThroughput(id)
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return {
-    [id]: JSON.parse(dataFromRedis),
-  }
-}
-
-const setThroughput = async data => {
-  const { id, data: dataToSet } = data
-  const redisKey = createRedisKeyForThroughput(id)
-  const dataToRedis = {
-    data: dataToSet.years,
-    totals: dataToSet.totals,
-    stTotals: dataToSet.stTotals,
-    status: 'DONE',
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: dataToRedis,
-  }
-}
-
-const patchThroughput = async data => {
-  const [id, dataToPatch] = Object.entries(data)[0]
-  const redisKey = createRedisKeyForThroughput(id)
-  const { status } = dataToPatch
-  const dataFromRedis = JSON.parse(await redisClient.getAsync(redisKey))
-  const patchedData = {
-    ...dataFromRedis,
-    status,
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(patchedData))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: patchedData,
-  }
-}
-
-const getBasicStats = async (id, yearType) => {
-  const redisKey = createRedisKeyForBasicStats(id, yearType)
+const getBasicStats = async (id, yearType, specialGroups) => {
+  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
 }
 
-const setBasicStats = async (data, yearType) => {
+const setBasicStats = async (data, yearType, specialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForBasicStats(id, yearType)
+  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
   const dataToRedis = {
     ...data,
     status: 'DONE',
@@ -111,16 +28,16 @@ const setBasicStats = async (data, yearType) => {
   return dataToRedis
 }
 
-const getCreditStats = async (id, yearType) => {
-  const redisKey = createRedisKeyForCreditStats(id, yearType)
+const getCreditStats = async (id, yearType, specialGroups) => {
+  const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
 }
 
-const setCreditStats = async (data, yearType) => {
+const setCreditStats = async (data, yearType, specialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForCreditStats(id, yearType)
+  const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
   const dataToRedis = {
     ...data,
     status: 'DONE',
@@ -131,16 +48,16 @@ const setCreditStats = async (data, yearType) => {
   return dataToRedis
 }
 
-const getGraduationStats = async (id, yearType) => {
-  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+const getGraduationStats = async (id, yearType, specialGroups) => {
+  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
 }
 
-const setGraduationStats = async (data, yearType) => {
+const setGraduationStats = async (data, yearType, specialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
   const dataToRedis = {
     ...data,
     status: 'DONE',
@@ -151,16 +68,16 @@ const setGraduationStats = async (data, yearType) => {
   return dataToRedis
 }
 
-const getStudytrackStats = async id => {
-  const redisKey = createRedisKeyForStudytrackStats(id)
+const getStudytrackStats = async (id, graduated, specialGroups) => {
+  const redisKey = createRedisKeyForStudytrackStats(id, graduated, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
 }
 
-const setStudytrackStats = async data => {
+const setStudytrackStats = async (data, graduated, specialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForStudytrackStats(id)
+  const redisKey = createRedisKeyForStudytrackStats(id, graduated, specialGroups)
   const dataToRedis = {
     ...data,
     status: 'DONE',
@@ -172,12 +89,6 @@ const setStudytrackStats = async data => {
 }
 
 module.exports = {
-  getProductivity,
-  setProductivity,
-  patchProductivity,
-  getThroughput,
-  setThroughput,
-  patchThroughput,
   getBasicStats,
   setBasicStats,
   getCreditStats,

services/backend/src/services/cache.js

@@ -1,12 +0,0 @@
-const LRU = require('lru-cache')
-
-// Max 25 users can be stored in the cache
-const userDataCache = new LRU({
-  max: 25,
-  length: () => 1,
-  maxAge: 1000 * 60 * 60,
-})
-
-module.exports = {
-  userDataCache,
-}

services/backend/src/services/courses.js

@@ -13,7 +13,7 @@ const {
 const { parseCredit } = require('./parseCredits')
 const Op = Sequelize.Op
 const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
-const _ = require('lodash')
+const { sortMainCode, getSortRank } = require('../util/utils')
 
 const byNameOrCode = (searchTerm, language) =>
   Course.findAll({
@@ -59,7 +59,15 @@ const findOneByCode = code => {
   })
 }
 
-const creditsForCourses = async (codes, anonymizationSalt) => {
+const creditsForCourses = async (codes, anonymizationSalt, unifyOpenUniCourses) => {
+  let is_open = false
+
+  if (unifyOpenUniCourses) {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
   const credits = await Credit.findAll({
     include: [
       {
@@ -105,9 +113,11 @@ const creditsForCourses = async (codes, anonymizationSalt) => {
       student_studentnumber: {
         [Op.ne]: null,
       },
+      [Op.or]: [{ is_open }, { is_open: null }],
     },
     order: [['attainment_date', 'ASC']],
   })
+
   const parsedCredits = credits.map(credit => parseCredit(credit, anonymizationSalt))
   return parsedCredits
 }
@@ -144,47 +154,20 @@ const unifyOpenUniversity = code => {
   return regexresult[1]
 }
 
-const allCodeAltenatives = async code => {
-  let course = await Course.findAll({
-    raw: true,
-    attributes: ['id', 'code', 'substitutions'],
-    where: {
-      code: code,
-    },
+const allCodeAlternatives = async code => {
+  const course = await Course.findOne({
+    where: { code: code },
   })
 
-  const allSubstitutions = _.flatten(course.map(c => c.substitutions))
-
-  let subcodes = []
-  if (allSubstitutions) {
-    subcodes = await Course.findAll({
-      raw: true,
-      attributes: ['code'],
-      where: {
-        id: {
-          [Op.in]: allSubstitutions,
-        },
-      },
-    })
-  }
-
-  const courses = [...course, ...subcodes]
-  const temp = courses
-    .map(c => c.code)
-    .map(c => {
-      if (c.match(/^AY?/)) return [c, 4] // open university codes come last
-      if (c.match(/^\d/)) return [c, 2] // old numeric codes come second
-      if (c.match(/^[A-Za-z]/)) return [c, 1] // new letter based codes come first
-      return [c, 3] // unknown, comes before open uni?
-    })
-    .sort((a, b) => a[1] - b[1])
-    .map(c => c[0])
-
-  return allSubstitutions ? _.uniq(temp) : [code]
+  return sortMainCode([...course.substitutions, code])
 }
 
 const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anonymizationSalt) => {
-  let codes = await allCodeAltenatives(coursecode)
+  const courseForSubs = await Course.findOne({
+    where: { code: coursecode },
+  })
+
+  let codes = courseForSubs.substitutions ? sortMainCode([...courseForSubs.substitutions, coursecode]) : [coursecode]
 
   if (isOpenUniCourseCode(coursecode) && !unifyOpenUniCourses) {
     codes = [coursecode]
@@ -195,7 +178,7 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
   }
 
   const [credits, course] = await Promise.all([
-    creditsForCourses(codes, anonymizationSalt),
+    creditsForCourses(codes, anonymizationSalt, unifyOpenUniCourses),
     Course.findOne({
       where: {
         code: coursecode,
@@ -204,6 +187,7 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
   ])
 
   const counter = new CourseYearlyStatsCounter()
+
   for (let credit of credits) {
     const {
       studentnumber,
@@ -357,10 +341,6 @@ const codeLikeTerm = code =>
 
 const byNameAndOrCodeLike = async (name, code) => {
   let rawCourses = await Course.findAll({
-    include: {
-      model: Organization,
-      required: true,
-    },
     where: {
       ...nameLikeTerm(name),
       ...codeLikeTerm(code),
@@ -369,9 +349,9 @@ const byNameAndOrCodeLike = async (name, code) => {
 
   const courses = rawCourses
     .map(course => {
-      return { ...course.dataValues, organizations: course.organizations.map(o => o.id) }
+      return { ...course.dataValues }
     })
-    .sort(a => (a.code.match(/^[A-Za-z]{3}[0-9]{1}/) ? -1 : 1))
+    .sort((x, y) => getSortRank(y.code) - getSortRank(x.code))
 
   let substitutionGroupIndex = 0
   const visited = []
@@ -379,7 +359,11 @@ const byNameAndOrCodeLike = async (name, code) => {
   const organizeSubgroups = course => {
     if (visited.includes(course.code)) return
 
-    let temp = courses.filter(c => (course.substitutions ? course.substitutions.includes(c.id) : false))
+    let temp = []
+    if (course.substitutions !== null) {
+      temp = courses.filter(c => course.substitutions.includes(c.code))
+    }
+
     temp.unshift(course)
     temp.forEach(cu => {
       if (visited.includes(course.code)) return
@@ -407,6 +391,6 @@ module.exports = {
   byCodes,
   maxYearsToCreatePopulationFrom,
   unifyOpenUniversity,
-  allCodeAltenatives,
+  allCodeAlternatives,
   findOneByCode,
 }

services/backend/src/services/mailservice.js

@@ -1,6 +1,6 @@
 const axios = require('axios')
 const { isProduction, pateToken } = require('../conf-backend')
-const logger = require('../util/logger')
+const { ApplicationError } = require('../util/customErrors')
 
 const pateClient = axios.create({
   baseURL: 'https://pate.toska.cs.helsinki.fi',
@@ -20,33 +20,22 @@ const baseSettings = {
 }
 
 const sendEmail = async (options = {}) => {
-  if (!isProduction) {
-    const errorMessage = 'Skipped sending email in non-production environment'
-    logger.error({ message: errorMessage, meta: options })
-    return { error: errorMessage }
-  }
-
-  if (!pateToken) {
-    const errorMessage = 'Skipped sending email since pate token was not available'
-    logger.error({ message: errorMessage, meta: options })
-    return { error: errorMessage }
-  }
-
-  const { data } = await pateClient.post('/', options)
-
-  return data
+  if (!isProduction) throw new ApplicationError('Email sending is disabled in development mode.')
+  if (!pateToken) throw new ApplicationError('Email sending failed because pate token is missing.')
+  await pateClient.post('/', options)
 }
 
-const sendFeedbackToToska = ({ feedbackContent, userId, userEmail, userFullName }) => {
-  const userDetails = `Sent by ${userFullName}, userid: ${userId}, email: ${userEmail}`
+const sendFeedbackToToska = async ({ feedbackContent, user }) => {
+  const { name, userId, email } = user
+  const userDetails = `Sent by ${name}, userid: ${userId}, email: ${email}`
   const text = [feedbackContent, userDetails].join('<br />')
-  return sendEmail({
+  await sendEmail({
     template,
     emails: [
       {
         to: 'Toska <grp-toska@helsinki.fi>',
-        replyTo: userEmail,
-        subject: `Oodikone feedback from ${userFullName}`,
+        replyTo: email,
+        subject: `Oodikone feedback from ${name}`,
         text,
       },
     ],

services/backend/src/services/newStudyprogramme.js

@@ -12,12 +12,17 @@ const {
   SemesterEnrollment,
   Semester,
 } = require('../models')
-const { formatStudyright, formatStudent } = require('./studyprogrammeHelpers')
+const { formatStudyright, formatStudent, formatTransfer } = require('./studyprogrammeHelpers')
+const { getCurrentSemester } = require('./semesters')
 
 const whereStudents = studentnumbers => {
   return studentnumbers ? studentnumbers : { [Op.not]: null }
 }
 
+const sinceDate = since => {
+  return since ? { [Op.gte]: since } : { [Op.not]: null }
+}
+
 const studytrackStudents = async studentnumbers =>
   (
     await Student.findAll({
@@ -29,24 +34,8 @@ const studytrackStudents = async studentnumbers =>
     })
   ).map(formatStudent)
 
-const enrolledStudyrights = async studytrack => {
-  const studyrights = await Studyright.findAll({
-    attributes: ['studyrightid', 'studystartdate', 'enddate', 'prioritycode'],
-    include: [
-      {
-        model: StudyrightElement,
-        required: true,
-        where: { code: studytrack },
-      },
-      {
-        model: Student,
-        attributes: ['studentnumber'],
-        required: true,
-      },
-    ],
-  })
-
-  const studentnumbers = studyrights.map(s => s.student.studentnumber)
+const enrolledStudents = async (studytrack, since, studentnumbers) => {
+  const currentSemester = await getCurrentSemester()
 
   const students = await Student.findAll({
     attributes: ['studentnumber'],
@@ -71,6 +60,9 @@ const enrolledStudyrights = async studytrack => {
           {
             model: Semester,
             required: true,
+            where: {
+              semestercode: currentSemester.semestercode,
+            },
           },
         ],
         where: {
@@ -85,32 +77,73 @@ const enrolledStudyrights = async studytrack => {
     },
   })
 
-  let enrollments = new Set()
+  return students.filter(s => s.semester_enrollments?.length)
+}
 
-  students.forEach(student => {
-    student.semester_enrollments.forEach(e => {
-      const enrollmentStartDate = e.semester.startdate
-      const enrollmentEndDate = e.semester.enddate
-      const correctStudyRight = studyrights.find(
-        s =>
-          student.studentnumber === s.student.studentnumber &&
-          s.studystartdate <= enrollmentStartDate && // Studying in the programme should have started on the same day or earlier than the enrollment
-          s.enddate >= enrollmentEndDate && // Studying in the programme should have ended on the same day or later than the enrollment
-          (!s.canceldate || s.canceldate > enrollmentStartDate) && // Studyright should not be canceled OR the possible canceldate should be after the enrollmentStartDate
-          (s.prioritycode === 1 || s.prioritycode === 30) // Studyright should be primary or the student should have graduated
-      )
-      if (correctStudyRight) {
-        enrollments.add({
-          studentnumber: student.studentnumber,
-          startdate: e.semester.startdate,
-        })
-      }
-    })
+const absentStudents = async (studytrack, studentnumbers) => {
+  const currentSemester = await getCurrentSemester()
+  const students = await Student.findAll({
+    attributes: ['studentnumber'],
+    include: [
+      {
+        model: Studyright,
+        include: [
+          {
+            model: StudyrightElement,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        ],
+        attributes: ['studyrightid'],
+      },
+      {
+        model: SemesterEnrollment,
+        attributes: ['semestercode'],
+        where: {
+          enrollmenttype: 2,
+          semestercode: currentSemester.semestercode,
+        },
+      },
+    ],
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumbers,
+      },
+    },
   })
 
-  return enrollments
+  return students
 }
 
+const allStudyrights = async (studytrack, since, studentnumbers) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: {
+            model: ElementDetail,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        student_studentnumber: whereStudents(studentnumbers),
+      },
+    })
+  ).map(formatStudyright)
+
 const startedStudyrights = async (studytrack, since, studentnumbers) =>
   (
     await Studyright.findAll({
@@ -142,73 +175,100 @@ const startedStudyrights = async (studytrack, since, studentnumbers) =>
   ).map(formatStudyright)
 
 const graduatedStudyRights = async (studytrack, since, studentnumbers) =>
-  await Studyright.findAll({
-    include: {
-      model: StudyrightElement,
-      required: true,
-      include: {
-        model: ElementDetail,
-        required: true,
-        where: {
-          code: studytrack,
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: {
+            model: ElementDetail,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
         },
-      },
-    },
-    where: {
-      graduated: 1,
-      enddate: {
-        [Op.gte]: since,
-      },
-      student_studentnumber: whereStudents(studentnumbers),
-    },
-  })
-
-const cancelledStudyRights = async (studytrack, since) => {
-  return await Studyright.findAll({
-    include: {
-      model: StudyrightElement,
-      required: true,
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
       where: {
-        code: {
-          [Op.eq]: studytrack,
+        graduated: 1,
+        enddate: sinceDate(since),
+        student_studentnumber: whereStudents(studentnumbers),
+      },
+    })
+  ).map(formatStudyright)
+
+const followingStudyrights = async (since, studentnumbers) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: [
+            {
+              model: ElementDetail,
+              attributes: ['name'],
+            },
+          ],
+          attributes: ['code'],
         },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        studystartdate: {
+          [Op.gte]: since,
+        },
+        extentcode: 2,
+        student_studentnumber: whereStudents(studentnumbers),
       },
-    },
-    where: {
-      canceldate: {
-        [Op.gte]: since,
+    })
+  ).map(formatStudyright)
+
+const transfersAway = async (studytrack, since) =>
+  (
+    await Transfer.findAll({
+      where: {
+        transferdate: {
+          [Op.gte]: since,
+        },
+        sourcecode: studytrack,
       },
-    },
-  })
+      distinct: true,
+      col: 'studentnumber',
+    })
+  ).map(formatTransfer)
+
+const transfersTo = async (studytrack, since) =>
+  (
+    await Transfer.findAll({
+      where: {
+        transferdate: {
+          [Op.gte]: since,
+        },
+        targetcode: studytrack,
+      },
+      distinct: true,
+      col: 'studentnumber',
+    })
+  ).map(formatTransfer)
+
+const allTransfers = async (studytrack, since) => {
+  const transferredTo = await transfersTo(studytrack, since)
+  const transferredAway = await transfersAway(studytrack, since)
+  return [...transferredTo, ...transferredAway]
 }
 
-const transfersAway = async (studytrack, since) => {
-  return await Transfer.findAll({
-    where: {
-      transferdate: {
-        [Op.gte]: since,
-      },
-      sourcecode: studytrack,
-    },
-    distinct: true,
-    col: 'studentnumber',
-  })
-}
-
-const transfersTo = async (studytrack, since) => {
-  return await Transfer.findAll({
-    where: {
-      transferdate: {
-        [Op.gte]: since,
-      },
-      targetcode: studytrack,
-    },
-    distinct: true,
-    col: 'studentnumber',
-  })
-}
-
-const getProgrammesStudents = async studyprogramme =>
+const getProgrammesStudyrights = async studyprogramme =>
   (
     await Studyright.findAll({
       attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode'],
@@ -264,14 +324,11 @@ const getCreditsForStudyProgramme = async (provider, since) =>
 
 const getTransferredCredits = async (provider, since) =>
   await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'credittypecode', 'student_studentnumber'],
+    attributes: ['id', 'course_code', 'credits', 'attainment_date'],
     include: {
       model: Course,
       attributes: ['code'],
       required: true,
-      where: {
-        is_study_module: false,
-      },
       include: {
         model: Organization,
         required: true,
@@ -284,13 +341,16 @@ const getTransferredCredits = async (provider, since) =>
       credittypecode: {
         [Op.eq]: [9],
       },
+      isStudyModule: {
+        [Op.not]: true,
+      },
       attainment_date: {
         [Op.gte]: since,
       },
     },
   })
 
-const getThesisCredits = async (provider, since, thesisType) =>
+const getThesisCredits = async (provider, since, thesisType, studentnumbers) =>
   await Credit.findAll({
     attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
     include: {
@@ -320,18 +380,22 @@ const getThesisCredits = async (provider, since, thesisType) =>
       attainment_date: {
         [Op.gte]: since,
       },
+      student_studentnumber: whereStudents(studentnumbers),
     },
   })
 
 module.exports = {
   studytrackStudents,
-  enrolledStudyrights,
+  enrolledStudents,
+  absentStudents,
+  allStudyrights,
   startedStudyrights,
   graduatedStudyRights,
-  cancelledStudyRights,
+  followingStudyrights,
   transfersAway,
   transfersTo,
-  getProgrammesStudents,
+  allTransfers,
+  getProgrammesStudyrights,
   getCreditsForStudyProgramme,
   getTransferredCredits,
   getThesisCredits,

services/backend/src/services/populations.js

@@ -1,6 +1,7 @@
 const { Op } = require('sequelize')
 const moment = require('moment')
 const { sortBy } = require('lodash')
+const { sortMainCode } = require('../util/utils')
 
 const {
   Student,
@@ -19,7 +20,6 @@ const {
   dbConnections: { sequelize },
 } = require('../database/connection')
 const { Tag, TagStudent } = require('../models/models_kone')
-const { /*allCodeAltenatives, */ findOneByCode } = require('./courses')
 const { CourseStatsCounter } = require('./course_stats_counter')
 const { getPassingSemester, semesterEnd, semesterStart } = require('../util/semester')
 const { getAllProgrammes } = require('./studyrights')
@@ -363,8 +363,10 @@ const studentnumbersWithAllStudyrightElements = async (
   exchangeStudents,
   cancelledStudents,
   nondegreeStudents,
-  transferredStudents,
-  tag
+  transferredOutStudents,
+  tag,
+  transferredToStudents,
+  graduatedStudents
 ) => {
   // eslint-disable-line
 
@@ -497,11 +499,11 @@ const studentnumbersWithAllStudyrightElements = async (
     return studyRights.includes(newestStudytrack.code)
   })
 
-  const studentnumberlist = filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers
+  let studentnumberlist = filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers
 
   // fetch students that have transferred out of the programme and filter out these studentnumbers
-  if (!transferredStudents) {
-    const transferredOutStudents = (
+  if (!transferredOutStudents) {
+    const transfersOut = (
       await Transfer.findAll({
         attributes: ['studentnumber'],
         where: {
@@ -519,8 +521,63 @@ const studentnumbersWithAllStudyrightElements = async (
       })
     ).map(s => s.studentnumber)
 
-    const notTransferredStudents = studentnumberlist.filter(sn => !transferredOutStudents.includes(sn))
-    return notTransferredStudents
+    studentnumberlist = studentnumberlist.filter(sn => !transfersOut.includes(sn))
+  }
+
+  // fetch students that have transferred to the programme and filter out these studentnumbers
+  if (transferredToStudents) {
+    const transfersTo = (
+      await Transfer.findAll({
+        attributes: ['studentnumber'],
+        where: {
+          targetcode: {
+            [Op.in]: studyRights,
+          },
+          transferdate: {
+            [Op.gt]: startDate,
+          },
+          studentnumber: {
+            [Op.in]: studentnumberlist,
+          },
+        },
+        raw: true,
+      })
+    ).map(s => s.studentnumber)
+    studentnumberlist = studentnumberlist.filter(sn => !transfersTo.includes(sn))
+  }
+
+  // fetch students that have graduated from the programme and filter out these studentnumbers
+  if (graduatedStudents) {
+    const graduated = (
+      await Student.findAll({
+        attributes: ['studentnumber'],
+        include: [
+          {
+            model: Studyright,
+            include: [
+              {
+                model: StudyrightElement,
+                required: true,
+                where: {
+                  code: {
+                    [Op.in]: studyRights,
+                  },
+                },
+              },
+            ],
+            where: {
+              graduated: 1,
+            },
+          },
+        ],
+        where: {
+          studentnumber: {
+            [Op.in]: studentnumbers,
+          },
+        },
+      })
+    ).map(s => s.studentnumber)
+    studentnumberlist = studentnumberlist.filter(sn => !graduated.includes(sn))
   }
 
   return studentnumberlist
@@ -819,10 +876,6 @@ const findCourses = async (studentnumbers, beforeDate) => {
           },
         },
       },
-      /* {
-        attributes: ['discipline_id', 'name'],
-        model: Discipline
-      }, */
       {
         attributes: ['coursetypecode', 'name'],
         model: CourseType,
@@ -845,12 +898,24 @@ const parseCreditInfo = credit => ({
   date: credit.attainment_date,
 })
 
-const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+const findMainCourse = async code => {
+  const course = await Course.findOne({
+    where: { code: code },
+  })
 
-const unifyOpenUniversity = code => {
-  const regexresult = isOpenUniCourseCode(code)
-  if (!regexresult) return code
-  return regexresult[1]
+  let result = []
+
+  if (course.substitutions) {
+    const sortedCourses = sortMainCode([...course.substitutions, code])
+    result = await Course.findOne({
+      attributes: ['code', 'name'],
+      where: { code: sortedCourses[0] },
+    })
+  } else {
+    return course
+  }
+
+  return result
 }
 
 const bottlenecksOf = async (query, studentnumberlist) => {
@@ -921,6 +986,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
           transferredStudents,
           tag
         ))
+
       const allstudents = studentnumbers.reduce((numbers, num) => {
         numbers[num] = true
         return numbers
@@ -957,14 +1023,14 @@ const bottlenecksOf = async (query, studentnumberlist) => {
   for (const course of courses) {
     let { course_type } = course
     let maincourse = course
-    if (isOpenUniCourseCode) {
+    const sortedMaincourse = await findMainCourse(course.code)
+    if (sortedMaincourse) maincourse = sortedMaincourse
+
+    /*  if (isOpenUniCourseCode) {
       const nonOpen = unifyOpenUniversity(course.code)
       const response = await findOneByCode(nonOpen)
+ */
 
-      if (response) {
-        maincourse = response
-      }
-    }
     if (!stats[maincourse.code]) {
       stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudentslength)
     }
@@ -985,6 +1051,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
 
   bottlenecks.coursestatistics = Object.values(stats).map(coursestatistics => coursestatistics.getFinalStats())
   bottlenecks.allStudents = allstudentslength
+
   return bottlenecks
 }
 

services/backend/src/services/students.js

@@ -288,56 +288,71 @@ const bySearchTerm = async searchterm => {
   return matches.map(formatStudent)
 }
 
-const bySearchTermAndElements = async (searchterm, codes) => {
+const bySearchTermAndStudentNumbers = async (searchterm, studentnumbers) => {
   const terms = splitByEmptySpace(searchterm)
   const matches = await Student.findAll({
-    include: {
-      model: StudyrightElement,
-      required: true,
-      where: {
-        code: {
-          [Op.in]: codes,
+    where: {
+      [Op.and]: {
+        [Op.or]: [nameLike(terms), studentnumberLike(terms)],
+        studentnumber: {
+          [Op.in]: studentnumbers,
         },
       },
     },
-    where: {
-      [Op.or]: [nameLike(terms), studentnumberLike(terms)],
-    },
   })
   return matches.map(formatStudent)
 }
 
-const filterStudentnumbersByAccessrights = async (studentnumbers, codes) => {
-  const students = await Student.findAll({
-    attributes: ['studentnumber'],
-    include: {
-      attributes: [],
-      model: StudyrightElement,
-      required: true,
-      where: {
-        code: {
-          [Op.in]: codes,
+const filterStudentnumbersByAccessrights = async (studentnumbers, codes) =>
+  (
+    await Student.findAll({
+      attributes: ['studentnumber'],
+      include: {
+        attributes: [],
+        model: StudyrightElement,
+        required: true,
+        where: {
+          code: {
+            [Op.in]: codes,
+          },
         },
       },
-    },
-    where: {
-      studentnumber: {
-        [Op.in]: studentnumbers,
+      where: {
+        studentnumber: {
+          [Op.in]: studentnumbers,
+        },
       },
-    },
-    raw: true,
-  })
-  return students.map(student => student.studentnumber)
-}
+      raw: true,
+    })
+  ).map(({ studentnumber }) => studentnumber)
+
+const getStudentnumbersByElementdetails = async codes =>
+  (
+    await Student.findAll({
+      attributes: ['studentnumber'],
+      include: {
+        attributes: [],
+        model: StudyrightElement,
+        required: true,
+        where: {
+          code: {
+            [Op.in]: codes,
+          },
+        },
+      },
+      raw: true,
+    })
+  ).map(({ studentnumber }) => studentnumber)
 
 module.exports = {
   withId,
   bySearchTerm,
   createStudent,
   updateStudent,
-  bySearchTermAndElements,
+  bySearchTermAndStudentNumbers,
   filterStudentnumbersByAccessrights,
   findByCourseAndSemesters,
   findByTag,
   splitByEmptySpace,
+  getStudentnumbersByElementdetails,
 }

services/backend/src/services/studyGuidanceGroups.js

@@ -2,42 +2,42 @@ const { getImporterClient } = require('../util/importerClient')
 const importerClient = getImporterClient()
 const { StudyGuidanceGroupTag } = require('../models/models_kone')
 const logger = require('../util/logger')
+const _ = require('lodash')
 
-const getAllStudentsUserHasInGroups = async sisPersonId => {
+const getGroupsFromImporter = async sisPersonId => {
   if (!importerClient) {
-    return new Set()
+    return []
   }
+  const answerTimeout = new Promise(resolve => setTimeout(resolve, 2000))
   try {
-    const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
-    return new Set(
-      Object.values(data)
-        .map(group => group.members)
-        .flat()
-        .map(member => member.personStudentNumber)
-    )
+    const result = await Promise.race([importerClient.get(`/person-groups/person/${sisPersonId}`), answerTimeout])
+    if (!result) return []
+    return Object.values(result.data)
   } catch (error) {
     logger.error("Couldn't fetch users studyguidance groups")
-    return new Set()
+    return []
   }
 }
 
 const getAllGroupsAndStudents = async sisPersonId => {
-  if (!importerClient) {
-    return []
-  }
-  try {
-    const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
-    const tagsByGroupId = (await StudyGuidanceGroupTag.findAll()).reduce((acc, curr) => {
-      const { studyProgramme, year } = curr
-      return { ...acc, [curr.studyGuidanceGroupId]: { studyProgramme, year } }
-    }, {})
-    return Object.values(data).map(group => ({ ...group, tags: tagsByGroupId[group.id] }))
-  } catch (error) {
-    logger.error("Couldn't fetch users studyguidance groups")
-    return []
-  }
+  const tagsByGroupId = (await StudyGuidanceGroupTag.findAll()).reduce((acc, curr) => {
+    const { studyProgramme, year, studyGuidanceGroupId } = curr
+    return { ...acc, [studyGuidanceGroupId]: { studyProgramme, year } }
+  }, {})
+  return (await getGroupsFromImporter(sisPersonId)).map(group => ({
+    ...group,
+    tags: tagsByGroupId[group.id],
+  }))
 }
 
+const getAllStudentsUserHasInGroups = async sisPersonId =>
+  _.uniqBy(
+    (await getGroupsFromImporter(sisPersonId))
+      .map(group => group.members)
+      .flat()
+      .map(member => member.personStudentNumber)
+  )
+
 const changeGroupTags = async ({ groupId, tags }) => {
   const { studyProgramme, year } = tags
   const tagToUpdate = studyProgramme ? { studyProgramme } : { year }
@@ -56,8 +56,11 @@ const changeGroupTags = async ({ groupId, tags }) => {
   return result
 }
 
+const checkStudyGuidanceGroupsAccess = async hyPersonSisuId => (await getGroupsFromImporter(hyPersonSisuId)).length > 0
+
 module.exports = {
   getAllStudentsUserHasInGroups,
   getAllGroupsAndStudents,
   changeGroupTags,
+  checkStudyGuidanceGroupsAccess,
 }

services/backend/src/services/studyprogramme.js

@@ -1,902 +0,0 @@
-const sequelize = require('sequelize')
-const { Op } = sequelize
-const moment = require('moment')
-const { flatMap } = require('lodash')
-const {
-  Credit,
-  Student,
-  Course,
-  Organization,
-  Studyright,
-  StudyrightElement,
-  ElementDetail,
-  Transfer,
-} = require('../models')
-const { getAssociations, getAllProgrammes } = require('./studyrights')
-const { ThesisCourse, ThesisTypeEnums } = require('../models/models_kone')
-const { studentnumbersWithAllStudyrightElements } = require('./populations')
-const { semesterStart, semesterEnd } = require('../util/semester')
-const { mapToProviders } = require('../util/utils')
-
-const formatCredit = credit => {
-  const { id, credits, attainment_date } = credit
-  const year = attainment_date && attainment_date.getFullYear()
-  return { id, year, credits }
-}
-
-const getTransferredCredits = async (provider, since) =>
-  await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'credittypecode'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
-      where: {
-        is_study_module: false,
-      },
-      include: {
-        model: Organization,
-        attributes: [],
-        required: true,
-        where: {
-          code: provider,
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.eq]: [9],
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-    },
-  })
-
-const getCreditsForMajors = async (provider, since, studentnumbers) =>
-  await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
-      where: {
-        is_study_module: false,
-      },
-      include: {
-        model: Organization,
-        attributes: [],
-        required: true,
-        where: {
-          code: provider,
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.notIn]: [10, 9, 7],
-      },
-      isStudyModule: {
-        [Op.not]: true,
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-      student_studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-    },
-  })
-
-const getCreditsForProvider = async (provider, since) =>
-  (
-    await Credit.findAll({
-      attributes: ['id', 'course_code', 'credits', 'attainment_date'],
-      include: {
-        model: Course,
-        attributes: ['code'],
-        required: true,
-        where: {
-          is_study_module: false,
-        },
-        include: {
-          model: Organization,
-          attributes: [],
-          required: true,
-          where: {
-            code: provider,
-          },
-        },
-      },
-      where: {
-        credittypecode: {
-          [Op.notIn]: [10, 9, 7],
-        },
-        isStudyModule: {
-          [Op.not]: true,
-        },
-        attainment_date: {
-          [Op.gte]: since,
-        },
-      },
-    })
-  ).map(formatCredit)
-
-const productivityStatsFromCredits = credits => {
-  const stats = {}
-  credits.forEach(({ year, credits: creds }) => {
-    const stat = stats[year] || (stats[year] = { credits: 0, year })
-    stat.credits += creds
-  })
-  return stats
-}
-const productivityStatsForProvider = async (providercode, since) => {
-  const credits = await getCreditsForProvider(providercode, since)
-  return productivityStatsFromCredits(credits)
-}
-
-const formatGraduatedStudyright = ({ studyrightid, enddate, studystartdate }) => {
-  const year = enddate && enddate.getFullYear()
-  const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
-  return { studyrightid, year, timeToGraduation }
-}
-
-const findGraduated = async (studytrack, since) =>
-  (
-    await Studyright.findAll({
-      include: {
-        model: StudyrightElement,
-        attributes: [],
-        required: true,
-        include: {
-          model: ElementDetail,
-          attributes: [],
-          required: true,
-          where: {
-            code: studytrack,
-          },
-        },
-      },
-      where: {
-        graduated: 1,
-        enddate: {
-          [Op.gte]: since,
-        },
-      },
-    })
-  ).map(formatGraduatedStudyright)
-
-const graduatedStatsFromStudyrights = studyrights => {
-  const stats = {}
-  let graduationTimes = []
-  studyrights.forEach(({ year, timeToGraduation }) => {
-    const graduated = stats[year] ? stats[year].graduated : 0
-    stats[year] = {
-      graduated: graduated + 1,
-      timesToGraduation: stats[year]
-        ? [...stats[year].timesToGraduation, timeToGraduation || 0]
-        : [timeToGraduation || 0],
-    }
-    graduationTimes = [...graduationTimes, timeToGraduation || 0]
-  })
-  return stats
-}
-
-const graduatedStatsForStudytrack = async (studytrack, since) => {
-  const studyrights = await findGraduated(studytrack, since)
-  return graduatedStatsFromStudyrights(studyrights)
-}
-
-const findProgrammeThesisCredits = async code => {
-  const thesiscourses = await ThesisCourse.findAll({
-    model: ThesisCourse,
-    where: {
-      programmeCode: code,
-    },
-  })
-  const courseCodeToThesisCourse = thesiscourses.reduce((acc, tc) => {
-    acc[tc.courseCode] = tc
-    return acc
-  }, {})
-
-  const credits = await Credit.findAll({
-    include: {
-      model: Course,
-      required: true,
-      where: {
-        code: {
-          [Op.in]: thesiscourses.map(tc => tc.courseCode),
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.eq]: 4,
-      },
-    },
-  })
-
-  return credits.map(credit => {
-    const { id, course, attainment_date } = credit
-    const { code } = course
-    const year = attainment_date && attainment_date.getFullYear()
-    return { id, code, type: courseCodeToThesisCourse[code].thesisType, year }
-  })
-}
-
-const thesisProductivityFromCredits = credits => {
-  const stats = {}
-  credits.forEach(({ type, year }) => {
-    const yearstat = stats[year] || (stats[year] = { mThesis: 0, bThesis: 0 })
-    if (type === ThesisTypeEnums.MASTER) {
-      yearstat.mThesis++
-    } else if (type === ThesisTypeEnums.BACHELOR) {
-      yearstat.bThesis++
-    } else {
-      return
-    }
-  })
-  return stats
-}
-
-const thesisProductivityForStudytrack = async code => {
-  const credits = await findProgrammeThesisCredits(code)
-  return thesisProductivityFromCredits(credits)
-}
-
-const combineStatistics = (creditStats, studyrightStats, thesisStats, creditsForMajors, transferredCredits) => {
-  const allYears = [
-    ...new Set(flatMap([creditStats, studyrightStats, thesisStats, creditsForMajors, transferredCredits], Object.keys)),
-  ]
-  const stats = {}
-  allYears.forEach(year => {
-    const thesis = thesisStats[year] || {}
-    stats[year] = {}
-    stats[year] = creditStats[year] || { credits: 0, year }
-    stats[year].graduated = studyrightStats[year] ? studyrightStats[year].graduated : 0
-    // stats[year].medianGraduationTime = studyrightStats[year] ? studyrightStats[year].medianGraduationTime : 0
-    stats[year].bThesis = thesis.bThesis || 0
-    stats[year].mThesis = thesis.mThesis || 0
-    stats[year].creditsForMajors = creditsForMajors[year] || 0
-    stats[year].transferredCredits = transferredCredits[year] || 0
-  })
-  return Object.values(stats)
-}
-
-const productivityStatsForStudytrack = async (studytrack, since) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const year = 1950
-  const startDate = `${year}-${semesterStart['FALL']}`
-  const endDate = `${moment(new Date(), 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`
-  // This includes ALL students: exchange students, the ones that have transferred to program, the ones
-  // with non-degree studyright and the ones that have cancelled their studyright
-  const studentnumbers = await studentnumbersWithAllStudyrightElements(
-    [studytrack],
-    startDate,
-    endDate,
-    true, // exchange students
-    true, // cancelled students
-    true, // non-degree students
-    true // transferred to students
-  )
-  const promises = [
-    graduatedStatsForStudytrack(studytrack, since),
-    productivityStatsForProvider(providercode, since),
-    thesisProductivityForStudytrack(studytrack),
-    productivityCreditsFromStudyprogrammeStudents(studytrack, since, studentnumbers),
-    transferredCreditsForProductivity(studytrack, since),
-  ]
-  const [studyrightStats, creditStats, thesisStats, creditsForMajors, transferredCredits] = await Promise.all(promises)
-  return {
-    id: studytrack,
-    status: null,
-    data: combineStatistics(creditStats, studyrightStats, thesisStats, creditsForMajors, transferredCredits),
-  }
-}
-
-const creditsAfter = (studentnumbers, startDate) => {
-  return Promise.all(
-    studentnumbers.map(student =>
-      Credit.sum('credits', {
-        where: {
-          student_studentnumber: {
-            [Op.eq]: student,
-          },
-          attainment_date: {
-            [Op.gte]: startDate,
-          },
-          isStudyModule: {
-            [Op.not]: true,
-          },
-          credittypecode: {
-            [Op.notIn]: [10, 9, 7],
-          },
-        },
-      })
-    )
-  )
-}
-
-const thesesFromClass = async (studentnumbers, startDate, code) => {
-  const thesiscourses = await ThesisCourse.findAll({
-    where: {
-      programmeCode: code,
-    },
-  })
-  const credits = await Credit.findAll({
-    include: {
-      model: Course,
-      required: true,
-      where: {
-        code: {
-          [Op.in]: thesiscourses.map(tc => tc.courseCode),
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.ne]: 10,
-      },
-      attainment_date: {
-        [Op.gte]: startDate,
-      },
-      student_studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-    },
-  })
-
-  const courseCodeToThesisCourse = thesiscourses.reduce((acc, tc) => {
-    acc[tc.courseCode] = tc
-    return acc
-  }, {})
-  const courseCodes = credits.map(c => c.course.code)
-  const theses = courseCodes.reduce((acc, code) => {
-    const thesisType = courseCodeToThesisCourse[code].thesisType
-    acc[thesisType] = acc[thesisType] ? acc[thesisType] + 1 : 1
-    return acc
-  }, {})
-  return theses
-}
-
-const graduationsFromClass = async (studentnumbers, studytrack) => {
-  return await Studyright.findAll({
-    include: {
-      model: StudyrightElement,
-      attributes: [],
-      required: true,
-      where: {
-        code: {
-          [Op.eq]: studytrack,
-        },
-      },
-    },
-    where: {
-      student_studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-      graduated: {
-        [Op.eq]: 1,
-      },
-    },
-  })
-}
-
-const genderCodeToValue = code => {
-  if (code === '1') return 'Male'
-  if (code === '2') return 'Female'
-  if (code === '3') return 'Other'
-  if (code === '0') return 'Unknown'
-  return 'Unknown'
-}
-
-const gendersFromClass = async studentnumbers => {
-  return (
-    await Student.findAll({
-      attributes: [[sequelize.fn('count', sequelize.col('gender_code')), 'count'], 'gender_code'],
-      where: {
-        studentnumber: {
-          [Op.in]: studentnumbers,
-        },
-      },
-      group: ['gender_code'],
-      raw: true,
-    })
-  ).reduce((acc, curr) => {
-    acc[genderCodeToValue(curr.gender_code)] = curr.count
-    return acc
-  }, {})
-}
-
-const nationalitiesFromClass = async studentnumbers => {
-  return (
-    await Student.findAll({
-      where: {
-        studentnumber: {
-          [Op.in]: studentnumbers,
-        },
-      },
-      attributes: ['home_country_en'],
-    })
-  ).reduce((acc, { home_country_en }) => {
-    const country = home_country_en || 'Unknown'
-    if (!acc[country]) acc[country] = 0
-    acc[country] += 1
-    return acc
-  }, {})
-}
-
-const tranferredToStudyprogram = async (studentnumbers, startDate, studytrack, endDate) => {
-  return await Transfer.count({
-    where: {
-      studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-      transferdate: {
-        [Op.between]: [startDate, endDate],
-      },
-      targetcode: studytrack,
-    },
-    distinct: true,
-    col: 'studentnumber',
-  })
-}
-
-const formatCreditsForProductivity = credits => {
-  return credits.map(formatCredit).reduce(function (acc, curr) {
-    var key = curr['year']
-    if (!acc[key]) {
-      acc[key] = []
-    }
-    acc[key] = Number(acc[key]) + Number(curr.credits)
-    return acc
-  }, {})
-}
-
-const productivityCreditsFromStudyprogrammeStudents = async (studytrack, startDate, studentnumbers) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const credits = await getCreditsForMajors(providercode, startDate, studentnumbers)
-  const formattedStudentCredits = formatCreditsForProductivity(credits)
-  return formattedStudentCredits
-}
-
-const transferredCreditsForProductivity = async (studytrack, since) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const credits = await getTransferredCredits(providercode, since)
-  const formattedCredits = formatCreditsForProductivity(credits)
-  return formattedCredits
-}
-
-const cancelledStudyright = async (studentnumbers, startDate, studytrack, endDate) => {
-  return await Studyright.count({
-    include: {
-      model: StudyrightElement,
-      attributes: [],
-      required: true,
-      where: {
-        code: {
-          [Op.eq]: studytrack,
-        },
-      },
-    },
-    where: {
-      student_studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-      startdate: {
-        [Op.between]: [startDate, endDate],
-      },
-      canceldate: {
-        [Op.not]: null,
-      },
-    },
-  })
-}
-
-const startedStudyright = async (studentnumbers, startDate, studytrack, endDate) => {
-  return await Studyright.count({
-    include: {
-      model: StudyrightElement,
-      attributes: [],
-      required: true,
-      where: {
-        code: {
-          [Op.eq]: studytrack,
-        },
-      },
-    },
-    where: {
-      student_studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-      studystartdate: {
-        [Op.between]: [startDate, endDate],
-      },
-    },
-  })
-}
-
-const optionData = async (startDate, endDate, code, level) => {
-  const programmes = await getAllProgrammes()
-  // This includes ALL students: exchange students, the ones that have transferred to program, the ones
-  // with non-degree studyright and the ones that have cancelled their studyright
-  const students = await studentnumbersWithAllStudyrightElements(
-    [code],
-    startDate,
-    endDate,
-    true, // exchange students
-    true, // cancelled students
-    true, // non-degree students
-    true // transferred to students
-  )
-
-  let graduated
-  let currentExtent
-  let optionExtent
-
-  if (level === 'BSC') {
-    graduated = { graduated: 1 }
-    currentExtent = 1
-    optionExtent = 2
-  } else if (level === 'MSC') {
-    graduated = {}
-    currentExtent = 2
-    optionExtent = 1
-  } else {
-    throw new Error('Invalid study level ' + level)
-  }
-
-  const currentStudyrights = await Studyright.findAll({
-    include: [
-      {
-        model: StudyrightElement,
-        where: {
-          code: code,
-        },
-        attributes: ['startdate'],
-      },
-    ],
-    where: {
-      ...graduated,
-      extentcode: currentExtent,
-      student_studentnumber: {
-        [Op.in]: students,
-      },
-    },
-    attributes: ['studentStudentnumber', 'givendate', 'studystartdate'],
-  })
-
-  const currentStudyrightsMap = currentStudyrights
-    .filter(b => b.studystartdate)
-    .reduce((obj, studyright) => {
-      const acualDate = new Date(Math.max(+studyright.studystartdate, +studyright.studyright_elements[0].startdate))
-      obj[studyright.studentStudentnumber] = { givendate: studyright.givendate, startdate: acualDate }
-      return obj
-    }, {})
-
-  const options = await Studyright.findAll({
-    include: [
-      {
-        model: StudyrightElement,
-        where: {
-          studentnumber: {
-            [Op.in]: students,
-          },
-          code: {
-            [Op.in]: programmes.map(p => p.code),
-          },
-        },
-        include: [
-          {
-            model: ElementDetail,
-            attributes: ['name'],
-          },
-        ],
-        attributes: ['code'],
-      },
-    ],
-    where: {
-      extentcode: optionExtent,
-      student_studentnumber: {
-        [Op.in]: students,
-      },
-    },
-    order: [[StudyrightElement, 'startdate', 'DESC']],
-    attributes: ['studentStudentnumber', 'startdate', 'givendate'],
-  })
-
-  const data = {}
-  const years = new Set()
-
-  options
-    .filter(m => m.studentStudentnumber in currentStudyrightsMap)
-    .filter(m => m.givendate.getTime() === currentStudyrightsMap[m.studentStudentnumber].givendate.getTime())
-    .forEach(b => {
-      const date = currentStudyrightsMap[b.studentStudentnumber].startdate
-      const year =
-        date.getMonth() > 6 || (date.getMonth() === 6 && date.getDate() == 31)
-          ? date.getFullYear()
-          : date.getFullYear() - 1
-      years.add(year)
-      const code = b.studyright_elements[0].code
-
-      if (!data[code]) {
-        data[code] = {}
-        data[code].name = b.studyright_elements[0].element_detail.name
-        data[code].total = 0
-      }
-      if (!data[code][year]) {
-        data[code][year] = 0
-      }
-      data[code][year] += 1
-      data[code].total += 1
-    })
-
-  const dataAsArray = Object.keys(data).map(code => {
-    const { name, total } = data[code]
-    const output = { code, name, total }
-    years.forEach(y => (output[y] = data[code][y] || 0))
-    return output
-  })
-  return { data: dataAsArray, years: Array.from(years).sort() }
-}
-
-const statsForClass = async (studentnumbers, startDate, studyprogramme, endDate) => {
-  return Promise.all([
-    creditsAfter(studentnumbers, startDate),
-    graduationsFromClass(studentnumbers, studyprogramme),
-    thesesFromClass(studentnumbers, startDate, studyprogramme),
-    gendersFromClass(studentnumbers),
-    tranferredToStudyprogram(studentnumbers, startDate, studyprogramme, endDate),
-    nationalitiesFromClass(studentnumbers),
-    cancelledStudyright(studentnumbers, startDate, studyprogramme, endDate),
-    startedStudyright(studentnumbers, startDate, studyprogramme, endDate),
-  ])
-}
-
-const getYears = since => {
-  const years = []
-  for (let i = since; i <= new Date().getFullYear(); i++) {
-    years.push(i)
-  }
-  return years
-}
-
-const throughputStatsForStudytrack = async (studyprogramme, since) => {
-  const associations = await getAssociations()
-  const studyprogrammeYears = associations.programmes[studyprogramme]
-    ? associations.programmes[studyprogramme].enrollmentStartYears
-    : {}
-
-  const totals = {
-    credits: {
-      mte30: 0,
-      mte60: 0,
-      mte90: 0,
-      mte120: 0,
-      mte150: 0,
-    },
-    genders: {},
-    thesisM: 0,
-    thesisB: 0,
-    students: 0,
-    graduated: 0,
-    inTargetTime: 0,
-    transferred: 0,
-    nationalities: {},
-    cancelled: 0,
-    started: 0,
-  }
-
-  const stTotals = {}
-
-  const years = getYears(since)
-  // studyprogramme starts with K if bachelors and M if masters
-  const graduationTimeLimit = studyprogramme[0] === 'K' ? 36 : 24
-  const median = values => {
-    if (values.length === 0) return 0
-
-    values.sort((a, b) => a - b)
-
-    var half = Math.floor(values.length / 2)
-
-    if (values.length % 2) return values[half]
-
-    return (values[half - 1] + values[half]) / 2.0
-  }
-  let allGraduationTimes = []
-  const arr = await Promise.all(
-    years.map(async year => {
-      const startDate = `${year}-${semesterStart['FALL']}`
-      const endDate = `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`
-      const studytracks = studyprogrammeYears[year] ? Object.keys(studyprogrammeYears[year].studyTracks) : []
-      const studytrackdata = await studytracks.reduce(async (acc, curr) => {
-        const previousData = await acc
-        const studentnumbers = await studentnumbersWithAllStudyrightElements(
-          [studyprogramme, curr],
-          startDate,
-          endDate,
-          true, // exchange students
-          true, // cancelled students
-          true, // non-degree students
-          true // transferred to students
-        )
-        const creditsForStudyprogramme = await productivityCreditsFromStudyprogrammeStudents(
-          studyprogramme,
-          startDate,
-          studentnumbers
-        )
-        const [credits, graduated, theses, genders, transferredTo, nationalities, cancelled, started] =
-          await statsForClass(studentnumbers, startDate, studyprogramme, endDate)
-
-        delete genders[null]
-        const creditValues = credits.reduce(
-          (acc, curr) => {
-            acc.mte30 = curr >= 30 ? acc.mte30 + 1 : acc.mte30
-            acc.mte60 = curr >= 60 ? acc.mte60 + 1 : acc.mte60
-            acc.mte90 = curr >= 90 ? acc.mte90 + 1 : acc.mte90
-            acc.mte120 = curr >= 120 ? acc.mte120 + 1 : acc.mte120
-            acc.mte150 = curr >= 150 ? acc.mte150 + 1 : acc.mte150
-            return acc
-          },
-          { mte30: 0, mte60: 0, mte90: 0, mte120: 0, mte150: 0 }
-        )
-
-        if (!stTotals[curr]) {
-          stTotals[curr] = {
-            credits: {
-              mte30: 0,
-              mte60: 0,
-              mte90: 0,
-              mte120: 0,
-              mte150: 0,
-            },
-            genders: {},
-            thesisM: 0,
-            thesisB: 0,
-            students: 0,
-            graduated: 0,
-            inTargetTime: 0,
-            transferred: 0,
-            nationalities: {},
-            cancelled: 0,
-            started: 0,
-          }
-        }
-
-        Object.keys(stTotals[curr].credits).forEach(key => {
-          stTotals[curr].credits[key] += creditValues[key]
-        })
-        Object.keys(genders).forEach(genderKey => {
-          stTotals[curr].genders[genderKey] = stTotals[curr].genders[genderKey]
-            ? stTotals[curr].genders[genderKey] + Number(genders[genderKey])
-            : Number(genders[genderKey])
-        })
-        Object.keys(nationalities).forEach(nationality => {
-          if (!stTotals[curr].nationalities[nationality]) stTotals[curr].nationalities[nationality] = 0
-          stTotals[curr].nationalities[nationality] += nationalities[nationality]
-        })
-        const graduationTimes = graduated.map(g => moment(g.enddate).diff(g.studystartdate, 'months'))
-        const inTargetTime = graduationTimes.filter(time => time <= graduationTimeLimit).length
-        allGraduationTimes = [...allGraduationTimes, ...graduationTimes]
-
-        stTotals[curr].thesisM = theses.MASTER ? stTotals[curr].thesisM + theses.MASTER : stTotals[curr].thesisM
-        stTotals[curr].thesisB = theses.BACHELOR ? stTotals[curr].thesisB + theses.BACHELOR : stTotals[curr].thesisB
-        stTotals[curr].students = stTotals[curr].students + credits.length
-        stTotals[curr].graduated = stTotals[curr].graduated + graduated.length
-        stTotals[curr].medianGraduationTime = median(allGraduationTimes)
-        stTotals[curr].inTargetTime = stTotals[curr].inTargetTime + inTargetTime
-        stTotals[curr].transferred = stTotals[curr].transferred + transferredTo
-        stTotals[curr].cancelled += cancelled
-        stTotals[curr].started += started
-        return {
-          ...previousData,
-          [curr]: {
-            year: `${year}-${year + 1}`,
-            credits: credits.map(cr => (cr === null ? 0 : cr)),
-            creditsForStudyprogramme: creditsForStudyprogramme,
-            graduated: graduated.length,
-            medianGraduationTime: median(graduationTimes),
-            inTargetTime,
-            thesisM: theses.MASTER || 0,
-            thesisB: theses.BACHELOR || 0,
-            genders,
-            creditValues,
-            transferred: transferredTo,
-            nationalities,
-            cancelled,
-            started,
-          },
-        }
-      }, {})
-
-      const studentnumbers = await studentnumbersWithAllStudyrightElements(
-        [studyprogramme],
-        startDate,
-        endDate,
-        true, // exchange students
-        true, // cancelled students
-        true, // non-degree students
-        true // transferred to students
-      )
-      const creditsForStudyprogramme = await productivityCreditsFromStudyprogrammeStudents(
-        studyprogramme,
-        startDate,
-        studentnumbers
-      )
-      const [credits, graduated, theses, genders, transferredTo, nationalities, cancelled, started] =
-        await statsForClass(studentnumbers, startDate, studyprogramme, endDate)
-      // theres so much shit in the data that transefferFrom doesnt rly mean anything
-      delete genders[null]
-      const creditValues = credits.reduce(
-        (acc, curr) => {
-          acc.mte30 = curr >= 30 ? acc.mte30 + 1 : acc.mte30
-          acc.mte60 = curr >= 60 ? acc.mte60 + 1 : acc.mte60
-          acc.mte90 = curr >= 90 ? acc.mte90 + 1 : acc.mte90
-          acc.mte120 = curr >= 120 ? acc.mte120 + 1 : acc.mte120
-          acc.mte150 = curr >= 150 ? acc.mte150 + 1 : acc.mte150
-          return acc
-        },
-        { mte30: 0, mte60: 0, mte90: 0, mte120: 0, mte150: 0 }
-      )
-      Object.keys(totals.credits).forEach(key => {
-        totals.credits[key] += creditValues[key]
-      })
-      Object.keys(genders).forEach(genderKey => {
-        totals.genders[genderKey] = totals.genders[genderKey]
-          ? totals.genders[genderKey] + Number(genders[genderKey])
-          : Number(genders[genderKey])
-      })
-      Object.keys(nationalities).forEach(nationality => {
-        if (!totals.nationalities[nationality]) totals.nationalities[nationality] = 0
-        totals.nationalities[nationality] += nationalities[nationality]
-      })
-      const graduationTimes = graduated.map(g => moment(g.enddate).diff(g.studystartdate, 'months'))
-      const inTargetTime = graduationTimes.filter(time => time <= graduationTimeLimit).length
-      allGraduationTimes = [...allGraduationTimes, ...graduationTimes]
-
-      totals.thesisM = theses.MASTER ? totals.thesisM + theses.MASTER : totals.thesisM
-      totals.thesisB = theses.BACHELOR ? totals.thesisB + theses.BACHELOR : totals.thesisB
-      totals.students = totals.students + credits.length
-      totals.graduated = totals.graduated + graduated.length
-      totals.medianGraduationTime = median(allGraduationTimes)
-      totals.inTargetTime = totals.inTargetTime + inTargetTime
-      totals.transferred = totals.transferred + transferredTo
-      totals.cancelled += cancelled
-      totals.started += started
-      return {
-        year: `${year}-${year + 1}`,
-        credits: credits.map(cr => (cr === null ? 0 : cr)),
-        creditsForStudyprogramme: creditsForStudyprogramme,
-        graduated: graduated.length,
-        medianGraduationTime: median(graduationTimes),
-        inTargetTime,
-        thesisM: theses.MASTER || 0,
-        thesisB: theses.BACHELOR || 0,
-        genders,
-        creditValues,
-        transferred: transferredTo,
-        nationalities,
-        cancelled,
-        started,
-        studytrackdata,
-      }
-    })
-  )
-  return { id: studyprogramme, status: null, data: { years: arr, totals, stTotals } }
-}
-
-module.exports = {
-  getCreditsForProvider,
-  productivityStatsFromCredits,
-  productivityStatsForProvider,
-  findGraduated,
-  graduatedStatsFromStudyrights,
-  combineStatistics,
-  productivityStatsForStudytrack,
-  throughputStatsForStudytrack,
-  findProgrammeThesisCredits,
-  thesisProductivityFromCredits,
-  thesisProductivityForStudytrack,
-  optionData,
-}

services/backend/src/services/studyprogrammeBasics.js

@@ -0,0 +1,200 @@
+const { indexOf } = require('lodash')
+const {
+  getCorrectStudentnumbers,
+  getStatsBasis,
+  defineYear,
+  getYearsArray,
+  getStartDate,
+  tableTitles,
+} = require('./studyprogrammeHelpers')
+const {
+  allStudyrights,
+  graduatedStudyRights,
+  startedStudyrights,
+  transfersAway,
+  transfersTo,
+} = require('./newStudyprogramme')
+const { getYearStartAndEndDates } = require('../util/semester')
+
+// const getAllStats = async ({ studyprogramme, years, isAcademicYear, includeAllSpecials }) => {
+//   const { graphStats, tableStats } = getStatsBasis(years)
+
+//   await Promise.all(
+//     years.map(async year => {
+//       const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+//       const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+//         codes: [studyprogramme],
+//         startDate,
+//         endDate,
+//         includeAllSpecials,
+//       })
+//       const all = await allStudyrights(studyprogramme, null, studentnumbersOfTheYear)
+
+//       graphStats[indexOf(years, year)] = all.length
+//       tableStats[year] += all.length
+//     })
+//   )
+//   return { graphStats, tableStats }
+// }
+
+const getStartedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+      const studyrights = await startedStudyrights(studyprogramme, since, studentnumbersOfTheYear)
+
+      studyrights.forEach(({ studystartdate }) => {
+        const startYear = defineYear(studystartdate, isAcademicYear)
+        graphStats[indexOf(years, startYear)] += 1
+        tableStats[startYear] += 1
+      })
+    })
+  )
+  return { graphStats, tableStats }
+}
+
+const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+
+      const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
+      studyrights.forEach(({ enddate }) => {
+        const graduationYear = defineYear(enddate, isAcademicYear)
+        graphStats[indexOf(years, graduationYear)] += 1
+        tableStats[graduationYear] += 1
+      })
+    })
+  )
+  return { graphStats, tableStats }
+}
+
+const getCancelledStats = async ({ studyprogramme, years, isAcademicYear }) => {
+  const studyrights = await allStudyrights(studyprogramme)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ canceldate }) => {
+    if (canceldate !== null && canceldate !== undefined) {
+      const cancelYear = defineYear(canceldate, isAcademicYear)
+      graphStats[indexOf(years, cancelYear)] += 1
+      tableStats[cancelYear] += 1
+    }
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredAwayStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const studyrights = await transfersAway(studyprogramme, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredToStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const studyrights = await transfersTo(studyprogramme, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getBasicStatsForStudytrack = async ({ studyprogramme, yearType, specialGroups }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const started = await getStartedStats(queryParameters)
+  const graduated = await getGraduatedStats(queryParameters)
+  const cancelled = await getCancelledStats(queryParameters)
+  const transferredAway = await getTransferredAwayStats(queryParameters)
+  const transferredTo = await getTransferredToStats(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const titles = tableTitles['basics'][specialGroups]
+  const tableStats = reversedYears.map(year =>
+    includeAllSpecials
+      ? [
+          year,
+          started.tableStats[year],
+          graduated.tableStats[year],
+          cancelled.tableStats[year],
+          transferredAway.tableStats[year],
+          transferredTo.tableStats[year],
+        ]
+      : [year, started.tableStats[year], graduated.tableStats[year]]
+  )
+
+  return {
+    id: studyprogramme,
+    years,
+    graphStats: includeAllSpecials
+      ? [
+          {
+            name: 'Started',
+            data: started.graphStats,
+          },
+          {
+            name: 'Graduated',
+            data: graduated.graphStats,
+          },
+          {
+            name: 'Cancelled',
+            data: cancelled.graphStats,
+          },
+          {
+            name: 'Transferred away',
+            data: transferredAway.graphStats,
+          },
+          {
+            name: 'Transferred To',
+            data: transferredTo.graphStats,
+          },
+        ]
+      : [
+          {
+            name: 'Started',
+            data: started.graphStats,
+          },
+          {
+            name: 'Graduated',
+            data: graduated.graphStats,
+          },
+        ],
+    tableStats,
+    titles,
+  }
+}
+
+module.exports = {
+  getBasicStatsForStudytrack,
+}

services/backend/src/services/studyprogrammeCredits.js

@@ -0,0 +1,128 @@
+const { indexOf } = require('lodash')
+
+const { mapToProviders } = require('../util/utils')
+const {
+  getStatsBasis,
+  defineYear,
+  isMajorStudentCredit,
+  getYearsArray,
+  getStartDate,
+  isSpecialGroupCredit,
+  tableTitles,
+} = require('./studyprogrammeHelpers')
+const {
+  getProgrammesStudyrights,
+  getCreditsForStudyProgramme,
+  getTransferredCredits,
+  allTransfers,
+} = require('./newStudyprogramme')
+
+// Fetches all the credits for the studyprogramme and divides them into major-students and non-major students credits
+// Division is done on the basis that whether the student had a primary studyright to the programme on the attainment_date
+// If special groups are excluded, the transfer students are filtered away from the major-students as well
+const getRegularCreditStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  let studyrights = await getProgrammesStudyrights(studyprogramme, since)
+  const transfers = (await allTransfers(studyprogramme, since)).map(t => t.studyrightid)
+
+  if (!includeAllSpecials) {
+    studyrights.filter(s => !transfers.includes(s.studyrightid))
+  }
+
+  const credits = await getCreditsForStudyProgramme(providercode, since)
+
+  let majors = getStatsBasis(years)
+  let nonMajors = getStatsBasis(years)
+
+  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
+    const studyright = studyrights.find(studyright => studyright.studentnumber == student_studentnumber)
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+
+    if (!includeAllSpecials && isSpecialGroupCredit(studyright, attainment_date, transfers)) {
+      return
+    }
+
+    if (isMajorStudentCredit(studyright, attainment_date)) {
+      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      majors.tableStats[attainmentYear] += credits || 0
+    } else {
+      nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      nonMajors.tableStats[attainmentYear] += credits || 0
+    }
+  })
+
+  return { majors, nonMajors }
+}
+
+// Fetches all the credits with the type "transferred / hyväksiluettu" and divides them by year
+const getTransferredCreditStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const credits = await getTransferredCredits(providercode, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  credits.forEach(({ attainment_date, credits }) => {
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+    graphStats[indexOf(years, attainmentYear)] += credits || 0
+    tableStats[attainmentYear] += credits || 0
+  })
+
+  return { graphStats, tableStats }
+}
+
+// Fetches all credits for the studytrack and combines them into the statistics for the table
+// and graph in the studyprogramme overview
+const getCreditStatsForStudytrack = async ({ studyprogramme, yearType, specialGroups }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const { majors, nonMajors } = await getRegularCreditStats(queryParameters)
+  const transferred = await getTransferredCreditStats(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const titles = tableTitles['credits'][specialGroups]
+
+  const tableStats = reversedYears.map(year =>
+    includeAllSpecials
+      ? [
+          year,
+          majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
+          majors.tableStats[year],
+          nonMajors.tableStats[year],
+          transferred.tableStats[year],
+        ]
+      : [
+          year,
+          majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
+          majors.tableStats[year],
+          transferred.tableStats[year],
+        ]
+  )
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    graphStats: [
+      {
+        name: 'Major students credits',
+        data: majors.graphStats,
+      },
+      {
+        name: 'Non-major students credits',
+        data: nonMajors.graphStats,
+      },
+      {
+        name: 'Transferred credits',
+        data: transferred.graphStats,
+      },
+    ],
+    titles,
+  }
+}
+
+module.exports = {
+  getCreditStatsForStudytrack,
+}

services/backend/src/services/studyprogrammeGraduations.js

@@ -0,0 +1,205 @@
+const moment = require('moment')
+const { indexOf, uniqBy } = require('lodash')
+
+const { mapToProviders } = require('../util/utils')
+const {
+  getCorrectStudentnumbers,
+  getStatsBasis,
+  defineYear,
+  getYearsObject,
+  getAcademicYearsObject,
+  getYearsArray,
+  getMedian,
+  getMean,
+  getStartDate,
+  getThesisType,
+} = require('./studyprogrammeHelpers')
+const {
+  graduatedStudyRights,
+  transfersAway,
+  transfersTo,
+  getThesisCredits,
+  followingStudyrights,
+} = require('./newStudyprogramme')
+const { getYearStartAndEndDates } = require('../util/semester')
+
+const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+
+      const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
+      studyrights.forEach(({ enddate }) => {
+        const graduationYear = defineYear(enddate, isAcademicYear)
+        graphStats[indexOf(years, graduationYear)] += 1
+        tableStats[graduationYear] += 1
+      })
+    })
+  )
+  return { graphStats, tableStats }
+}
+
+const getThesisStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const thesisType = getThesisType(studyprogramme)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+
+      const credits = await getThesisCredits(providercode, since, thesisType, studentnumbersOfTheYear)
+
+      credits?.forEach(({ attainment_date }) => {
+        const attainmentYear = defineYear(attainment_date, isAcademicYear)
+        graphStats[indexOf(years, attainmentYear)] += 1
+        tableStats[attainmentYear] += 1
+      })
+    })
+  )
+
+  return { graphStats, tableStats }
+}
+
+const getGraduationTimeStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  let graduationAmounts = getYearsObject(years)
+  let graduationTimes = getYearsObject(years, true)
+  let totalAmounts = isAcademicYear ? getAcademicYearsObject(years) : getYearsObject(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+
+      const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
+      totalAmounts[year] = studentnumbersOfTheYear.length
+
+      studyrights.forEach(({ enddate, studystartdate }) => {
+        const graduationYear = defineYear(enddate, isAcademicYear)
+        const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+        graduationAmounts[graduationYear] += 1
+        graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
+      })
+    })
+  )
+  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
+  const comparisonValue = studyprogramme.includes('KH') ? 72 : 48
+
+  const medians = getYearsObject(years, true)
+  const means = getYearsObject(years, true)
+
+  // HighCharts graph require the data to have this format (ie. actual value, "empty value")
+  years.forEach(year => {
+    const median = getMedian(graduationTimes[year])
+    const mean = getMean(graduationTimes[year])
+    medians[year] = [
+      ['', median],
+      ['', comparisonValue - median],
+    ]
+    means[year] = [
+      ['', mean],
+      ['', comparisonValue - mean],
+    ]
+  })
+  return { medians, means, graduationAmounts, totalAmounts }
+}
+
+const getProgrammesAfterGraduation = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const graduated = await graduatedStudyRights(studyprogramme, since)
+  const graduatedStudents = graduated.map(s => s.studentnumber)
+  let studyrightsAfterThisProgramme = await followingStudyrights(since, graduatedStudents)
+  const studyprogrammeCodes = uniqBy(studyrightsAfterThisProgramme, 'code').map(s => ({ code: s.code, name: s.name }))
+
+  const stats = {}
+  studyprogrammeCodes.forEach(c => (stats[c.code] = { ...c, ...getYearsObject(years) }))
+
+  if (!includeAllSpecials) {
+    const transferredAway = await transfersAway(studyprogramme, since)
+    const transferredTo = await transfersTo(studyprogramme, since)
+    const transfers = [...transferredAway, ...transferredTo].map(s => s.studyrightid)
+    studyrightsAfterThisProgramme.filter(s => !transfers.includes(s.studyrightid) && !s.canceldate)
+  }
+
+  studyrightsAfterThisProgramme.forEach(({ studystartdate, code }) => {
+    const startYear = defineYear(studystartdate, isAcademicYear)
+    if (years.includes(startYear) && code) {
+      stats[code][startYear] += 1
+    }
+  })
+
+  const tableStats = Object.values(stats)
+    .map(p => [p.code, p.name['fi'], ...years.map(year => p[year])])
+    .filter(p => p[0].includes('MH'))
+
+  const graphStats = Object.values(stats)
+    .map(p => ({ name: p.name['fi'], code: p.code, data: years.map(year => p[year]) }))
+    .filter(p => p.code.includes('MH'))
+
+  return { tableStats, graphStats }
+}
+
+const getGraduationStatsForStudytrack = async ({ studyprogramme, yearType, specialGroups }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const thesis = await getThesisStats(queryParameters)
+  const graduated = await getGraduatedStats(queryParameters)
+  const graduationTimeStats = await getGraduationTimeStats(queryParameters)
+  const programmesAfterGraduation = await getProgrammesAfterGraduation(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+
+  const titles = ['', 'Graduated', 'Wrote thesis']
+  const programmesAfterTitles = ['Code', 'Programme', ...years]
+  const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    titles,
+    graphStats: [
+      {
+        name: 'Graduated students',
+        data: graduated.graphStats,
+      },
+      {
+        name: 'Wrote thesis',
+        data: thesis.graphStats,
+      },
+    ],
+    graduationMedianTime: graduationTimeStats.medians,
+    graduationMeanTime: graduationTimeStats.means,
+    graduationAmounts: graduationTimeStats.graduationAmounts,
+    totalAmounts: graduationTimeStats.totalAmounts,
+    programmesAfterTableStats: programmesAfterGraduation.tableStats,
+    programmesAfterGraphStats: programmesAfterGraduation.graphStats,
+    programmesAfterTitles,
+  }
+}
+
+module.exports = {
+  getGraduationStatsForStudytrack,
+}

services/backend/src/services/studyprogrammeHelpers.js

@@ -1,16 +1,60 @@
 const { mean } = require('lodash')
+const { studentnumbersWithAllStudyrightElements } = require('./populations')
 
 // Helper functions
+
+const getCorrectStudentnumbers = async ({ codes, startDate, endDate, includeAllSpecials, includeGraduated = true }) => {
+  let studentnumbers = []
+  const exchangeStudents = includeAllSpecials
+  const cancelledStudents = includeAllSpecials
+  const nondegreeStudents = includeAllSpecials
+  const transferredOutStudents = includeAllSpecials
+  const transferredToStudents = !includeAllSpecials
+  const graduatedStudents = !includeGraduated
+
+  studentnumbers = await studentnumbersWithAllStudyrightElements(
+    codes,
+    startDate,
+    endDate,
+    exchangeStudents,
+    cancelledStudents,
+    nondegreeStudents,
+    transferredOutStudents,
+    null,
+    transferredToStudents,
+    graduatedStudents
+  )
+
+  return studentnumbers
+}
+
 const formatStudyright = studyright => {
-  const { studyrightid, studystartdate, enddate, graduated, prioritycode, extentcode, student } = studyright
+  const {
+    studyrightid,
+    studystartdate,
+    enddate,
+    canceldate,
+    graduated,
+    prioritycode,
+    extentcode,
+    student,
+    studyright_elements,
+  } = studyright
+
   return {
     studyrightid,
     studystartdate,
     enddate,
+    canceldate,
     graduated,
     prioritycode,
     extentcode,
     studentnumber: student.studentnumber,
+    code: studyright_elements?.length ? studyright_elements[0].code : null,
+    name:
+      studyright_elements?.length && studyright_elements[0].element_detail && studyright_elements[0].element_detail.name
+        ? studyright_elements[0].element_detail.name
+        : null,
   }
 }
 
@@ -24,6 +68,16 @@ const formatStudent = student => {
   }
 }
 
+const formatTransfer = transfer => {
+  const { sourcecode, transfercode, transferdate, studyrightid } = transfer
+  return {
+    sourcecode,
+    transfercode,
+    transferdate,
+    studyrightid,
+  }
+}
+
 const getYearsArray = (since, isAcademicYear) => {
   const years = []
   for (let i = since; i <= new Date().getFullYear(); i++) {
@@ -41,6 +95,14 @@ const getYearsObject = (years, emptyArrays = false) => {
   return yearsObject
 }
 
+const getAcademicYearsObject = (years, emptyArrays = false) => {
+  let yearsObject = {}
+  for (const year of years) {
+    yearsObject = { ...yearsObject, [`${year} - ${year + 1}`]: emptyArrays ? [] : 0 }
+  }
+  return yearsObject
+}
+
 const getStatsBasis = years => {
   return {
     graphStats: new Array(years.length).fill(0),
@@ -51,10 +113,21 @@ const getStatsBasis = years => {
 const isMajorStudentCredit = (studyright, attainment_date) =>
   studyright &&
   (studyright.prioritycode === 1 || studyright.prioritycode === 30) && // Is studyright state = MAIN or state = GRADUATED
+  studyright.studystartdate && // The student has started studying in the programme
   studyright.studystartdate <= attainment_date && // Has the credit been attained after studying in the programme started
   studyright.enddate >= attainment_date && // Has the credit been attained before the studyright ended
   (!studyright.canceldate || studyright.canceldate >= attainment_date) // If the studyright was cancelled, was the credit attained before it was cancelled
 
+const isSpecialGroupCredit = (studyright, attainment_date, transfers) => {
+  if (!studyright) return true // If there is no studyright matching the credit, is not a major student credit
+  if (studyright.canceldate) return true // Cancelled studyrights are in the special groups
+  if (studyright.studystartdate > attainment_date) return true // Credits before the studyright started are not major student credits
+  if (studyright.enddate && attainment_date > studyright.enddate) return true // Credits after studyright are not major student credits
+  if ([7, 9, 16, 34, 33, 99, 14, 13].includes(studyright.extentcode)) return true // Excludes non-degree studyrights and exchange students
+  if (transfers.includes(studyright.studyrightid)) return true // Excludes both transfers in and out of the programme
+  return false
+}
+
 const getMedian = values => {
   if (values.length === 0) return 0
   values.sort((a, b) => a - b)
@@ -79,8 +152,8 @@ const defineYear = (date, isAcademicYear) => {
 const getStartDate = (studyprogramme, isAcademicYear) => {
   if ((studyprogramme.includes('KH') || studyprogramme.includes('MH')) && isAcademicYear) return new Date('2017-08-01')
   if (studyprogramme.includes('KH') || studyprogramme.includes('MH')) return new Date('2017-01-01')
-  if (isAcademicYear) return new Date('2000-08-01')
-  return new Date('2000-01-01')
+  if (isAcademicYear) return new Date('2017-08-01')
+  return new Date('2017-01-01')
 }
 
 // There are 9 course_unit_types
@@ -114,7 +187,7 @@ const getPercentage = (value, total) => {
 
 const getEmptyArray = length => new Array(length).fill(0)
 
-const getCreditGraphStats = years => ({
+const getBachelorCreditGraphStats = years => ({
   lte30: {
     name: 'Less than 30 credits',
     data: getEmptyArray(years.length),
@@ -141,21 +214,117 @@ const getCreditGraphStats = years => ({
   },
 })
 
-const creditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'mte150']
+const getMasterCreditGraphStats = years => ({
+  lte200: {
+    name: 'Less than 200 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte220: {
+    name: '200-219 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte240: {
+    name: '220-239 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte260: {
+    name: '240-259 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte280: {
+    name: '260-279 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte280: {
+    name: 'More than 280 credits',
+    data: getEmptyArray(years.length),
+  },
+})
+
+const getCreditGraphStats = (studyprogramme, years) =>
+  studyprogramme.includes('KH') ? getBachelorCreditGraphStats(years) : getMasterCreditGraphStats(years)
+
+const bachelorCreditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'mte150']
+const masterCreditThresholds = ['lte200', 'lte220', 'lte240', 'lte260', 'lte280', 'mte280']
+const bachelorCreditAmounts = [30, 60, 90, 120, 150, 150]
+const masterCreditAmounts = [200, 220, 240, 260, 280, 280]
+
+const getCreditThresholds = studyprogramme => {
+  if (studyprogramme.includes('KH')) {
+    return { creditThresholdKeys: bachelorCreditThresholds, creditThresholdAmounts: bachelorCreditAmounts }
+  }
+  return { creditThresholdKeys: masterCreditThresholds, creditThresholdAmounts: masterCreditAmounts }
+}
+
+const tableTitles = {
+  basics: {
+    SPECIAL_EXCLUDED: ['', 'Started studying', 'Graduated'],
+    SPECIAL_INCLUDED: ['', 'Started studying', 'Graduated', 'Cancelled', 'Transferred away', 'Transferred to'],
+  },
+  credits: {
+    SPECIAL_EXCLUDED: ['', 'Total', 'Major students credits', 'Transferred credits'],
+    SPECIAL_INCLUDED: ['', 'Total', 'Major students credits', 'Non major students credits', 'Transferred credits'],
+  },
+  creditProgress: {
+    bachelor: [
+      '',
+      'All',
+      '< 30 credits',
+      '30-59 credits',
+      '60-89 credits',
+      '90-119 credits',
+      '120-149 credits',
+      '> 150 credits',
+    ],
+    master: [
+      '',
+      'All',
+      '< 200 credits',
+      '200-219 credits',
+      '220-239 credits',
+      '240-259 credits',
+      '260-279 credits',
+      '> 280 credits',
+    ],
+  },
+  studytracks: [
+    '',
+    'All',
+    'Started studying',
+    'Currently enrolled',
+    'Absent',
+    'Cancelled',
+    'Graduated',
+    'Men',
+    'Women',
+    'Finnish',
+  ],
+}
+
+const getCreditProgressTableTitles = studyprogramme =>
+  studyprogramme.includes('KH') ? tableTitles.bachelorCreditProgress : tableTitles.masterCreditProgress
 
 module.exports = {
+  getCorrectStudentnumbers,
   formatStudyright,
   formatStudent,
+  formatTransfer,
   getYearsArray,
   getYearsObject,
+  getAcademicYearsObject,
   getStatsBasis,
   isMajorStudentCredit,
+  isSpecialGroupCredit,
   getMedian,
   getMean,
   defineYear,
   getStartDate,
   getThesisType,
   getPercentage,
+  getBachelorCreditGraphStats,
+  getMasterCreditGraphStats,
   getCreditGraphStats,
-  creditThresholds,
+  getCreditThresholds,
+  tableTitles,
+  getCreditProgressTableTitles,
 }

services/backend/src/services/studyprogrammeStats.js

@@ -1,315 +0,0 @@
-const moment = require('moment')
-const { indexOf } = require('lodash')
-
-const { mapToProviders } = require('../util/utils')
-const {
-  getStatsBasis,
-  defineYear,
-  isMajorStudentCredit,
-  getYearsObject,
-  getYearsArray,
-  getMedian,
-  getMean,
-  getStartDate,
-  getThesisType,
-} = require('./studyprogrammeHelpers')
-const {
-  graduatedStudyRights,
-  startedStudyrights,
-  cancelledStudyRights,
-  transfersAway,
-  transfersTo,
-  getProgrammesStudents,
-  getCreditsForStudyProgramme,
-  getTransferredCredits,
-  getThesisCredits,
-} = require('./newStudyprogramme')
-
-// const getEnrolledStats = async (studytrack, since, years, isAcademicYear) => {
-//   const students = await enrolledStudyrights(studytrack)
-//   const { graphStats, tableStats } = getStatsBasis(years)
-
-//   students.forEach(({ startdate }) => {
-//     if (startdate.getMonth() === 7) {
-//       const enrolledYear = defineYear(startdate, isAcademicYear)
-
-//       graphStats[indexOf(years, enrolledYear)] += 1
-//       tableStats[enrolledYear] += 1
-//     }
-//   })
-
-//   return { graphStats, tableStats }
-// }
-
-const getGraduatedStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await graduatedStudyRights(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ enddate }) => {
-    const graduationYear = defineYear(enddate, isAcademicYear)
-    graphStats[indexOf(years, graduationYear)] += 1
-    tableStats[graduationYear] += 1
-  })
-  return { graphStats, tableStats }
-}
-
-const getStartedStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await startedStudyrights(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ studystartdate }) => {
-    const startYear = defineYear(studystartdate, isAcademicYear)
-    graphStats[indexOf(years, startYear)] += 1
-    tableStats[startYear] += 1
-  })
-  return { graphStats, tableStats }
-}
-
-const getCancelledStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await cancelledStudyRights(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ canceldate }) => {
-    const cancelYear = defineYear(canceldate, isAcademicYear)
-    graphStats[indexOf(years, cancelYear)] += 1
-    tableStats[cancelYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getTransferredAwayStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await transfersAway(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ transferdate }) => {
-    const transferYear = defineYear(transferdate, isAcademicYear)
-    graphStats[indexOf(years, transferYear)] += 1
-    tableStats[transferYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getTransferredToStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await transfersTo(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ transferdate }) => {
-    const transferYear = defineYear(transferdate, isAcademicYear)
-    graphStats[indexOf(years, transferYear)] += 1
-    tableStats[transferYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getRegularCreditStats = async (studytrack, since, years, isAcademicYear) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const studyrights = await getProgrammesStudents(studytrack)
-  const credits = await getCreditsForStudyProgramme(providercode, since)
-
-  let majors = getStatsBasis(years)
-  let nonMajors = getStatsBasis(years)
-
-  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
-    const studyright = studyrights.find(studyright => studyright.studentnumber == student_studentnumber)
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-
-    if (isMajorStudentCredit(studyright, attainment_date)) {
-      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
-      majors.tableStats[attainmentYear] += credits || 0
-    } else {
-      nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
-      nonMajors.tableStats[attainmentYear] += credits || 0
-    }
-  })
-
-  return { majors, nonMajors }
-}
-
-const getTransferredCreditStats = async (studytrack, since, years, isAcademicYear) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const credits = await getTransferredCredits(providercode, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  credits.forEach(({ attainment_date, credits }) => {
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-    graphStats[indexOf(years, attainmentYear)] += credits || 0
-    tableStats[attainmentYear] += credits || 0
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getThesisStats = async (studytrack, since, years, isAcademicYear) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const thesisType = getThesisType(studytrack)
-  const credits = await getThesisCredits(providercode, since, thesisType)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  credits.forEach(({ attainment_date }) => {
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-    graphStats[indexOf(years, attainmentYear)] += 1
-    tableStats[attainmentYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getGraduationTimeStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await graduatedStudyRights(studytrack, since)
-  let graduationAmounts = getYearsObject(years)
-  let graduationTimes = getYearsObject(years, true)
-
-  studyrights.forEach(({ enddate, studystartdate }) => {
-    const graduationYear = defineYear(enddate, isAcademicYear)
-    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
-    graduationAmounts[graduationYear] += 1
-    graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
-  })
-
-  const medians = getYearsObject(years, true)
-  const means = getYearsObject(years, true)
-
-  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
-  const comparisonValue = studytrack.includes('KH') ? 72 : 48
-
-  // HighCharts graph require the data to have this format (ie. actual value, "empty value")
-  years.forEach(year => {
-    const median = getMedian(graduationTimes[year])
-    const mean = getMean(graduationTimes[year])
-    medians[year] = [
-      ['', median],
-      ['', comparisonValue - median],
-    ]
-    means[year] = [
-      ['', mean],
-      ['', comparisonValue - mean],
-    ]
-  })
-  return { medians, means, graduationAmounts }
-}
-
-const getBasicStatsForStudytrack = async ({ studyprogramme, yearType }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-  const started = await getStartedStats(studyprogramme, since, years, isAcademicYear)
-  const graduated = await getGraduatedStats(studyprogramme, since, years, isAcademicYear)
-  const cancelled = await getCancelledStats(studyprogramme, since, years, isAcademicYear)
-  const transferredAway = await getTransferredAwayStats(studyprogramme, since, years, isAcademicYear)
-  const transferredTo = await getTransferredToStats(studyprogramme, since, years, isAcademicYear)
-
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const tableStats = reversedYears.map(year => [
-    year,
-    started.tableStats[year],
-    graduated.tableStats[year],
-    cancelled.tableStats[year],
-    transferredAway.tableStats[year],
-    transferredTo.tableStats[year],
-  ])
-
-  return {
-    id: studyprogramme,
-    years,
-    graphStats: [
-      {
-        name: 'Started',
-        data: started.graphStats,
-      },
-      {
-        name: 'Graduated',
-        data: graduated.graphStats,
-      },
-      {
-        name: 'Cancelled',
-        data: cancelled.graphStats,
-      },
-      {
-        name: 'Transferred away',
-        data: transferredAway.graphStats,
-      },
-      {
-        name: 'Transferred To',
-        data: transferredTo.graphStats,
-      },
-    ],
-    tableStats,
-  }
-}
-
-const getCreditStatsForStudytrack = async ({ studyprogramme, yearType }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-  const { majors, nonMajors } = await getRegularCreditStats(studyprogramme, since, years, isAcademicYear)
-  const transferred = await getTransferredCreditStats(studyprogramme, since, years, isAcademicYear)
-
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const tableStats = reversedYears.map(year => [
-    year,
-    majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
-    majors.tableStats[year],
-    nonMajors.tableStats[year],
-    transferred.tableStats[year],
-  ])
-
-  return {
-    id: studyprogramme,
-    years,
-    tableStats,
-    graphStats: [
-      {
-        name: 'Major students credits',
-        data: majors.graphStats,
-      },
-      {
-        name: 'Non-major students credits',
-        data: nonMajors.graphStats,
-      },
-      {
-        name: 'Transferred credits',
-        data: transferred.graphStats,
-      },
-    ],
-  }
-}
-
-const getGraduationStatsForStudytrack = async ({ studyprogramme, yearType }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-  const thesis = await getThesisStats(studyprogramme, since, years, isAcademicYear)
-  const graduated = await getGraduatedStats(studyprogramme, since, years, isAcademicYear)
-  const graduationTimeStats = await getGraduationTimeStats(studyprogramme, since, years, isAcademicYear)
-
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
-
-  return {
-    id: studyprogramme,
-    years,
-    tableStats,
-    graphStats: [
-      {
-        name: 'Graduated students',
-        data: graduated.graphStats,
-      },
-      {
-        name: 'Wrote thesis',
-        data: thesis.graphStats,
-      },
-    ],
-    graduationMedianTime: graduationTimeStats.medians,
-    graduationMeanTime: graduationTimeStats.means,
-    graduationAmounts: graduationTimeStats.graduationAmounts,
-  }
-}
-
-module.exports = {
-  getBasicStatsForStudytrack,
-  getCreditStatsForStudytrack,
-  getGraduationStatsForStudytrack,
-}

services/backend/src/services/studyprogrammeUpdates.js

@@ -0,0 +1,84 @@
+const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics')
+const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
+const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
+const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
+const {
+  setBasicStats,
+  setCreditStats,
+  setGraduationStats,
+  setStudytrackStats,
+} = require('../services/analyticsService')
+
+const updateBasicView = async code => {
+  const yearTypes = ['ACADEMIC_YEAR', 'CALENDAR_YEAR']
+  const special = ['SPECIAL_INCLUDED', 'SPECIAL_EXCLUDED']
+
+  yearTypes.forEach(async yearType => {
+    special.forEach(async specialGroups => {
+      const basicStats = await getBasicStatsForStudytrack({
+        studyprogramme: code,
+        yearType,
+        specialGroups,
+      })
+      await setBasicStats(basicStats, yearType, specialGroups)
+
+      const creditStats = await getCreditStatsForStudytrack({
+        studyprogramme: code,
+        yearType,
+        specialGroups,
+      })
+      await setCreditStats(creditStats, yearType, specialGroups)
+
+      const graduationStats = await getGraduationStatsForStudytrack({
+        studyprogramme: code,
+        yearType,
+        specialGroups,
+      })
+      await setGraduationStats(graduationStats, yearType, specialGroups)
+    })
+  })
+
+  return 'OK'
+}
+
+const updateStudytrackView = async code => {
+  const specialIncluded = 'SPECIAL_INCLUDED'
+  const specialExcluded = 'SPECIAL_EXCLUDED'
+  const graduatedIncluded = 'GRADUATED_INCLUDED'
+  const graduatedExcluded = 'GRADUATED_EXCLUDED'
+
+  const statsSpecialAndGraduated = await getStudytrackStatsForStudyprogramme({
+    studyprogramme: code,
+    specialGroups: specialIncluded,
+    graduated: graduatedIncluded,
+  })
+  await setStudytrackStats(statsSpecialAndGraduated, graduatedIncluded, specialIncluded)
+
+  const statsSpecialNotGraduated = await getStudytrackStatsForStudyprogramme({
+    studyprogramme: code,
+    specialGroups: specialIncluded,
+    graduated: graduatedExcluded,
+  })
+  await setStudytrackStats(statsSpecialNotGraduated, graduatedExcluded, specialIncluded)
+
+  const statsNotSpecialAndGraduated = await getStudytrackStatsForStudyprogramme({
+    studyprogramme: code,
+    specialGroups: specialExcluded,
+    graduated: graduatedIncluded,
+  })
+  await setStudytrackStats(statsNotSpecialAndGraduated, graduatedIncluded, specialExcluded)
+
+  const statsNotSpecialNotGraduated = await getStudytrackStatsForStudyprogramme({
+    studyprogramme: code,
+    specialGroups: specialExcluded,
+    graduated: graduatedExcluded,
+  })
+  await setStudytrackStats(statsNotSpecialNotGraduated, graduatedExcluded, specialExcluded)
+
+  return 'OK'
+}
+
+module.exports = {
+  updateBasicView,
+  updateStudytrackView,
+}

services/backend/src/services/studytrackStats.js

@@ -1,83 +1,143 @@
 const { indexOf } = require('lodash')
+const moment = require('moment')
 
 const { getAssociations } = require('./studyrights')
-const { studentnumbersWithAllStudyrightElements } = require('./populations')
 const {
+  getMedian,
+  getMean,
   getStartDate,
   getYearsArray,
   getPercentage,
-  getCreditGraphStats,
   getYearsObject,
-  creditThresholds,
+  getAcademicYearsObject,
+  getCorrectStudentnumbers,
+  getCreditThresholds,
+  getCreditGraphStats,
+  tableTitles,
 } = require('./studyprogrammeHelpers')
-const { studytrackStudents, startedStudyrights, graduatedStudyRights } = require('./newStudyprogramme')
+const {
+  studytrackStudents,
+  allStudyrights,
+  startedStudyrights,
+  enrolledStudents,
+  absentStudents,
+  graduatedStudyRights,
+} = require('./newStudyprogramme')
 const { getAcademicYearDates } = require('../util/semester')
 
-const getStudentData = students => {
-  const data = { female: 0, male: 0, finnish: 0, lte30: 0, lte60: 0, lte90: 0, lte120: 0, lte150: 0, mte150: 0 }
+const getStudentData = (students, thresholdKeys, thresholdAmounts) => {
+  let data = { female: 0, male: 0, finnish: 0 }
+  thresholdKeys.forEach(t => (data[t] = 0))
+
   students.forEach(({ gender_code, home_country_en, creditcount }) => {
     data.male += gender_code === '1' ? 1 : 0
     data.female += gender_code === '2' ? 1 : 0
     data.finnish += home_country_en === 'Finland' ? 1 : 0
-    data.lte30 += creditcount < 30 ? 1 : 0
-    data.lte60 += creditcount >= 30 && creditcount < 60 ? 1 : 0
-    data.lte90 += creditcount >= 60 && creditcount < 90 ? 1 : 0
-    data.lte120 += creditcount >= 90 && creditcount < 120 ? 1 : 0
-    data.lte150 += creditcount >= 120 && creditcount < 150 ? 1 : 0
-    data.mte150 += creditcount >= 150 ? 1 : 0
+    data[thresholdKeys[0]] += creditcount < thresholdAmounts[0] ? 1 : 0
+    data[thresholdKeys[1]] += creditcount >= thresholdAmounts[0] && creditcount < thresholdAmounts[1] ? 1 : 0
+    data[thresholdKeys[2]] += creditcount >= thresholdAmounts[1] && creditcount < thresholdAmounts[2] ? 1 : 0
+    data[thresholdKeys[3]] += creditcount >= thresholdAmounts[2] && creditcount < thresholdAmounts[3] ? 1 : 0
+    data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
+    data[thresholdKeys[5]] += creditcount >= thresholdAmounts[5] ? 1 : 0
   })
   return data
 }
 
+const getGraduationTimeStats = async ({
+  year,
+  graduated,
+  studyprogramme,
+  track,
+  graduationMeanTime,
+  graduationMedianTime,
+  graduationAmounts,
+}) => {
+  // Count how long each student took to graduate
+  let graduationTimes = []
+  graduated.forEach(({ enddate, studystartdate }) => {
+    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+    graduationAmounts[track][year] += 1
+    graduationTimes = [...graduationTimes, timeToGraduation]
+  })
+
+  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
+  const comparisonValue = studyprogramme.includes('KH') ? 72 : 48
+
+  // HighCharts graph requires the data to have this format (ie. actual value, "empty value")
+  const median = getMedian(graduationTimes)
+  const mean = getMean(graduationTimes)
+  graduationMedianTime[track][year] = [
+    ['', median],
+    ['', comparisonValue - median],
+  ]
+  graduationMeanTime[track][year] = [
+    ['', mean],
+    ['', comparisonValue - mean],
+  ]
+}
+
 // Goes through the programme and all its studytracks for the said year and adds the wanted stats to the data objects
-const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studytrackNames, year, years, data }) => {
-  const { mainStatsByYear, mainStatsByTrack, creditGraphStats, creditTableStats, emptyTracks } = data
+const getStudytrackDataForTheYear = async ({
+  studyprogramme,
+  special,
+  includeGraduated,
+  studytracks,
+  studytrackNames,
+  year,
+  years,
+  data,
+  creditThresholdKeys,
+  creditThresholdAmounts,
+}) => {
+  const {
+    mainStatsByYear,
+    mainStatsByTrack,
+    creditGraphStats,
+    creditTableStats,
+    graduationMeanTime,
+    graduationMedianTime,
+    graduationAmounts,
+    totalAmounts,
+    emptyTracks,
+  } = data
   const { startDate, endDate } = getAcademicYearDates(year)
 
   await Promise.all(
     studytracks.map(async track => {
       const codes = studyprogramme === track ? [studyprogramme] : [studyprogramme, track]
-      const studentnumbers = await studentnumbersWithAllStudyrightElements(
+
+      const studentnumbers = await getCorrectStudentnumbers({
         codes,
         startDate,
         endDate,
-        true,
-        true,
-        true,
-        true
-      )
+        includeAllSpecials: special,
+        includeGraduated,
+      })
 
-      // All the stats are counted for the students who actually started studying
+      // Get all the studyrights and students for the calculations
+      const all = await allStudyrights(track, null, studentnumbers)
+      const students = await studytrackStudents(studentnumbers)
+      const studentData = getStudentData(students, creditThresholdKeys, creditThresholdAmounts)
       const started = await startedStudyrights(track, startDate, studentnumbers)
-      const startedStudentnumbers = [...new Set(started.map(s => s.studentnumber))]
-      const students = await studytrackStudents(startedStudentnumbers)
-      const studentData = getStudentData(students)
-      const graduated = await graduatedStudyRights(track, startDate, studentnumbers)
+      const enrolled = await enrolledStudents(track, startDate, studentnumbers)
+      const absent = await absentStudents(track, studentnumbers)
+      const cancelled = all.filter(s => s.canceldate !== null && s.canceldate !== undefined)
+      const graduated = await graduatedStudyRights(track, null, studentnumbers)
 
       // If the track has no stats for that year, it should be removed from the table and dropdown options
-      if (started.length === 0) {
+      if (all.length === 0) {
         emptyTracks.has(track) ? emptyTracks.set(track, emptyTracks.get(track) + 1) : emptyTracks.set(track, 1)
         return
       }
 
       // Count stats for creditgraph for the year per track
-      creditThresholds.forEach(
+      creditThresholdKeys.forEach(
         threshold => (creditGraphStats[track][threshold].data[indexOf(years, year)] = studentData[threshold])
       )
-
       // Count stats for the credit progress table for the year per track
       creditTableStats[track] = [
         ...creditTableStats[track],
-        [
-          `${year} - ${year + 1}`,
-          started.length,
-          studentData.lte30,
-          studentData.lte60,
-          studentData.lte90,
-          studentData.lte120,
-          studentData.lte150,
-          studentData.mte150,
-        ],
+        [`${year} - ${year + 1}`, all.length, ...creditThresholdKeys.map(threshold => studentData[threshold])],
       ]
 
       // Count stats for the main studytrack table grouped by tracks
@@ -85,16 +145,24 @@ const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studyt
         ...mainStatsByTrack[track],
         [
           `${year} - ${year + 1}`,
+          all.length,
+          getPercentage(all.length, all.length),
           started.length,
-          getPercentage(started.length, started.length),
-          studentData.male,
-          getPercentage(studentData.male, started.length),
-          studentData.female,
-          getPercentage(studentData.female, started.length),
-          studentData.finnish,
-          getPercentage(studentData.finnish, started.length),
+          getPercentage(started.length, all.length),
+          enrolled.length,
+          getPercentage(enrolled.length, all.length),
+          absent.length,
+          getPercentage(absent.length, all.length),
+          cancelled.length,
+          getPercentage(cancelled.length, all.length),
           graduated.length,
-          getPercentage(graduated.length, started.length),
+          getPercentage(graduated.length, all.length),
+          studentData.male,
+          getPercentage(studentData.male, all.length),
+          studentData.female,
+          getPercentage(studentData.female, all.length),
+          studentData.finnish,
+          getPercentage(studentData.finnish, all.length),
         ],
       ]
 
@@ -102,19 +170,41 @@ const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studyt
       mainStatsByYear[year] = [
         ...mainStatsByYear[year],
         [
-          studytrackNames[track]?.name['fi'] || `${year} - ${year + 1}`,
+          studytrackNames[track]?.name['fi']
+            ? `${studytrackNames[track]?.name['fi']}, ${track}`
+            : `${year} - ${year + 1}`,
+          all.length,
+          getPercentage(all.length, all.length),
           started.length,
-          getPercentage(started.length, started.length),
-          studentData.male,
-          getPercentage(studentData.male, started.length),
-          studentData.female,
-          getPercentage(studentData.female, started.length),
-          studentData.finnish,
-          getPercentage(studentData.finnish, started.length),
+          getPercentage(started.length, all.length),
+          enrolled.length,
+          getPercentage(enrolled.length, all.length),
+          absent.length,
+          getPercentage(absent.length, all.length),
+          cancelled.length,
+          getPercentage(cancelled.length, all.length),
           graduated.length,
-          getPercentage(graduated.length, started.length),
+          getPercentage(graduated.length, all.length),
+          studentData.male,
+          getPercentage(studentData.male, all.length),
+          studentData.female,
+          getPercentage(studentData.female, all.length),
+          studentData.finnish,
+          getPercentage(studentData.finnish, all.length),
         ],
       ]
+
+      // Count stats for the graduation time charts grouped by year
+      totalAmounts[track][`${year} - ${year + 1}`] = await all.length
+      await getGraduationTimeStats({
+        year: `${year} - ${year + 1}`,
+        graduated,
+        studyprogramme,
+        track,
+        graduationMedianTime,
+        graduationMeanTime,
+        graduationAmounts,
+      })
     })
   )
 }
@@ -133,25 +223,45 @@ const getStudytrackOptions = (studyprogramme, studytrackNames, studytracks, empt
 }
 
 // Creates empty objects for each statistic type, which are then updated with the studytrack data
-const getEmptyStatsObjects = (years, studytracks) => {
+const getEmptyStatsObjects = (years, studytracks, studyprogramme) => {
   const mainStatsByYear = getYearsObject(years, true)
   const mainStatsByTrack = {}
   const creditGraphStats = {}
   const creditTableStats = {}
+  const graduationMedianTime = {}
+  const graduationMeanTime = {}
+  const graduationAmounts = {}
+  const totalAmounts = {}
   const emptyTracks = new Map()
 
   studytracks.forEach(async track => {
     mainStatsByTrack[track] = []
-    creditGraphStats[track] = await getCreditGraphStats(years)
+    creditGraphStats[track] = getCreditGraphStats(studyprogramme, years)
     creditTableStats[track] = []
+    graduationMedianTime[track] = getAcademicYearsObject(years, true)
+    graduationMeanTime[track] = getAcademicYearsObject(years, true)
+    graduationAmounts[track] = getAcademicYearsObject(years)
+    totalAmounts[track] = getAcademicYearsObject(years)
   })
 
-  return { mainStatsByYear, mainStatsByTrack, creditGraphStats, creditTableStats, emptyTracks }
+  return {
+    mainStatsByYear,
+    mainStatsByTrack,
+    creditGraphStats,
+    creditTableStats,
+    graduationMedianTime,
+    graduationMeanTime,
+    graduationAmounts,
+    totalAmounts,
+    emptyTracks,
+  }
 }
 
 // Combines all the data for the Populations and Studytracks -view
-const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {
+const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, graduated, specialGroups }) => {
   const isAcademicYear = true
+  const special = specialGroups === 'SPECIAL_INCLUDED'
+  const includeGraduated = graduated === 'GRADUATED_INCLUDED'
   const since = getStartDate(studyprogramme, isAcademicYear)
   const years = getYearsArray(since.getFullYear())
 
@@ -162,17 +272,22 @@ const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {
 
   const studytrackNames = associations.studyTracks
 
-  const data = getEmptyStatsObjects(years, studytracks)
+  const data = getEmptyStatsObjects(years, studytracks, studyprogramme)
+  const { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(studyprogramme)
 
   await Promise.all(
     years.map(async year => {
       return await getStudytrackDataForTheYear({
         studyprogramme,
+        special,
+        includeGraduated,
         studytracks,
         studytrackNames,
         year,
         years,
         data,
+        creditThresholdKeys,
+        creditThresholdAmounts,
       })
     })
   )
@@ -186,7 +301,14 @@ const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {
     mainStatsByYear: data.mainStatsByYear,
     creditTableStats: data.creditTableStats,
     creditGraphStats: data.creditGraphStats,
+    graduationMedianTime: data.graduationMedianTime,
+    graduationMeanTime: data.graduationMeanTime,
+    graduationAmounts: data.graduationAmounts,
+    totalAmounts: data.totalAmounts,
     studytrackOptions,
+    includeGraduated,
+    populationTitles: tableTitles['studytracks'],
+    creditTableTitles: tableTitles['creditProgress'][studyprogramme.includes('KH') ? 'bachelor' : 'master'],
   }
 }
 

services/backend/src/services/trends/statusGraduated.js

@@ -93,7 +93,9 @@ const calculateStatusGraduated = async (unixMillis, showByYear) => {
       // init faculty
       if (!acc[curr.faculty_code]) {
         const org = orgs.find(o => o.code === curr.faculty_code)
-        acc[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        if (org) {
+          acc[curr.faculty_code] = { name: org.name, yearly: {}, drill: {} }
+        }
       }
       if (!acc[curr.faculty_code]['yearly'][data.year])
         acc[curr.faculty_code]['yearly'][data.year] = { acc: 0, total: 0 }

services/backend/src/services/units.js

@@ -1,41 +0,0 @@
-const Sequelize = require('sequelize')
-const { ElementDetail, StudyrightElement } = require('../models')
-const Op = Sequelize.Op
-
-const hasStudent = async (code, studentnumber) =>
-  StudyrightElement.findOne({
-    where: {
-      studentnumber,
-      code,
-    },
-  })
-
-const parseUnitFromElement = element => ({
-  id: element.code,
-  name: element.name.fi,
-  enabled: true,
-  type: element.type,
-})
-
-const getUnitsFromElementDetails = async () => {
-  const elementdetails = await ElementDetail.findAll({
-    where: {
-      type: {
-        [Op.or]: ['20'],
-      },
-    },
-  })
-  return elementdetails.map(parseUnitFromElement)
-}
-
-const getUnitFromElementDetail = async id => {
-  const element = await ElementDetail.findByPk(id)
-  return parseUnitFromElement(element)
-}
-
-module.exports = {
-  hasStudent,
-  getUnitsFromElementDetails,
-  getUnitFromElementDetail,
-  parseUnitFromElement,
-}

services/backend/src/services/userService.js

@@ -1,173 +1,280 @@
-const UnitService = require('./units')
-const elementDetailService = require('./elementdetails')
-const { filterStudentnumbersByAccessrights } = require('./students')
-const { userDataCache } = require('./cache')
-const { getImporterClient } = require('../util/importerClient')
-const logger = require('../util/logger')
+const Sequelize = require('sequelize')
+const { Op } = Sequelize
+const LRU = require('lru-cache')
+const { getStudentnumbersByElementdetails } = require('./students')
 const { facultiesAndProgrammesForTrends } = require('../services/organisations')
 const { sendNotificationAboutNewUser } = require('../services/mailservice')
-const User = require('./users/users')
-const AccessGroup = require('./users/accessgroups')
+const { checkStudyGuidanceGroupsAccess, getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')
 const _ = require('lodash')
+const { User, UserElementDetails, AccessGroup, UserFaculties, sequelizeUser } = require('../models/models_user')
 
-// Private helpers
-const enrichProgrammesFromFaculties = faculties =>
-  facultiesAndProgrammesForTrends.filter(f => faculties.includes(f.faculty_code)).map(f => f.programme_code)
+const courseStatisticsGroup = 'grp-oodikone-basic-users'
+const hyOneGroup = 'hy-one'
+const toskaGroup = 'grp-toska'
 
-const enrichWithProgrammes = toEnrich => {
-  if (Array.isArray(toEnrich)) {
-    return toEnrich.map(obj => enrichWithProgrammes(obj))
-  }
-  if (toEnrich.rights) {
-    return {
-      ...toEnrich,
-      rights: _.union(toEnrich.rights, enrichProgrammesFromFaculties(toEnrich.faculties)),
-    }
-  }
+// Max 25 users can be stored in the cache
+const userDataCache = new LRU({
+  max: 25,
+  length: () => 1,
+  maxAge: 1000 * 60 * 60,
+})
 
-  if (toEnrich.elementdetails) {
-    return {
-      ...toEnrich,
-      elementdetails: _.union(
-        toEnrich.elementdetails,
-        enrichProgrammesFromFaculties(toEnrich.faculty.map(f => f.faculty_code))
-      ),
-    }
+// Stuff to work with old userservice based user-db (separate microservice, now deprecated)
+const userIncludes = [
+  {
+    separate: true,
+    model: UserElementDetails,
+    as: 'programme',
+  },
+  {
+    model: AccessGroup,
+    as: 'accessgroup',
+    attributes: ['id', 'group_code', 'group_info'],
+  },
+  {
+    separate: true,
+    model: UserFaculties,
+    as: 'faculty',
+  },
+]
+
+const accessGroupsByCodes = codes =>
+  AccessGroup.findAll({
+    where: {
+      group_code: {
+        [Op.in]: codes,
+      },
+    },
+  })
+
+const byUsername = async username =>
+  await User.findOne({
+    where: {
+      username,
+    },
+    include: userIncludes,
+  })
+
+const byId = async id =>
+  await User.findOne({
+    where: {
+      id,
+    },
+    include: userIncludes,
+  })
+
+// Crud things from old userservice
+
+// some fields that frontend needs
+const formatUserForAdminView = user => ({
+  ...user.get(),
+  elementdetails: user.programme.map(p => p.elementDetailCode),
+  is_enabled: true, // this is probably not needed: is here mainly because old userservice created users for every logged in user, even if they hadn't correct iamgroups
+})
+
+const addProgrammes = async (id, codes) => {
+  for (const code of codes) {
+    await UserElementDetails.upsert({ userId: id, elementDetailCode: code })
   }
-  return toEnrich
-}
-const checkStudyGuidanceGroupsAccess = async hyPersonSisuId => {
-  if (!hyPersonSisuId) {
-    logger.error('Not possible to get groups without personId header')
-    return false
-  }
-  const importerClient = getImporterClient()
-  if (!importerClient) return false
-  const answerTimeout = new Promise(resolve => setTimeout(resolve, 2000))
-  const result = await Promise.race([importerClient.get(`/person-groups/person/${hyPersonSisuId}`), answerTimeout])
-  if (!result) return false
-  const { data } = result
-  return data && Object.values(data).length > 0
+  const user = await byId(id)
+  userDataCache.del(user.username)
 }
 
-const byUsernameData = async username => {
-  // OK, enriched
-  const user = await User.byUsernameMinified(username)
-  const roles = user.accessgroup.map(({ group_code }) => group_code)
-  const rights = User.getUserProgrammes(user)
-  const faculties = user.faculty.map(({ faculty_code }) => faculty_code)
-  const data = {
-    email: user.email,
-    full_name: user.full_name,
-    roles,
-    rights,
-    faculties,
+const removeProgrammes = async (id, codes) => {
+  for (const code of codes) {
+    await UserElementDetails.destroy({
+      where: { userId: id, elementDetailCode: code },
+    })
   }
-  return enrichWithProgrammes(data)
+  const user = await byId(id)
+  userDataCache.del(user.username)
 }
 
-// Exported helpers
-const getUserDataFor = async username => {
-  // no need to enrich
-  let userData = userDataCache.get(username)
-  if (!userData) {
-    userData = await byUsernameData(username)
-    userDataCache.set(username, userData)
-  }
+const modifyRights = async (username, rights) => {
+  const rightsToAdd = Object.entries(rights)
+    .map(([code, val]) => {
+      if (val === true) {
+        return code
+      }
+    })
+    .filter(code => code)
+  const rightsToRemove = Object.entries(rights)
+    .map(([code, val]) => {
+      if (val === false) {
+        return code
+      }
+    })
+    .filter(code => code)
 
-  return {
-    ...userData,
-    faculties: new Set(userData.faculties),
-  }
+  const user = await byUsername(username)
+  const accessGroupsToAdd = await accessGroupsByCodes(rightsToAdd)
+  const accessGroupsToRemove = await accessGroupsByCodes(rightsToRemove)
+
+  await user.addAccessgroup(accessGroupsToAdd)
+  await user.removeAccessgroup(accessGroupsToRemove)
+  userDataCache.del(username)
 }
 
-const getUnitsFromElementDetails = async username => {
-  // No need to enrich
-  let userData = userDataCache.get(username)
-  if (!userData) {
-    userData = await byUsernameData(username)
-    userDataCache.set(username, userData)
-  }
-  const elementDetails = await elementDetailService.byCodes(userData.rights)
-  return elementDetails.map(element => UnitService.parseUnitFromElement(element))
-}
-
-const getStudentsUserCanAccess = async (studentnumbers, roles, userId) => {
-  // No need to enrich
-  let studentsUserCanAccess
-  if (roles?.includes('admin')) {
-    studentsUserCanAccess = new Set(studentnumbers)
-  } else {
-    const unitsUserCanAccess = await getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentsUserCanAccess = new Set(await filterStudentnumbersByAccessrights(studentnumbers, codes))
-  }
-  return studentsUserCanAccess
-}
-
-// FOR ROUTES 1-to-1 mapping
 const modifyAccess = async body => {
   const { username, accessgroups } = body
-  await User.modifyRights(username, accessgroups)
-  const user = await User.byUsername(username)
-  return enrichWithProgrammes(User.getUserData(user))
+  await modifyRights(username, accessgroups)
+  userDataCache.del(username)
+  return formatUserForAdminView(await byUsername(username))
 }
 
-const getAccessGroups = async () => enrichWithProgrammes(await AccessGroup.findAll())
-
-const enableElementDetails = async (uid, codes) => {
-  await User.addProgrammes(uid, codes)
-  const user = await User.byId(uid)
-  return enrichWithProgrammes(User.getUserData(user))
+const enableElementDetails = async (id, codes) => {
+  await addProgrammes(id, codes)
+  const user = await byId(id)
+  userDataCache.del(user.username)
+  return formatUserForAdminView(user)
 }
 
-const findAll = async () => enrichWithProgrammes((await User.findAll()).map(User.getUserData))
-
-const setFaculties = async (uid, faculties) => {
-  await User.setFaculties(uid, faculties)
-  const user = await User.byId(uid)
-  return enrichWithProgrammes(User.getUserData(user))
+const setFaculties = async (id, faculties) => {
+  await sequelizeUser.transaction(async transaction => {
+    await UserFaculties.destroy({ where: { userId: id }, transaction })
+    for (const faculty of faculties) {
+      await UserFaculties.create({ userId: id, faculty_code: faculty }, { transaction })
+    }
+  })
+  const user = await byId(id)
+  userDataCache.del(user.username)
+  return formatUserForAdminView(user)
 }
 
-const removeElementDetails = async (uid, codes) => {
-  await User.removeProgrammes(uid, codes)
-  const user = await User.byId(uid)
-  return enrichWithProgrammes(User.getUserData(user))
+const removeElementDetails = async (id, codes) => {
+  await removeProgrammes(id, codes)
+  const user = await byId(id)
+  userDataCache.del(user.userId)
+  return formatUserForAdminView(user)
 }
 
 const updateUser = async (username, fields) => {
-  userDataCache.del(username)
-  const uid = username
-  const user = await User.byUsername(uid)
+  const user = await byUsername(username)
   if (!user) {
-    throw new Error(`User ${uid} not found`)
+    throw new Error(`User ${username} not found`)
   }
-  await User.updateUser(user, fields)
-  const returnedUser = await User.byUsername(username)
-  return enrichWithProgrammes(User.getUserData(returnedUser))
+  await User.upsert({ id: user.id, ...fields })
+  user.update(fields)
+
+  userDataCache.del(username)
+
+  return formatUserForAdminView(await byUsername(username))
 }
 
-const getLoginDataWithoutToken = async (uid, full_name, hyGroups, email, hyPersonSisuId) => {
-  const hasStudyGuidanceGroupAccess = await checkStudyGuidanceGroupsAccess(hyPersonSisuId)
-  const result = await User.loginWithoutToken(
-    uid,
-    full_name,
-    hyGroups,
-    email,
-    hyPersonSisuId,
-    hasStudyGuidanceGroupAccess
+const getAccessGroups = async () => await AccessGroup.findAll()
+
+// newer logic
+const enrichProgrammesFromFaculties = faculties =>
+  facultiesAndProgrammesForTrends.filter(f => faculties.includes(f.faculty_code)).map(f => f.programme_code)
+
+const findAll = async () =>
+  (
+    await User.findAll({
+      include: userIncludes,
+    })
+  ).map(user => ({
+    ...user.get(),
+    is_enabled: true,
+    elementdetails: _.uniqBy([
+      ...user.programme.map(p => p.elementDetailCode),
+      ...enrichProgrammesFromFaculties(user.faculty.map(f => f.faculty_code)),
+    ]),
+  }))
+
+const formatUser = async userFromDb => {
+  const [accessGroups, programmes, faculties] = [
+    ['accessgroup', 'group_code'],
+    ['programme', 'elementDetailCode'],
+    ['faculty', 'faculty_code'],
+  ].map(([field, code]) => userFromDb[field].map(entry => entry[code]))
+
+  const rights = _.uniqBy([...programmes, ...enrichProgrammesFromFaculties(faculties)])
+
+  const {
+    dataValues: { id, username: userId, full_name: name, email, language, sisu_person_id: sisPersonId },
+  } = userFromDb
+
+  const studentsUserCanAccess = _.uniqBy(
+    (await Promise.all([getStudentnumbersByElementdetails(rights), getAllStudentsUserHasInGroups(sisPersonId)])).flat()
   )
-  return result
+
+  // attribute naming is a bit confusing, but it's used so widely in oodikone, that it's not probably worth changing
+  return {
+    id, // our internal id, not the one from sis, (e.g. 101)
+    userId, // acually username (e.g. mluukkai)
+    name,
+    language,
+    sisPersonId, //
+    email,
+    is_enabled: true, // this is probably not needed: is here mainly because old userservice created users for every logged in user, even if they hadn't correct iamgroups
+    rights,
+    roles: accessGroups,
+    studentsUserCanAccess, // studentnumbers used in various parts of backend. For admin this is usually empty, since no programmes / faculties are set.
+    isAdmin: accessGroups.includes('admin'),
+  }
 }
 
-const getMockedUser = async (uid, asUser) => await User.superlogin(uid, asUser)
+const getMockedUser = async ({ userToMock, mockedBy }) => {
+  if (userDataCache.has(userToMock)) {
+    return userDataCache.get(userToMock)
+  }
+  const toReturn = {
+    ...(await formatUser(await byUsername(userToMock))),
+    mockedBy,
+  }
+  userDataCache.set(userToMock, toReturn)
+  return toReturn
+}
 
 const getUser = async ({ username, name, email, iamGroups, sisId }) => {
-  const { payload: user, isNew } = await getLoginDataWithoutToken(username, name, iamGroups, email, sisId)
-  if (isNew) {
-    await sendNotificationAboutNewUser({ userId: username, userFullName: name })
+  if (userDataCache.has(username)) {
+    return userDataCache.get(username)
   }
-  return user
+
+  const isNewUser = !(await User.findOne({ where: { username } }))
+  const language = 'fi'
+
+  await User.upsert({
+    full_name: name,
+    username,
+    email,
+    language,
+    sisu_person_id: sisId,
+    last_login: new Date(),
+  })
+
+  const userFromDb = await byUsername(username)
+  const formattedUser = await formatUser(userFromDb)
+  const { roles: currentAccessGroups } = formattedUser
+
+  // Modify accessgroups based on current groups and iamGroups
+  let newAccessGroups = []
+  if (await checkStudyGuidanceGroupsAccess(sisId)) newAccessGroups.push('studyGuidanceGroups')
+  if (iamGroups.includes(courseStatisticsGroup)) newAccessGroups.push('courseStatistics')
+  if (iamGroups.includes(hyOneGroup) || currentAccessGroups.includes('teachers')) newAccessGroups.push('teachers')
+  if (iamGroups.includes(toskaGroup) || currentAccessGroups.includes('admin')) newAccessGroups.push('admin')
+  if (_.difference(newAccessGroups, currentAccessGroups).length > 0) {
+    userFromDb.setAccessgroup(
+      (
+        await AccessGroup.findAll({
+          where: {
+            group_code: {
+              [Op.in]: newAccessGroups,
+            },
+          },
+        })
+      ).map(({ id }) => id)
+    )
+  }
+
+  if (isNewUser) await sendNotificationAboutNewUser({ userId: username, userFullName: name })
+
+  const toReturn = {
+    ...formattedUser,
+    roles: newAccessGroups,
+  }
+  userDataCache.set(username, toReturn)
+  return toReturn
 }
 
 module.exports = {
@@ -177,11 +284,7 @@ module.exports = {
   findAll,
   modifyAccess,
   getAccessGroups,
-  getUnitsFromElementDetails,
   setFaculties,
-  getUserDataFor,
-  getStudentsUserCanAccess,
-  getLoginDataWithoutToken,
   getUser,
   getMockedUser,
 }

services/backend/src/services/users/accessgroups.js

@@ -1,17 +0,0 @@
-const { AccessGroup } = require('../../models/models_user')
-const { Op } = require('sequelize')
-
-const byId = id => AccessGroup.findByPk(id)
-
-const findAll = () => AccessGroup.findAll()
-
-const byCodes = codes =>
-  AccessGroup.findAll({
-    where: {
-      group_code: {
-        [Op.in]: codes,
-      },
-    },
-  })
-
-module.exports = { byId, byCodes, findAll }

services/backend/src/services/users/hygroups.js

@@ -1,24 +0,0 @@
-const { HyGroup } = require('../../models/models_user')
-const { Op } = require('sequelize')
-
-const byId = id => HyGroup.findByPk(id)
-
-const findAll = () => HyGroup.findAll()
-
-const byCodes = codes =>
-  HyGroup.findAll({
-    where: {
-      code: {
-        [Op.in]: codes,
-      },
-    },
-  })
-
-const create = code => HyGroup.create({ code })
-
-const addGroups = async (uid, codes) => {
-  const user = await byId(uid)
-  const groups = await byCodes(codes)
-  await user.addHygroups(groups)
-}
-module.exports = { byId, byCodes, findAll, create, addGroups }

services/backend/src/services/users/users.js

@@ -1,339 +0,0 @@
-const Sequelize = require('sequelize')
-const _ = require('lodash')
-const moment = require('moment')
-const {
-  User,
-  UserElementDetails,
-  AccessGroup,
-  HyGroup,
-  UserFaculties,
-  sequelizeUser,
-} = require('../../models/models_user')
-const AccessService = require('./accessgroups')
-const HyGroupService = require('./hygroups')
-const Op = Sequelize.Op
-
-// move these to conf
-const courseStatisticsGroup = 'grp-oodikone-basic-users'
-const hyOneGroup = 'hy-one'
-const requiredGroup =
-  process.env.NODE_ENV === 'staging'
-    ? ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
-    : ['grp-oodikone-users', 'grp-oodikone-basic-users']
-
-// help for refactoring away from token
-const generateTokenData = async (uid, mockedBy = null) => {
-  const user = await byUsername(uid)
-  const userData = getUserData(user)
-  const programmes = userData.elementdetails
-  const payload = {
-    id: user.id,
-    userId: uid, // username
-    name: user.full_name,
-    enabled: userData.is_enabled,
-    language: user.language,
-    mockedBy,
-    rights: programmes,
-    roles: user.accessgroup,
-    createdAt: moment().toISOString(),
-    sisPersonId: user.sisu_person_id,
-  }
-  return payload
-}
-
-const createMissingGroups = async (group, service) => {
-  const savedGroups = await service.findAll()
-  group.forEach(async code => {
-    if (!savedGroups.map(sg => sg.code).includes(code)) {
-      await service.create(code)
-    }
-  })
-}
-
-const updateGroups = async (user, hyGroups) => {
-  let hyGroupsToBeUpdated = (await user.getHy_group()).map(hg => hg.code)
-  let hyGroupsToAdd = []
-  let hyGroupsToDelete = []
-
-  hyGroups.forEach(async hyGroup => {
-    if (!hyGroupsToBeUpdated.includes(hyGroup)) {
-      hyGroupsToAdd = hyGroupsToAdd.concat(hyGroup)
-    }
-  })
-  hyGroupsToBeUpdated.forEach(async hyGroup => {
-    if (!hyGroups.includes(hyGroup)) {
-      hyGroupsToDelete = hyGroupsToDelete.concat(hyGroup)
-    }
-  })
-
-  await user.addHy_group(await HyGroupService.byCodes(hyGroupsToAdd))
-  await user.removeHy_group(await HyGroupService.byCodes(hyGroupsToDelete))
-}
-
-const loginBase = async (uid, full_name, hyGroups, mail, hyPersonSisuId, hasStudyGuidanceGroupAccess) => {
-  let user = await byUsername(uid)
-  let isNew = false
-  const lastLogin = new Date()
-  await createMissingGroups(hyGroups, HyGroupService)
-
-  if (!user) {
-    user = await createUser(uid, full_name, mail, hyPersonSisuId, lastLogin)
-
-    const userHyGroups = await HyGroupService.byCodes(hyGroups)
-    await user.addHy_group(userHyGroups)
-
-    isNew = true
-  } else {
-    user = await updateUser(user, { full_name, email: mail, sisu_person_id: hyPersonSisuId, last_login: lastLogin })
-    await updateGroups(user, hyGroups)
-  }
-
-  await determineAccessToCourseStats(user, hyGroups)
-  await determineAccessToTeachersForOne(user, hyGroups)
-  await determineAccessToStudyGuidanceGroups(user, hasStudyGuidanceGroupAccess)
-
-  return isNew
-}
-
-const loginWithoutToken = async (
-  uid,
-  full_name,
-  hyGroups,
-  affiliations,
-  mail,
-  hyPersonSisuId,
-  hasStudyGuidanceGroupAccess
-) => {
-  const isNew = await loginBase(
-    uid,
-    full_name,
-    hyGroups,
-    affiliations,
-    mail,
-    hyPersonSisuId,
-    hasStudyGuidanceGroupAccess
-  )
-  const payload = await generateTokenData(uid)
-  return { payload, isNew }
-}
-
-const superlogin = async (uid, asUser) => {
-  const user = await byUsername(uid)
-  if (user && user.accessgroup.map(r => r.group_code).includes('admin')) {
-    const token = await generateTokenData(asUser, uid)
-    return token
-  }
-  return undefined
-}
-
-const userIncludes = [
-  {
-    separate: true,
-    model: UserElementDetails,
-    as: 'programme',
-  },
-  {
-    model: AccessGroup,
-    as: 'accessgroup',
-    attributes: ['id', 'group_code', 'group_info'],
-  },
-  {
-    required: false,
-    model: HyGroup,
-    as: 'hy_group',
-    where: { code: requiredGroup },
-  },
-  {
-    separate: true,
-    model: UserFaculties,
-    as: 'faculty',
-  },
-]
-
-const getUserData = user => {
-  if (user == null) return null
-  const newuser = user.get()
-  newuser.elementdetails = getUserProgrammes(newuser)
-  const hyGroups = newuser.hy_group.map(e => e.code)
-  newuser.is_enabled = requiredGroup === null || _.intersection(hyGroups, requiredGroup).length > 0
-  newuser.hy_group = null
-  return newuser
-}
-
-const byUsernameMinified = async username => {
-  const userMinified = await User.findOne({
-    where: {
-      username,
-    },
-    include: [
-      {
-        separate: true,
-        model: UserElementDetails,
-        as: 'programme',
-        attributes: ['elementDetailCode'],
-      },
-      {
-        model: AccessGroup,
-        as: 'accessgroup',
-        attributes: ['group_code'],
-      },
-      {
-        separate: true,
-        model: UserFaculties,
-        as: 'faculty',
-        attributes: ['faculty_code'],
-      },
-    ],
-  })
-  return userMinified
-}
-
-const byUsername = async username => {
-  const user = await User.findOne({
-    where: {
-      username: {
-        [Op.eq]: username,
-      },
-    },
-    include: userIncludes,
-  })
-  return user
-}
-
-const createUser = async (username, fullname, email, hyPersonSisuId, lastLogin) => {
-  return User.create({
-    username: username,
-    full_name: fullname,
-    email,
-    sisu_person_id: hyPersonSisuId,
-    last_login: lastLogin,
-  })
-}
-
-const updateUser = async (userObject, values) => {
-  await User.upsert({ id: userObject.id, ...values })
-  return userObject.update(values)
-}
-
-const byId = async id => {
-  const user = await User.findOne({
-    where: {
-      id: {
-        [Op.eq]: id,
-      },
-    },
-    include: userIncludes,
-  })
-  return user
-}
-
-const getUserProgrammes = user => user.programme.map(p => p.elementDetailCode)
-
-const getUserAccessGroups = async username => {
-  const user = await byUsername(username)
-  return await user.getAccessgroup()
-}
-
-const findAll = async () => {
-  const users = await User.findAll({
-    include: userIncludes,
-  })
-  return users
-}
-
-const addProgrammes = async (uid, codes) => {
-  for (const code of codes) {
-    await UserElementDetails.upsert({ userId: uid, elementDetailCode: code })
-  }
-}
-
-const removeProgrammes = async (uid, codes) => {
-  for (const code of codes) {
-    await UserElementDetails.destroy({
-      where: { userId: uid, elementDetailCode: code },
-    })
-  }
-}
-
-const setFaculties = async (uid, faculties) => {
-  await sequelizeUser.transaction(async transaction => {
-    await UserFaculties.destroy({ where: { userId: uid }, transaction })
-    for (const faculty of faculties) {
-      await UserFaculties.create({ userId: uid, faculty_code: faculty }, { transaction })
-    }
-  })
-}
-
-const modifyRights = async (username, rights) => {
-  const rightsToAdd = Object.entries(rights)
-    .map(([code, val]) => {
-      if (val === true) {
-        return code
-      }
-    })
-    .filter(code => code)
-  const rightsToRemove = Object.entries(rights)
-    .map(([code, val]) => {
-      if (val === false) {
-        return code
-      }
-    })
-    .filter(code => code)
-
-  const user = await byUsername(username)
-  const accessGroupsToAdd = await AccessService.byCodes(rightsToAdd)
-  const accessGroupsToRemove = await AccessService.byCodes(rightsToRemove)
-
-  await user.addAccessgroup(accessGroupsToAdd)
-  await user.removeAccessgroup(accessGroupsToRemove)
-}
-const getRoles = async user => {
-  const foundUser = byUsername(user)
-  return foundUser.accessgroup
-}
-
-const determineAccessToCourseStats = async (user, hyGroups) => {
-  const accessGroups = (user && user.accessgroup) || []
-  const alreadyAccess = accessGroups.some(({ group_code }) => group_code === 'courseStatistics')
-  if (hyGroups.includes(courseStatisticsGroup) && !alreadyAccess) {
-    await modifyRights(user.username, { courseStatistics: true })
-  } else if (!hyGroups.includes(courseStatisticsGroup) && alreadyAccess) {
-    await modifyRights(user.username, { courseStatistics: false })
-  }
-}
-
-const determineAccessToTeachersForOne = async (user, hyGroups) => {
-  const accessGroups = (user && user.accessgroup) || []
-  const alreadyAccess = accessGroups.some(({ group_code }) => group_code === 'teachers')
-  if (hyGroups.includes(hyOneGroup) && !alreadyAccess) {
-    await modifyRights(user.username, { teachers: true })
-  }
-}
-
-const determineAccessToStudyGuidanceGroups = async (user, hasStudyGuidanceGroupAccess) => {
-  const accessGroups = (user && user.accessgroup) || []
-  const alreadyAccess = accessGroups.some(({ group_code }) => group_code === 'studyGuidanceGroups')
-  if (hasStudyGuidanceGroupAccess && !alreadyAccess) {
-    await modifyRights(user.username, { studyGuidanceGroups: true })
-  } else if (!hasStudyGuidanceGroupAccess && alreadyAccess) {
-    await modifyRights(user.username, { studyGuidanceGroups: false })
-  }
-}
-
-module.exports = {
-  byUsername,
-  byUsernameMinified,
-  updateUser,
-  findAll,
-  byId,
-  getUserProgrammes,
-  addProgrammes,
-  removeProgrammes,
-  superlogin,
-  modifyRights,
-  getUserAccessGroups,
-  getRoles,
-  getUserData,
-  setFaculties,
-  loginWithoutToken,
-}

services/backend/src/util/semester.js

@@ -36,9 +36,27 @@ const getAcademicYearDates = year => {
   }
 }
 
+const getYearStartAndEndDates = (year, isAcademicYear) => {
+  if (isAcademicYear) {
+    const firstYear = year.slice(0, 4)
+    return {
+      startDate: new Date(`${firstYear}-${semesterStart['FALL']}`).toISOString(),
+      endDate: new Date(
+        `${moment(firstYear, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`
+      ).toISOString(),
+    }
+  }
+
+  return {
+    startDate: new Date(`${year}-01-01`).toISOString(),
+    endDate: new Date(`${year}-12-31`).toISOString(),
+  }
+}
+
 module.exports = {
   getPassingSemester,
   getAcademicYearDates,
+  getYearStartAndEndDates,
   semesterEnd,
   semesterStart,
 }

services/backend/src/util/utils.js

@@ -11,6 +11,34 @@ const mapToProviders = elementDetails =>
     return r
   })
 
+// sort substitutions so that main code is first
+const newLetterBasedCode = /^[A-Za-z]/ // new letter based codes come first
+const oldNumericCode = /^\d/ // old numeric codes come second
+const openUniCode = /^AY?(.+?)(?:en|fi|sv)?$/ // open university codes come last
+const openUniCodeA = /A\d/ // open university with just A come last
+const digi = /DIGI-A?(.+?)(?:en|fi|sv)?$/ // digi-a goes on top courses goes third
+const bscsCode = /BSCS??/
+
+const codeRegexes = [openUniCodeA, openUniCode, bscsCode, oldNumericCode, newLetterBasedCode, digi]
+
+const getSortRank = code => {
+  for (let i = 0; i < codeRegexes.length; i++) {
+    if (codeRegexes[i].test(code)) {
+      return i
+    }
+  }
+  return 3 // if no hit, put before open uni courses
+}
+
+const sortMainCode = codeArray => {
+  if (!codeArray) return []
+  return codeArray.sort(function (x, y) {
+    return getSortRank(y) - getSortRank(x)
+  })
+}
+
 module.exports = {
   mapToProviders,
+  sortMainCode,
+  getSortRank,
 }

services/frontend/package-lock.json

@@ -6837,9 +6837,9 @@
       }
     },
     "follow-redirects": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz",
-      "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg=="
+      "version": "1.14.8",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz",
+      "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA=="
     },
     "for-in": {
       "version": "1.0.2",

services/frontend/src/apiConnection/index.js

@@ -10,7 +10,7 @@ export const getHeaders = () => {
     uid: 'mluukkai',
     displayName: 'Matti Luukkainen',
     'shib-session-id': 'mock-session',
-    hyGroupCn: 'grp-oodikone-users;grp-oodikone-basic-users',
+    hyGroupCn: 'grp-oodikone-users;grp-oodikone-basic-users;grp-toska',
     mail: 'grp-toska+mockmluukkai@helsinki.fi',
     hyPersonSisuId: 'hy-hlo-1441871',
     shib_logout_url: 'https://helsinki.fi/shibboleth-sp/Logout',

services/frontend/src/common/InfoToolTips/courseStatistics.js

@@ -1,7 +1,7 @@
 export default {
-  0: {
-    Attempts: `
-      Table - Attempts
+  Tables: {
+    ATTEMPTS: `
+      **Table - Attempts**
 
       Taulukko näyttää suorituskerrat lukuvuositasolla.
       Samalla opiskelijalla voi olla useampia suorituskertoja.
@@ -13,8 +13,8 @@ export default {
       Arvosanajakaumaa katsoessasi voit muuttaa luvut prosenteiksi, klikkaamalla painiketta "Relative".
       Mahdollinen sarake "Other passed" sisältää sekä arvosanan "Hyv." saaneet että kurssin hyväksilukeneet.
     `,
-    Students: `
-      Table - Students
+    STUDENT: `
+      **Table - Students**
 
       Taulukko näyttää yksittäiset opiskelijat, jotka ovat kurssia suorittaneet.
       Jokainen opiskelija on jaoteltu luokkiin: "läpäissyt ensiyrittämällä", "läpäissyt lopulta" tai "ei koskaan läpäissyt kurssia".
@@ -26,17 +26,17 @@ export default {
       Total-rivillä näkyy niiden opiskelijoiden kokonaismäärä, jotka ovat kurssista joskus arvosanan saaneet.
     `,
   },
-  1: {
-    Attempts: `
-      Pass rate chart - Attempts
+  PassRate: {
+    ATTEMPTS: `
+      **Pass rate chart - Attempts**
 
       Graafi näyttää suorituskerrat lukuvuositasolla jaoteltuina hyväksyttyihin ja hylättyihin suorituksiin.
       Hyväksytyt suoritukset sisältävät arvosanat 1-5 sekä arvosanan Hyv.
       Hylätyt suoritukset sisältävät arvosanat Hyl. ja 0, sekä suoritukset, joille opettaja on antanut merkinnän
       "Luop" (Luopunut suorituksesta) tai "Eisa" (Ei saapunut tenttiin).
     `,
-    Students: `
-      Pass rate chart - Students
+    STUDENT: `
+      **Pass rate chart - Students**
 
       Graafi näyttää yksittäiset opiskelijat, jotka ovat kurssia suorittaneet.
       Jokainen opiskelija on jaoteltu luokkiin: "läpäissyt ensiyrittämällä", "läpäissyt lopulta" tai "ei koskaan läpäissyt kurssia".
@@ -46,18 +46,9 @@ export default {
       myöhemmin keväällä 2019, tulisi merkityksi lukuvuodelle 2017-2018 pylvääseen "passed eventually".        
     `,
   },
-  2: {
-    Attempts: `
-      Grade distribution
-  
-      Graafi näyttää suorituskertojen arvosanajakauman eri lukuvuosille. Mahdollinen kategoria
-      "Other passed" sisältää sekä arvosanan "Hyv." että kurssin hyväksiluvut.
-    `,
-    Students: `
-      Grade distribution
+  GradeDistribution: `
+    **Grade distribution**
 
-      Graafi näyttää suorituskertojen arvosanajakauman eri lukuvuosille. Mahdollinen kategoria
-      "Other passed" sisältää sekä arvosanan "Hyv." että kurssin hyväksiluvut.
-    `,
-  },
+    Graafi näyttää suorituskertojen arvosanajakauman eri lukuvuosille. Mahdollinen kategoria
+  `,
 }

services/frontend/src/common/InfoToolTips/filters.js

@@ -6,7 +6,8 @@ export default {
   },
   transferred: {
     label: 'Include and exclude students from this program',
-    short: 'Mahdollisuus suodattaa opiskelijat, jotka ovat siirtyneet tutkinto-ohjelmaan',
+    short:
+      'Mahdollisuus suodattaa opiskelijat, jotka ovat siirtyneet nykyiseen 2017 alkaneeseen tutkinto-ohjelmaan vanhasta tutkinto-ohjelmasta',
   },
   courseCredits: {
     label: null,

services/frontend/src/common/InfoToolTips/index.jsx

@@ -1,5 +1,4 @@
 import PopulationStatistics from './populationStatistics'
-import PopulationOverview from './populationOverview'
 import CoolDataScience from './coolDataScience'
 import CourseStatistics from './courseStatistics'
 import CoursePopulation from './coursePopulation'
@@ -7,7 +6,6 @@ import Studyprogramme from './studyprogramme'
 
 export default {
   PopulationStatistics,
-  PopulationOverview,
   CoolDataScience,
   CourseStatistics,
   CoursePopulation,

services/frontend/src/common/InfoToolTips/populationOverview.js

@@ -1,24 +0,0 @@
-export default {
-  PopulationProgress: `Tämä koontisivu sisältää koulutusohjelman kaikkien opiskelijoiden perustiedot lukuvuosittain. Luvut perustuvat opinto-oikeuksiin kyseisessä ohjelmassa.
-
-  Population progress:
-  Vuosittain ohjelmassa aloittaneet, ohjelman keskeyttäneet / peruneet sekä ohjelmaan siirtyneet opiskelijat. 
-  Huomattavaa on, että sama opiskelija voi olla kahdessa luokassa, jos hän on esimerkiksi aloittanut opinnot jo, mutta keskeyttänyt ne lukuvuoden aikana.
-  Tarkemmat tiedot saat esille klikkaamalla nuolta kunkin lukuvuoden kohdalla.`,
-  YearlyProductivity: `
-  Yearly productivity:
-  Koulutusohjelman kunakin kalenterivuonna tuottamat opintopisteet, valmistuneiden määrän, ohjelman pääaineopiskelijoiden
-  opintopisteet sekä muiden kuin pääaineopiskelijoiden opintopisteet. Mukana on myös kuhunkin koulutusohjelmaan hyväksiluetut opintopisteet.
-  
-  Vanhojen koulutusohjelmien lakkauttaminen ja opiskelijoiden siirtäminen uusiin ohjelmiin voi aiheuttaa tavallista suuremmat opintopistemäärät lukuvuodelle 2020-2021.`,
-  Year: `Lukuvuosi`,
-  Students: `Opiskelijat, jotka ovat koulutusohjelmassa aloittaneet, opinto-oikeutensa peruneet sekä koulutusohjelmaan tai siitä pois siirtyneet.`,
-  Started: `Koulutusohjelmassa kunakin lukuvuonna aloittaneet opiskelijat. EI sisällä ohjelmaan siirtyneitä opiskelijoita.`,
-  Cancelled: `Koulutusohjelmaan oikeuttavan opinto-oikeutensa kunakin lukuvuonna peruttaneet`,
-  Graduated: `Tiettynä lukuvuonna aloittaneista opiskelijoista on valmistunut`,
-  Transferred: `Koulutusohjelmaan kyseisenä lukuvuonna siirtyneet opiskelijat. 
-  
-  Vanhojen koulutusohjelmien lakkauttaminen aiheuttaa tavallista suuremmat siirtoluvut lukuvuodelle 2020-2021.`,
-  Credits: `Kunakin vuonna aloittaneista opiskelijoista ne, jotka ovat ylittäneet tietyn opintopisterajapyykin`,
-  Thesis: `Loppututkielman kirjoittaneiden määrä`,
-}

services/frontend/src/common/InfoToolTips/populationStatistics.js

@@ -71,6 +71,10 @@ export default {
   - **Study programme:** haluttu koulutusohjelma.
   - **Select tag (Optional):** itsetehty tai valmis tägi.
   - **Study track (Optional):** opintosuunta.`,
+  TagAndTrackMovedIntoFilters: `
+  Tägien ja opintosuuntien valinnat ovat siirtyneet tästä omiksi filtereikseen.
+  Voit tehdä valinnat siirryttyäsi tarkastelemaan ohjelman tilastoja.
+  `,
   Students: `**Students**
 
   Taulukko näyttää oletusarvoisesti vain opiskelijanumerot, mutta harmaata liukukytkintä klikkaamalla saa näkyviin opiskelijoiden nimet.
@@ -133,4 +137,5 @@ export default {
   ProgrammeDistributionCoursePopulation: `Näyttää sen koulutusohjelman, jossa opiskelija oli suorittaessaan kurssin`,
   GradeDistributionCoursePopulation: `Näyttää korkeimman arvosanan, jonka opiskelija on saanut valitulla aikavälillä.`,
   CreditDistributionCoursePopulation: `Näyttää tiedekunta-/ohjelmankohtaisen opintopistejakauman.`,
+  LanguageDistributionCoursePopulation: `Näyttää kielijakauman kurssin suorituskielistä.`,
 }

services/frontend/src/common/InfoToolTips/studyprogramme.js

@@ -1,43 +1,74 @@
 export default {
+  YearToggle: `
+  Academic year: Näyttää tilastot akateemisille vuosille jaoteltuna, esimerkiksi 1.8.2019-31.7.2020
+  Calendar year: Näyttää tilastot kalenterivuosille jaoteltuna, esimerkiksi 1.1.2019-31.12.2019 
+  `,
+  StudentToggle: `
+  All studyrights included: Valinnassa ovat mukana vaihto-opiskelijat, erillisoikeudella opiskelevat, opintonsa keskeyttäneet opiskelijat, ohjelmaan siirtyneet opiskelijat sekä ohjelmasta pois siirtyneet opiskelijat.
+  
+  Special studyrights excluded: Valinnasta ja kaikista luvuista on poissuljettu kaikki edellä mainitut erikoisryhmät.
+  `,
+  GradToggle: `
+  Graduated included: Valinnassa ovat mukana ohjelmasta jo valmistuneet opiskelijat.
+  
+  Graduated excluded: Valinnasta on poistettu ohjelmasta jo valmistuneet opiskelijat.
+  `,
   StudentsOfTheStudyprogramme: `
-  **Started**: Sisältää kyseisenä vuonna aloitetut opinto-oikeudet riippumatta siitä, minä vuonna kyseinen opiskeluoikeus on myönnetty. Opiskelija on myös voinut keskeyttää opintonsa tai siirtyä pois ohjelmasta.\n
+  **All**: Sisältää kaikki kyseisenä vuonna alkaneet opinto-oikeudet riippumatta siitä, onko opintoja aloitettu tai onko niitä aloitettu ko. vuonna. Mukana myös ohjelmaan siirtyneet opiskelijat.
+
+  **Started studying**: Sisältää kyseisenä vuonna aloittaneet opiskelijat (aloitetut opinto-oikeudet) riippumatta siitä, minä vuonna kyseinen opiskeluoikeus on myönnetty. Opiskelija on myös voinut myöhemmin keskeyttää opintonsa tai siirtyä pois ohjelmasta.\n
   **Graduated**: Sisältää kyseisenä vuonna valmistuneet opiskelijat riippumatta siitä, minä vuonna opiskeluoikeus ohjelmaan on myönnetty. Jaottelu on tehty opinto-oikeuden päättymispäivän (=valmistumispäivän) perusteella.
+  
   **Cancelled**: Sisältää kyseisenä vuonna perutut opiskeluoikeudet riippumatta siitä, onko kyseinen opiskelija ikinä edes alkanut opiskelemaan ohjelmassa tai opiskelun aloitusvuodesta.
   
-  **Transferred away**: Sisältää kyseisenä vuonna pois ohjelmasta siirtyneet opinto-oikeudet. Sama opiskelija voi esiintyä tilastoissa useana vuonna. Kyseiset opiskelijat ovat saattaneet aloittaa ohjelmassa tai eivät koskaan ole aloittaneet ohjelmassa. 
+  **Transferred away**: Sisältää kyseisenä vuonna pois ohjelmasta siirtyneet opiskelijat (opinto-oikeudet). Sama opiskelija voi esiintyä tilastoissa useana vuonna, mikäli hän on siirtynyt ohjelmaan ja siitä pois useasti.
   
-  **Transferred to**: Sisältää kyseisenä vuonna ohjelmaan siirtyneet opinto-oikeudet.
+  **Transferred to**: Sisältää kyseisenä vuonna ohjelmaan siirtyneet opiskelijat (opinto-oikeudet).
 
   HUOM! Ohjelmaan siirtyneiden määrä on erityisen suuri vuonna 2020, jolloin vanhoista koulutusohjelmista luovuttiin ja opiskelijat siirtyivät uusiin ohjelmiin.
   `,
   CreditsProducedByTheStudyprogramme: `
-  Sisältää opintopisteet suoritusvuosittain (Sisussa näkyvä kurssin suorituspäivä)
+  Sisältää opintopisteet suoritusvuosittain. Suoritukset on jaoteltu Sisussa näkyvän kurssin suorituspäivän mukaan.
 
-  **Major students' credits**: Sisältää ne opintopisteet, joiden suorittajalla on suoritushetkellä ollut ensisijainen opinto-oikeus kyseiseen koulutusohjelmaan.
+  **Major students' credits**: Sisältää ne opintopisteet, joiden suorittajalla on suoritushetkellä ollut ensisijainen opinto-oikeus kyseiseen koulutusohjelmaan. Sisältää myös ohjelmaan siirtyneet ensisijaiset opiskelijat.
 
-  **Non-major students' credits**: Sisältää koulutusohjelman tuottamat opintopisteet, joiden suorittajalla EI ole ollut suoritushetkellä ensisijaista opinto-oikeutta kyseiseen koulutusohjelmaan.
+  **Non-major students' credits**: Sisältää koulutusohjelman tuottamat opintopisteet, joiden suorittajalla EI ole ollut suoritushetkellä ensisijaista opinto-oikeutta kyseiseen koulutusohjelmaan. Sisältää
+  esimerkiksi vaihto-opiskelijat ja erillis-opinto-oikeudella opiskelevat. Sisältää myös opintopisteet, jotka opiskelija on suorittanut koulutusohjelmaan ennen opintojen aloittamista koulutusohjelmassa tai opintojen päätyttyä.
   
-  **Transferred credits**: Sisältää opintopisteet, jotka on suoritettu kyseisenä vuonna, ja hyväksiluettu tähän koulutusohjelmaan.
+  **Transferred credits**: Sisältää opintopisteet, jotka on suoritettu kyseisenä vuonna, ja hyväksiluettu tähän koulutusohjelmaan. Mukana ovat kaikenlaisilla opinto-oikeuksilla hyväksiluetut opintopisteet.
   `,
   GraduatedAndThesisWritersOfTheProgramme: `
-  **Wrote thesis**: Sisältää kyseisenä vuonna kandidaatintutkielman hyväksytysti suorittaneet opiskelijat. Mukana ei ole mahdollisia arvosanan korotuksia.
+  **Wrote thesis**: Sisältää kyseisenä vuonna kandidaatintutkielman tai pro gradu-tutkielman hyväksytysti suorittaneet opiskelijat. Mukana ei ole mahdollisia arvosanan korotuksia.
 
   **Graduated**: Sisältää kyseisenä vuonna valmistuneet opiskelijat riippumatta siitä, minä vuonna opiskeluoikeus ohjelmaan on myönnetty. Jaottelu on tehty opinto-oikeuden päättymispäivän (=valmistumispäivän) perusteella.
   `,
   AverageGraduationTimes: `
-  **Median time**: Näyttää kyseisenä vuonna valmistuneiden opintojen keston **mediaanin** kuukausina.
+  **Median time**: Näyttää kyseisenä vuonna valmistuneiden opiskelijoiden opintojen keston **mediaanin** kuukausina. Luku sisältää mahdolliset poissaolokuukaudet.
 
-  **Mean time**: Näyttää kyseisenä vuonna valmistuneiden opintojen keston **keskiarvon** kuukausina.
+  **Mean time**: Näyttää kyseisenä vuonna valmistuneiden opiskelijoiden opintojen keston **keskiarvon** kuukausina. Luku sisältää mahdolliset poissaolokuukaudet.
   
-  Mukana ovat **vain valmistuneet opiskelijat**. Kandiohjelmissa tavoiteaika on 36 kk, maisteriohjelmissa 24 kk`,
-  StudytrackOverview: `
-  Taulukko näyttää kyseisenä lukuvuonna aloittaneet pääaineopiskelijat. Taulukko näyttää myös näistä opiskelijoista miesten, naisten ja suomalaisten osuudet sekä valmistuneiden määrän. 
+  n = **valmistuneet opiskelijat / kaikki ohjelmassa kyseisenä vuonna alkaneet opinto-oikeudet**. Kandiohjelmissa tavoiteaika on 36 kk, maisteriohjelmissa 24 kk`,
+  ProgrammesAfterGraduation: `
+  Sisältää maisteriohjelmat, joissa tämän kandiohjelman opiskelijat ovat aloittaneet valmistuttuaan kandeiksi. Vuosijaottelu on tehty maisteriohjelmassa aloittamisen perusteella.
+
+  Mukana ovat **vain valmistuneet opiskelijat ja vain maisteriohjelmat**.
+  `,
+  StudytrackOverview: `  
+  Kuvaa koulutusohjelman kyseiseen ohjelmaan opinto-oikeudella varustettujen opiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
+  
+  HUOM!
+  **All** Sisältää kaikki opiskelijat, joiden opinto-oikeudella on aloituspäivä kyseisenä vuonna, riippumatta siitä, onko opiskelija aloittanut opiskelemaan ko. vuonna.
+  **Started studying** Sisältää kaikki opiskelijat, jotka ovat aloittaneet opiskelemaan koulutusohjelmassa kyseisenä lukuvuonna.
+  
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, tutkinnonsuoritusoikeudesta luopuneet ja erillisopinto-oikeudella opiskelevat.
+  Taulukko näyttää myös näistä opiskelijoista miesten, naisten ja suomalaisten osuudet sekä valmistuneiden määrän. 
 
   Yläosan valikosta on mahdollista valita tarkasteluun yhden opintosuunnan opiskelijat. Sekä opiskelijoiden yleistiedot että edistyminen opintopisteittäin
   kuvaavat tällöin kyseisen opintosuunnan opiskelijoita.
   `,
   StudytrackProgress: `
-  Lähtökohtaisesti kuvaa koulutusohjelman kaikkien pääaineoopiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
+  Kuvaa koulutusohjelman kyseiseen ohjelmaan opinto-oikeudella varustettujen opiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, tutkinnonsuoritusoikeudesta luopuneet ja erillisopinto-oikeudella opiskelevat.   
 
   Mikäli yläosan valikosta valittuna on jokin tietty opintosuunta, myös nämä graafit ja taulukot kuvaavat kyseisen opintosuunnan opiskelijoita.    
   `,

services/frontend/src/common/index.js

@@ -18,8 +18,8 @@ export const images = {
   irtomikko,
 }
 
-export const checkUserAccess = (requiredRoles, userRoles) => {
-  return intersection(userRoles, requiredRoles).length > 0
+export const checkUserAccess = (requiredRoles, roles) => {
+  return intersection(requiredRoles, roles).length > 0
 }
 
 export const containsOnlyNumbers = str => str.match('^\\d+$')
@@ -160,7 +160,7 @@ export const getNewestProgramme = (studyrights, studentNumber, studentToTargetCo
         elementDetails[srE.code] &&
         elementDetails[srE.code].type === 20 &&
         (studentToTargetCourseDateMap && studentNumber
-          ? moment(studentToTargetCourseDateMap[studentNumber]).isBetween(moment(srE.startdate), moment(srE.enddate))
+          ? moment(studentToTargetCourseDateMap[studentNumber]).isBetween(srE.startdate, srE.enddate, 'day', '[]')
           : true)
     )
     if (studyrightElements.length > 0) {
@@ -179,6 +179,15 @@ export const getNewestProgramme = (studyrights, studentNumber, studentToTargetCo
   if (programme) {
     return programme
   }
+
+  if (studentToTargetCourseDateMap) {
+    return {
+      name: { en: 'No programme at the time of attainment', fi: 'Ei ohjelmaa suorituksen hetkellä' },
+      startdate: '',
+      code: '00000',
+    }
+  }
+
   return { name: { en: 'No programme', fi: 'Ei ohjelmaa' }, startdate: '', code: '00000' }
 }
 

services/frontend/src/components/CoolDataScience/Status.jsx

@@ -59,6 +59,7 @@ const StatusContainer = ({
   max1,
   showYearlyValues,
   showRelativeValues,
+  showCountingFrom,
   showByYear,
   showStudentCounts,
   clickable,
@@ -84,11 +85,31 @@ const StatusContainer = ({
   let yearlyValues = null
 
   if (showYearlyValues) {
-    yearlyValues = _.orderBy(Object.entries(stats.yearly), ([y]) => y, ['desc']).map(([year, yearStats]) => ({
-      label: showByYear ? year : `${year}-${`${Number(year) + 1}`.slice(-2)}`,
-      accumulated: showStudentCounts && stats.type === 'course' ? yearStats.accStudents : yearStats.acc,
-      total: showStudentCounts && stats.type === 'course' ? yearStats.totalStudents : yearStats.total,
-    }))
+    yearlyValues = _.chain(stats.yearly)
+      .toPairs()
+      .map(([year, stats]) => ({
+        dateRange: {
+          start: showByYear ? moment([year]).startOf('year') : moment([year]).month(7).date(1),
+          end: showByYear
+            ? moment([year]).startOf('year')
+            : moment([parseInt(year, 10) + 1])
+                .month(6)
+                .date(31),
+        },
+        stats,
+      }))
+      .filter(({ dateRange: { start } }) => moment(showCountingFrom).isSameOrAfter(start))
+      .orderBy(({ dateRange: { start } }) => start, ['desc'])
+      .map(({ dateRange, stats }) => ({
+        label: _.chain([dateRange.start, dateRange.end])
+          .map(date => date.year())
+          .uniq()
+          .join('-')
+          .value(),
+        accumulated: showStudentCounts && stats.type === 'course' ? stats.accStudents : stats.acc,
+        total: showStudentCounts && stats.type === 'course' ? stats.totalStudents : stats.total,
+      }))
+      .value()
   }
 
   return (
@@ -108,7 +129,7 @@ const StatusContainer = ({
 }
 
 const StatusContent = ({ data, settings }) => {
-  const { showByYear, showRelativeValues, showYearlyValues, showStudentCounts } = settings
+  const { showByYear, showRelativeValues, showYearlyValues, showStudentCounts, showCountingFrom } = settings
 
   return (
     <DrillStack
@@ -124,6 +145,7 @@ const StatusContent = ({ data, settings }) => {
           showRelativeValues={showRelativeValues}
           showYearlyValues={showYearlyValues}
           showStudentCounts={showStudentCounts}
+          showCountingFrom={showCountingFrom}
           min1={-2 * medianDiff}
           max1={2 * medianDiff}
           showByYear={showByYear}

services/frontend/src/components/CoursePopulation/index.jsx

@@ -1,6 +1,8 @@
 import React, { useEffect, useState, useMemo, useRef } from 'react'
 import { withRouter } from 'react-router-dom'
 import { connect } from 'react-redux'
+import moment from 'moment'
+import _ from 'lodash'
 import { Segment, Header, Accordion } from 'semantic-ui-react'
 import scrollToComponent from 'react-scroll-to-component'
 import InfoBox from 'components/Info/InfoBox'
@@ -28,6 +30,7 @@ import {
   creditsEarnedFilter,
   startYearAtUniFilter,
   programmeFilter,
+  studyTrackFilter,
 } from '../FilterView/filters'
 import FilterView from '../FilterView'
 import useLanguage from '../LanguagePicker/useLanguage'
@@ -39,7 +42,6 @@ const CoursePopulation = ({
   getElementDetails,
   studentData,
   pending,
-  elementDetails,
   history,
   courseData,
   getSemestersDispatch,
@@ -190,7 +192,7 @@ const CoursePopulation = ({
       content: {
         content: (
           <div ref={languageDistRef}>
-            <InfoBox content={infotooltips.PopulationStatistics.GradeDistributionCoursePopulation} />
+            <InfoBox content={infotooltips.PopulationStatistics.LanguageDistributionCoursePopulation} />
             <CoursePopulationLanguageDist from={dateFrom} to={dateTo} samples={studentData.students} codes={codes} />
           </div>
         ),
@@ -291,6 +293,17 @@ const CoursePopulation = ({
 
   const courses = JSON.parse(queryParamsFromUrl(history.location).coursecodes)
 
+  const studyRightPredicate = (student, sre) => {
+    const date = _.chain(student)
+      .get('courses')
+      .filter(c => courses.includes(c.course_code))
+      .map('date')
+      .max()
+      .value()
+
+    return moment(date).isBetween(sre.startdate, sre.enddate, 'day', '[]')
+  }
+
   return (
     <FilterView
       name="CoursePopulation"
@@ -300,13 +313,28 @@ const CoursePopulation = ({
         courseFilter({ courses: courseStatistics?.coursestatistics }),
         creditsEarnedFilter,
         startYearAtUniFilter,
-        programmeFilter({ courses, elementDetails }),
+        programmeFilter({
+          additionalModes: [
+            {
+              key: 'attainment',
+              label: 'Attainment',
+              predicate: studyRightPredicate,
+              description: 'Student had an active study right at the time of course attainment.',
+            },
+          ],
+        }),
         gradeFilter({
           courseCodes: courses,
           from: dateFrom,
           to: dateTo,
         }),
+        studyTrackFilter({
+          activeAt: dateFrom,
+        }),
       ]}
+      initialOptions={{
+        [programmeFilter.key]: { mode: 'attainment', selectedProgrammes: [] },
+      }}
       students={studentData.students ?? []}
     >
       {filtered => (

services/frontend/src/components/CourseStatistics/ResultTabs/HelpButton.jsx

@@ -0,0 +1,31 @@
+import React from 'react'
+import ReactMarkdown from 'react-markdown'
+
+import { Popup, Button, Icon } from 'semantic-ui-react'
+
+import infotooltips from '../../../common/InfoToolTips'
+
+export default ({ tab, viewMode }) => {
+  let content = infotooltips.CourseStatistics[tab]
+
+  if (viewMode) {
+    content = content[viewMode]
+  }
+
+  content = content.replace(/\n +/g, '\n')
+
+  return (
+    <Popup
+      on="click"
+      wide="very"
+      trigger={
+        <Button icon labelPosition="left">
+          <Icon name="question circle outline" />
+          Help
+        </Button>
+      }
+    >
+      <ReactMarkdown>{content}</ReactMarkdown>
+    </Popup>
+  )
+}

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/Tables/attempts.jsx

@@ -1,10 +1,10 @@
 import React from 'react'
 import qs from 'query-string'
+import _, { uniq } from 'lodash'
 import { Link } from 'react-router-dom'
-import { arrayOf, number, oneOfType, shape, string, bool } from 'prop-types'
 import { connect } from 'react-redux'
 import { Header, Icon, Item } from 'semantic-ui-react'
-import { uniq } from 'lodash'
+
 import SortableTable from '../../../../SortableTable'
 import { defineCellColor, getGradeSpread, getThesisGradeSpread, isThesisGrades, THESIS_GRADE_KEYS } from '../util'
 
@@ -40,7 +40,7 @@ const getTableData = (stats, notThesisGrades, isRelative) =>
       passRate: stat.attempts.passRate,
       attempts,
       rowObfuscated,
-      ...gradeSpread,
+      ..._.mapValues(gradeSpread, x => x[0]),
     }
   })
 
@@ -69,14 +69,13 @@ const getGradeColumns = (notThesisGrades, addHTAndTT) => {
 }
 
 const AttemptsTable = ({
-  stats,
-  name,
+  data: { stats, name },
+  settings: { showGrades },
   alternatives,
   separate,
   isRelative,
   userHasAccessToAllStats,
   headerVisible = false,
-  showGrades,
 }) => {
   const {
     attempts: { grades },
@@ -147,7 +146,7 @@ const AttemptsTable = ({
       <SortableTable
         defaultdescending
         getRowKey={s => s.code}
-        tableProps={{ celled: true, structured: true }}
+        tableProps={{ celled: true, style: { width: 'auto' } }}
         columns={columns}
         data={data}
       />
@@ -158,19 +157,4 @@ const AttemptsTable = ({
   )
 }
 
-AttemptsTable.propTypes = {
-  stats: arrayOf(shape({})).isRequired,
-  name: oneOfType([number, string]).isRequired,
-  alternatives: arrayOf(string).isRequired,
-  separate: bool,
-  isRelative: bool.isRequired,
-  userHasAccessToAllStats: bool.isRequired,
-  headerVisible: bool.isRequired,
-  showGrades: bool.isRequired,
-}
-
-AttemptsTable.defaultProps = {
-  separate: false,
-}
-
 export default connect(null)(AttemptsTable)

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/Tables/student.jsx

@@ -1,4 +1,4 @@
-import React from 'react'
+import React, { useMemo } from 'react'
 import qs from 'query-string'
 import { Link } from 'react-router-dom'
 import { Header, Icon, Item } from 'semantic-ui-react'
@@ -10,7 +10,7 @@ import { defineCellColor } from '../util'
 
 const formatPercentage = p => `${(p * 100).toFixed(2)} %`
 
-const StudentTable = ({ stats, name, alternatives, separate, userHasAccessToAllStats, headerVisible = false }) => {
+const getColumns = (showDetails, userHasAccessToAllStats, alternatives, separate) => {
   const showPopulation = (yearcode, years) => {
     const queryObject = {
       from: yearcode,
@@ -23,6 +23,86 @@ const StudentTable = ({ stats, name, alternatives, separate, userHasAccessToAllS
     return `/coursepopulation?${searchString}`
   }
 
+  const columns = [
+    {
+      key: 'TIME',
+      title: 'Time',
+      getRowVal: s => s.code,
+      getRowContent: s => (
+        <div style={{ whiteSpace: 'nowrap' }}>
+          {s.name}
+          {s.name === 'Total' && !userHasAccessToAllStats && <strong>*</strong>}
+          {s.name !== 'Total' && userHasAccessToAllStats && (
+            <Item as={Link} to={showPopulation(s.code, s.name, s)}>
+              <Icon name="level up alternate" />
+            </Item>
+          )}
+        </div>
+      ),
+      getCellProps: s => defineCellColor(s),
+    },
+    {
+      key: 'TOTAL',
+      title: 'Total Students',
+      getRowVal: s => (s.rowObfuscated ? '5 or less students' : s.students.total),
+      getCellProps: s => defineCellColor(s),
+    },
+    {
+      key: 'PASS_RATE',
+      title: 'Pass-%',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.passRate),
+      getRowContent: s => (s.rowObfuscated ? 'NA' : formatPercentage(s.students.passRate || 0)),
+      getCellProps: s => defineCellColor(s),
+    },
+    {
+      key: 'PASS_FIRST',
+      title: 'On First Attempt',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedFirst || 0),
+      getCellProps: s => defineCellColor(s),
+      headerProps: { style: { borderLeft: '0' } },
+      onlyInDetailedView: true,
+    },
+    {
+      key: 'PASS_EVENTUALLY',
+      title: 'Eventually',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedEventually || 0),
+      getCellProps: s => defineCellColor(s),
+      headerProps: { style: { borderLeft: '0' } },
+      onlyInDetailedView: true,
+    },
+    {
+      key: 'FAIL_RATE',
+      title: 'Fail-%',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.failRate),
+      getRowContent: s => (s.rowObfuscated ? 'NA' : formatPercentage(s.students.failRate || 0)),
+      getCellProps: s => defineCellColor(s),
+    },
+    {
+      key: 'NEVER PASSED',
+      title: 'Yet to Pass',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.neverPassed || 0),
+      getCellProps: s => defineCellColor(s),
+      headerProps: { style: { borderLeft: '0' } },
+      onlyInDetailedView: true,
+    },
+  ]
+
+  return columns.filter(column => showDetails || !column.onlyInDetailedView)
+}
+
+const StudentTable = ({
+  data: { name, stats },
+  settings: { showDetails },
+  alternatives,
+  separate,
+  userHasAccessToAllStats,
+  headerVisible = false,
+}) => {
+  const columns = useMemo(
+    () => getColumns(showDetails, userHasAccessToAllStats, alternatives, separate),
+    [showDetails, userHasAccessToAllStats, alternatives, separate]
+  )
+
   return (
     <div>
       {headerVisible && (
@@ -33,88 +113,8 @@ const StudentTable = ({ stats, name, alternatives, separate, userHasAccessToAllS
       <SortableTable
         defaultdescending
         getRowKey={s => s.code}
-        tableProps={{ celled: true, structured: true }}
-        columns={[
-          {
-            key: 'TIME',
-            title: 'Time',
-            getRowVal: s => s.code,
-            getRowContent: s => (
-              <div>
-                {s.name}
-                {s.name === 'Total' && !userHasAccessToAllStats && <strong>*</strong>}
-                {s.name !== 'Total' && userHasAccessToAllStats && (
-                  <Item as={Link} to={showPopulation(s.code, s.name, s)}>
-                    <Icon name="level up alternate" />
-                  </Item>
-                )}
-              </div>
-            ),
-            getCellProps: s => defineCellColor(s),
-            headerProps: { rowSpan: 2, width: 3 },
-          },
-          {
-            key: 'TOTAL',
-            title: 'All students',
-            getRowVal: s => (s.rowObfuscated ? '5 or less students' : s.students.total),
-            getCellProps: s => defineCellColor(s),
-            headerProps: { rowSpan: 2, width: 3 },
-          },
-          {
-            key: 'PASSED',
-            title: 'Passed',
-            parent: true,
-            headerProps: { colSpan: 3, width: 5 },
-          },
-          {
-            key: 'PASS_FIRST',
-            title: 'first try',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedFirst || 0),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 2 },
-            child: true,
-          },
-          {
-            key: 'PASS_EVENTUALLY',
-            title: 'passed eventually',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedEventually || 0),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 2 },
-            child: true,
-          },
-          {
-            key: 'PASS_RATE',
-            title: 'percentage',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.passRate),
-            getRowContent: s => (s.rowObfuscated ? 'NA' : formatPercentage(s.students.passRate || 0)),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 1 },
-            child: true,
-          },
-          {
-            key: 'FAIL',
-            title: 'Failed',
-            parent: true,
-            headerProps: { colSpan: 3, width: 5 },
-          },
-          {
-            key: 'NEVER PASSED',
-            title: 'never passed',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.neverPassed || 0),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 2 },
-            child: true,
-          },
-          {
-            key: 'FAIL_RATE',
-            title: 'percentage',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.failRate),
-            getRowContent: s => (s.rowObfuscated ? 'NA' : formatPercentage(s.students.failRate || 0)),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 1 },
-            child: true,
-          },
-        ]}
+        tableProps={{ celled: true, fixed: true, style: { width: 'auto' } }}
+        columns={columns}
         data={stats}
       />
       {!userHasAccessToAllStats && (

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/distribution.jsx

@@ -1,12 +1,11 @@
 /* eslint-disable quotes */
 import React from 'react'
-import { Grid } from 'semantic-ui-react'
-import { bool } from 'prop-types'
+import { Menu, Radio } from 'semantic-ui-react'
+import HelpButton from '../HelpButton'
 import { gradeGraphOptions } from '../../../../constants'
 import {
   getDataObject,
   getMaxValueOfSeries,
-  dataSeriesType,
   getGradeSpread,
   getThesisGradeSpread,
   isThesisSeries,
@@ -71,61 +70,47 @@ const getGradeSeries = series => {
       }
 }
 
-const Distribution = ({ primary, comparison, isRelative, userHasAccessToAllStats }) => {
-  const primaryStats = primary.stats.filter(stat => stat.name !== 'Total' || isRelative)
-
-  const statYears = primaryStats.map(year => year.name)
-  const comparisonStats = comparison ? comparison.stats.filter(stat => stat.name !== 'Total' || isRelative) : []
-  const primaryGrades = primaryStats.flatMap(s => s.attempts.grades)
-  const comparisonGrades = comparisonStats.flatMap(s => s.attempts.grades)
-
-  const gradeGraphSeries = getGradeSeries(primaryGrades)
-  const comparisonGraphSeries = getGradeSeries(comparisonGrades)
-
-  const maxGradeValue = isRelative ? 1 : getMaxValueOfSeries(gradeGraphSeries.absolute)
-
-  const primaryDistributionOptions = comparison
-    ? gradeGraphOptions(statYears, maxGradeValue, 'Primary Grades')
-    : gradeGraphOptions(statYears, maxGradeValue, 'Grades')
-  const comparisonDistributionOptions = gradeGraphOptions(statYears, maxGradeValue, 'Comparison Grades')
+export const DistributionSettings = ({ value, onChange }) => {
+  const { isRelative } = value
 
   return (
-    <>
-      <Grid.Row>
-        <Grid.Column>
-          <StackedBarChart
-            options={primaryDistributionOptions}
-            series={isRelative ? gradeGraphSeries.relative : gradeGraphSeries.absolute}
-          />
-        </Grid.Column>
-      </Grid.Row>
-      {comparison && (
-        <Grid.Row>
-          <Grid.Column>
-            <StackedBarChart
-              options={comparisonDistributionOptions}
-              series={isRelative ? comparisonGraphSeries.relative : comparisonGraphSeries.absolute}
-            />
-          </Grid.Column>
-        </Grid.Row>
-      )}
-      {!userHasAccessToAllStats && (
-        <span className="totalsDisclaimer">* Years with 5 students or less are shown as 0 in the chart</span>
-      )}
-    </>
+    <Menu secondary style={{ marginBottom: 0 }}>
+      <Menu.Item>
+        <Radio
+          toggle
+          label="Show relative"
+          checked={isRelative}
+          onChange={() => onChange({ ...value, isRelative: !isRelative })}
+        />
+        <Menu.Item>
+          <HelpButton tab="GradeDistribution" />
+        </Menu.Item>
+      </Menu.Item>
+    </Menu>
   )
 }
 
-Distribution.propTypes = {
-  primary: dataSeriesType.isRequired,
-  comparison: dataSeriesType,
-  isRelative: bool,
-  userHasAccessToAllStats: bool.isRequired,
-}
+export const Distribution = ({ data, settings: { isRelative }, userHasAccessToAllStats }) => {
+  const stats = data.stats.filter(stat => stat.name !== 'Total' || isRelative)
 
-Distribution.defaultProps = {
-  comparison: undefined,
-  isRelative: false,
-}
+  const statYears = stats.map(year => year.name)
+  const grades = stats.flatMap(s => s.attempts.grades)
 
-export default Distribution
+  const gradeGraphSeries = getGradeSeries(grades)
+
+  const maxGradeValue = isRelative ? 1 : getMaxValueOfSeries(gradeGraphSeries.absolute)
+
+  const primaryDistributionOptions = gradeGraphOptions(statYears, maxGradeValue, 'Grades')
+
+  return (
+    <div>
+      <StackedBarChart
+        options={primaryDistributionOptions}
+        series={isRelative ? gradeGraphSeries.relative : gradeGraphSeries.absolute}
+      />
+      {!userHasAccessToAllStats && (
+        <span className="totalsDisclaimer">* Years with 5 students or less are shown as 0 in the chart</span>
+      )}
+    </div>
+  )
+}

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/passRate.jsx

@@ -1,17 +1,10 @@
 /* eslint-disable quotes */
 import React from 'react'
-import { Grid } from 'semantic-ui-react'
-import { bool } from 'prop-types'
+import { Menu, Radio } from 'semantic-ui-react'
 import StackedBarChart from '../../../StackedBarChart'
 import { passRateAttemptGraphOptions, passRateStudGraphOptions } from '../../../../constants'
-import {
-  viewModeNames,
-  getDataObject,
-  getMaxValueOfSeries,
-  dataSeriesType,
-  viewModeType,
-  absoluteToRelative,
-} from './util'
+import HelpButton from '../HelpButton'
+import { viewModeNames, getDataObject, getMaxValueOfSeries, absoluteToRelative } from './util'
 
 const getPassRateAttemptSeriesFromStats = stats => {
   const all = []
@@ -70,62 +63,62 @@ const getPassRateStudSeriesFromStats = stats => {
   }
 }
 
-const PassRate = ({ primary, comparison, viewMode, isRelative = false, userHasAccessToAllStats }) => {
-  const isAttemptsMode = viewMode === viewModeNames.ATTEMPTS
-
-  const primaryStats = primary.stats.filter(stat => stat.name !== 'Total')
-  const statYears = primaryStats.map(year => year.name)
-  const comparisonStats = comparison ? comparison.stats : []
-  const passGraphSerieFn = isAttemptsMode ? getPassRateAttemptSeriesFromStats : getPassRateStudSeriesFromStats
-
-  const passGraphSerie = passGraphSerieFn(primaryStats)
-  const comparisonGraphSerie = passGraphSerieFn(comparisonStats)
-
-  const maxPassRateVal = isRelative ? 1 : getMaxValueOfSeries(passGraphSerie.absolute)
-  const graphOptionsFn = isAttemptsMode ? passRateAttemptGraphOptions : passRateStudGraphOptions
-  const primaryGraphOptions = comparison
-    ? graphOptionsFn(statYears, maxPassRateVal, 'Primary pass rate chart', isRelative)
-    : graphOptionsFn(statYears, maxPassRateVal, 'Pass rate chart', isRelative)
-  const comparisonGraphOptions = graphOptionsFn(statYears, maxPassRateVal, 'Comparison pass rate chart', isRelative)
+export const PassRateSettings = ({ value, onChange }) => {
+  const { viewMode, isRelative } = value
 
   return (
-    <>
-      <Grid.Row>
-        <Grid.Column>
-          <StackedBarChart
-            options={primaryGraphOptions}
-            series={isRelative ? passGraphSerie.relative : passGraphSerie.absolute}
-          />
-        </Grid.Column>
-      </Grid.Row>
-      {comparison && (
-        <Grid.Row>
-          <Grid.Column>
-            <StackedBarChart
-              options={comparisonGraphOptions}
-              series={isRelative ? comparisonGraphSerie.relative : comparisonGraphSerie.absolute}
-            />
-          </Grid.Column>
-        </Grid.Row>
-      )}
-      {!userHasAccessToAllStats && (
-        <span className="totalsDisclaimer">* Years with 5 students or less are shown as 0 in the chart</span>
-      )}
-    </>
+    <Menu secondary style={{ marginBottom: 0 }}>
+      {Object.entries(viewModeNames).map(([key, name]) => (
+        <Menu.Item
+          active={viewMode === key}
+          name={name}
+          onClick={() =>
+            onChange({
+              ...value,
+              viewMode: key,
+            })
+          }
+        />
+      ))}
+      <Menu.Item>
+        <Radio
+          toggle
+          label="Show relative"
+          checked={isRelative}
+          onChange={() => onChange({ ...value, isRelative: !isRelative })}
+        />
+      </Menu.Item>
+      <Menu.Item>
+        <HelpButton tab="PassRate" viewMode={viewMode} />
+      </Menu.Item>
+    </Menu>
   )
 }
 
-PassRate.propTypes = {
-  primary: dataSeriesType.isRequired,
-  comparison: dataSeriesType,
-  viewMode: viewModeType.isRequired,
-  isRelative: bool,
-  userHasAccessToAllStats: bool.isRequired,
-}
+export const PassRate = ({ data, settings: { viewMode, isRelative }, userHasAccessToAllStats }) => {
+  const isAttemptsMode = viewMode === 'ATTEMPTS'
 
-PassRate.defaultProps = {
-  comparison: undefined,
-  isRelative: false,
-}
+  const stats = data.stats.filter(stat => stat.name !== 'Total')
+  const statYears = stats.map(year => year.name)
+  // const comparisonStats = comparison ? comparison.stats : []
+  const passGraphSerieFn = isAttemptsMode ? getPassRateAttemptSeriesFromStats : getPassRateStudSeriesFromStats
 
-export default PassRate
+  const passGraphSerie = passGraphSerieFn(stats)
+  // const comparisonGraphSerie = passGraphSerieFn(comparisonStats)
+
+  const maxPassRateVal = isRelative ? 1 : getMaxValueOfSeries(passGraphSerie.absolute)
+  const graphOptionsFn = isAttemptsMode ? passRateAttemptGraphOptions : passRateStudGraphOptions
+  const primaryGraphOptions = graphOptionsFn(statYears, maxPassRateVal, 'Pass rate chart', isRelative)
+
+  return (
+    <div>
+      <StackedBarChart
+        options={primaryGraphOptions}
+        series={isRelative ? passGraphSerie.relative : passGraphSerie.absolute}
+      />
+      {!userHasAccessToAllStats && (
+        <span className="totalsDisclaimer">* Years with 5 students or less are shown as 0 in the chart</span>
+      )}
+    </div>
+  )
+}

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/tables.jsx

@@ -1,85 +1,61 @@
-import React, { Fragment } from 'react'
-import { connect } from 'react-redux'
-import { Grid } from 'semantic-ui-react'
-import { string, arrayOf, bool } from 'prop-types'
-import { dataSeriesType, viewModeNames, viewModeType } from './util'
+import React from 'react'
+import { useSelector } from 'react-redux'
+import { Menu, Radio } from 'semantic-ui-react'
+import { viewModeNames } from './util'
+import HelpButton from '../HelpButton'
 import StudentTable from './Tables/student'
 import AttemptsTable from './Tables/attempts'
 
-const Tables = ({
-  primary,
-  comparison,
-  viewMode,
-  alternatives,
-  separate,
-  isRelative,
-  showGrades,
-  userHasAccessToAllStats,
-}) => {
-  const getViewMode = statistics => {
-    const { name, stats } = statistics
-    const headerVisible = !!comparison
-
-    switch (viewMode) {
-      case viewModeNames.ATTEMPTS:
-        return (
-          <AttemptsTable
-            separate={separate}
-            name={name}
-            stats={stats}
-            alternatives={alternatives}
-            isRelative={isRelative}
-            userHasAccessToAllStats={userHasAccessToAllStats}
-            headerVisible={headerVisible}
-            showGrades={showGrades}
-          />
-        )
-      case viewModeNames.STUDENT:
-        return (
-          <StudentTable
-            separate={separate}
-            name={name}
-            stats={stats}
-            alternatives={alternatives}
-            userHasAccessToAllStats={userHasAccessToAllStats}
-            headerVisible={headerVisible}
-          />
-        )
-      default:
-        return null
-    }
-  }
+export const TablesSettings = ({ value, onChange }) => {
+  const { viewMode, showDetails, showGrades } = value
 
   return (
-    <>
-      <Grid.Column id="PrimaryDataTable">{getViewMode(primary)}</Grid.Column>
-      {comparison && <Grid.Column id="ComparisonDataTable">{getViewMode(comparison)}</Grid.Column>}
-    </>
+    <Menu secondary style={{ marginBottom: 0 }}>
+      {Object.entries(viewModeNames).map(([key, name]) => (
+        <Menu.Item
+          active={viewMode === key}
+          data-cy={`viewMode-${name}`}
+          name={name}
+          onClick={() =>
+            onChange({
+              ...value,
+              viewMode: key,
+            })
+          }
+        />
+      ))}
+      <Menu.Item>
+        <Radio
+          toggle
+          label="Show details"
+          disabled={viewMode !== 'STUDENT'}
+          checked={showDetails}
+          onChange={() => onChange({ ...value, showDetails: !showDetails })}
+        />
+      </Menu.Item>
+      <Menu.Item>
+        <Radio
+          toggle
+          label="Show grades"
+          data-cy="gradeToggle"
+          disabled={viewMode !== 'ATTEMPTS'}
+          checked={showGrades}
+          onChange={() => onChange({ ...value, showGrades: !showGrades })}
+        />
+      </Menu.Item>
+      <Menu.Item>
+        <HelpButton tab="Tables" viewMode={viewMode} />
+      </Menu.Item>
+    </Menu>
   )
 }
 
-Tables.propTypes = {
-  primary: dataSeriesType.isRequired,
-  comparison: dataSeriesType,
-  viewMode: viewModeType.isRequired,
-  alternatives: arrayOf(string).isRequired,
-  separate: bool,
-  isRelative: bool.isRequired,
-  showGrades: bool.isRequired,
-  userHasAccessToAllStats: bool.isRequired,
+export const Tables = props => {
+  const alternatives = useSelector(state => state.courseStats.data[state.singleCourseStats.selectedCourse].alternatives)
+
+  const viewModes = { ATTEMPTS: AttemptsTable, STUDENT: StudentTable }
+
+  const Content = viewModes[props.settings.viewMode]
+
+  return <Content {...props} alternatives={alternatives} />
 }
-
-Tables.defaultProps = {
-  comparison: null,
-  separate: false,
-}
-
-const mapStateToProps = state => {
-  const { selectedCourse } = state.singleCourseStats
-
-  return {
-    alternatives: state.courseStats.data[selectedCourse].alternatives,
-  }
-}
-
-export default connect(mapStateToProps)(Tables)

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/util.js

@@ -1,8 +1,8 @@
 import { arrayOf, number, oneOfType, shape, string, oneOf } from 'prop-types'
 
 export const viewModeNames = {
-  ATTEMPTS: 'Attempts',
   STUDENT: 'Students',
+  ATTEMPTS: 'Attempts',
 }
 
 export const getDataObject = (name, data, stack) => ({ name, data, stack })

services/frontend/src/components/CourseStatistics/ResultTabs/index.jsx

@@ -1,62 +1,91 @@
-import React, { useEffect, useState } from 'react'
-import { Tab, Grid, Radio, Menu } from 'semantic-ui-react'
+import React, { useState } from 'react'
+import { Tab, Segment, Menu, Icon } from 'semantic-ui-react'
 import { withRouter } from 'react-router-dom'
-import { shape, bool } from 'prop-types'
-import { dataSeriesType, viewModeNames } from './Panes/util'
-import PassRate from './Panes/passRate'
-import Distribution from './Panes/distribution'
-import Tables from './Panes/tables'
-import InfoBox from '../../Info/InfoBox'
+import { PassRate, PassRateSettings } from './Panes/passRate'
+import { Distribution, DistributionSettings } from './Panes/distribution'
+import { Tables, TablesSettings } from './Panes/tables'
 import { useTabs } from '../../../common/hooks'
 
 import './resultTabs.css'
 import TSA from '../../../common/tsa'
-import infotooltips from '../../../common/InfoToolTips'
 
 const ANALYTICS_CATEGORY = 'Course Statistics'
 const sendAnalytics = (action, name, value) => TSA.Matomo.sendEvent(ANALYTICS_CATEGORY, action, name, value)
 
-const paneViewIndex = {
-  TABLE: 0,
-  PASS_RAGE: 1,
-  GRADE_DISTRIBUTION: 2,
+const PaneContent = ({ component: Component, settings: SettingsComponent, initialSettings, datasets, ...rest }) => {
+  const [settings, setSettings] = useState(initialSettings)
+  const [splitDirection, setSplitDirection] = useState('column')
+
+  return (
+    <Tab.Pane>
+      <Segment basic>
+        <div style={{ display: 'flex', marginBottom: '2em' }}>
+          <SettingsComponent value={settings} onChange={setSettings} />
+          <div style={{ flexGrow: 1 }} />
+          {datasets.filter(i => i).length > 1 && (
+            <div style={{ display: 'flex', alignItems: 'center', gap: '1em' }}>
+              <label>Split direction: </label>
+              <Menu style={{ margin: 0 }}>
+                <Menu.Item active={splitDirection === 'row'} onClick={() => setSplitDirection('row')}>
+                  <Icon name="arrows alternate horizontal" />
+                </Menu.Item>
+                <Menu.Item active={splitDirection === 'column'} onClick={() => setSplitDirection('column')}>
+                  <Icon name="arrows alternate vertical" />
+                </Menu.Item>
+              </Menu>
+            </div>
+          )}
+        </div>
+        <div style={{ display: 'flex', flexDirection: splitDirection, gap: '2em' }}>
+          {datasets
+            .filter(i => i)
+            .map(data => (
+              <div style={{ flexGrow: 1, flexBasis: 1 }}>
+                <h3>{data.name}</h3>
+                <Component data={data} settings={settings} {...rest} />
+              </div>
+            ))}
+        </div>
+      </Segment>
+    </Tab.Pane>
+  )
 }
 
-const ResultTabs = props => {
-  const [tab, setTab] = useTabs('cs_tab', 0, props.history)
-  const [viewMode, setViewMode] = useState(viewModeNames.ATTEMPTS)
-  const [selectedView, setSelectedView] = useState(true)
-  const [isRelative, setIsRelative] = useState(false)
-  const [showGrades, setShowGrades] = useState(false)
+const ResultTabs = ({ primary, comparison, history }) => {
+  const [tab, setTab] = useTabs('cs_tab', 0, history)
+  // const [viewMode, setViewMode] = useState(viewModeNames.STUDENT)
+  // const [selectedView, setSelectedView] = useState(false)
+  // const [isRelative, setIsRelative] = useState(false)
+  // const [showGrades, setShowGrades] = useState(false)
+  const { userHasAccessToAllStats } = primary
 
-  const handleModeChange = newViewMode => {
+  /* const handleModeChange = newViewMode => {
     sendAnalytics(`Current view mode '${newViewMode}'`, 'Course statistics')
     setViewMode(newViewMode)
-  }
+  } */
 
-  useEffect(() => {
+  /* useEffect(() => {
     const newViewMode = selectedView ? viewModeNames.ATTEMPTS : viewModeNames.STUDENT
     handleModeChange(newViewMode)
-  }, [selectedView])
+  }, [selectedView]) */
 
   const handleTabChange = (...params) => {
-    const resetViewMode = params[1].activeIndex === paneViewIndex.TABLE && viewMode === viewModeNames.ATTEMPTS
     const { activeIndex } = params[1]
     const currentTab = params[1].panes[activeIndex]
     sendAnalytics(`Current tab '${currentTab.menuItem.content}'`, 'Course statistics')
     setTab(...params)
-    setViewMode(resetViewMode ? viewMode : viewModeNames.ATTEMPTS)
+    // setViewMode(resetViewMode ? viewMode : viewModeNames.ATTEMPTS)
   }
 
-  const getRadioButton = (firstLabel, secondLabel, value, setValue) => (
+  /* const getRadioButton = (firstLabel, secondLabel, value, setValue) => (
     <div className="toggleContainer">
       <label className="toggleLabel">{firstLabel}</label>
       <Radio toggle data-cy="gradeToggle" checked={value} onChange={() => setValue(!value)} />
       <label className="toggleLabel">{secondLabel}</label>
     </div>
-  )
+  ) */
 
-  const renderViewModeSelector = () => {
+  /* const renderViewModeSelector = () => {
     const isTogglePane = tab !== 0
     const getButtonMenu = () => (
       <Menu secondary>
@@ -72,7 +101,7 @@ const ResultTabs = props => {
       return (
         <div className="chartToggleContainer">
           {tab === 1 && getRadioButton('Student', 'Attempts', selectedView, setSelectedView)}
-          {(tab === 2 || props.comparison) && getRadioButton('Absolute', 'Relative', isRelative, setIsRelative)}
+          {(tab === 2 || comparison) && getRadioButton('Absolute', 'Relative', isRelative, setIsRelative)}
         </div>
       )
     }
@@ -84,16 +113,13 @@ const ResultTabs = props => {
         <InfoBox content={infotooltips.CourseStatistics[tab][viewMode]} />
       </div>
     )
-  }
+  } */
 
-  const getPanes = () => {
-    const { primary, comparison, separate } = props
-    const { userHasAccessToAllStats } = primary
-
-    const paneMenuItems = [
+  /* const getPanes = () => {
+    return [
       {
         menuItem: { key: 'Table', icon: 'table', content: 'Table' },
-        renderFn: () => (
+        render: () => (
           <Tables
             separate={separate}
             comparison={comparison}
@@ -107,7 +133,7 @@ const ResultTabs = props => {
       },
       {
         menuItem: { key: 'pass', icon: 'balance', content: 'Pass rate chart' },
-        renderFn: () => (
+        render: () => (
           <PassRate
             comparison={comparison}
             primary={primary}
@@ -119,7 +145,7 @@ const ResultTabs = props => {
       },
       {
         menuItem: { key: 'grade', icon: 'chart bar', content: 'Grade distribution chart' },
-        renderFn: () => (
+        render: () => (
           <Distribution
             comparison={comparison}
             primary={primary}
@@ -128,7 +154,7 @@ const ResultTabs = props => {
           />
         ),
       },
-    ]
+    ];
 
     return paneMenuItems.map(p => {
       const { menuItem, renderFn } = p
@@ -142,25 +168,52 @@ const ResultTabs = props => {
         ),
       }
     })
-  }
+  } */
+
+  const paneTypes = [
+    {
+      label: 'Tables',
+      icon: 'table',
+      initialSettings: { showDetails: false, viewMode: 'STUDENT' },
+      settings: TablesSettings,
+      component: Tables,
+    },
+    {
+      label: 'Pass rate chart',
+      icon: 'balance',
+      initialSettings: { viewMode: 'STUDENT' },
+      settings: PassRateSettings,
+      component: PassRate,
+    },
+    {
+      label: 'Grade distribution chart',
+      icon: 'chart bar',
+      initialSettings: { isRelative: false, viewMode: 'STUDENT' },
+      settings: DistributionSettings,
+      component: Distribution,
+    },
+  ]
+
+  const panes = paneTypes.map(
+    ({ icon, label, initialSettings, component: Component, settings: SettingsComponent }) => ({
+      menuItem: { icon, content: label },
+      render: () => (
+        <PaneContent
+          component={Component}
+          settings={SettingsComponent}
+          userHasAccessToAllStats={userHasAccessToAllStats}
+          initialSettings={initialSettings}
+          datasets={[primary, comparison]}
+        />
+      ),
+    })
+  )
 
   return (
     <div>
-      <Tab id="CourseStatPanes" panes={getPanes()} onTabChange={handleTabChange} activeIndex={tab} />
+      <Tab id="CourseStatPanes" panes={panes} onTabChange={handleTabChange} activeIndex={tab} />
     </div>
   )
 }
 
-ResultTabs.propTypes = {
-  primary: dataSeriesType.isRequired,
-  comparison: dataSeriesType,
-  history: shape({}).isRequired,
-  separate: bool,
-}
-
-ResultTabs.defaultProps = {
-  comparison: undefined,
-  separate: false,
-}
-
 export default withRouter(ResultTabs)

services/frontend/src/components/CourseStatistics/SearchForm/searchFormUtils.js

@@ -1,8 +1,7 @@
 import { orderBy } from 'lodash'
 
 // handle special case where the course's actual code starts with "A" so it is mistakenly taken as Open Uni course
-// const isAvoin = code => !!code.match(/^[A][0-9]|^AY/)
-const isAvoinTwo = course => course.organizations.some(c => c === 'hy-org-48645785')
+const isAvoin = course => !!course.code.match(/^[A][0-9]|^AY/)
 
 const sortAlternatives = alternatives => {
   const result = orderBy(
@@ -31,7 +30,7 @@ const filterCourseSearchResults = (courses, unifyOpenUniCourses) => {
   const mergedCourses = {}
   const sortedCourses = sortAlternatives(courses)
   sortedCourses.forEach(course => {
-    const groupId = isAvoinTwo(course) && !unifyOpenUniCourses ? course.code : course.subsId
+    const groupId = isAvoin(course) && !unifyOpenUniCourses ? course.code : course.subsId
 
     if (!(course.max_attainment_date && course.min_attainment_date)) {
       return

services/frontend/src/components/CourseStatistics/courseStatisticsUtils.js

@@ -32,8 +32,6 @@ const getActiveYears = course => {
   return `${startYearText} — ${endYearText}`
 }
 
-const userHasAccessToAllCourseStats = (roles, rights) => {
-  return (roles && ['admin'].some(role => roles.includes(role))) || rights.length > 0
-}
+const userHasAccessToAllCourseStats = (roles, rights) => roles.includes('admin') || rights.length > 0
 
 export { getActiveYears, getYearText, userHasAccessToAllCourseStats }

services/frontend/src/components/CourseStatistics/index.jsx

@@ -28,7 +28,7 @@ const MENU = {
 const CourseStatistics = () => {
   const history = useHistory()
   const dispatch = useDispatch()
-  const { rights, userRoles } = useGetAuthorizedUserQuery()
+  const { rights, roles } = useGetAuthorizedUserQuery()
   const courseStats = useSelector(({ courseStats }) => courseStats)
   const { pending: loading } = courseStats
   const courses = Object.keys(courseStats.data)
@@ -57,7 +57,7 @@ const CourseStatistics = () => {
     setSelected(coursecode)
   }
 
-  const userHasAccessToAllStats = userHasAccessToAllCourseStats(userRoles, rights)
+  const userHasAccessToAllStats = userHasAccessToAllCourseStats(roles, rights)
 
   const getPanes = () => {
     let panes = [
@@ -109,7 +109,8 @@ const CourseStatistics = () => {
       setActiveIndex(activeIndex)
     }
   }
-  if (!checkUserAccess(['courseStatistics', 'admin'], userRoles) && rights.length < 1)
+
+  if (!checkUserAccess(['courseStatistics', 'admin'], roles) && rights.length < 1)
     return (
       <div className="segmentContainer">
         <Message

services/frontend/src/components/CreditAccumulationGraphHighCharts/index.jsx

@@ -226,7 +226,7 @@ const CreditAccumulationGraphHighCharts = ({ students, singleStudent, absences,
   const seriesData = useMemo(() => createStudentCreditLines(students, singleStudent), [students, singleStudent])
 
   if (singleStudent) {
-    const starting = _.chain(students[0].studyRights || students[0].courses)
+    const starting = _.chain(students[0].studyrights || students[0].courses)
       .map(elem => new Date(elem.startdate || elem.date))
       .sortBy()
       .head()

services/frontend/src/components/CustomPopulation/index.jsx

@@ -27,6 +27,7 @@ import {
   tagsFilter,
   creditDateFilter,
   enrollmentStatusFilter,
+  studyTrackFilter,
 } from '../FilterView/filters'
 import useLanguage from '../LanguagePicker/useLanguage'
 import CustomPopulationSearch from './CustomPopulationSearch'
@@ -37,7 +38,6 @@ const selectCustomPopulationData = createSelector(
   state => state.populations.data,
   (semesters, populations) => ({
     allSemesters: semesters?.semesters ?? [],
-    elementDetails: populations?.elementDetails?.data ?? [],
     custompop: populations?.students ?? [],
   })
 )
@@ -46,7 +46,7 @@ const CustomPopulation = () => {
   const { language } = useLanguage()
   const dispatch = useDispatch()
 
-  const { allSemesters, elementDetails, custompop } = useSelector(selectCustomPopulationData)
+  const { allSemesters, custompop } = useSelector(selectCustomPopulationData)
 
   const { data: courseStats } = useGetStudentListCourseStatisticsQuery({
     studentNumbers: custompop.map(s => s.studentNumber),
@@ -68,17 +68,15 @@ const CustomPopulation = () => {
       transferredToProgrammeFilter,
       startYearAtUniFilter,
       tagsFilter,
-      programmeFilter({
-        courses: courseStats?.coursestatistics ? courseStats?.coursestatistics.map(c => c.course.code) : [],
-        elementDetails,
-      }),
+      programmeFilter,
       creditDateFilter,
       enrollmentStatusFilter({
         allSemesters: allSemesters ?? [],
         language,
       }),
+      studyTrackFilter,
     ],
-    [courseStats, elementDetails, allSemesters, language]
+    [courseStats, allSemesters, language]
   )
 
   return (

services/frontend/src/components/FilterView/FilterTray/index.jsx

@@ -20,31 +20,33 @@ const FilterTray = () => {
 
   const haveOptionsBeenChanged = filters.some(({ key }) => areOptionsDirty(key))
 
-  const filterSet = filters.map(filter => {
-    const { key, title, isActive, render, info } = filter
-    const ctx = getContextByKey(key)
+  const filterSet = filters
+    .sort((a, b) => (a.title ?? a.key).localeCompare(b.title ?? b.key))
+    .map(filter => {
+      const { key, title, isActive, render, info } = filter
+      const ctx = getContextByKey(key)
 
-    const props = {
-      options: ctx.options,
-      onOptionsChange: options => setFilterOptions(key, options),
-      withoutSelf: () => withoutFilter(key),
-    }
+      const props = {
+        options: ctx.options,
+        onOptionsChange: options => setFilterOptions(key, options),
+        withoutSelf: () => withoutFilter(key),
+      }
 
-    return (
-      <div key={key}>
-        <FilterCard
-          title={title ?? key}
-          active={isActive(filterOptions[key])}
-          onClear={() => resetFilter(key)}
-          info={info}
-          filter={filter}
-          options={ctx.options}
-        >
-          {render(props, ctx)}
-        </FilterCard>
-      </div>
-    )
-  })
+      return (
+        <div key={key}>
+          <FilterCard
+            title={title ?? key}
+            active={isActive(filterOptions[key])}
+            onClear={() => resetFilter(key)}
+            info={info}
+            filter={filter}
+            options={ctx.options}
+          >
+            {render(props, ctx)}
+          </FilterCard>
+        </div>
+      )
+    })
 
   return (
     <>

services/frontend/src/components/FilterView/filters/common/FilterCard.jsx

@@ -1,15 +1,39 @@
-import React, { useState } from 'react'
+import React, { useState, useRef } from 'react'
 import { Icon, Header } from 'semantic-ui-react'
+import _ from 'lodash'
 import WithHelpTooltip from '../../../Info/InfoWithHelpTooltip'
 
 import './FilterCard.css'
 
+const useChange = value => {
+  const prevValue = useRef()
+
+  if (prevValue.current === undefined) {
+    prevValue.current = value
+    return true
+  }
+
+  const change = !_.isEqual(value, prevValue.current)
+
+  prevValue.current = value
+
+  return change
+}
+
 const FilterCard = ({ filter, options, children, onClear }) => {
   const title = filter.title ?? filter.key
   const active = filter.isActive(options)
   const { info, key } = filter
 
-  const [open, setOpen] = useState(false)
+  const hasChanged = useChange(options)
+
+  const [manuallyOpened, setManuallyOpened] = useState(null)
+
+  if (hasChanged && manuallyOpened === false) {
+    setManuallyOpened(null)
+  }
+
+  const open = manuallyOpened !== null ? manuallyOpened : active
 
   let header = (
     <div
@@ -20,7 +44,7 @@ const FilterCard = ({ filter, options, children, onClear }) => {
         flexGrow: 1,
       }}
       className="filter-card-header"
-      onClick={() => setOpen(!open)}
+      onClick={() => setManuallyOpened(!open)}
       data-cy={`${key}-header`}
     >
       <Icon name={open ? 'caret down' : 'caret right'} style={{ color: 'black', flexShrink: 0 }} />
@@ -63,9 +87,9 @@ const FilterCard = ({ filter, options, children, onClear }) => {
   }
 
   return (
-    <div style={{ margin: '1rem 0' }} data-cy={`${key}-filter-card`}>
+    <div style={{ margin: '1rem 0' }} data-cy={`${key}-filter-card`} data-open={open}>
       <div style={{ marginBottom: '1rem' }}>{header}</div>
-      {open && children}
+      {open && <div onClick={() => setManuallyOpened(true)}>{children}</div>}
     </div>
   )
 }

services/frontend/src/components/FilterView/filters/grade.jsx

@@ -84,6 +84,7 @@ export default createFilter({
       )
     ), */
       fp.map(([sn, courses]) => [sn, getHighestGradeOfCourseBetweenRange(courses, args.from, args.to)]),
+      fp.filter(([, grade]) => grade !== undefined),
       fp.groupBy(([, { grade }]) => grade),
       fp.mapValues(fp.map(([sn]) => sn)),
       grades => ({ grades })

services/frontend/src/components/FilterView/filters/index.js

@@ -11,6 +11,7 @@ import admissionType from './admissionType'
 import tags from './tags'
 import date from './date'
 import enrollmentStatus from './enrollmentStatus'
+import studyTrack from './studyTrack'
 
 export const genderFilter = gender
 export const ageFilter = age
@@ -25,3 +26,4 @@ export const admissionTypeFilter = admissionType
 export const tagsFilter = tags
 export const creditDateFilter = date
 export const enrollmentStatusFilter = enrollmentStatus
+export const studyTrackFilter = studyTrack

services/frontend/src/components/FilterView/filters/programmes.jsx

@@ -1,76 +1,186 @@
 import React, { useMemo } from 'react'
 import { Dropdown } from 'semantic-ui-react'
 import fp from 'lodash/fp'
-import { getTextIn, getStudentToTargetCourseDateMap, getNewestProgramme } from '../../../common'
+import _ from 'lodash'
+import moment from 'moment'
+import { getTextIn } from '../../../common'
 import useLanguage from '../../LanguagePicker/useLanguage'
 import createFilter from './createFilter'
 
-const ProgrammeFilterCard = ({ options, onOptionsChange, programmes }) => {
+const ProgrammeFilterCard = ({
+  options,
+  onOptionsChange,
+  programmes,
+  withoutSelf,
+  studyRightPredicate,
+  studentToProgrammeMap,
+  additionalModes,
+}) => {
   const { language } = useLanguage()
   const { selectedProgrammes } = options
   const name = 'programmeFilterCard'
 
+  const visibleProgrammes = fp.flow(
+    fp.flatMap(student =>
+      _.get(studentToProgrammeMap, student.studentNumber, []).map(programme => ({ student, programme }))
+    ),
+    fp.groupBy('student.studentNumber'),
+    fp.pickBy(fp.some(({ programme }) => options.selectedProgrammes.every(pcode => programme.code === pcode))),
+    fp.values,
+    fp.flatten,
+    fp.filter(({ student, programme }) => studyRightPredicate(student, programme)),
+    fp.map('programme'),
+    fp.reduce((acc, details) => {
+      if (!acc[details.code]) {
+        acc[details.code] = { ...details, studentCount: 0 }
+      }
+
+      acc[details.code].studentCount += 1
+
+      return acc
+    }, {}),
+    fp.values
+  )(withoutSelf())
+
   const dropdownOptions = useMemo(
     () =>
-      programmes
+      _.chain(visibleProgrammes)
+        .concat(selectedProgrammes.map(code => programmes.find(p => p.code === code)))
         .map(program => ({
           key: `programme-filter-value-${program.code}`,
           text: getTextIn(program.name, language),
           value: program.code,
+          content: (
+            <>
+              {getTextIn(program.name, language)}{' '}
+              <span style={{ color: 'rgb(136, 136, 136)', whiteSpace: 'nowrap' }}>
+                ({program.studentCount} students)
+              </span>
+            </>
+          ),
         }))
-        .sort((a, b) => a.text.localeCompare(b.text)),
+        .uniqBy('value')
+        .sort((a, b) => a.text.localeCompare(b.text))
+        .value(),
     [programmes, language]
   )
 
   const handleChange = (_, { value }) => {
     onOptionsChange({
+      ...options,
       selectedProgrammes: value,
     })
   }
 
+  const setMode = mode => {
+    onOptionsChange({
+      ...options,
+      mode,
+    })
+  }
+
+  const builtInModes = [
+    {
+      key: 'any',
+      value: 'any',
+      label: 'Past or Present',
+      description: 'Student has had a study right at any point in time.',
+    },
+    {
+      key: 'active',
+      value: 'active',
+      label: 'Active Study Right',
+      description: 'Student has a currently active study right.',
+    },
+  ]
+
   return (
-    <Dropdown
-      options={dropdownOptions}
-      placeholder="Select Programme"
-      onChange={handleChange}
-      button
-      value={selectedProgrammes}
-      name={name}
-      closeOnChange
-      multiple
-    />
+    <>
+      <Dropdown
+        multiple
+        closeOnChange
+        fluid
+        name={name}
+        onChange={handleChange}
+        options={dropdownOptions}
+        placeholder="Select Programme"
+        search
+        selection
+        value={selectedProgrammes}
+        data-cy="Programme-filter-dropdown"
+      />
+      <div style={{ marginTop: '0.5em' }}>
+        Mode:{' '}
+        <Dropdown
+          data-cy="Programme-filter-mode-selector"
+          compact
+          inline
+          placeholder="Type"
+          value={options.mode ?? 'active'}
+          onChange={(_, { value }) => setMode(value)}
+          options={[...builtInModes, ...additionalModes].map(mode => ({
+            key: mode.key,
+            value: mode.key,
+            text: mode.label,
+            content: (
+              <>
+                {mode.label}
+                {mode.description && (
+                  <>
+                    <br />
+                    <span
+                      style={{
+                        fontWeight: 'normal',
+                        marginTop: '0.4em',
+                        color: '#5b5b5b',
+                        maxWidth: '13em',
+                        whiteSpace: 'normal',
+                        display: 'inline-block',
+                        fontSize: '0.9em',
+                      }}
+                    >
+                      {mode.description}
+                    </span>
+                  </>
+                )}
+              </>
+            ),
+          }))}
+        />
+      </div>
+    </>
   )
 }
 
-const createStudentToProgrammeMap = (courses, students, elementDetails) => {
-  if (!elementDetails) {
-    return {}
-  }
+const getStudentProgrammes = fp.flow(
+  fp.get('studyrights'),
+  fp.flatMap('studyright_elements'),
+  fp.filter(['element_detail.type', 20])
+)
 
-  const studentToTargetCourseDateMap = getStudentToTargetCourseDateMap(students, courses)
+const createStudentToProgrammeMap = (students, studyRightPredicate) => {
+  const studentProgrammePairs = []
 
-  return fp.flow(
-    fp.map(student => [
-      student,
-      getNewestProgramme(student.studyrights, student.studentNumber, studentToTargetCourseDateMap, elementDetails),
-    ]),
-    fp.reduce(
-      ({ programmeMap, studentToProgrammeMap }, [student, programme]) => {
-        return {
-          programmeMap: { ...programmeMap, [programme.code]: programme },
-          studentToProgrammeMap: {
-            ...studentToProgrammeMap,
-            [student.studentNumber]: programme.code,
-          },
-        }
-      },
-      { programmeMap: {}, studentToProgrammeMap: {} }
-    ),
-    ({ programmeMap, studentToProgrammeMap }) => ({
-      programmes: Object.values(programmeMap),
-      studentToProgrammeMap,
+  students.forEach(student => {
+    getStudentProgrammes(student).forEach(programme => {
+      studentProgrammePairs.push({ student, programme })
     })
-  )(students)
+  })
+
+  const programmes = fp.flow(fp.map('programme.element_detail'), fp.uniqBy('code'))(studentProgrammePairs)
+
+  const studentToProgrammeMap = fp.flow(
+    fp.filter(({ student, programme }) => studyRightPredicate(student, programme)),
+    fp.groupBy('student.studentNumber'),
+    fp.mapValues(fp.map('programme.element_detail'))
+  )(studentProgrammePairs)
+
+  return { programmes, studentToProgrammeMap }
+}
+
+const MODE_PREDICATES = {
+  any: () => true,
+  active: (_, sre) => moment().isBetween(sre.startdate, sre.enddate, 'day', '[]'),
 }
 
 const filter = createFilter({
@@ -78,14 +188,40 @@ const filter = createFilter({
 
   defaultOptions: {
     selectedProgrammes: [],
+    mode: 'active',
   },
 
-  precompute: ({ students, args }) => createStudentToProgrammeMap(args.courses, students, args.elementDetails ?? []),
+  precompute: ({ students, options, args }) => {
+    let predicate = () => true
+
+    if (args?.studyRightPredicate) {
+      predicate = args.studyRightPredicate
+    }
+
+    if (options.mode) {
+      let modePredicate = MODE_PREDICATES[options.mode]
+
+      const additional = _.get(args, 'additionalModes', []).find(mode => mode.key === options.mode)?.predicate
+
+      if (!modePredicate && additional) {
+        modePredicate = additional
+      }
+
+      if (modePredicate) {
+        const prevPredicate = predicate
+        predicate = (student, sre) => prevPredicate(student, sre) && modePredicate(student, sre)
+      }
+    }
+
+    return createStudentToProgrammeMap(students, predicate)
+  },
 
   isActive: ({ selectedProgrammes }) => selectedProgrammes.length > 0,
 
   filter({ studentNumber }, { selectedProgrammes }, { precomputed: { studentToProgrammeMap } }) {
-    return selectedProgrammes.some(pcode => pcode === studentToProgrammeMap[studentNumber])
+    return selectedProgrammes.every(pcode =>
+      _.get(studentToProgrammeMap, studentNumber, []).some(({ code }) => code === pcode)
+    )
   },
 
   selectors: {
@@ -104,7 +240,15 @@ const filter = createFilter({
     },
   },
 
-  render: (props, { precomputed }) => <ProgrammeFilterCard {...props} programmes={precomputed.programmes} />,
+  render: (props, { precomputed, args }) => (
+    <ProgrammeFilterCard
+      {...props}
+      programmes={precomputed.programmes}
+      studentToProgrammeMap={precomputed.studentToProgrammeMap}
+      additionalModes={args?.additionalModes ?? []}
+      studyRightPredicate={args?.studyRightPredicate ?? (() => true)}
+    />
+  ),
 })
 
 export default filter

services/frontend/src/components/FilterView/filters/studyTrack.jsx

@@ -0,0 +1,85 @@
+import React from 'react'
+import { Form, Dropdown } from 'semantic-ui-react'
+import { getTextIn } from 'common'
+import moment from 'moment'
+import _ from 'lodash'
+import createFilter from './createFilter'
+
+const StudyTrackFilterCard = ({ options, onOptionsChange, withoutSelf, activeAt }) => {
+  const { selected } = options
+
+  const activeAtMoment = activeAt ? moment(activeAt) : moment()
+
+  const dropdownOptions = _.chain(withoutSelf())
+    .flatMap(student => student.studyrights)
+    .flatMap(sr => sr.studyright_elements)
+    .filter(sre => sre.element_detail.type === 30 && activeAtMoment.isBetween(sre.startdate, sre.enddate, 'day', '[]'))
+    .map(sre => sre.element_detail)
+    .keyBy('code')
+    .values()
+    .map(({ code, name }) => ({
+      key: code,
+      value: code,
+      text: `${getTextIn(name)} (${code})`,
+      content: (
+        <>
+          {getTextIn(name)}{' '}
+          <span style={{ whiteSpace: 'nowrap', color: 'rgb(136, 136, 136)', fontSize: '0.8em' }}>({code})</span>
+        </>
+      ),
+    }))
+    .value()
+
+  const handleChange = (_, { value }) => {
+    onOptionsChange({
+      selected: value,
+    })
+  }
+
+  return (
+    <Form>
+      <Dropdown
+        options={dropdownOptions}
+        value={selected}
+        onChange={handleChange}
+        placeholder="Choose study track"
+        className="mini"
+        selection
+        clearable
+        button
+        search
+        multiple
+        fluid
+        data-cy="StudyTrack-filter-dropdown"
+      />
+    </Form>
+  )
+}
+
+export default createFilter({
+  key: 'StudyTrack',
+
+  title: 'Study Track',
+
+  defaultOptions: {
+    selected: [],
+  },
+
+  isActive: ({ selected }) => selected.length > 0,
+
+  filter: (student, { selected, args }) => {
+    const activeAt = _.get(args, 'activeAt', moment())
+
+    return student.studyrights
+      .flatMap(sr => sr.studyright_elements)
+      .filter(
+        sre =>
+          sre.element_detail.type === 30 &&
+          (!activeAt || moment(activeAt).isBetween(sre.startdate, sre.enddate, 'day', '[]'))
+      )
+      .map(sre => sre.element_detail.code)
+      .some(code => selected.includes(code))
+  },
+
+  render: (props, { args }) => <StudyTrackFilterCard {...props} activeAt={_.get(args, 'activeAt')} />,
+})

services/frontend/src/components/FilterView/filters/tags.jsx

@@ -1,5 +1,5 @@
 import React from 'react'
-import { Form, Dropdown } from 'semantic-ui-react'
+import { Form, Dropdown, Message } from 'semantic-ui-react'
 import fp from 'lodash/fp'
 import createFilter from './createFilter'
 
@@ -14,7 +14,13 @@ const TagsFilterCard = ({ options, onOptionsChange, withoutSelf }) => {
     value: tag,
   }))
 
-  if (Object.entries(countsByTag).length === 0) return null
+  if (Object.entries(countsByTag).length === 0) {
+    return (
+      <Message color="orange" size="tiny">
+        No tags have been defined for any of the selected students.
+      </Message>
+    )
+  }
 
   return (
     <div className="card-content">

services/frontend/src/components/FilterView/index.jsx

@@ -131,7 +131,9 @@ const FilterView = ({ children, name, filters: pFilters, students, displayTray:
   return (
     <FilterViewContext.Provider value={value}>
       <div style={{ display: 'flex', alignItems: 'start', justifyContent: 'center', gap: '0.75rem' }}>
-        <div style={{ alignSelf: 'flex-start', position: 'sticky', top: '1rem' }}>{displayTray && <FilterTray />}</div>
+        <div style={{ zIndex: 1, alignSelf: 'flex-start', position: 'sticky', top: '1rem' }}>
+          {displayTray && <FilterTray />}
+        </div>
         <div style={{ flexGrow: 1 }}>{typeof children === 'function' ? children(filteredStudents) : children}</div>
       </div>
     </FilterViewContext.Provider>

services/frontend/src/components/Frontpage/OoodiToOodikone.jsx

@@ -0,0 +1,46 @@
+import React, { useState, useEffect } from 'react'
+import oodiTXT from '../../static/oodi.txt'
+import './oodiToOodikone.css'
+
+const OodiToOodikone = () => {
+  const [oodis, setOodis] = useState([])
+  useEffect(() => {
+    const getOodis = async () => {
+      try {
+        const data = await fetch(oodiTXT)
+        setOodis(
+          (await data.text())
+            .split('**')
+            .map(o => o.trim())
+            .filter(oodi => !!oodi)
+        )
+      } catch (error) {
+        setOodis([])
+      }
+    }
+    getOodis()
+  }, [])
+
+  if (!oodis || oodis.length === 0) {
+    return null
+  }
+  const oodi = oodis[Math.floor(Math.random() * oodis.length)].split('\n')
+  return (
+    <blockquote cite="http://www.helsinki.fi/discovery">
+      <div style={{ textAlign: 'center' }}>
+        {oodi.map((l, index) => (
+          // eslint-disable-next-line react/no-array-index-key
+          <p key={index} style={{ marginBottom: '0.6rem' }}>
+            {l}
+          </p>
+        ))}
+      </div>
+
+      <p style={{ marginTop: '1.3em', textAlign: 'center' }}>
+        - <a href="http://www.helsinki.fi/discovery">R. U. Nokone</a>
+      </p>
+    </blockquote>
+  )
+}
+
+export default OodiToOodikone

services/frontend/src/components/Frontpage/index.jsx

@@ -1,62 +1,22 @@
-import React, { useEffect, useState } from 'react'
+import React from 'react'
 import { Container, Header, Image, Divider, List } from 'semantic-ui-react'
 import moment from 'moment'
 import { useGetAuthorizedUserQuery } from 'redux/auth'
-import { images, checkUserAccess } from '../common'
-import { useTitle } from '../common/hooks'
-import { builtAt } from '../conf'
-import oodiTXT from '../static/oodi.txt'
-
-const OodiToOodikone = () => {
-  const [oodis, setOodis] = useState([])
-  useEffect(() => {
-    const getOodis = async () => {
-      try {
-        const data = await fetch(oodiTXT)
-        setOodis(
-          (await data.text())
-            .split('**')
-            .map(o => o.trim())
-            .filter(oodi => !!oodi)
-        )
-      } catch (error) {
-        setOodis([])
-      }
-    }
-    getOodis()
-  }, [])
-
-  if (!oodis || oodis.length === 0) {
-    return null
-  }
-  const oodi = oodis[Math.floor(Math.random() * oodis.length)].split('\n')
-  return (
-    <div style={{ margin: 'auto', width: '50%' }}>
-      {oodi.map((l, index) => (
-        // eslint-disable-next-line react/no-array-index-key
-        <p key={index} style={{ textAlign: 'center', fontStyle: 'italic', margin: 0 }}>
-          {index === 0 && '\u201C'}
-          {l}
-          {index === oodi.length - 1 && '\u201D'}
-        </p>
-      ))}
-      <p style={{ marginTop: '1.3em', textAlign: 'center' }}>
-        - <a href="http://www.helsinki.fi/discovery">R. U. Nokone</a>
-      </p>
-    </div>
-  )
-}
+import { images, checkUserAccess } from '../../common'
+import { useTitle } from '../../common/hooks'
+import { builtAt } from '../../conf'
+import OodiToOodikone from './OoodiToOodikone'
 
 const FrontPage = () => {
-  const { rights, userRoles } = useGetAuthorizedUserQuery()
+  const { rights, roles } = useGetAuthorizedUserQuery()
   useTitle()
 
   const showItems = {
-    populations: userRoles.includes('admin') || rights.length > 0,
-    studyProgramme: userRoles.includes('admin') || rights.length > 0,
-    students: checkUserAccess(['studyGuidanceGroups', 'admin'], userRoles) || rights.length > 0,
-    courseStatistics: checkUserAccess(['courseStatistics', 'admin'], userRoles) || rights.length > 0,
-    teachers: checkUserAccess(['teachers', 'admin'], userRoles),
+    populations: roles.includes('admin') || rights.length > 0,
+    studyProgramme: roles.includes('admin') || rights.length > 0,
+    students: checkUserAccess(['studyGuidanceGroups', 'admin'], roles) || rights.length > 0,
+    courseStatistics: checkUserAccess(['courseStatistics', 'admin'], roles) || rights.length > 0,
+    teachers: checkUserAccess(['teachers', 'admin'], roles),
     trends: true,
     feedback: true,
   }

services/frontend/src/components/Frontpage/oodiToOodikone.css

@@ -0,0 +1,31 @@
+blockquote {
+  font-style: italic;
+  line-height: 1.618;
+  font-size: 1.2em;
+  width: 30em;
+  position: relative;
+  padding: 40px 20px;
+}
+blockquote::before {
+  content: open-quote;
+  top: 0;
+  left: 0;
+}
+
+blockquote::after {
+  content: close-quote;
+  bottom: 0;
+  right: 0;
+}
+
+blockquote::before,
+blockquote::after {
+  display: block;
+  width: 60px;
+  height: 60px;
+  line-height: 1.618;
+  font-size: 2em;
+  border-radius: 100%;
+  text-align: center;
+  position: absolute;
+}

services/frontend/src/components/LanguagePicker/useLanguage.jsx

@@ -7,22 +7,25 @@ const LanguageContext = createContext([[], () => {}])
 LanguageContext.displayName = 'Language'
 
 export const LanguageProvider = ({ children }) => {
-  const [state, setState] = useState(LANGUAGE_CODES[0])
+  let language = 'fi'
+  const [state, setState] = useState(language)
   const user = useGetAuthorizedUserQuery()
+  if (user) {
+    language = user.language
+  }
 
   // Load selected language.
   useEffect(() => {
     if (LANGUAGE_CODES.includes(user?.language)) {
       setState(user.language)
     }
-  }, [user])
+  }, [language])
 
   return <LanguageContext.Provider value={[state, setState]}>{children}</LanguageContext.Provider>
 }
 
 export default () => {
   const [state, setState] = useContext(LanguageContext)
-
   const setLanguage = newLanguage => {
     if (!LANGUAGE_CODES.includes(newLanguage)) {
       throw new Error('Illegal language code passed to useLanguage hook!')