Delete highcharts

from months to semesters like frontend expects

.github/actions/lint/action.yaml

@@ -4,9 +4,9 @@ description: 'Runs ESLint and Stylelint for frontend and backend'
 runs:
   using: composite
   steps:
-    - uses: actions/setup-node@v5
+    - uses: actions/setup-node@v6
       with:
-        node-version: 22
+        node-version: 24
     - name: 'Install dependencies'
       shell: bash
       run: npm ci && npm ci --prefix services/shared && npm ci --prefix services/frontend && npm ci --prefix services/backend

.github/actions/run-tests/action.yaml

@@ -19,7 +19,7 @@ runs:
   using: composite
   steps:
     - name: 'Login to toska Docker registry'
-      uses: docker/login-action@v3
+      uses: docker/login-action@v4
       with:
         registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
         username: ${{ inputs.username }}
@@ -27,10 +27,10 @@ runs:
 
     - name: 'Run services, pull rest of the needed images'
       shell: bash
-      run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull
+      run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull
 
     - name: 'Run cypress'
-      uses: cypress-io/github-action@v6
+      uses: cypress-io/github-action@v7
       with:
         browser: chrome
         spec: ${{ inputs.spec }}
@@ -38,7 +38,7 @@ runs:
         CYPRESS_baseUrl: http://localhost:1337
 
     - name: 'Upload video if tests fail'
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       if: failure()
       with:
         name: 'Cypress videos - ${{ inputs.test_name }}'
@@ -47,4 +47,4 @@ runs:
     - name: 'Print logs on failure'
       shell: bash
       if: failure()
-      run: docker compose --file docker-compose.ci.yml logs
+      run: docker compose --file docker-compose.ci.yml --profile oodikone logs

.github/workflows/production.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: 'Get current time'
         id: get_current_time
@@ -49,7 +49,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Extract release version
         id: extract_version
@@ -85,7 +85,7 @@ jobs:
       - release_backend
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Extract release version
         id: extract_version
@@ -100,9 +100,9 @@ jobs:
       - name: Create Sentry release
         uses: getsentry/action-release@v3
         env:
-          SENTRY_URL: https://toska.cs.helsinki.fi/
+          SENTRY_URL: https://toska.it.helsinki.fi/
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_TOKEN }}
-          SENTRY_ORG: toska
+          SENTRY_ORG: sentry
           SENTRY_PROJECT: oodikone-frontend
         with:
           environment: production
@@ -119,7 +119,7 @@ jobs:
       - release_backend
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Extract release version
         id: extract_version
@@ -134,9 +134,9 @@ jobs:
       - name: Create Sentry release
         uses: getsentry/action-release@v3
         env:
-          SENTRY_URL: https://toska.cs.helsinki.fi/
+          SENTRY_URL: https://toska.it.helsinki.fi/
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_TOKEN }}
-          SENTRY_ORG: toska
+          SENTRY_ORG: sentry
           SENTRY_PROJECT: oodikone-backend
         with:
           environment: production
@@ -157,13 +157,13 @@ jobs:
     if: success()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
-          softa-url: https://oodikone.cs.helsinki.fi/
+          softa-url: https://oodikone.helsinki.fi/
           deployment-target: production
 
   report_failure:
@@ -178,7 +178,7 @@ jobs:
     if: failure()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:

.github/workflows/pull_request.yaml

@@ -17,14 +17,43 @@ jobs:
     name: 'Lint packages'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Lint backend, frontend and shared
         uses: ./.github/actions/lint
 
+  test_api:
+    needs:
+      - lint_packages
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install node-24 for cypress
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v4
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+
+      - name: Start oodikone detached
+        run: npm run oodikone:ci:setup
+
+      - name: Run api-tests
+        run: npm run oodikone:ci:api
+
   test_end_to_end:
     runs-on: ubuntu-latest
     needs:
       - lint_packages
+      - test_api
     strategy:
       fail-fast: false
       matrix:
@@ -59,7 +88,7 @@ jobs:
             spec: cypress/e2e/University.js
     name: ${{ matrix.name }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:

.github/workflows/staging.yaml

@@ -29,7 +29,7 @@ jobs:
     needs: test
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Get current time
         id: get_current_time
@@ -64,7 +64,7 @@ jobs:
     needs: test
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Build backend image
         id: build-image
@@ -99,7 +99,7 @@ jobs:
     if: success()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
@@ -118,7 +118,7 @@ jobs:
     if: failure()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:

.github/workflows/test.yaml

@@ -16,7 +16,7 @@ jobs:
     name: 'Lint packages'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Lint backend, frontend and shared
         uses: ./.github/actions/lint
 
@@ -25,10 +25,10 @@ jobs:
     name: 'Build backend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: docker/setup-buildx-action@v3
+      - uses: actions/checkout@v6
+      - uses: docker/setup-buildx-action@v4
       - name: 'Build image and export it'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           build-args: |
             SENTRY_RELEASE=${{ github.sha }}
@@ -41,7 +41,7 @@ jobs:
         env:
           DOCKER_BUILD_SUMMARY: false
       - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: backend-image
           path: /tmp/backend.tar
@@ -50,7 +50,7 @@ jobs:
     name: 'Build frontend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Get current time
         id: get_current_time
         run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
@@ -76,7 +76,7 @@ jobs:
       user_db_migrations: ${{ steps.check_migrations.outputs.user_db_migrations }}
       sis_db_migrations: ${{ steps.check_migrations.outputs.sis_db_migrations }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Check if new migrations have been added
         id: check_migrations
         uses: dorny/paths-filter@v3
@@ -98,9 +98,9 @@ jobs:
     runs-on: ubuntu-latest
     if: needs.check_migration_files.outputs.sis_db_migrations == 'true'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
@@ -120,15 +120,15 @@ jobs:
     runs-on: ubuntu-latest
     if: needs.check_migration_files.outputs.kone_db_migrations == 'true'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Download backend image'
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v8
         with:
           name: backend-image
       - name: 'Load built backend image'
         run: docker load --input backend.tar
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
@@ -148,7 +148,7 @@ jobs:
     runs-on: ubuntu-latest
     if: needs.check_migration_files.outputs.user_db_migrations == 'true'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: dorny/paths-filter@v3
         id: check_migrations
         with:
@@ -157,13 +157,13 @@ jobs:
               - 'services/backend/src/database/migrations_user/**'
               - '.github/workflows/test.yaml'
       - name: 'Download backend image'
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v8
         with:
           name: backend-image
       - name: 'Load built backend image'
         run: docker load --input backend.tar
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
@@ -174,14 +174,104 @@ jobs:
           sleep 30 # wait for backend to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
 
-  # === End-to-end tests ===
-  test_end_to_end:
+  # === Other tests ===
+  test_service_exit_statuses:
+    name: 'Check services run correctly'
     needs:
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
-    # Always run the tests, even if some of the needed jobs failed or were skipped
-    if: always()
+      - lint_packages
+    if: always() && !cancelled() && needs.lint_packages.result == 'success'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v4
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+
+      - name: 'Run services, pull rest of the needed images'
+        run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull
+
+      - name: 'Check that exit statuses are not errors'
+        run: |
+          docker ps -a
+          if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
+            echo 'Some process had nonzero exit code';
+            exit 1;
+          fi
+
+      - name: 'Save Docker images for downstream jobs'
+        run: |
+          docker save \
+            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db \
+            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db \
+            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db \
+            redis:7.2.6 \
+            postgres:15.10 \
+            "quay.io/toska/jami:production" \
+            nginx:latest \
+          | zstd -T0 -3 -o /tmp/docker-images.tar.zst
+
+      - name: 'Upload Docker images artifact'
+        uses: actions/upload-artifact@v7
+        with:
+          name: docker-images
+          path: /tmp/docker-images.tar.zst
+          retention-days: 1
+
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker compose --file docker-compose.ci.yml --profile oodikone logs
+
+  # === Backend api-tests ===
+  test_api:
+    needs:
+      - test_service_exit_statuses
+    if: always() && !cancelled() && needs.test_service_exit_statuses.result == 'success'
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install node-24 for cypress
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: 'Download Docker images'
+        uses: actions/download-artifact@v8
+        with:
+          name: docker-images
+          path: /tmp
+
+      - name: 'Load Docker images'
+        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
+
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v4
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+
+      - name: Start oodikone (without frontend and backend)
+        run: npm run oodikone:ci:setup
+
+      - name: Run api-tests
+        run: npm run oodikone:ci:api
+
+  # === End-to-end tests ===
+  test_end_to_end:
+    needs:
+      - test_service_exit_statuses
+      - test_api
+    if: always() && !cancelled() && needs.test_service_exit_statuses.result == 'success'
     strategy:
       fail-fast: false
       matrix:
@@ -219,7 +309,21 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
+
+      - name: Install node-24 for cypress
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: 'Download Docker images'
+        uses: actions/download-artifact@v8
+        with:
+          name: docker-images
+          path: /tmp
+
+      - name: 'Load Docker images'
+        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
 
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
@@ -228,36 +332,3 @@ jobs:
           password: ${{ secrets.password }}
           spec: ${{ matrix.spec }}
           test_name: ${{ matrix.name }}
-
-  # === Other tests ===
-  test_service_exit_statuses:
-    name: 'Check services run correctly'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    if: always()
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v5
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
-        with:
-          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-
-      - name: 'Run services, pull rest of the needed images'
-        run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull
-
-      - name: 'Check that exit statuses are not errors'
-        run: |
-          docker ps -a
-          if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
-            echo 'Some process had nonzero exit code';
-            exit 1;
-          fi
-
-      - name: 'Print logs on failure'
-        if: failure()
-        run: docker compose --file docker-compose.ci.yml logs

.github/workflows/updater.yaml

@@ -1,29 +1,22 @@
 name: 'Test (and release) Updater'
 
 on:
-  push:
-    branches:
-      - master
+  release:
+    types: [published]
     paths:
       - '.github/workflows/updater.yaml'
       - 'updater/sis-updater-worker/**'
       - 'updater/sis-updater-scheduler/**'
-  pull_request:
-    branches:
-      - master
-    paths:
-      - 'updater/sis-updater-worker/**'
-      - 'updater/sis-updater-scheduler/**'
 
 jobs:
   lint_scheduler_and_worker:
     name: 'Lint scheduler and worker'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: actions/setup-node@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-node@v6
         with:
-          node-version: 22
+          node-version: 24
       - name: 'Install dependencies'
         run: npm ci && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker
       - name: 'Run linter'
@@ -33,10 +26,10 @@ jobs:
     name: 'Build Updater worker'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: docker/setup-buildx-action@v3
+      - uses: actions/checkout@v6
+      - uses: docker/setup-buildx-action@v4
       - name: 'Build the image'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: updater/sis-updater-worker/
           push: false
@@ -45,7 +38,7 @@ jobs:
         env:
           DOCKER_BUILD_SUMMARY: false
       - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: updater-worker-image
           path: /tmp/worker.tar
@@ -54,10 +47,10 @@ jobs:
     name: 'Build Updater scheduler'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: docker/setup-buildx-action@v3
+      - uses: actions/checkout@v6
+      - uses: docker/setup-buildx-action@v4
       - name: 'Build the image'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: updater/sis-updater-scheduler/
           push: false
@@ -66,7 +59,7 @@ jobs:
         env:
           DOCKER_BUILD_SUMMARY: false
       - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: updater-scheduler-image
           path: /tmp/scheduler.tar
@@ -79,15 +72,15 @@ jobs:
       - build_worker
       - build_scheduler
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
       - name: 'Download built images'
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v8
         with:
           pattern: 'updater-{worker,scheduler}-image'
           merge-multiple: true
@@ -122,9 +115,8 @@ jobs:
     name: 'Build and release Updater worker'
     runs-on: ubuntu-latest
     needs: test_updater
-    if: github.event_name == 'push'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Build worker image
         id: build-image
@@ -152,9 +144,8 @@ jobs:
     name: 'Build and release Updater scheduler'
     runs-on: ubuntu-latest
     needs: test_updater
-    if: github.event_name == 'push'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Build scheduler image
         id: build-image

.gitignore

@@ -65,9 +65,6 @@ instrumented/
 .vscode/launch.json
 .vscode/terminals.json
 
-# Funidata specific
-environments/
-
 # Target of shared symlink
 services/**/src/shared
 

.nycrc

@@ -1,5 +0,0 @@
-{
-  "all": true,
-  "include": ["services/frontend/src/**/*.js", "services/frontend/src/**/*.jsx"],
-  "extends": "./nyc-config.js"
-}

.stylelintrc

@@ -1,6 +1,6 @@
 {
   "extends": ["stylelint-config-standard"],
   "rules": {
-    "selector-class-pattern": "^[a-z][a-zA-Z-]*$"
+    "selector-class-pattern": "^(?:[a-z][a-zA-Z-]*|Mui[A-Za-z-]+)$"
   }
 }

README.md

@@ -112,8 +112,6 @@ The `documentation` directory contains some notes on the sis-db schema and is me
 
 For internal meeting notes and sensitive information, see Toska GitLab.
 
-[SortableTable documentation](./services/frontend/src/components/SortableTable/README.md)
-
 ### Basics
 
 The development environment is entirely configured in the `docker-compose.yml` file located in this repository. The file defines the services for Oodikone's two main components: **oodikone** and **updater**.

anonyymioodi/README.md

@@ -6,14 +6,14 @@ The quick guide should be enough for most purposes: updating schema, data, or th
 
 ## Quick guide to updating a test database
 
-1. Ensure you have the latest test-db. Just pulling images might not replace your existing volume, so you may need to remove it manually. This command might work: `docker volume rm oodikone_sis-db-data` (replace `oodikone_sis-db-data` with the correct volume name). The new image, including data, should be pulled the next time you launch Oodikone normally.
+1. Ensure you have the latest test-db. Just pulling images might not replace your existing volume, so you may need to remove it manually. This command might work: `docker volume rm oodikone-sis-db-data` (replace `oodikone-sis-db-data` with the correct volume name). The new image, including data, should be pulled the next time you launch Oodikone normally.
 
 2. Do your dev work as usual with the test database, for example, using `npm run both`. Edit data as needed, (for example, run the updater via the frontend), and then run tests to see if they pass (follow the instructions in the [README](../README.md) to run the tests more quickly).
 
 3. If you're updating `sis-db`:
 
    - Modify the scheduler limit in [config.js](../updater/sis-updater-scheduler/src/config.js) by setting `module.exports.DEV_SCHEDULE_COUNT = null`.
-   - Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone_sis-updater-worker_1`).
+   - Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone-sis-updater-worker`).
 
 4. When you're satisfied with the database, create a dump:
 
@@ -67,6 +67,6 @@ The quick guide should be enough for most purposes: updating schema, data, or th
 
 5. Navigate to <http://localhost:3000/updater> and click the three buttons (Update meta, Update students, Update curriculums) one by one. This will populate `sis-db` with data from the new `sis-importer-db`. You might want to do each update a couple of times, as the database tables affect each other, and not all necessary data might be generated on the first try.
 
-6. Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone_sis-updater-worker_1`). If you see any errors, try clicking the buttons again.
+6. Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone-sis-updater-worker`). If you see any errors, try clicking the buttons again.
 
 7. Run all the tests to ensure everything is working as expected (follow the instructions in the [README](../README.md) to run the tests more quickly). If any tests fail, update the test files accordingly.

cypress.config.js

@@ -3,7 +3,6 @@
 const { defineConfig } = require('cypress')
 
 module.exports = defineConfig({
-  experimentalStudio: process.env.NODE_ENV === 'development',
   projectId: 'c3jsph',
   defaultCommandTimeout: 30000,
   requestTimeout: 30000,
@@ -14,7 +13,6 @@ module.exports = defineConfig({
     // We've imported your old cypress plugins here.
     // You may want to clean this up later by importing these.
     setupNodeEvents(on, config) {
-      require('@cypress/code-coverage/task')(on, config)
       require('./cypress/plugins/index.js')(on, config)
       return config
     },

cypress/e2e/Completed_courses.js

@@ -17,32 +17,32 @@ const openSearch = () => {
 }
 
 const selectSavedCourselist = name => {
-  cy.get('[data-cy=history-search]').click()
-  cy.get('[data-cy=history-search]').type(name)
-  cy.get('[data-cy=history-search]').type('{downarrow}')
-  cy.get('[data-cy=history-search]').type('{enter}')
+  cy.cs('history-search').click()
+  cy.cs('history-search').type(name)
+  cy.cs('history-search').type('{downarrow}')
+  cy.cs('history-search').type('{enter}')
 }
 
 const createCourseList = (courseCodes, courseListName) => {
   openSearch()
   openCompletedCoursesModal()
-  cy.get('[data-cy=course-list-input]').type(courseCodes.join('\n'))
-  cy.get('[data-cy=search-name]').type(courseListName)
-  cy.get('[data-cy=save-courselist]').click()
+  cy.cs('course-list-input').type(courseCodes.join('\n'))
+  cy.cs('search-name').type(courseListName)
+  cy.cs('save-courselist').click()
 }
 
 const generateCourseListName = () => `TEST-course-list-${new Date().getTime()}`
 
 const deleteAllPreviousSearches = () => {
-  cy.get('[data-cy=history-search]').click()
+  cy.cs('history-search').click()
 
   cy.get('.MuiAutocomplete-popper li').then($options => {
     if ($options.length > 0) {
       cy.wrap($options).each(($option, index, $list) => {
         cy.wrap($option).click()
-        cy.get('[data-cy=delete-courselist]').click()
+        cy.cs('delete-courselist').click()
         if (index < $list.length - 1) {
-          cy.get('[data-cy=history-search]').click()
+          cy.cs('history-search').click()
         }
       })
     } else {
@@ -57,7 +57,7 @@ describe('When search modal is opened', () => {
     hasLanded()
     cy.url().should('include', '/completedcoursessearch')
     cy.contains('Search completed courses of students')
-    cy.get('[data-cy=open-completed-courses-modal-button]').click()
+    cy.cs('open-completed-courses-modal-button').click()
   })
 
   it('Modal opens correctly', () => {
@@ -74,8 +74,8 @@ describe('When search modal is opened', () => {
     cy.init('/completedcoursessearch?courseList=TKT10001&courseList=TKT10002&studentList=433237&studentList=457144')
     hasLanded()
     openCompletedCoursesModal()
-    cy.contains('[data-cy=student-no-input]', '433237, 457144')
-    cy.contains('[data-cy=course-list-input]', 'TKT10001, TKT10002')
+    cy.cs('student-no-input').contains('433237, 457144')
+    cy.cs('course-list-input').contains('TKT10001, TKT10002')
   })
 
   describe('When a search is executed with invalid input', () => {
@@ -83,16 +83,16 @@ describe('When search modal is opened', () => {
       cy.init('/completedcoursessearch')
       hasLanded()
       openCompletedCoursesModal()
-      cy.get('[data-cy=student-no-input]').type('1')
-      cy.get('[data-cy=course-list-input]').type('1')
-      cy.get('[data-cy=completed-courses-search-button]').click()
-      cy.get('[data-cy=rights-notification]').should(
+      cy.cs('student-no-input').type('1')
+      cy.cs('course-list-input').type('1')
+      cy.cs('completed-courses-search-button').click()
+      cy.cs('rights-notification').should(
         'contain.text',
         'The information for the following students could not be displayed'
       )
-      cy.contains('[data-cy=rights-notification]', '1')
-      cy.contains('[data-cy=completed-courses-table-div]', 'Student number')
-      cy.contains('[data-cy=completed-courses-table-div]', '1').should('not.exist')
+      cy.cs('rights-notification').contains('1')
+      cy.cs('completed-courses-table-div').contains('Student number')
+      cy.cs('completed-courses-table-div').contains('1').should('not.exist')
     })
   })
 
@@ -102,9 +102,9 @@ describe('When search modal is opened', () => {
       hasLanded()
       openCompletedCoursesModal()
       cy.fixture('completedCoursesData').then(({ studentSet1, coursesSet1 }) => {
-        cy.get('[data-cy=student-no-input]').type(studentSet1.join('\n'))
-        cy.get('[data-cy=course-list-input]').type(coursesSet1.join('\n'))
-        cy.get('[data-cy=completed-courses-search-button]').click()
+        cy.cs('student-no-input').type(studentSet1.join('\n'))
+        cy.cs('course-list-input').type(coursesSet1.join('\n'))
+        cy.cs('completed-courses-search-button').click()
       })
     })
 
@@ -120,10 +120,10 @@ describe('When search modal is opened', () => {
     it('Finds correct students and courses', () => {
       cy.fixture('completedCoursesData').then(({ studentSet1, coursesSet1 }) => {
         for (const studentNumber of studentSet1) {
-          cy.contains('[data-cy=completed-courses-table-div]', studentNumber)
+          cy.cs('completed-courses-table-div').contains(studentNumber)
         }
         for (const courseCode of coursesSet1) {
-          cy.contains('[data-cy=completed-courses-table-div]', courseCode)
+          cy.cs('completed-courses-table-div').contains(courseCode)
         }
       })
     })
@@ -150,7 +150,7 @@ describe('Courselist saving-related functions work', () => {
     openSearch()
     openCompletedCoursesModal()
     selectSavedCourselist(courseList)
-    cy.contains('[data-cy=course-list-input]', courses.join(', '))
+    cy.cs('course-list-input').contains(courses.join(', '))
   })
 
   it('Course list can be deleted', () => {
@@ -159,7 +159,7 @@ describe('Courselist saving-related functions work', () => {
     openSearch()
     openCompletedCoursesModal()
     selectSavedCourselist(courseList)
-    cy.get('[data-cy=delete-courselist]').click()
+    cy.cs('delete-courselist').click()
     cy.contains('You have no previous searches.')
   })
 
@@ -169,11 +169,11 @@ describe('Courselist saving-related functions work', () => {
     openSearch()
     openCompletedCoursesModal()
     selectSavedCourselist(courseList)
-    cy.get('[data-cy=course-list-input]').type(',TKT10001')
-    cy.get('[data-cy=save-courselist]').click()
+    cy.cs('course-list-input').type(',TKT10001')
+    cy.cs('save-courselist').click()
     openSearch()
     openCompletedCoursesModal()
     selectSavedCourselist(courseList)
-    cy.contains('[data-cy=course-list-input]', 'CSM14204, TKT10004, TKT10001')
+    cy.cs('course-list-input').contains('CSM14204, TKT10004, TKT10001')
   })
 })

cypress/e2e/Course_statistics.js

@@ -33,6 +33,7 @@ const checkGradeTable = gradesTableContents => {
 }
 
 const checkTableContents = contents => {
+  cy.get('table tbody').scrollIntoView()
   cy.get('table tbody').within(() => {
     contents.forEach((values, trIndex) => {
       cy.get('tr')
@@ -117,7 +118,7 @@ describe('Basic user', () => {
 
     searchByCourseCode(coursecode1)
     cy.wait('@courses').its('response.statusCode').should('be.oneOf', [200, 304])
-    cy.contains('td', coursecode1).click()
+    cy.cs(`course-${coursecode1}`).click()
 
     cy.wait('@yearlystats').its('response.statusCode').should('be.oneOf', [200, 304])
     cy.url().should('contain', `courseCodes=%5B%22${coursecode1}%22%5D`)
@@ -135,7 +136,7 @@ describe('Basic user', () => {
 
     searchByCourseCode(coursecode2)
     cy.wait('@courses').its('response.statusCode').should('be.oneOf', [200, 304])
-    cy.contains('td', coursecode2).click()
+    cy.cs(`course-${coursecode2}`).click()
 
     cy.wait('@yearlystats').its('response.statusCode').should('be.oneOf', [200, 304])
     cy.url().should('contain', `courseCodes=%5B%22${coursecode2}%22%5D`)
@@ -154,7 +155,7 @@ describe('Basic user', () => {
 
       searchByCourseCode(coursecode)
       cy.wait('@courses').its('response.statusCode').should('be.oneOf', [200, 304])
-      cy.contains('td', `${coursecode},`).click()
+      cy.cs(`course-${coursecode}`).click()
 
       cy.wait('@yearlystats').its('response.statusCode').should('be.oneOf', [200, 304])
       cy.url().should('contain', `courseCodes=%5B%22${coursecode}%22%5D`)
@@ -179,7 +180,7 @@ describe('Basic user', () => {
 
       searchByCourseCode(coursecode)
       cy.wait('@courses').its('response.statusCode').should('be.oneOf', [200, 304])
-      cy.contains('td', coursecode).click()
+      cy.cs(`course-${coursecode}`).click()
 
       cy.wait('@yearlystats').its('response.statusCode').should('be.oneOf', [200, 304])
       cy.url().should('contain', `courseCodes=%5B%22${coursecode}%22%5D`)
@@ -208,7 +209,7 @@ describe('Basic user', () => {
 
       searchByCourseCode(coursecode)
       cy.wait('@courses').its('response.statusCode').should('be.oneOf', [200, 304])
-      cy.contains('td', coursecode).click()
+      cy.cs(`course-${coursecode}`).click()
 
       cy.wait('@yearlystats').its('response.statusCode').should('be.oneOf', [200, 304])
       cy.url().should('contain', `courseCodes=%5B%22${coursecode}%22%5D`)
@@ -222,7 +223,7 @@ describe('Basic user', () => {
 
       cy.wait('@coursestats').its('response.statusCode').should('be.oneOf', [200, 304])
       cy.contains(
-        'Population of course Tieteellisen kirjoittamisen seminaarin alkuopetus: Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot 2011-2018 (open and normal)'
+        'Population of course Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot 2011-2018 (open and normal)'
       )
       cy.contains('Showing 4 out of 4 students')
 
@@ -241,7 +242,7 @@ describe('Basic user', () => {
 
       searchByCourseCode(coursecode)
       cy.wait('@courses').its('response.statusCode').should('be.oneOf', [200, 304])
-      cy.contains('td', coursecode).click()
+      cy.cs(`course-${coursecode}`).click()
 
       cy.wait('@yearlystats').its('response.statusCode').should('be.oneOf', [200, 304])
       cy.url().should('contain', `courseCodes=%5B%22${coursecode}%22%5D`)
@@ -281,8 +282,8 @@ describe('Basic user', () => {
 
       searchByCourseCode('TKT')
       cy.wait('@courses').its('response.statusCode').should('be.oneOf', [200, 304])
-      cy.contains('td', /^TKT20001/).click()
-      cy.contains('td', /^TKT10002/).click()
+      cy.cs(`course-TKT20001`).click()
+      cy.cs(`course-TKT10002`).click()
 
       cy.contains('Fetch statistics').should('be.enabled')
       cy.contains('Fetch statistics').click()
@@ -292,15 +293,16 @@ describe('Basic user', () => {
 
       openSummaryTab()
       cy.contains('Tietorakenteet ja algoritmit').click()
-      cy.contains('Ohjelmoinnin perusteet').should('not.exist')
-      cy.contains('TKT20001')
-      cy.contains('58131')
+      cy.contains('TKT10002').should('exist')
+      cy.contains('TKT20001').should('exist')
+      cy.contains('58131').should('not.exist')
 
       openSummaryTab()
       cy.contains('Ohjelmoinnin perusteet').click()
+      cy.contains('TKT10002').should('exist')
+      cy.contains('TKT20001').should('exist')
       cy.contains('Käyttöjärjestelmät').should('not.exist')
-      cy.contains('TKT10002')
-      cy.contains('581325')
+      cy.contains('581325').should('not.exist')
       openSummaryTab()
     })
   })
@@ -308,12 +310,12 @@ describe('Basic user', () => {
   it('On searches with multiple courses, has correct links on the Course tab', () => {
     cy.intercept({
       pathname: '/api/v2/coursesmulti',
-      query: { code: 'TKT' },
+      query: { code: 'BSCS' },
     }).as('courses')
 
     cy.intercept({
       pathname: '/api/v3/courseyearlystats',
-      query: { 'codes[]': 'TKT20001', 'codes[]': 'TKT10002' },
+      query: { 'codes[]': 'BSCS1003', 'codes[]': 'BSCS1001' },
     }).as('yearlystats')
 
     cy.intercept({
@@ -324,16 +326,16 @@ describe('Basic user', () => {
     cy.cs('select-multiple-courses-toggle').click()
     cy.cs('select-multiple-courses-toggle').should('have.class', 'Mui-checked')
 
-    searchByCourseCode('TKT')
+    searchByCourseCode('BSCS')
     cy.wait('@courses').its('response.statusCode').should('be.oneOf', [200, 304])
-    cy.contains('td', /^TKT20001/).click()
-    cy.contains('td', /^TKT10002/).click()
+    cy.cs(`course-BSCS1003`).click()
+    cy.cs(`course-BSCS1001`).click()
 
     cy.contains('Fetch statistics').should('be.enabled')
     cy.contains('Fetch statistics').click()
 
     cy.wait('@yearlystats').its('response.statusCode').should('be.oneOf', [200, 304])
-    cy.url().should('contain', 'courseCodes=%5B%22TKT10002%22%2C%22TKT20001%22%5D')
+    cy.url().should('contain', 'courseCodes=%5B%22BSCS1001%22%2C%22BSCS1003%22%5D')
 
     cy.cs('course-population-for-2023-2024').click()
     cy.contains('Population of course Introduction to Programming 2023-2024 (open and normal)')
@@ -343,9 +345,11 @@ describe('Basic user', () => {
     //       necessary if the test is flaky.
     cy.go('back')
     cy.cs('CourseSelector').click()
-    cy.cs('CourseSelectorOptionTKT20001').click()
+    cy.cs('CourseSelectorOptionBSCS1003').click()
+    // Check the link has updated correctly
+    cy.cs('course-population-for-2022-2023').invoke('attr', 'href').should('include', 'BSCS1003')
     cy.cs('course-population-for-2022-2023').click()
-    cy.contains('Population of course Tietorakenteet ja algoritmit')
+    cy.contains('Population of course Data Structures and Algorithms 2022-2023 (open and normal)')
   })
 
   it('Searching course by name displays right courses', () => {
@@ -353,7 +357,7 @@ describe('Basic user', () => {
     searchByCourseName('tietokantojen perusteet')
 
     cy.contains('Tietokantojen perusteet')
-    cy.contains('td', 'TKT10004, BSCS2001, 581328, A581328, AYTKT10004').click()
+    cy.cs(`course-TKT10004`).click()
 
     cy.contains('Search for courses').should('not.exist')
 
@@ -370,7 +374,7 @@ describe('Basic user', () => {
     cy.contains('Search for courses')
     searchByCourseName('tietorakenteet ja algoritmit')
     cy.contains('Tietorakenteet ja algoritmit')
-    cy.contains('td', 'TKT20001, BSCS1003, 58131, AYTKT20001').click()
+    cy.cs(`course-TKT20001`).click()
     cy.contains('Search for courses').should('not.exist')
 
     cy.contains('TKT20001 • Tietorakenteet ja algoritmit')
@@ -380,7 +384,7 @@ describe('Basic user', () => {
     clickNewQueryButton()
     cy.contains('Please enter at least 5 characters for course name or 2 characters for course code.')
     searchByCourseName('tietorakenteet ja algoritmit')
-    cy.contains('td', 'TKT20001, BSCS1003, 58131, AYTKT20001').click()
+    cy.cs(`course-TKT20001`).click()
 
     cy.contains('Search for courses').should('not.exist')
     cy.contains('TKT20001 • Tietorakenteet ja algoritmit')
@@ -393,7 +397,6 @@ describe('Basic user', () => {
     cy.contains('Search for courses')
     searchByCourseCode('TKT')
     cy.cs('select-multiple-courses-toggle').should('not.have.class', 'Mui-checked')
-    cy.contains('TKT10004, BSCS2001, 581328, A581328, AYTKT10004')
     cy.contains('Select all search results').should('not.exist')
     cy.cs('select-multiple-courses-toggle').click()
     cy.cs('select-multiple-courses-toggle').should('have.class', 'Mui-checked')
@@ -404,7 +407,7 @@ describe('Basic user', () => {
     cy.contains('Search for courses')
     searchByCourseName('tietokantojen perusteet')
 
-    cy.contains('td', /^TKT10004/).click()
+    cy.cs(`course-TKT10004`).click()
     cy.contains('Search for courses').should('not.exist')
 
     cy.contains('TKT10004 • Tietokantojen perusteet')
@@ -448,7 +451,7 @@ describe('Basic user', () => {
 
     it('Population of course shows grades for each student', () => {
       searchByCourseCode('TKT20001')
-      cy.contains('td', 'TKT20001, BSCS1003, 58131, AYTKT20001').click()
+      cy.cs(`course-TKT20001`).click()
       cy.contains('TKT20001 • Tietorakenteet ja algoritmit')
       cy.contains('AYTKT20001 • Avoin yo: Tietorakenteet ja algoritmit')
       cy.contains('BSCS1003 • Data Structures and Algorithms')
@@ -456,13 +459,13 @@ describe('Basic user', () => {
       cy.cs('course-population-for-2019-2020').click()
       cy.contains('Population of course Tietorakenteet ja algoritmit 2019-2020 (open and normal)')
       cy.contains('Students (33)').click()
-      cy.contains('td', '394776').siblings().eq(1).contains('3')
-      cy.contains('td', '497388').siblings().eq(1).contains('2')
+      cy.contains('td', '394776').siblings().eq(2).contains('3')
+      cy.contains('td', '497388').siblings().eq(2).contains('2')
     })
 
     it("In 'Course population' view, student numbers of students that the user isn't allowed to see are hidden", () => {
       searchByCourseCode('TKT20001')
-      cy.contains('td', 'TKT20001, BSCS1003, 58131, AYTKT20001').click()
+      cy.cs(`course-TKT20001`).click()
       cy.contains('TKT20001 • Tietorakenteet ja algoritmit')
       cy.contains('AYTKT20001 • Avoin yo: Tietorakenteet ja algoritmit')
       cy.contains('BSCS1003 • Data Structures and Algorithms')
@@ -476,7 +479,7 @@ describe('Basic user', () => {
 
   it('Language distribution is correct', () => {
     searchByCourseCode('TKT20003')
-    cy.contains('td', 'TKT20003, 582219').click()
+    cy.cs(`course-TKT20003`).click()
     cy.contains('TKT20003 • Käyttöjärjestelmät')
     cy.contains('582219 • Käyttöjärjestelmät')
     cy.cs('course-population-for-2021-2022').click()
@@ -494,7 +497,7 @@ describe('Basic user', () => {
         cy.cs('combine-substitutions-toggle').should('have.class', 'Mui-checked').click()
         cy.cs('combine-substitutions-toggle').should('not.have.class', 'Mui-checked')
         searchByCourseCode('TKT10002')
-        cy.contains('td', /^TKT10002$/).click()
+        cy.cs(`course-TKT10002`).click()
         cy.contains('Search for courses').should('not.exist')
         cy.contains('TKT10002 • Ohjelmoinnin perusteet')
       })
@@ -727,7 +730,7 @@ describe('Basic user', () => {
         cy.url().should('include', '/coursestatistics')
         cy.contains('Search for courses')
         searchByCourseCode('TKT10002')
-        cy.contains('td', /^TKT10002, BSCS1001, 581325, A581325, AYTKT10002$/).click()
+        cy.cs(`course-TKT10002`).click()
         cy.contains('Search for courses').should('not.exist')
         cy.contains('TKT10002 • Ohjelmoinnin perusteet')
         cy.contains('AYTKT10002 • Avoin yo: Ohjelmoinnin perusteet')

cypress/e2e/Custom_population.js

@@ -1,13 +1,9 @@
 /// <reference types="cypress" />
 
-const openCustomPopupForm = () => {
-  cy.get('[data-cy="custom-pop-search-button"]').click()
-}
-
 const fillName = () => {
   const name = `TEST-${new Date().getTime()}`
-  cy.contains('Insert name for this custom population if you wish to save it')
-  cy.get('[data-cy="custom-population-name-input"]').type(name)
+  cy.contains('Insert a name for this custom population if you wish to save it')
+  cy.cs('custom-population-name-input').type(name)
   return name
 }
 
@@ -16,38 +12,33 @@ const save = () => {
 }
 
 const selectSavedPopulation = name => {
-  cy.get('[data-cy="history-search"]').children().eq(0).type(`${name}{enter}`)
+  cy.cs('history-search').type(`${name}{enter}`)
 }
 
 const deleteAllSearches = () => {
-  cy.get('[data-cy="history-search"]').children().eq(0).click()
-  cy.contains('Saved populations')
-    .parent()
-    .parent()
-    .parent()
-    .parent()
-    .get('.dropdown')
-    .then(dropdown => {
-      const searchItems = dropdown.find('div[role=option] > span[class=text]')
-      for (let i = 0; i < searchItems.length; i++) {
-        if (searchItems[i].textContent.includes('TEST-')) {
-          cy.get('[data-cy="history-search"]').children().eq(0).type(`${searchItems[i].textContent}{enter}`)
-          cy.get('button').contains('Delete').click()
-        }
-      }
+  cy.cs('history-search').click()
+  cy.get('.MuiAutocomplete-popper > .MuiPaper-root')
+    .find('li[role=option]')
+    .then($lis => {
+      const labels = [...$lis].map(li => li.textContent?.trim() || '')
+
+      labels.forEach((label, index) => {
+        if (index !== 0) cy.cs('history-search').click()
+        cy.contains('.MuiAutocomplete-popper li[role=option]', label).click()
+        cy.contains('button', 'Delete').click()
+      })
     })
 }
 
 const fillForm = (content, separator) => {
-  cy.get('[data-cy="student-number-input"]').type(content.join(separator))
+  cy.cs('student-number-input').type(content.join(separator))
 }
 
 const search = () => {
-  cy.get('[data-cy="search-button"]').click()
+  cy.cs('search-button').click()
 }
 
 const searchFor = (studentnumbers, separator) => {
-  openCustomPopupForm()
   fillForm(studentnumbers, separator)
   search()
 }
@@ -74,7 +65,7 @@ const containsSpecificStudents = (studentnumbers = []) => {
 }
 
 const checkRightsNotification = studentNumbers => {
-  cy.get('[data-cy="rights-notification"]').within(() => {
+  cy.cs('rights-notification').within(() => {
     cy.contains(
       'The following students information could not be displayed. This could be either because they do not exist, or you do not have the right to view their information.'
     )
@@ -96,7 +87,6 @@ describe('Custom population tests', () => {
     cy.visit('/custompopulation')
     cy.url().should('include', '/custompopulation')
     cy.contains('Custom population')
-    cy.get('button').contains('Custom population').click()
     deleteAllSearches()
   })
 
@@ -126,7 +116,7 @@ describe('Custom population tests', () => {
       cy.contains('Credit accumulation').should('not.exist')
       cy.contains('Programme distribution').should('not.exist')
       cy.contains('Courses of population').should('not.exist')
-      cy.get('button').contains('Custom population')
+      cy.get('button').contains('Back to search form')
     })
 
     it("Doesn't return students user has no right to", () => {
@@ -138,16 +128,15 @@ describe('Custom population tests', () => {
       cy.contains('Credit accumulation').should('not.exist')
       cy.contains('Programme distribution').should('not.exist')
       cy.contains('Courses of population').should('not.exist')
-      cy.get('button').contains('Custom population')
+      cy.get('button').contains('Back to search form')
     })
   })
 
   describe('Custom population search saving', () => {
     it('Saves a custom population search', { retries: 2 }, () => {
       cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1: students }) => {
-        openCustomPopupForm()
         cy.contains(
-          'Insert student numbers you wish to use for population. Separate each number with a comma, semicolon, space, or newline.'
+          'Insert student numbers to use for the population. Each student number needs to be separated with a comma, a semicolon, a space, or a line break.'
         )
         const name = fillName()
         // A comma and a newline on purpose, the page should be able to handle it
@@ -157,17 +146,16 @@ describe('Custom population tests', () => {
         // Round 1
         selectSavedPopulation(name)
         search()
-        cy.contains(`Custom population "${name}"`)
+        cy.contains(`Custom population: ${name}`)
         containsAmountOfStudents(students.length)
         containsSpecificStudents(students)
 
         // Round 2
         cy.visit('/custompopulation')
         cy.contains('Custom population')
-        openCustomPopupForm()
         selectSavedPopulation(name)
         search()
-        cy.contains(`Custom population "${name}"`)
+        cy.contains(`Custom population: ${name}`)
         containsAmountOfStudents(students.length)
         containsSpecificStudents(students)
       })
@@ -176,28 +164,25 @@ describe('Custom population tests', () => {
     it('Updates a custom population search', () => {
       cy.fixture('customPopulations').then(
         ({ studentNumbersForCSStudentsSet1: students1, studentNumbersForCSStudentsSet2: students2 }) => {
-          openCustomPopupForm()
           const name = fillName()
           fillForm(students1, ' ')
           save()
 
           selectSavedPopulation(name)
           search()
-          cy.contains(`Custom population "${name}"`)
+          cy.contains(`Custom population: ${name}`)
           containsAmountOfStudents(students1.length)
           containsSpecificStudents(students1)
-          cy.contains('Custom population')
-          openCustomPopupForm()
+          cy.contains('Back to search form').click()
           selectSavedPopulation(name)
           fillForm(['\n', ...students2], ', ')
           save()
 
           cy.visit('/custompopulation')
           cy.contains('Custom population')
-          openCustomPopupForm()
           selectSavedPopulation(name)
           search()
-          cy.contains(`Custom population "${name}"`)
+          cy.contains(`Custom population: ${name}`)
           containsAmountOfStudents(students1.length + students2.length)
           containsSpecificStudents([...students1, ...students2])
         }

cypress/e2e/Degree_programme_overview.js

@@ -93,31 +93,14 @@ describe('Degree programme overview', () => {
     it('Basic information contains correct credits', () => {
       const years = getEmptyYears()
       const tableContents = [
-        ...years.map(year => [year, 0, 0, 0, 0, 0]),
-        [2023, 1519, 1519, 0, 0, 222],
-        [2022, 3235, 3205, 0, 30, 209],
-        [2021, 5133, 5108, 0, 25, 428],
-        [2020, 5801, 5796, 0, 5, 94],
-        [2019, 5305, 5305, 0, 0, 162],
-        [2018, 3442, 3432, 0, 10, 21],
-        [2017, 1211, 1211, 0, 0, 189],
-      ]
-
-      cy.checkTableStats(tableContents, 'credits-produced-by-the-study-programme')
-    })
-
-    it("Toggling 'Show special categories' on displays additional information", () => {
-      cy.cs('special-categories-toggle').click()
-      const years = getEmptyYears()
-      const tableContents = [
-        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0]),
-        [2023, 1519, 1519, 0, 0, 0, 0, 222],
-        [2022, 3235, 3205, 0, 30, 0, 0, 209],
-        [2021, 5133, 5108, 0, 25, 0, 0, 428],
-        [2020, 5801, 5796, 0, 5, 0, 0, 94],
-        [2019, 5305, 5305, 0, 0, 0, 0, 162],
-        [2018, 3442, 3432, 0, 10, 0, 0, 21],
-        [2017, 1211, 1211, 0, 0, 0, 0, 189],
+        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+        [2023, 1519, 1519, 0, 0, 0, 0, 0, 222, 0],
+        [2022, 3235, 3205, 0, 0, 30, 0, 0, 209, 0],
+        [2021, 5133, 5108, 0, 0, 25, 0, 0, 428, 25],
+        [2020, 5801, 5796, 0, 0, 5, 0, 0, 94, 0],
+        [2019, 5305, 5305, 0, 0, 0, 0, 0, 162, 0],
+        [2018, 3442, 3432, 0, 0, 10, 0, 0, 21, 0],
+        [2017, 1211, 1211, 0, 0, 0, 0, 0, 189, 0],
       ]
 
       cy.checkTableStats(tableContents, 'credits-produced-by-the-study-programme')
@@ -190,14 +173,14 @@ describe('Degree programme overview', () => {
       cy.checkTableStats(studentTableContents, 'students-of-the-study-programme')
 
       const creditTableContents = [
-        ...years.map(year => [year, 0, 0, 0, 0, 0]),
-        ['2023 - 2024', 160, 160, 0, 0, 67],
-        ['2022 - 2023', 2725, 2720, 0, 5, 337],
-        ['2021 - 2022', 4092, 4042, 0, 50, 198],
-        ['2020 - 2021', 5420, 5415, 0, 5, 321],
-        ['2019 - 2020', 6043, 6043, 0, 0, 101],
-        ['2018 - 2019', 4856, 4851, 0, 5, 107],
-        ['2017 - 2018', 2350, 2345, 0, 5, 26],
+        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+        ['2023 - 2024', 160, 160, 0, 0, 0, 0, 0, 67, 0],
+        ['2022 - 2023', 2725, 2720, 0, 0, 5, 0, 0, 337, 0],
+        ['2021 - 2022', 4092, 4042, 0, 0, 50, 0, 0, 198, 0],
+        ['2020 - 2021', 5420, 5415, 0, 0, 5, 0, 0, 321, 25],
+        ['2019 - 2020', 6043, 6043, 0, 0, 0, 0, 0, 101, 0],
+        ['2018 - 2019', 4856, 4851, 0, 0, 5, 0, 0, 107, 0],
+        ['2017 - 2018', 2350, 2345, 0, 0, 5, 0, 0, 26, 0],
       ]
 
       cy.checkTableStats(creditTableContents, 'credits-produced-by-the-study-programme')
@@ -231,10 +214,10 @@ describe('Degree programme overview', () => {
         cy.contains('2022')
         cy.contains('48 graduated').trigger('mouseover')
         cy.contains('48 students graduated in year 2022')
-        cy.contains('median study time: 44.5 months')
-        cy.contains('22 graduated on time')
-        cy.contains('12 graduated max year overtime')
-        cy.contains('14 graduated over year late')
+        cy.contains('median study time: 6 semesters')
+        cy.contains('26 graduated on time')
+        cy.contains('10 graduated max year overtime')
+        cy.contains('12 graduated over year late')
       })
 
       cy.cs('programmes-before-or-after-stacked-bar-chart-section')
@@ -262,7 +245,7 @@ describe('Degree programme overview', () => {
         cy.contains('2021')
         cy.contains('2 graduated').trigger('mouseover')
         cy.contains('2 students graduated in year 2021')
-        cy.contains('median study time: 25 months')
+        cy.contains('median study time: 4.5 semesters')
         cy.contains('1 graduated on time')
         cy.contains('1 graduated max year overtime')
         cy.contains('0 graduated over year late')
@@ -274,9 +257,9 @@ describe('Degree programme overview', () => {
         cy.contains('2023')
         cy.contains('11 graduated').trigger('mouseover')
         cy.contains('11 students graduated in year 2023')
-        cy.contains('median study time: 69 months')
-        cy.contains('3 graduated on time')
-        cy.contains('7 graduated max year overtime')
+        cy.contains('median study time: 12 semesters')
+        cy.contains('4 graduated on time')
+        cy.contains('6 graduated max year overtime')
         cy.contains('1 graduated over year late')
       })
     })
@@ -286,11 +269,11 @@ describe('Degree programme overview', () => {
     beforeEach(() => {
       cy.intercept(
         'GET',
-        'api/v2/studyprogrammes/KH50_001/studytrackstats?graduated=GRADUATED_INCLUDED&special_groups=SPECIAL_INCLUDED&combined_programme='
+        'api/v2/studyprogrammes/KH50_001/studytrackstats?special_groups=SPECIAL_INCLUDED&combined_programme='
       ).as('stQuery')
       cy.init('/study-programme')
       cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click()
-      cy.cs('study-tracks-and-class-statistics-tab').click()
+      cy.cs('StudyTracksAndClassStatisticsTab').click()
     })
 
     describe('Info boxes', () => {
@@ -319,7 +302,7 @@ describe('Degree programme overview', () => {
 
     it('Students of the degree programme are shown correctly', () => {
       const tableContents = [
-        // [Year, All, Started studying, Present, Absent, Inactive, Graduated, Men, Women, Other/Unknown, Finland, Other]
+        // [Year, All, Started studying, Present, Absent, Passive, Graduated, Men, Women, Other/Unknown, Finland, Other]
         ['2023 - 2024', 8, 8, 0, 0, 8, 0, 5, 3, 0, 8, 1],
         ['2022 - 2023', 26, 25, 0, 0, 24, 2, 19, 7, 0, 25, 1],
         ['2021 - 2022', 38, 29, 0, 0, 33, 5, 29, 9, 0, 36, 4],
@@ -398,44 +381,46 @@ describe('Degree programme overview', () => {
     it('Student progress data is shown correctly', () => {
       const years = getEmptyYears(true)
       const tableContents = [
-        // [Year, All, < 30 credits, 30–60 credits, 60–90 credits, 90–120 credits, 120–150 credits, 150–180 credits, ≥ 180 credits]
-        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0]),
-        ['2023 - 2024', 8, 8, 0, 0, 0, 0, 0, 0],
-        ['2022 - 2023', 26, 9, 9, 4, 3, 0, 1, 0],
-        ['2021 - 2022', 38, 8, 6, 11, 8, 5, 0, 0],
-        ['2020 - 2021', 30, 2, 1, 3, 7, 4, 4, 9],
-        ['2019 - 2020', 35, 1, 0, 2, 1, 0, 4, 27],
-        ['2018 - 2019', 45, 0, 1, 1, 2, 0, 3, 38],
-        ['2017 - 2018', 47, 0, 1, 3, 0, 1, 2, 40],
-        ['Total', 229, 28, 18, 24, 21, 10, 14, 114],
+        // [Year, All, < 30 credits, 30–60 credits, 60–90 credits, 90–120 credits, 120–150 credits, 150–180 credits, ≥ 180 credits, Graduated]
+        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+        ['2023 - 2024', 8, 8, 0, 0, 0, 0, 0, 0, 0],
+        ['2022 - 2023', 26, 9, 8, 4, 3, 0, 0, 0, 2],
+        ['2021 - 2022', 38, 8, 4, 9, 8, 4, 0, 0, 5],
+        ['2020 - 2021', 30, 2, 1, 2, 5, 0, 0, 1, 19],
+        ['2019 - 2020', 35, 1, 0, 2, 1, 0, 0, 4, 27],
+        ['2018 - 2019', 45, 0, 1, 1, 2, 0, 0, 2, 39],
+        ['2017 - 2018', 47, 0, 1, 3, 0, 0, 1, 0, 42],
+        ['Total', 229, 28, 15, 21, 19, 4, 1, 7, 134],
       ]
 
       cy.checkTableStats(tableContents, 'study-programme-progress')
     })
 
     it('Progress section', () => {
+      cy.cs('programme-progress-bar-chart-section').scrollIntoView()
+
       cy.cs('programme-progress-bar-chart-section')
         .should('contain', 'Less than 30 credits')
         .should('contain', '30–60 credits')
         .should('contain', 'At least 180 credits')
-        .should('contain', '49.8%') // The percentage for total, at least 180 credits, to check that the graph renders
+        .should('contain', '58.5%') // The percentage for total graduated, to check that the graph renders
 
-      cy.cs('programme-progress-bar-chart-section').contains('49.8%').trigger('mouseover', { force: true })
-      cy.contains('At least 180 credits: 114')
+      cy.cs('programme-progress-bar-chart-section').contains('58.5%').trigger('mouseover', { force: true })
+      cy.contains('Graduated: 134')
     })
 
     it('Average graduation times section', () => {
       cy.cs('average-graduation-times-section').within(() => {
         cy.contains('2020 - 2021')
-        cy.get('[aria-label="2020 - 2021, 15. On time."]').trigger('mouseover')
-        cy.contains('On time: 15')
+        cy.get('[aria-label="2020 - 2021, 16. On time."]').trigger('mouseover')
+        cy.contains('On time: 16')
         cy.contains("Click a bar to view that year's study track level breakdown")
         cy.get('[aria-label="2019 - 2020, 22. On time."]').click()
         cy.contains('Year 2019 - 2020 by start year')
-        cy.get('[aria-label="MAT-MAT, 14. On time."]').trigger('mouseover')
+        cy.get('[aria-label="MAT-MAT, 13. On time."]').trigger('mouseover')
         cy.contains('Matematiikka')
         cy.contains('MAT-MAT')
-        cy.contains('On time: 14')
+        cy.contains('On time: 13')
       })
 
       cy.cs('graduation-time-toggle').click()
@@ -444,9 +429,9 @@ describe('Degree programme overview', () => {
         cy.contains('2020 - 2021')
         cy.contains('19 graduated').trigger('mouseover')
         cy.contains('From class of 2020 - 2021, 19/30 students have graduated')
-        cy.contains('median study time: 33 months')
-        cy.contains('15 graduated on time')
-        cy.contains('3 graduated max year overtime')
+        cy.contains('median study time: 6 semesters')
+        cy.contains('16 graduated on time')
+        cy.contains('2 graduated max year overtime')
         cy.contains('1 graduated over year late')
       })
     })
@@ -465,7 +450,7 @@ describe('Degree programme overview', () => {
       it('Students of the study track are shown correctly', () => {
         cy.cs('study-track-overview-section').contains('Students of the study track MAT-MAT by starting year')
         const tableContents = [
-          // [Year, All, Started studying, Present, Absent, Inactive, Graduated, Men, Women, Other/Unknown, Finland, Other]
+          // [Year, All, Started studying, Present, Absent, Passive, Graduated, Men, Women, Other/Unknown, Finland, Other]
           ['2022 - 2023', 3, 3, 0, 0, 1, 2, 2, 1, 0, 3, 0],
           ['2021 - 2022', 5, 1, 0, 0, 1, 4, 3, 2, 0, 5, 0],
           ['2020 - 2021', 13, 10, 0, 0, 1, 12, 7, 6, 0, 12, 3],
@@ -519,7 +504,7 @@ describe('Degree programme overview', () => {
     beforeEach(() => {
       cy.init('/study-programme')
       cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click()
-      cy.cs('programme-courses-tab').click()
+      cy.cs('ProgrammeCoursesTab').click()
 
       cy.cs('by-credit-type-section')
       cy.cs('by-credit-type-tab')
@@ -528,14 +513,21 @@ describe('Degree programme overview', () => {
 
     describe('By credit type tab', () => {
       beforeEach(() => {
-        cy.get('tbody').within(() => {
+        cy.get('thead').within(() => {
           cy.get('tr')
             .eq(0)
+            .within(() => {
+              cy.get('th').eq(3).click()
+            })
+        })
+        cy.get('tbody').within(() => {
+          cy.get('tr')
+            .eq(8)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('341')
+              cy.get('td').eq(1).contains('MAT11004')
+              cy.get('td').eq(2).contains('Differentiaalilaskenta')
+              cy.get('td').eq(3).contains('1170')
             })
         })
       })
@@ -548,12 +540,12 @@ describe('Degree programme overview', () => {
 
         cy.get('tbody').within(() => {
           cy.get('tr')
-            .eq(0)
+            .eq(9)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('121')
+              cy.get('td').eq(1).contains('MAT21009')
+              cy.get('td').eq(2).contains('Kandidaatintutkielma')
+              cy.get('td').eq(3).contains('414')
             })
         })
       })
@@ -568,12 +560,12 @@ describe('Degree programme overview', () => {
 
         cy.get('tbody').within(() => {
           cy.get('tr')
-            .eq(0)
+            .eq(5)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('311')
+              cy.get('td').eq(1).contains('MAT11001')
+              cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
+              cy.get('td').eq(3).contains('1220')
             })
         })
       })
@@ -632,31 +624,67 @@ describe('Degree programme overview', () => {
       })
 
       it("'Show credits/Show students' toggle works", () => {
-        cy.get('thead tr').within(() => {
-          cy.get('th').should('have.length', 8)
-          const headers = ['Type', 'Code', 'Name', 'Total', 'Major', 'Non-major', 'Non-degree', 'Transferred']
-          headers.forEach((header, index) => {
-            cy.get('th').eq(index).contains(header)
+        cy.get('thead tr')
+          .eq(0)
+          .within(() => {
+            cy.get('th').should('have.length', 6)
+            const headers = ['Type', 'Code', 'Name', 'Total credits', 'Credits produced by', 'Transferred credits']
+            headers.forEach((header, index) => {
+              cy.get('th').eq(index).contains(header)
+            })
+          })
+        cy.get('thead tr')
+          .eq(1)
+          .within(() => {
+            cy.get('th').should('have.length', 7)
+            const headers = [
+              'Degree students',
+              'Open university (hetu)',
+              'Open university (no hetu)',
+              'Exchange students',
+              'Other universities',
+              'Separate studies',
+              'Other',
+            ]
+            headers.forEach((header, index) => {
+              cy.get('th').eq(index).contains(header)
+            })
           })
-        })
 
         cy.cs('show-credits-students-toggle').click()
 
+        cy.get('thead tr')
+          .eq(0)
+          .within(() => {
+            cy.get('th').should('have.length', 7)
+            const headers = [
+              'Type',
+              'Code',
+              'Name',
+              'Total students',
+              'Breakdown of total',
+              'Breakdown of passed',
+              'Not included in total or passed',
+            ]
+            headers.forEach((header, index) => {
+              cy.get('th').eq(index).contains(header)
+            })
+          })
         cy.get('thead tr')
           .eq(1)
           .within(() => {
             cy.get('th').should('have.length', 10)
             const headers = [
-              'Type',
-              'Code',
-              'Name',
-              'Total',
               'Passed',
-              'Not',
-              'Major',
-              'Non-major',
-              'Non-degree',
-              'Transferred',
+              'Not completed',
+              'Degree students',
+              'Open university students (with hetu)',
+              'Open university students (without hetu)',
+              'Exchange students',
+              'Other university students',
+              'Separate studies',
+              'Other students',
+              'Students with transferred credits',
             ]
             headers.forEach((header, index) => {
               cy.get('th').eq(index).contains(header)
@@ -665,12 +693,12 @@ describe('Degree programme overview', () => {
 
         cy.get('tbody').within(() => {
           cy.get('tr')
-            .eq(0)
+            .eq(5)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('341')
+              cy.get('td').eq(1).contains('MAT11001')
+              cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
+              cy.get('td').eq(3).contains('375')
             })
         })
       })
@@ -682,7 +710,7 @@ describe('Degree programme overview', () => {
       })
 
       // FIXME: Flaky
-      it('test', () => {
+      it.skip('test', () => {
         // TODO: Implement some real tests for this tab
         cy.contains('From')
       })
@@ -693,7 +721,7 @@ describe('Degree programme overview', () => {
     beforeEach(() => {
       cy.init('/study-programme')
       cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click()
-      cy.cs('degree-courses-tab').click()
+      cy.cs('DegreeCoursesTab').click()
     })
 
     it('Curriculum section', () => {
@@ -729,7 +757,7 @@ describe('Degree programme overview', () => {
     beforeEach(() => {
       cy.init('/study-programme')
       cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click({ force: true })
-      cy.cs('tags-tab').click()
+      cy.cs('TagsTab').click()
     })
 
     it('info box', () => {
@@ -856,7 +884,7 @@ describe('Degree programme overview', () => {
     })
 
     it('can access basic information', () => {
-      cy.cs('basic-information-tab').click()
+      cy.cs('BasicInformationTab').click()
 
       cy.cs('students-of-the-study-programme-section')
       cy.cs('credits-produced-by-the-study-programme-section')
@@ -866,7 +894,7 @@ describe('Degree programme overview', () => {
     })
 
     it('can access study tracks', () => {
-      cy.cs('study-tracks-and-class-statistics-tab').click()
+      cy.cs('StudyTracksAndClassStatisticsTab').click()
 
       cy.cs('study-track-overview-section')
       cy.cs('progress-of-students-section')
@@ -874,9 +902,9 @@ describe('Degree programme overview', () => {
     })
 
     it("doesn't see other tabs", () => {
-      cy.cs('degree-courses-tab').should('not.exist')
-      cy.cs('tags-tab').should('not.exist')
-      cy.cs('update-statistics-tab').should('not.exist')
+      cy.cs('DegreeCoursesTab').should('not.exist')
+      cy.cs('TagsTab').should('not.exist')
+      cy.cs('UpdateStatisticsTab').should('not.exist')
     })
   })
 })

cypress/e2e/Filters.js

@@ -273,12 +273,12 @@ describe('Population Statistics with Bachelor + Master', { testIsolation: false
     checkFilteredStudentCount(0)
 
     // Non-Bachelor
-    cy.get('input[name="studyRightStatusFilter"][value="Inactive Bachelor study right"]').click()
-    cy.get('input[name="studyRightStatusFilter"][value="Inactive Bachelor study right"]').should('be.checked')
+    cy.get('input[name="studyRightStatusFilter"][value="Passive Bachelor study right"]').click()
+    cy.get('input[name="studyRightStatusFilter"][value="Passive Bachelor study right"]').should('be.checked')
     checkFilteredStudentCount(5)
     // Non-Master (combined)
-    cy.get('input[name="studyRightStatusFilter"][value="Inactive Master study right"]').click()
-    cy.get('input[name="studyRightStatusFilter"][value="Inactive Master study right"]').should('be.checked')
+    cy.get('input[name="studyRightStatusFilter"][value="Passive Master study right"]').click()
+    cy.get('input[name="studyRightStatusFilter"][value="Passive Master study right"]').should('be.checked')
     checkFilteredStudentCount(24)
 
     cy.cs('studyRightStatusFilter-clear').click()
@@ -385,7 +385,6 @@ describe('Custom Population Statistics', { testIsolation: false }, () => {
 
   before(() => {
     cy.init('/custompopulation')
-    cy.cs('custom-pop-search-button').click()
     cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
       const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
       defaultAmountOfStudents = students.length

cypress/e2e/Language_center.js

@@ -1,42 +1,23 @@
 /// <reference types="cypress" />
 
-const hasLanded = () => {
-  cy.contains('This view displays amounts')
-  cy.contains('Show number of')
-}
-
 const chooseSemester = (semester, fromOrTo) => {
-  cy.get(`[data-cy="semester-${fromOrTo}"]`).click()
-  cy.get(`[data-cy="semester-${fromOrTo}"]`).within(() => {
-    cy.get('div.visible.menu.transition').within(() => {
-      cy.contains('div.item', semester).click()
-    })
-    cy.get('div.menu.transition').should('not.have.class', 'visible')
-  })
+  cy.cs(`semester-${fromOrTo}`).click()
+  cy.cs(`select-opt-${semester}`).click()
+  cy.cs(`select-opt-${semester}`).should('not.have.class', 'visible')
 }
 
 const checkNumbers = (numbers, numberOfColumns, mode) => {
-  const offset = mode === 'semesters' ? 2 : 1
-  cy.get('table > tbody > tr:first').within(() => {
-    cy.get('td').should('have.length', numberOfColumns)
-    cy.get('td').eq(0).contains('All courses total')
-    numbers.forEach((number, index) => {
-      cy.get('td')
-        .eq(index + offset)
-        .contains(number)
-    })
-  })
-}
+  const offset = 1 // skip first col with codes / names
+  cy.cs(`ooditable-${mode}`).find('table').find('tbody').as('tbody')
+  cy.get('@tbody').find('tr').first().as('totalRow')
+  cy.get('@totalRow').find('td').should('have.length', numberOfColumns)
+  cy.get('@totalRow').find('td').eq(0).should('contain', 'All courses total')
 
-const checkStyle = (styles, numberOfColumns) => {
-  cy.get('table > tbody > tr:first').within(() => {
-    cy.get('td').should('have.length', numberOfColumns)
-    styles.forEach((style, index) => {
-      cy.get('td')
-        .eq(index + 2)
-        .should('have.attr', 'style')
-        .and('include', style)
-    })
+  numbers.forEach((number, index) => {
+    cy.get('@totalRow')
+      .find('td')
+      .eq(index + offset)
+      .should('contain', number)
   })
 }
 
@@ -44,8 +25,7 @@ describe('When language center is opened', () => {
   describe('as an admin user', () => {
     beforeEach(() => {
       cy.init('/languagecenterview', 'admin')
-      cy.get('[data-cy="completions-button"]').click()
-      hasLanded()
+      cy.cs('completions-button').click()
     })
 
     describe('Faculties tab', () => {
@@ -64,7 +44,7 @@ describe('When language center is opened', () => {
         cy.get('table > tbody > tr:first').within(() => {
           cy.get('td').should('have.length', 15)
           cy.get('td').eq(0).contains('All courses total')
-          const numbers = [36, 9, 33, 756, 15, 40, 6, 48, 1, 24, 1059, 49, 0, 2076]
+          const numbers = [2076, 36, 9, 33, 756, 15, 40, 6, 48, 1, 24, 1059, 49, 0]
           numbers.forEach((number, index) => {
             cy.get('td')
               .eq(index + 1)
@@ -73,16 +53,14 @@ describe('When language center is opened', () => {
         })
       })
 
-      // FIXME: Flaky
       it('Faculties tab "exceeding" button works', () => {
-        cy.get('[data-cy="difference-button"]').click()
-        checkNumbers([...new Array(12).fill(0), 1, 221], 15)
+        cy.cs('difference-button').click()
+        checkNumbers([65, 2, 0, 3, 13, 0, 1, 1, 2, 0, 1, 30, 11, 1], 15, 'faculties')
       })
 
-      // FIXME: Flaky
       it('Faculties tab semester selector changes numbers', () => {
         chooseSemester('Syksy 2020', 'from')
-        checkNumbers([28, 6, 27, 496, 4, 10, 2, 33, 0, 9, 535, 34, 0, 1184], 15)
+        checkNumbers([1184, 28, 6, 27, 496, 4, 10, 2, 33, 0, 9, 535, 34, 0], 15, 'faculties')
       })
     })
 
@@ -93,23 +71,8 @@ describe('When language center is opened', () => {
         chooseSemester('Kevät 2024', 'to')
       })
 
-      // FIXME: Flaky
       it('Semester tab shows numbers', () => {
-        checkNumbers([69, 26, 298, 58, 343, 98, 438, 138, 310, 90, 123, 74, 10, 1, 2076], 17, 'semesters')
-      })
-
-      // FIXME: Flaky
-      it('Coloring mode works on semester tab', () => {
-        cy.contains('Compare to average of course').click()
-        const expectedAlphas = [0.04, 0.016, 0.17, 0.03, 0.192, 0.055, 0.247, 0.08, 0.173, 0.05, 0.07, 0.043, 0.004, 0]
-
-        checkStyle(
-          expectedAlphas.map(alpha => `background-color: rgba(0, 170, 0, ${alpha})`),
-          17
-        )
-        cy.get('table > tbody > tr:first').within(() => {
-          cy.get('td').eq(16).should('have.attr', 'style').and('not.include', 'background-color')
-        })
+        checkNumbers([2076, 69, 26, 298, 58, 343, 98, 438, 138, 310, 90, 123, 74, 10, 1], 16, 'semesters')
       })
     })
   })

cypress/e2e/Openuni.js

@@ -15,7 +15,9 @@ const deleteAllSearches = () => {
   })
 }
 
-describe('Open uni population tests', () => {
+// Skipping the whole test suite as it is unclear whether the feature has any use or will be
+// removed alltogether.
+describe.skip('Open uni population tests', () => {
   describe('Open uni form can be used', () => {
     // When openUniUser is created, change it here
     beforeEach(() => {

cypress/e2e/Population_statistics.js

@@ -252,8 +252,8 @@ describe('Population statistics tests', () => {
         cy.visit(pathToMathBSc2020)
         cy.contains('Courses of class').click()
         cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
-        cy.get('[data-cy=toggle-group-module-MAT110]').click()
-        cy.contains('td', 'MAT11001').siblings().find('i.level.up').click()
+        cy.cs('toggle-group-module-MAT110').click()
+        cy.contains('td', 'MAT11001').siblings().find('[data-testid="NorthEastIcon"]').click()
         cy.wait('@coursePage')
         cy.url().should('include', '/coursestatistics')
         cy.contains('MAT11001 • Johdatus yliopistomatematiikkaan')
@@ -319,7 +319,7 @@ describe('Population statistics tests', () => {
       })
 
       it("'General tab' is usable", () => {
-        cy.cs('student-table-tabs').within(() => {
+        cy.cs('ooditable-general').within(() => {
           cy.contains('522142')
           cy.contains('Tilastotiede')
           cy.contains('Matematiikka')
@@ -330,18 +330,19 @@ describe('Population statistics tests', () => {
         })
       })
 
-      // FIXME: Flaky
       it("'Courses tab' is usable", () => {
         cy.cs('student-table-tabs').within(() => {
           cy.contains('Courses').click()
+        })
+        cy.cs('ooditable-courses').within(() => {
           cy.contains('MAT12001')
           cy.contains('MAT21001')
         })
       })
 
       it("'Modules tab' Displays correct modules based on the selected programme", () => {
-        cy.cs('student-table-tabs').within(() => {
-          cy.contains('Modules').click()
+        cy.cs('student-table-tabs').contains('Modules').click()
+        cy.cs('ooditable-modules').within(() => {
           cy.contains('MAT011')
           cy.contains('MAT110')
         })
@@ -349,18 +350,17 @@ describe('Population statistics tests', () => {
         cy.cs('curriculum-picker').click()
         cy.contains('2023–2026').click()
         cy.contains('Courses of class').click()
-        cy.cs('student-table-tabs').within(() => {
+        cy.cs('ooditable-modules').within(() => {
           cy.contains('MAT011').should('not.exist')
           cy.contains('MAT110')
         })
       })
 
-      it.skip("Empty 'tags' tab has a link to the page where tags can be created", () => {
-        // TODO: This fails in the pipeline, but works locally. Investigate.
+      it("Empty 'tags' tab has a link to the page where tags can be created", () => {
         cy.cs('student-table-tabs').within(() => {
           cy.contains('Tags').click()
-          cy.contains('No tags defined. You can define them here.').find('a').click()
         })
+        cy.contains('No tags defined. You can define them here.').find('a').click()
         cy.url().should('include', '/study-programme/KH50_001?tab=4')
         cy.contains('Matemaattisten tieteiden kandiohjelma')
         cy.contains('Create new tag')
@@ -383,17 +383,15 @@ describe('Population statistics tests', () => {
       cy.contains(nonExisting).should('not.exist')
       cy.contains('button', 'Check student numbers').click()
       cy.contains('Check for student numbers')
-      cy.get('textarea').type(existing)
-      cy.get('textarea').type('{enter}')
-      cy.get('textarea').type(nonExisting)
+      cy.cs('check-student-numbers').type(`${existing}{enter}${nonExisting}`)
       cy.contains('button', 'Check students').click()
-      cy.contains('#checkstudentsresults', 'Results').within(() => {
-        cy.contains('Student numbers in list and in Sisu').click()
-        cy.contains('#found', existing)
-        cy.contains('Student numbers in list but not in Sisu').click()
-        cy.contains('#notfound', nonExisting)
-        cy.contains('Student numbers in Sisu but not in list').click()
-        cy.contains('#notsearched', '457144')
+      cy.get('#checkstudentsresults').within(() => {
+        cy.cs('found-title').click()
+        cy.cs('found-data').contains(existing)
+        cy.cs('not-found-title').click()
+        cy.cs('not-found-data').contains(nonExisting)
+        cy.cs('not-searched-title').click()
+        cy.cs('not-searched-data').contains('457144')
       })
       cy.contains('button', 'Close').click()
       cy.contains('Student numbers in list and in Sisu').should('not.exist')

cypress/e2e/Study_guidance_groups.js

@@ -31,7 +31,7 @@ describe('Study guidance group tests', () => {
 
         const studyGuidanceGroupsTableContent = [
           ['MAT kandit 2020', 3, 'Matemaattisten tieteiden kandiohjelma', '2020 - 2021'],
-          ['Oma ohjausryhmä', 2, 'Add degree programme', 'Add year'],
+          ['Oma ohjausryhmä', 2, 'Add degree programme', 'Add starting year'],
         ]
 
         cy.checkTableStats(studyGuidanceGroupsTableContent, 'study-guidance-group-overview')
@@ -52,7 +52,7 @@ describe('Study guidance group tests', () => {
     it('has a working back button', () => {
       cy.init('/studyguidancegroups/sgg-cypress-1', 'onlystudyguidancegroups')
 
-      cy.get('button.ui.button').contains('Back').click()
+      cy.get('button').contains('Back to groups').click()
       cy.url().should('eq', `${Cypress.config().baseUrl}/studyguidancegroups`)
     })
 
@@ -100,9 +100,9 @@ describe('Study guidance group tests', () => {
       })
 
       it('shows the correct labels', () => {
-        cy.get('.ui.medium.header').contains('MAT kandit 2020')
-        cy.get('.ui.blue.tag.label:first').contains('Matemaattisten tieteiden kandiohjelma')
-        cy.get('.ui.blue.tag.label:last').contains('2020 - 2021')
+        cy.get('h4').contains('MAT kandit 2020')
+        cy.get('.MuiChip-root:first').contains('Matemaattisten tieteiden kandiohjelma')
+        cy.get('.MuiChip-root:last').contains('2020 - 2021')
       })
 
       it('shows the correct panes', () => {
@@ -113,17 +113,17 @@ describe('Study guidance group tests', () => {
         cy.contains('Students (3)')
       })
 
-      it("clicking the 'Show starting from associated year' button activates the 'Date of Course Credits' filter", () => {
-        cy.get('[data-cy="CreditDate-filter-card"][data-active="false"]')
+      it("clicking the 'Show credits starting from the associated academic year' filter", () => {
+        cy.get('[data-cy="ParticipationDate-filter-card"][data-active="false"]')
         cy.cs('Credit accumulation (for 3 students)').click()
-        cy.get('.ui.primary.button').contains('Show starting from associated year').click()
-        cy.get('[data-cy="CreditDate-filter-card"][data-active="true"]')
+        cy.get('label').contains('Show credits starting from the associated academic year').click()
+        cy.get('[data-cy="ParticipationDate-filter-card"][data-active="true"]')
         cy.get('.date-picker')
           .eq(0)
           .within(() => cy.get('input').should('have.value', '01.08.2020'))
         cy.contains('Reset All Filters')
-        cy.get('.ui.primary.button').contains('Show all credits').click()
-        cy.get('[data-cy="CreditDate-filter-card"][data-active="false"]')
+        cy.get('label').contains('Show credits starting from the associated academic year').click()
+        cy.get('[data-cy="ParticipationDate-filter-card"][data-active="false"]')
       })
 
       it.skip('students table has the correct tabs', () => {

cypress/e2e/Teachers.js

@@ -10,17 +10,19 @@ describe('Teachers page tests', () => {
   const teacher1 = 'Luokkanen Liisa Viljami'
   const teacher2 = 'Perälä Juhani Susanna'
 
+  const statisticsHeaders = ['Name', 'Credits', 'Credits transferred', 'Passed']
+
   it('Check Statistics', () => {
-    cy.get(':nth-child(1) > .ui > .search').click()
+    cy.cs('semester-start').click()
     cy.contains('Syksy 2020').click()
     cy.cs('course-providers').click()
     cy.contains('Matemaattisten tieteiden kandiohjelma').click()
-    cy.cs('course-providers').children('.icon').click()
-    cy.contains('button', 'Search').click()
+    cy.get('body').click(0, 0) // Click outside of the select
+    cy.cs('search-statistics').click()
+
     cy.contains('Teacher')
-    const headers = ['Name', 'Credits', 'Credits transferred', 'Passed']
     cy.get('table thead tr th').should('have.length', 4)
-    headers.forEach((header, index) => {
+    statisticsHeaders.forEach((header, index) => {
       cy.get('table thead tr th').eq(index).should('contain', header)
     })
     cy.contains('td', teacher1).siblings().eq(0).contains('235')
@@ -30,15 +32,15 @@ describe('Teachers page tests', () => {
   })
 
   it('Teacher search works', () => {
-    cy.get('.borderless > :nth-child(3)').click()
-    cy.get('.prompt').type(teacher1.split(' ')[0])
+    cy.cs('Search').click()
+    cy.cs('teacher-search').type(teacher1.split(' ')[0])
     cy.get('table tbody tr').should('have.length', 4)
-    cy.get('table tbody tr').eq(2).contains('td', teacher1)
+    cy.get('table tbody tr').eq(3).contains('td', teacher1)
   })
 
   it('Can check teacher page', () => {
-    cy.get('.borderless > :nth-child(3)').click()
-    cy.get('.prompt').type(teacher2)
+    cy.cs('Search').click()
+    cy.cs('teacher-search').type(teacher2)
 
     // Prevent opening in new tab
     cy.contains('a', teacher2).invoke('removeAttr', 'target').click()
@@ -46,19 +48,17 @@ describe('Teachers page tests', () => {
     cy.contains(teacher2)
     cy.contains('Syksy 2023').click()
     cy.contains('Kevät 2019').click()
-    cy.get('.ok-sortable-table tbody tr')
-      .eq(0)
-      .within(() => {
-        const rowContent = ['MAT12004', 'Tilastollinen päättely I', '120', '0', '92.31%']
-        rowContent.forEach((content, index) => {
-          cy.get('td').eq(index).contains(content)
-        })
+    cy.contains('tr', 'MAT12004').within(() => {
+      const rowContent = ['MAT12004', 'Tilastollinen päättely I', '120', '0', '92.31%']
+      rowContent.forEach((content, index) => {
+        cy.get('td').eq(index).contains(content)
       })
+    })
   })
 
   it('Check leaderboad works', () => {
-    cy.get('.borderless > :nth-child(2)').click()
-    cy.get(':nth-child(1) > .ui > .search').click()
+    cy.cs('Leaderboard').click()
+    cy.cs('academic-year').click()
     cy.contains('2020-2021').click()
   })
 })

cypress/e2e/University.js

@@ -39,12 +39,6 @@ describe('University view', () => {
       checkProgressTables()
     })
 
-    it("'Graduated included / Graduated excluded' toggle works", () => {
-      cy.cs('graduated-toggle').click()
-      checkProgressBarCharts()
-      checkProgressTables()
-    })
-
     it("'All study rights / Special study rights excluded' toggle works", () => {
       cy.cs('study-right-toggle').click()
       checkProgressBarCharts()

cypress/support/e2e.js

@@ -13,7 +13,6 @@
 // https://on.cypress.io/configuration
 // ***********************************************************
 
-require('@cypress/code-coverage/support')
 require('./commands')
 
 Cypress.on('uncaught:exception', error => {

docker-compose.ci.yml

@@ -25,6 +25,8 @@ services:
     build:
       context: services
       dockerfile: backend/Dockerfile
+    profiles:
+      - oodikone
 
   frontend:
     <<: *defaults
@@ -32,6 +34,8 @@ services:
     build:
       context: services
       dockerfile: frontend/Dockerfile
+    profiles:
+      - oodikone
 
   kone-db:
     <<: *defaults
@@ -47,6 +51,7 @@ services:
     command: >
       sh -c 'redis-server --appendonly yes && redis-cli flushall'
     image: redis:7.2.6
+    container_name: redis
     ports:
       - 6379:6379
 
@@ -101,6 +106,8 @@ services:
       - 1337:80
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf
+    profiles:
+      - oodikone
 
   # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
   sis-importer-db:
@@ -114,6 +121,7 @@ services:
     <<: *defaults
     build:
       context: updater/sis-updater-worker
+    container_name: sis-updater-worker
     depends_on:
       - redis
       - sis-db

docker-compose.fd.ci.yml

@@ -1,47 +0,0 @@
-services:
-  oodikone-backend:
-    build:
-      context: ${OODIKONE_REPO:-.}/services
-      dockerfile: backend/Dockerfile
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        SERVICE_PROVIDER: fd
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/oodikone-backend:${DOCKER_IMAGE_TAG:-dev}
-
-  oodikone-frontend:
-    build:
-      context: ${OODIKONE_REPO:-.}/services
-      dockerfile: frontend/Dockerfile
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        SERVICE_PROVIDER: fd
-        LANGUAGE_CENTER_VIEW_ENABLED: 'false' # currently no intention to be enabled
-        SIS_URL: 'https://sis-qa.funidata.fi'
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/oodikone-frontend:${DOCKER_IMAGE_TAG:-dev}
-
-  updater-scheduler:
-    build:
-      context: ${OODIKONE_REPO:-.}/updater/sis-updater-scheduler
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/updater-scheduler:${DOCKER_IMAGE_TAG:-dev}
-
-  updater-worker:
-    build:
-      context: ${OODIKONE_REPO:-.}/updater/sis-updater-worker
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        SERVICE_PROVIDER: fd
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/updater-worker:${DOCKER_IMAGE_TAG:-dev}

docker-compose.fd.yml

@@ -1,190 +0,0 @@
-x-defaults: &defaults
-  restart: unless-stopped
-
-x-build_defaults: &build_defaults
-  dockerfile: dev.Dockerfile
-
-services:
-  adminer:
-    <<: *defaults
-    container_name: adminer
-    environment:
-      ADMINER_DESIGN: pepa-linha
-    image: adminer:4.8.1
-    ports:
-      - 5050:8080
-    volumes:
-      - ./adminer-plugins:/var/www/html/plugins-enabled
-
-  backend:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/backend
-    depends_on:
-      - kone-db
-      - redis
-      - sis-db
-      - user-db
-    env_file:
-      - .env
-    environment:
-      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
-      DB_URL_USER: postgres://postgres@user-db:5432/user-db
-      FRONT_URL: http://localhost:3000
-      REDIS: redis
-      SECRET_TOKEN: dev
-      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
-      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      IMPORTER_DB_API_URL: 'https://localhost:3005'
-      JAMI_URL: 'http://jami:3003/'
-      SERVICE_PROVIDER: fd
-      SIS_HOST: ${SIS_HOST}
-      SIS_GRAPHQL_APPLICATION_ACCOUNT: ${SIS_GRAPHQL_APPLICATION_ACCOUNT}
-      SIS_GRAPHQL_APPLICATION_KEY: ${SIS_GRAPHQL_APPLICATION_KEY}
-      IMPORTER_DB_API_USER: dev_user
-      IMPORTER_DB_API_PASSWORD: dev_password
-      ROOT_ORG_ID: ${ROOT_ORG_ID}
-      LANGUAGE_CENTER_VIEW_ENABLED: 'false'
-    ports:
-      - 8080:8080
-    volumes:
-      - backend-nodemod:/opt/app-root/backend/node_modules
-      - ./environments/testu:/opt/app-root/backend/src/environment
-      - ./services/backend/:/opt/app-root/backend
-      - ./services/shared/:/opt/app-root/shared
-
-  frontend:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/frontend
-    container_name: frontend
-    ports:
-      - 3000:3000
-    environment:
-      REACT_APP_SERVICE_PROVIDER: fd
-      REACT_APP_LANGUAGE_CENTER_VIEW_ENABLED: 'false'
-      REACT_APP_SIS_URL: 'https://sis-qa.funidata.fi'
-    volumes:
-      - frontend-nodemod:/opt/app-root/frontend/node_modules
-      - ./services/frontend/:/opt/app-root/frontend
-      - ./services/shared/:/opt/app-root/shared
-
-  kone-db:
-    <<: *defaults
-    container_name: kone-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: kone-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
-    ports:
-      - 12349:5432
-    volumes:
-      - kone-db-data:/data
-
-  redis:
-    <<: *defaults
-    command: 'redis-server --appendonly yes'
-    container_name: redis
-    image: redis:6.2.16
-    ports:
-      - '6379:6379'
-    volumes:
-      - redis-data:/data
-
-  sis-db:
-    <<: *defaults
-    container_name: sis-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: sis-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
-    volumes:
-      - sis-db-data:/data
-    shm_size: 2gb
-    ports:
-      - 12345:5432
-
-  sis-updater-scheduler:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: updater/sis-updater-scheduler
-    container_name: sis-updater-scheduler
-    extra_hosts:
-      - 'host.docker.internal:host-gateway'
-    depends_on:
-      - redis
-    environment:
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      SECRET_TOKEN: dev
-      SIS_IMPORTER_HOST: ${SIS_IMPORTER_DB_HOST:-host.docker.internal}
-      SIS_IMPORTER_PORT: ${SIS_IMPORTER_DB_PORT:-15234}
-      SIS_IMPORTER_DATABASE: importer-db
-      SIS_IMPORTER_PASSWORD: dev
-      SIS_IMPORTER_USER: dev
-      SERVICE_PROVIDER: fd
-    ports:
-      - 8082:8082
-    volumes:
-      - ./updater/sis-updater-scheduler/:/opt/app-root/src
-
-  sis-updater-worker:
-    <<: *defaults
-    deploy:
-      replicas: 2
-      mode: replicated
-    build:
-      <<: *build_defaults
-      context: updater/sis-updater-worker
-    depends_on:
-      - redis
-      - sis-db
-    extra_hosts:
-      - 'host.docker.internal:host-gateway'
-    environment:
-      DB_URL: postgres://postgres@sis-db:5432/sis-db
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      SIS_IMPORTER_HOST: ${SIS_IMPORTER_DB_HOST:-host.docker.internal}
-      SIS_IMPORTER_PORT: ${SIS_IMPORTER_DB_PORT:-15234}
-      SIS_IMPORTER_DATABASE: importer-db
-      SIS_IMPORTER_PASSWORD: dev
-      SIS_IMPORTER_USER: dev
-      SIS_PASSWORD: postgres
-      ROOT_ORG_ID: ${ROOT_ORG_ID}
-      SERVICE_PROVIDER: fd
-    volumes:
-      - ./updater/sis-updater-worker/:/opt/app-root/src
-
-  user-db:
-    <<: *defaults
-    container_name: user-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: user-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
-    volumes:
-      - user-db-data:/data
-    ports:
-      - 12347:5432
-
-volumes:
-  backend-nodemod:
-  frontend-nodemod:
-  kone-db-data:
-  redis-data:
-  sis-db-data:
-  user-db-data:
-
-networks:
-  default:
-    name: oodikone
-  importer:
-    name: importer_network
-    external: true

docker-compose.test.yml

@@ -1,5 +1,26 @@
 services:
-  frontend:
+  api-tests:
     volumes:
-      - frontend-nodemod:/opt/app-root/frontend/node_modules
-      - ./services/frontend/instrumented:/opt/app-root/frontend/src
+      - backend-tests-nodemod:/opt/app-root/backend/node_modules
+      - ./services/backend/:/opt/app-root/backend
+      - ./services/shared/:/opt/app-root/shared
+    environment:
+      CI: 'true'
+      NODE_ENV: 'test'
+      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
+      FRONT_URL: http://localhost:3000
+      REDIS: redis
+      SECRET_TOKEN: test
+      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
+      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
+      JAMI_URL: 'http://jami:3003/'
+      IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
+      CRYPT_KEY: 'e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'
+    build:
+      context: services/backend
+      dockerfile: dev.Dockerfile
+    command: npm run test
+
+volumes:
+  backend-tests-nodemod:

docker-compose.yml

@@ -21,6 +21,7 @@ services:
     build:
       <<: *build_defaults
       context: services/backend
+    container_name: backend
     depends_on:
       - kone-db
       - redis
@@ -132,6 +133,7 @@ services:
     build:
       <<: *build_defaults
       context: updater/sis-updater-worker
+    container_name: sis-updater-worker
     depends_on:
       - redis
       - sis-db
@@ -168,7 +170,7 @@ services:
       - npm_config_cache=/tmp
       - PORT=3003
       - NODE_ENV=dev
-      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
+      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/jami-db-real
     ports:
       - 3003:3003
     container_name: jami

documentation/combined_programme.md

@@ -36,7 +36,7 @@ All things listed below should work automatically, if new combinations are done.
 
 - _Students of the studyprogramme by starting year_
   - Graduated field separated for both programmes
-  - All = currently enrolled + absent + inactive + graduated licentiate
+  - All = currently enrolled + absent + passive + graduated licentiate
 - _Progress of students of the studyprogramme by starting year_
   - computed for whole bachelor + master/licentiate studyright period
 - _Average graduation times_
@@ -85,7 +85,7 @@ All things listed below should work automatically, if new combinations are done.
 
 - **Graduated from programme** - possibility to filter out students based on graduation either in bachelor's or master's.
 
-- **Studyright status** - active or inactive based on the chosen degree
+- **Studyright status** - active or passive based on the chosen degree
 
 - **Studytrack filter** (needs programme code) is not touched at the moment, thus **may need future manual work**.
 

documentation/numeric_types.md

@@ -55,11 +55,11 @@ The `semester_enrollments` column is an array of JSON objects. Each object conta
 
 The `type` field represents one of the following enrollment types. The `semester` field refers to a semester code, defined as `semestercode` in the model Semester.
 
-| value | meaning  | notes                            |
-| ----: | :------- | :------------------------------- |
-|     1 | Present  |                                  |
-|     2 | Absent   |                                  |
-|     3 | Inactive | neither 1 or 2 has been declared |
+| value | meaning | notes                            |
+| ----: | :------ | :------------------------------- |
+|     1 | Present |                                  |
+|     2 | Absent  |                                  |
+|     3 | Passive | neither 1 or 2 has been declared |
 
 ## Credit type code
 

documentation/resources.md

@@ -2,26 +2,19 @@
 
 A collection of useful tools, documentation and other resources.
 
-## AI tools
-
-- [ChatGPT](https://chatgpt.com)
-- [CurreChat](https://curre.helsinki.fi/chat)
-- [GitHub Copilot](https://github.com/features/copilot)
-  - Free for students!
-- [Microsoft Copilot](https://copilot.microsoft.com)
-
 ## Code style and maintainability
 
-- [CodeClimate documentation](https://docs.codeclimate.com/docs)
-  - [Advanced configuration](https://docs.codeclimate.com/docs/advanced-configuration)
 - [ESLint documentation](https://eslint.org/docs/v8.x)
 - [Prettier documentation](https://prettier.io/docs/en)
+- [qlty.sh](https://qlty.sh/gh/UniversityOfHelsinkiCS/projects/oodikone)
+  - Qlty.sh replaced Code climate which was introduced by Riku
+  - The usefulness of qlty.sh is sometimes questionable
 
 ## Database
 
 - [pgcli](https://www.pgcli.com)
   - A useful commandline tool to interact with the database
-  - Install: `$ pip install pgcli`
+  - Can be installed from some package managers or optionally via `$ pip install pgcli`
 - [Postgres documentation](https://www.postgresql.org/docs/15/index.html)
 - [Sequelize documentation](https://sequelize.org/docs/v6/)
 
@@ -29,8 +22,6 @@ A collection of useful tools, documentation and other resources.
 
 - [GitHub documentation](https://docs.github.com/en)
   - [GitHub Actions](https://docs.github.com/en/actions)
-- [GitHub Education](https://github.com/education/students)
-  - Free benefits for students!
 - [Mermaid.js documentation](https://mermaid.js.org/intro/syntax-reference.html)
 
 ## TypeScript
@@ -46,7 +37,6 @@ A collection of useful tools, documentation and other resources.
   - [Material Icons](https://mui.com/material-ui/material-icons/)
   - [MUI X Date pickers](https://mui.com/x/react-date-pickers/)
 - [Material React Table documentation](https://www.material-react-table.com/docs)
-- [Semantic UI React documentation](https://react.semantic-ui.com)
 
 <!--
 

eslint.config.mjs

@@ -62,7 +62,7 @@ export default tseslint.config(
       'camelcase': 'off', // TODO: Enable. old db/backend logic is still using snake_case every now and then and needs a big refactor
       'class-methods-use-this': 'off', // Disabled in favor of @typescript-eslint
       'consistent-return': 'off', // Disabled in favor of @typescript-eslint
-      'id-denylist': ['error', 'c', 'd', 'e', 'err', 't'],
+      'id-denylist': ['error', 'err'],
       'no-alert': 'error',
       'no-await-in-loop': 'error',
       'no-console': 'error',
@@ -75,7 +75,14 @@ export default tseslint.config(
       'prefer-const': 'error',
       'prefer-destructuring': 'off', // Disabled in favor of @typescript-eslint
       'dot-notation': 'off', // Disabled in favor of @typescript-eslint
-      'no-restricted-imports': ['error', { 'patterns': [{ 'regex': '^@mui/[^/]+$' }] }],
+      'no-restricted-imports': ['error',
+        {
+          'patterns':
+            [
+              { 'regex': '^@mui/[^/]+$', 'message': 'Import only the needed module. This has no effect in prod due to tree-shake, but speeds up development builds.' },
+              { 'regex': '^src/.*', 'message': 'Root imports will not resolve in production build (instead silently crash).' }
+            ]
+        }],
       'quotes': ['error', 'single',
         { avoidEscape: true, allowTemplateLiterals: false }
       ],
@@ -205,6 +212,17 @@ export default tseslint.config(
     ]
   },
 
+  // Override for backend api tests
+  {
+    files: [
+      "services/backend/tests**/*.ts"
+    ],
+    rules: {
+      "import-x/no-extraneous-dependencies": "off",
+      'no-console': 'off',
+    }
+  },
+
   // Overrides for CommonJS
   {
     files: [

nyc-config.cjs

@@ -1,6 +0,0 @@
-const { parserPlugins } = require('@istanbuljs/schema').defaults.nyc
-
-module.exports = {
-  cache: false,
-  parserPlugins: parserPlugins.concat('typescript', 'jsx'),
-}

package.json

@@ -2,7 +2,7 @@
   "name": "oodikone",
   "description": "An application for analyzing university data.",
   "engines": {
-    "node": ">=22.0.0"
+    "node": ">=24.0.0"
   },
   "scripts": {
     "cypress": "cypress",
@@ -16,7 +16,8 @@
     "cli": "./cli.sh",
     "oodikone": "./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
-    "oodikone:test": "./run.sh oodikone test up --detach",
+    "oodikone:ci:setup": "docker compose --file docker-compose.ci.yml --profile tests up --detach",
+    "oodikone:ci:api": "docker compose --file docker-compose.test.yml up --exit-code-from api-tests",
     "updater": "./run.sh updater anon up --detach",
     "updater:real": "./run.sh updater real up --detach",
     "both": "./run.sh both anon up --detach",
@@ -25,11 +26,12 @@
     "testupdater": "docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start",
     "flushredis": "docker exec -it redis redis-cli flushall",
     "prepare": "husky",
-    "rapodiff": "docker exec -t oodikone_backend_1 /opt/app-root/backend/node_modules/.bin/tsx /opt/app-root/backend/src/rapodiff/rapodiff.js",
+    "rapodiff": "docker exec -t oodikone-backend /opt/app-root/backend/node_modules/.bin/tsx /opt/app-root/backend/src/rapodiff/rapodiff.js",
     "install:local": "npm i && npm i --prefix services/shared && npm i --prefix services/frontend && npm i --prefix services/backend && npm i --prefix updater/sis-updater-scheduler && npm i --prefix updater/sis-updater-worker",
+    "i:both": "npm run install:both",
     "ci:local": "npm ci && npm ci --prefix services/shared && npm ci --prefix services/frontend && npm ci --prefix services/backend && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker",
-    "install:docker": "docker exec frontend npm i && docker exec oodikone_backend_1 npm i && docker exec sis-updater-scheduler npm i && docker exec oodikone_sis-updater-worker_1 npm i",
-    "ci:docker": "docker exec frontend npm ci && docker exec oodikone_backend_1 npm ci && docker exec sis-updater-scheduler npm ci && docker exec oodikone_sis-updater-worker_1 npm ci",
+    "install:docker": "docker exec frontend npm i && docker exec backend npm i && docker exec sis-updater-scheduler npm i && docker exec sis-updater-worker npm i",
+    "ci:docker": "docker exec frontend npm ci && docker exec backend npm ci && docker exec sis-updater-scheduler npm ci && docker exec sis-updater-worker npm ci",
     "install:both": "npm run install:local && npm run install:docker",
     "ci:both": "npm run ci:local && npm run ci:docker"
   },
@@ -43,10 +45,8 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@cypress/code-coverage": "^3.14.4",
     "@eslint/js": "^9.28.0",
-    "@istanbuljs/schema": "^0.1.3",
-    "cypress": "^15.0.0",
+    "cypress": "^15.9.0",
     "eslint": "^9.28.0",
     "eslint-config-prettier": "^10.1.5",
     "eslint-import-resolver-typescript": "^4.4.3",
@@ -58,13 +58,9 @@
     "globals": "^16.2.0",
     "husky": "^9.1.7",
     "lint-staged": "^16.1.0",
-    "nyc": "^17.1.0",
     "prettier": "3.5.3",
     "stylelint": "^16.20.0",
     "stylelint-config-standard": "^39.0.0",
     "typescript-eslint": "^8.34.0"
-  },
-  "dependencies": {
-    "lodash": "^4.17.21"
   }
 }

run.sh

@@ -35,7 +35,7 @@ parse_params() {
   option=${args[0]}
 
   #  parse arguments
-  [[ ("$option" != "oodikone" && "$option" != "updater" && "$option" != "both") ]] && \
+  [[ ("$option" != "oodikone" && "$option" != "updater" && "$option" != "both" && "$option" != "tests") ]] && \
     usage && die "Wrong option: $option"
 
   [[ ${#args[@]} -eq 1 ]] && usage && die "Wrong number of arguments"
@@ -47,7 +47,7 @@ parse_params() {
     compose_command=${args[*]:1}
   else
     version=${args[1]}
-    [[ "$version" != "anon" && "$version" != "real" && "$version" != "test" ]] && usage && \
+    [[ "$version" != "anon" && "$version" != "real" && "$version" != "api" ]] && usage && \
       die "Wrong version $version"
     compose_command=${args[*]:2}
   fi
@@ -70,8 +70,8 @@ parse_env() {
   if [[ "$version" == "real" ]]; then
     env="--file docker-compose.yml --file docker-compose.real.yml"
   fi
-  if [[ "$version" == "test" ]]; then
-    env="--file docker-compose.yml --file docker-compose.test.yml"
+  if [[ "$version" == "api" ]]; then
+    env="--file docker-compose.ci.yml"
   fi
   return 0
 }
@@ -85,9 +85,9 @@ parse_params "$@"
 parse_profiles
 parse_env
 if [[ "$compose_command" == "" ]]; then
-  final_command="docker-compose ${env}"
+  final_command="docker compose ${env}"
 else
-  final_command="docker-compose ${env} ${profiles} ${compose_command}"
+  final_command="docker compose ${env} ${profiles} ${compose_command}"
 fi
 
 msg "${BLUE}Running: ${final_command}${NOFORMAT}"

scripts/functions_for_setup.sh

@@ -11,7 +11,6 @@ PROJECT_ROOT="$(git rev-parse --show-toplevel)"
 DUMP_DIR="$PROJECT_ROOT/.databasedumps"
 S3_CONFIG_FILE=~/.s3cfg
 S3_BUCKET="s3://psyduck"
-DOCKER_COMPOSE=$PROJECT_ROOT/docker-compose.yml
 
 ## Following the naming convention in docker-compose, these are names for database
 ## services. The real data databases inside service will have names suffixed by "-real".
@@ -139,29 +138,6 @@ reset_databases() {
   successmsg "Database setup finished"
 }
 
-reset_jami_data() {
-  local database="jami-db"
-  local dump_destination="$DUMP_DIR/$database.sql.gz"
-  local s3_path="jami"
-
-  infomsg "Removing old data"
-
-
-
-  infomsg "Removing database and related volume"
-  docker volume rm oodikone_jami-data || warningmsg "This is okay, continuing"
-  docker-compose -f "$DOCKER_COMPOSE" down "$database" || warningmsg "This is okay, continuing"
-
-  infomsg "Starting postgres in the background"
-  docker-compose -f "$DOCKER_COMPOSE" up -d "$database"
-  retry docker-compose -f "$DOCKER_COMPOSE" exec "$database" pg_isready --dbname="postgres"
-
-  infomsg "Populating Jami"
-  docker exec -i "$database" /bin/bash -c "gunzip | psql -U postgres" < "$dump_destination" 2> /dev/null
-  docker-compose stop "$database"
-  msg ""
-}
-
 reset_all_real_data() {
   infomsg "Downloading real data dumps"
   for i in ${!DATABASES[*]}; do

scripts/instrument.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-
-# This script is used for instrumenting the frontend code
-# Counters are added for calculating the code coverage
-# Instrumented code is used for generating code coverage reports
-
-# Ensure the script exits if any command fails
-set -euo pipefail
-
-# Define the directories
-SOURCE_DIR="services/frontend/src"
-INSTRU_DIR="services/frontend/instrumented"
-
-# Print status
-echo -n "Instrumenting frontend code... "
-
-# Create the instrumented directory if it does not exist
-mkdir -p $INSTRU_DIR
-
-# Instrument the code
-nyc instrument --all --compact=false $SOURCE_DIR $INSTRU_DIR
-
-# Copy remaining files without overwriting instrumented files
-# 
-# Images, CSS and other static files are not instrumended
-# but they are still needed to run the application
-rsync -av --quiet --ignore-existing $SOURCE_DIR/ $INSTRU_DIR/
-
-# Print status
-echo "Done!"

services/.dockerignore

@@ -0,0 +1,7 @@
+*/.dockerignore
+*/.eslintrc.json
+*/.gitignore
+*/*Dockerfile
+*/build
+*/dist
+*/node_modules

services/backend/.dockerignore

@@ -1,3 +0,0 @@
-.dockerignore
-*Dockerfile
-node_modules/

services/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi9/nodejs-22-minimal AS build
+FROM registry.access.redhat.com/ubi9/nodejs-24-minimal AS build
 
 ENV TZ="Europe/Helsinki"
 
@@ -29,7 +29,7 @@ RUN npm ci --include=dev --prefix shared && npm ci --include=dev --prefix backen
 RUN npm run build --prefix shared && npm run build --prefix backend
 
 
-FROM registry.access.redhat.com/ubi9/nodejs-22-minimal
+FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 

services/backend/dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM node:22-alpine
+FROM node:24-alpine
 
 ENV TZ="Europe/Helsinki"
 
@@ -10,4 +10,4 @@ COPY . .
 RUN npm ci
 
 EXPOSE 8080
-CMD ["node_modules/.bin/tsx", "watch", "--clear-screen=false", "index.js"]
+CMD ["node_modules/.bin/nodemon", "--exec", "node --import tsx index.ts", "--ext", "ts,js,json", "--watch", "src", "--watch", "index.ts", "--legacy-watch"]

services/backend/package.json

@@ -2,12 +2,15 @@
   "name": "backend",
   "private": true,
   "scripts": {
-    "build": "tsc"
+    "build": "tsc",
+    "test": "node --import tsx --test --test-reporter spec --test-force-exit"
   },
   "dependencies": {
     "@oodikone/shared": "file:../shared",
-    "@sentry/node": "^10.5.0",
-    "@sentry/profiling-node": "^10.5.0",
+    "@sentry/node": "^10.43.0",
+    "@sentry/profiling-node": "^10.43.0",
+    "@types/lodash-es": "^4.17.12",
+    "@types/pg": "^8.18.0",
     "axios": "^1.9.0",
     "bullmq": "^5.53.2",
     "compression": "^1.8.0",
@@ -16,14 +19,14 @@
     "dayjs": "^1.11.13",
     "express": "^4.21.2",
     "express-async-errors": "^3.1.1",
-    "lodash": "^4.17.21",
+    "lodash-es": "^4.17.23",
     "lru-cache": "^11.1.0",
     "morgan": "^1.10.0",
     "papaparse": "^5.5.3",
-    "pg": "^8.16.0",
+    "pg": "^8.20.0",
     "redis": "^4.7.1",
     "reflect-metadata": "^0.2.2",
-    "sequelize": "^6.37.7",
+    "sequelize": "^6.37.8",
     "sequelize-typescript": "^2.1.6",
     "umzug": "^3.8.2",
     "unfuck-utf8-headers-middleware": "^1.0.1",
@@ -35,10 +38,11 @@
     "@types/compression": "^1.8.1",
     "@types/cors": "^2.8.19",
     "@types/express": "^4.17.23",
-    "@types/lodash": "^4.17.17",
     "@types/morgan": "^1.9.10",
-    "@types/node": "^22.15.31",
-    "@types/validator": "^13.15.1",
+    "@types/node": "^24.10.1",
+    "@types/supertest": "^6.0.3",
+    "nodemon": "^3.1.11",
+    "supertest": "^7.2.2",
     "tsx": "^4.19.4",
     "typescript": "^5.9.2"
   },

services/backend/src/config/index.ts

@@ -1,5 +1,6 @@
 // Node env to use
 export const isDev = process.env.NODE_ENV === 'development'
+export const isTest = process.env.NODE_ENV === 'test'
 export const isStaging = process.env.REACT_APP_STAGING === 'true'
 export const isProduction = !isStaging && process.env.NODE_ENV === 'production'
 
@@ -32,12 +33,6 @@ export const baseUrl = isDev ? '/api' : ''
 export const frontUrl = process.env.FRONT_URL
 export const backendPort = 8080
 
-// System run in whose environment
-export const serviceProvider = process.env.SERVICE_PROVIDER?.toLowerCase() ?? 'toska'
-
-// Optional logout-url configuration
-export const configLogoutUrl = serviceProvider === 'toska' ? undefined : process.env.LOGOUT_URL
-
 // Other stuff
 export const { DB_URL_KONE, DB_URL_USER, SECRET_TOKEN, SIS_DB_URL, SIS_UPDATER_URL, CRYPT_KEY } = process.env
 

services/backend/src/middleware/accessLogger.ts

@@ -1,5 +1,5 @@
 import { Request, Response } from 'express'
-import { omit } from 'lodash'
+import { omit } from 'lodash-es'
 import morgan from 'morgan'
 
 import { FormattedUser } from '../types'

services/backend/src/middleware/currentUser.ts

@@ -1,10 +1,9 @@
 import * as Sentry from '@sentry/node'
 import { NextFunction, Request, Response } from 'express'
-import { intersection } from 'lodash'
+import { intersection } from 'lodash-es'
 
-import { configLogoutUrl, isDev, requiredGroup, serviceProvider } from '../config'
-import { getMockedUser, getMockedUserFd, getOrganizationAccess, getUserFd, getUserToska } from '../services/userService'
-import { FormattedUser, IamAccess } from '../types'
+import { requiredGroup } from '../config'
+import { getMockedUser, getOrganizationAccess, getUser } from '../services/userService'
 import { ApplicationError } from '../util/customErrors'
 import logger from '../util/logger'
 
@@ -14,30 +13,6 @@ const hasRequiredIamGroup = (iamGroups: string[], iamRights: string[]) => {
   return intersection(iamGroups, requiredGroup).length > 0 || iamRights.length > 0
 }
 
-const getUser = async (
-  showAsUser: string | undefined,
-  username: string,
-  name: string,
-  email: string,
-  iamGroups: string[],
-  specialGroup: Record<string, boolean>,
-  sisId: string,
-  iamAccess: IamAccess
-) => {
-  if (showAsUser && specialGroup?.superAdmin) {
-    return await getMockedUser({ userToMock: showAsUser, mockedBy: username })
-  }
-  return await getUserToska({
-    username,
-    name,
-    email,
-    iamGroups,
-    specialGroup,
-    sisId,
-    iamAccess,
-  })
-}
-
 type Headers = {
   displayname?: string
   hygroupcn?: string
@@ -49,7 +24,7 @@ type Headers = {
   'x-show-as-user'?: string
 }
 
-const toskaUserMiddleware = async (req: Request, _res: Response, next: NextFunction) => {
+export default async (req: Request, _: Response, next: NextFunction) => {
   const {
     'shib-session-id': sessionId,
     'x-show-as-user': showAsUser,
@@ -61,20 +36,15 @@ const toskaUserMiddleware = async (req: Request, _res: Response, next: NextFunct
     uid: username,
   } = req.headers as Headers
 
-  const missingHeaders: string[] = []
-  if (!sessionId) missingHeaders.push('shib-session-id')
-  if (!username) missingHeaders.push('uid')
-
-  if (missingHeaders.length > 0) {
-    const reqUser = username ?? 'Anonymous user'
+  if (!username || !sessionId)
     throw new ApplicationError(
-      `${reqUser} requested ${req.url} without valid request headers. Missing: ${missingHeaders.join(', ')}`,
+      `${username ?? 'Anonymous user'} requested ${req.url} without authorization headers.`,
       403,
       { logoutUrl }
     )
-  }
 
   const iamGroups = parseIamGroups(hygroupcn!)
+
   const { iamAccess = {}, specialGroup = {} } = await getOrganizationAccess(sisId!, iamGroups)
   const iamRights = Object.keys(iamAccess)
 
@@ -83,49 +53,29 @@ const toskaUserMiddleware = async (req: Request, _res: Response, next: NextFunct
       message: 'User does not have required iam group',
       meta: { username, name, email, iamGroups, iamRights },
     })
+
     throw new ApplicationError(`User '${username}' does not have required iam group`, 403, { logoutUrl })
   }
 
-  const user = await getUser(showAsUser, username!, name!, email!, iamGroups, specialGroup, sisId!, iamAccess)
-  if (!user) {
-    throw new ApplicationError(`Username ${username} not found.`, 403, { logoutUrl })
-  }
+  const user =
+    showAsUser && specialGroup?.superAdmin
+      ? await getMockedUser({ userToMock: showAsUser, mockedBy: username })
+      : await getUser({
+          username,
+          name: name!,
+          email: email!,
+          iamGroups,
+          specialGroup,
+          sisId: sisId!,
+          iamAccess,
+        })
 
-  Sentry.setUser({ username: user.mockedBy ?? username! })
+  // Set username just-in-case there is error for invalid user
+  Sentry.setUser({ username: user?.mockedBy ?? username })
+  if (!user) throw new ApplicationError(`Username ${username} not found.`, 403, { logoutUrl })
 
   req.user = user
   req.logoutUrl = logoutUrl!
 
   next()
 }
-
-const fdUserMiddleware = async (req: Request, _res: Response, next: NextFunction) => {
-  const { remote_user: remoteUser, 'x-show-as-user': showAsUser } = req.headers
-
-  if (!remoteUser) {
-    throw new ApplicationError('Not enough data in request headers, remote_user was missing', 403, { configLogoutUrl })
-  }
-
-  let user: FormattedUser | null
-
-  // getMockedUser in production requires the superAdmin role, which is only available
-  // via iamGroups, so it's now only implemented for the dev environment
-  if (showAsUser && isDev) {
-    user = await getMockedUserFd({ userToMock: showAsUser as string, mockedBy: remoteUser as string })
-  } else {
-    user = await getUserFd({ username: remoteUser as string })
-  }
-
-  if (!user) {
-    throw new ApplicationError(`Could not grant access with the eppn ${remoteUser}.`, 403, { configLogoutUrl })
-  }
-
-  req.user = user
-  req.logoutUrl = configLogoutUrl
-
-  next()
-}
-
-const currentUserMiddleware = serviceProvider === 'toska' ? toskaUserMiddleware : fdUserMiddleware
-
-export default currentUserMiddleware

services/backend/src/middleware/headers.ts

@@ -1,10 +1,4 @@
 import headersMiddleware from 'unfuck-utf8-headers-middleware'
 
-import { serviceProvider } from '../config'
-
-const headersToska = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
-const headersOther = ['remote_user']
-
-const headers = serviceProvider === 'toska' ? headersToska : headersOther
-
+const headers = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
 export default headersMiddleware(headers)

services/backend/src/models/SISStudyRight.ts

@@ -71,6 +71,9 @@ export class SISStudyRightModel extends Model implements SISStudyRight {
   @Column(DataType.JSONB)
   declare semesterEnrollments: SISStudyRight['semesterEnrollments']
 
+  @Column(DataType.JSONB)
+  declare transferInfo: SISStudyRight['transferInfo']
+
   @HasMany(() => CreditModel, { foreignKey: 'studyright_id', sourceKey: 'id' })
   declare credits: SISStudyRight['credits']
 

services/backend/src/models/course.ts

@@ -73,6 +73,9 @@ export class CourseModel extends Model implements Course {
   @Column(DataType.JSONB)
   declare substitutions: Course['substitutions']
 
+  @Column(DataType.JSONB)
+  declare substitution_groups: Course['substitution_groups']
+
   @Column(DataType.STRING)
   declare course_unit_type: Course['course_unit_type']
 

services/backend/src/rapodiff/creditsDiff.js

@@ -1,4 +1,4 @@
-const { orderBy } = require('lodash')
+const { orderBy } = require('lodash-es')
 
 // dbConnections must be imported to avoid Sequelize errors even if it's not used
 // eslint-disable-next-line @typescript-eslint/no-unused-vars

services/backend/src/routes.ts

@@ -3,7 +3,7 @@ import compression from 'compression'
 import cors from 'cors'
 import { Express, json as jsonExpress } from 'express'
 
-import { frontUrl, languageCenterViewEnabled, serviceProvider } from './config'
+import { frontUrl, languageCenterViewEnabled } from './config'
 import accessLogger from './middleware/accessLogger'
 import * as auth from './middleware/auth'
 import currentUserMiddleware from './middleware/currentUser'
@@ -34,8 +34,7 @@ import tags from './routes/tags'
 import teachers from './routes/teachers'
 import university from './routes/university'
 import updater from './routes/updater'
-import usersToska from './routes/users'
-import usersFd from './routes/usersFd'
+import users from './routes/users'
 
 const routes = (app: Express, url: string) => {
   app.use(cors({ credentials: true, origin: frontUrl }))
@@ -66,12 +65,7 @@ const routes = (app: Express, url: string) => {
   app.use(`${url}/university`, university)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, teachers)
-  if (serviceProvider === 'toska') {
-    app.use(`${url}/users`, usersToska)
-  } else {
-    app.use(`${url}/users`, usersToska)
-    app.use(`${url}/users`, usersFd)
-  }
+  app.use(`${url}/users`, users)
   app.use(`${url}/feedback`, feedback)
   app.use(`${url}/custom-population-search`, customPopulationSearch)
   app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)

services/backend/src/routes/changelog.ts

@@ -1,20 +1,19 @@
 import axios from 'axios'
-import { Request, Response, Router } from 'express'
+import { Router } from 'express'
 
 import { Release } from '@oodikone/shared/types'
-import { isDev } from '../config'
+import { isDev, isTest } from '../config'
 
 const router = Router()
 
 const changelog: { data?: Release[] } = {}
 
-export type ChangelogResBody = Release[]
-
-router.get<never, ChangelogResBody>('/', async (_req: Request, res: Response) => {
+router.get<never, Release[]>('/', async (_, res) => {
   if (changelog.data) {
     return res.status(200).send(changelog.data)
   }
-  if (isDev) {
+
+  if (isDev || isTest) {
     const fakeRelease: Release[] = [
       {
         description: '**Feature 1**\n- Added a fancy new feature \n\n**Feature 2**\n- Fixed a bug\n- Fixed another bug',
@@ -35,8 +34,11 @@ router.get<never, ChangelogResBody>('/', async (_req: Request, res: Response) =>
         version: '0.0.1',
       },
     ]
+
+    changelog.data = fakeRelease
     return res.status(200).json(fakeRelease)
   }
+
   const response = await axios.get('https://api.github.com/repos/UniversityOfHelsinkiCS/oodikone/releases')
   const releasesFromAPI: Release[] = response.data.map((release: Record<string, any>) => ({
     description: release.body,
@@ -44,6 +46,7 @@ router.get<never, ChangelogResBody>('/', async (_req: Request, res: Response) =>
     title: release.name,
     version: release.tag_name,
   }))
+
   changelog.data = releasesFromAPI
   res.status(200).json(releasesFromAPI)
 })

services/backend/src/routes/closeToGraduation.ts

@@ -1,22 +1,24 @@
 import { Router } from 'express'
 
-import { findStudentsCloseToGraduation, getCloseToGraduationData } from '../services/populations/closeToGraduation'
+import type { CloseToGraduationResBody } from '@oodikone/shared/routes/closeToGraduation'
+import { getCloseToGraduationData } from '../services/populations/closeToGraduation'
 import { getAllStudentsUserHasInGroups } from '../services/studyGuidanceGroups'
 import { hasFullAccessToStudentData } from '../util'
 
 const router = Router()
 
-type CloseToGraduationResBody = Awaited<ReturnType<typeof findStudentsCloseToGraduation>>
-
 router.get<never, CloseToGraduationResBody>('/', async (req, res) => {
   const { user } = req
-  if (hasFullAccessToStudentData(user.roles)) {
-    const result = await getCloseToGraduationData()
-    return res.json(result)
+
+  if (!hasFullAccessToStudentData(user.roles)) {
+    const studentNumbers = await getAllStudentsUserHasInGroups(user.sisPersonId)
+    const result = await getCloseToGraduationData(studentNumbers)
+
+    res.json(result)
   }
-  const studentsInUsersGuidanceGroups = await getAllStudentsUserHasInGroups(user.sisPersonId)
-  const result = await getCloseToGraduationData(studentsInUsersGuidanceGroups)
-  res.json(result)
+
+  const result = await getCloseToGraduationData()
+  return res.json(result)
 })
 
 export default router

services/backend/src/routes/completedCoursesSearch.ts

@@ -1,5 +1,5 @@
 import { Router } from 'express'
-import { difference, intersection } from 'lodash'
+import { difference, intersection } from 'lodash-es'
 
 import { CanError } from '@oodikone/shared/routes'
 import { tryCatch } from '@oodikone/shared/util'
@@ -42,18 +42,11 @@ router.get<never, SearchResBody, SearchReqBody, SearchQuery>('/', async (req, re
     return res.status(400).json({ error: 'Courses must be of type array' })
   }
 
-  const answerTimeout = new Promise((_, reject) => setTimeout(() => reject(new Error('Request timeout')), 6000))
-
   // Teachers can also get rights to students via Importer if
   // the students have enrolled to their courses in last 8 months
   // (acual logic a bit different, see Importer)
   const { data: teacherRightsToStudents, error } = importerClient
-    ? await tryCatch<{ data: string[] }>(
-        Promise.race([
-          importerClient.post('/teacher-rights/', { teacherId, studentNumbers }),
-          answerTimeout as Promise<never>, // This will always reject with an Error
-        ])
-      )
+    ? await tryCatch<{ data: string[] }>(importerClient.post('/teacher-rights/', { teacherId, studentNumbers }))
     : { data: null, error: null }
 
   if (error) {

services/backend/src/routes/courseExclusions.ts

@@ -1,19 +1,20 @@
 import { Router } from 'express'
 
+import type { ExcludedCoursesResBody, ExcludedCoursesReqBody } from '@oodikone/shared/routes/courseExclusions'
 import { addExcludedCourses, removeExcludedCourses } from '../services/excludedCourses'
+import { hasFullAccessToStudentData } from '../util'
 
 const router = Router()
 
-export type ExcludedCoursesParams = { code: string }
-export type ExcludedCoursesResBody = void
-export type ExcludedCoursesReqBody = {
-  courseCodes: string[]
-  curriculumVersion: string
-}
+router.post<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async (req, res) => {
+  const { programmeCode, courseCodes, curriculumVersion } = req.body
+  const { roles, programmeRights } = req.user
+
+  const hasFullAccess = hasFullAccessToStudentData(roles)
+  const hasAccessToProgramme = programmeRights.map(({ code }) => code).includes(programmeCode)
+
+  if (!hasFullAccess && !hasAccessToProgramme) return res.status(403).end()
 
-router.post<ExcludedCoursesParams, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/:code', async (req, res) => {
-  const { code: programmeCode } = req.params
-  const { courseCodes, curriculumVersion } = req.body
   const result = await addExcludedCourses(courseCodes, curriculumVersion, programmeCode)
   if (!result) {
     res.status(400).end()
@@ -22,9 +23,15 @@ router.post<ExcludedCoursesParams, ExcludedCoursesResBody, ExcludedCoursesReqBod
   res.status(201).end()
 })
 
-router.delete<ExcludedCoursesParams, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/:code', async (req, res) => {
-  const { code: programmeCode } = req.params
-  const { courseCodes, curriculumVersion } = req.body
+router.delete<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async (req, res) => {
+  const { programmeCode, courseCodes, curriculumVersion } = req.body
+  const { roles, programmeRights } = req.user
+
+  const hasFullAccess = hasFullAccessToStudentData(roles)
+  const hasAccessToProgramme = programmeRights.map(({ code }) => code).includes(programmeCode)
+
+  if (!hasFullAccess && !hasAccessToProgramme) return res.status(403).end()
+
   const result = await removeExcludedCourses(courseCodes, curriculumVersion, programmeCode)
   if (!result) {
     res.status(400).end()

services/backend/src/routes/courses.ts

@@ -2,106 +2,80 @@ import crypto from 'crypto'
 import { Router } from 'express'
 
 import { CanError } from '@oodikone/shared/routes'
+import {
+  CoursesMultiResBody,
+  CoursesMultiReqBody,
+  CoursesMultiQuery,
+  CourseYearlyStatsReqBody,
+  CourseYearlyStatsQuery,
+} from '@oodikone/shared/routes/courses'
+import type { CourseWithSubsId } from '@oodikone/shared/types/course'
 import { getCourseYearlyStats } from '../services/courses'
-import { getCoursesByNameAndOrCode, getCoursesByCodes } from '../services/courses/courseFinders'
-import { CourseWithSubsId } from '../types/course'
+import { getCoursesByNameAndOrCode } from '../services/courses/courseFinders'
 import { getFullStudyProgrammeRights, hasFullAccessToStudentData, validateParamLength } from '../util'
 
 const router = Router()
 
-type CoursesMultiReqBody = never
-type CoursesMultiResBody = { courses: CourseWithSubsId[] }
-type CoursesMultiQuery = {
-  name: string
-  code: string
-  combineSubstitutions: string
-}
-
 router.get<never, CanError<CoursesMultiResBody>, CoursesMultiReqBody, CoursesMultiQuery>(
   '/v2/coursesmulti',
   async (req, res) => {
-    const { name, code, combineSubstitutions } = req.query
+    const { name, code, includeSpecial } = req.query
     if (!(validateParamLength(name, 5) || validateParamLength(code, 2))) {
       return res.status(400).json({ error: 'Query parameter name or code is invalid' })
     }
 
-    const courses: CourseWithSubsId[] = await getCoursesByNameAndOrCode(name, code)
+    const courses = await getCoursesByNameAndOrCode(name, code, includeSpecial === 'true')
 
-    if (combineSubstitutions === 'false') {
-      const courseCodes = courses.map(course => course.code)
-      const substitutions = [
-        ...new Set(
-          courses.flatMap(course => course.substitutions).filter(code => code !== null && !courseCodes.includes(code))
-        ),
-      ]
-      const substitutionDetails = await getCoursesByCodes(substitutions)
-      for (const substitution of substitutionDetails) {
-        courses.push(substitution.toJSON())
-      }
-    }
-
-    const mergedCourses: Record<string, CourseWithSubsId> = {}
+    const mergedCourses = new Map<string, CourseWithSubsId>()
 
     for (const course of courses) {
-      const groupId = combineSubstitutions === 'true' ? (course.subsId?.toString() ?? '') : course.code
-      if (!(course.max_attainment_date && course.min_attainment_date)) {
-        continue
+      if (!(course.max_attainment_date && course.min_attainment_date)) continue
+
+      if (!mergedCourses.has(course.code)) mergedCourses.set(course.code, structuredClone(course))
+      const mergedCourse = mergedCourses.get(course.code)!
+
+      if (mergedCourse.max_attainment_date < course.max_attainment_date) {
+        mergedCourse.max_attainment_date = course.max_attainment_date
       }
-      if (!mergedCourses[groupId]) {
-        mergedCourses[groupId] = {
-          ...course,
-          substitutions: combineSubstitutions === 'true' ? course.substitutions : [],
-        }
-      } else {
-        const mergedCourse = mergedCourses[groupId]
-        if (mergedCourse.max_attainment_date < course.max_attainment_date) {
-          mergedCourse.max_attainment_date = course.max_attainment_date
-        }
-        if (mergedCourse.min_attainment_date > course.min_attainment_date) {
-          mergedCourse.min_attainment_date = course.min_attainment_date
-        }
+      if (mergedCourse.min_attainment_date > course.min_attainment_date) {
+        mergedCourse.min_attainment_date = course.min_attainment_date
       }
     }
 
-    res.json({ courses: Object.values(mergedCourses) })
+    res.json({ courses: Array.from(mergedCourses.values()) })
   }
 )
 
 type CourseYearlyStatsResBody = Awaited<ReturnType<typeof getCourseYearlyStats>>
-type CourseYearlyStatsReqBody = never
-type CourseYearlyStatsQuery = {
-  codes: string[]
-  separate: string
-  combineSubstitutions: string
-}
 
 router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody, CourseYearlyStatsQuery>(
   '/v3/courseyearlystats',
   async (req, res) => {
-    const { roles, programmeRights } = req.user
+    const { codes, combineSubstitutions, separate } = req.query
 
-    const userHasFullAccessToStudentData = hasFullAccessToStudentData(roles)
-    const userHasCorrectRole = userHasFullAccessToStudentData === true || roles.includes('courseStatistics')
-    const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
-
-    // If user has rights to see at least one programme, then they are allowed to see all of them
-    if (!userHasCorrectRole && fullStudyProgrammeRights.length === 0) {
-      return res.status(403).json({ error: 'No programmes so no access to course stats' })
-    }
-
-    const { codes } = req.query
     if (!codes) {
       return res.status(422).send({ error: 'Missing required query parameters' })
     }
 
-    const combineSubstitutions = req.query.combineSubstitutions !== 'false'
-    const separate = req.query.separate === 'true'
+    const { roles, programmeRights } = req.user
+    const userHasFullAccessToStudentData = hasFullAccessToStudentData(roles)
+    const userHasAccessToCourseStats = userHasFullAccessToStudentData || roles.includes('courseStatistics')
+    const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
+
+    // If user has rights to see at least one programme, then they are allowed to see all of them
+    if (!userHasAccessToCourseStats && !fullStudyProgrammeRights.length) {
+      return res.status(403).json({ error: 'No valid rights provided' })
+    }
 
     // Student numbers should be obfuscated to all other users except admins,
     // fullSisuAccess users, and users with rights to any specific degree programmes
     const anonymize = !userHasFullAccessToStudentData && fullStudyProgrammeRights.length === 0
     const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
-    const results = await getCourseYearlyStats(codes, separate, anonymizationSalt, combineSubstitutions)
+
+    const useCombineSubstitutions = combineSubstitutions !== 'false'
+    const useSeparate = separate === 'true'
+
+    const results = await getCourseYearlyStats(codes, useSeparate, anonymizationSalt, useCombineSubstitutions)
     res.json(results)
   }
 )

services/backend/src/routes/faculties.ts

@@ -1,8 +1,18 @@
-import { Request, Response, Router } from 'express'
+import { Router } from 'express'
 
-import { Graduated, ProgrammeFilter, SpecialGroups, StatsType, YearType } from '@oodikone/shared/types'
+import type { CanError, GenericApplicationError } from '@oodikone/shared/routes'
+import type {
+  Graduated,
+  Name,
+  NameWithCode,
+  ProgrammeFilter,
+  SpecialGroups,
+  StatsType,
+  Unarray,
+  YearType,
+} from '@oodikone/shared/types'
 import * as auth from '../middleware/auth'
-import { getDegreeProgrammesOfFaculty, getFacultyCodeById } from '../services/faculty/faculty'
+import { getDegreeProgrammesOfFaculty, getFacultyCodeById, ProgrammesOfOrganization } from '../services/faculty/faculty'
 import { combineFacultyBasics } from '../services/faculty/facultyBasics'
 import { getFacultyCredits } from '../services/faculty/facultyCredits'
 import { countGraduationTimes } from '../services/faculty/facultyGraduationTimes'
@@ -18,242 +28,270 @@ import {
   setFacultyStudentStats,
   getFacultyProgressStats,
   setFacultyProgressStats,
+  type BasicData,
+  type ThesisWriterData,
+  GraduationData,
 } from '../services/faculty/facultyService'
 import { combineFacultyStudentProgress } from '../services/faculty/facultyStudentProgress'
 import { combineFacultyStudents } from '../services/faculty/facultyStudents'
 import { combineFacultyThesisWriters } from '../services/faculty/facultyThesisWriters'
 import { updateFacultyOverview, updateFacultyProgressOverview } from '../services/faculty/facultyUpdates'
-import logger from '../util/logger'
+import { ApplicationError } from '../util/customErrors'
 
 // Faculty uses a lot of tools designed for Degree programme.
 // Some of them have been copied here and slightly edited for faculty purpose.
 
 const router = Router()
 
-router.get('/', async (_req: Request, res: Response) => {
-  const faculties = await getFacultiesForFacultyList()
-  res.json(faculties)
-})
+type GetFacultyListResBody = {
+  id: string
+  code: string
+  name: Name
+}[]
 
-interface GetStatsRequest extends Request {
-  query: {
-    programme_filter: ProgrammeFilter
-    special_groups: SpecialGroups
-    year_type: YearType
-  }
+router.get<never, GetFacultyListResBody>('/', async (_req, res) => res.json(await getFacultiesForFacultyList()))
+
+type GetStatsParams = { id: string }
+type GetStatsReqBody = never
+type GetStatsResBody = BasicData
+type GetStatsQuery = {
+  programme_filter: ProgrammeFilter
+  special_groups: SpecialGroups
+  year_type: YearType
 }
 
-router.get('/:id/basicstats', auth.roles(['facultyStatistics']), async (req: GetStatsRequest, res: Response) => {
-  const facultyId = req.params.id
-  const code = await getFacultyCodeById(facultyId)
-  if (!code) {
-    return res.status(422).end()
-  }
+router.get<GetStatsParams, CanError<GetStatsResBody, GenericApplicationError>, GetStatsReqBody, GetStatsQuery>(
+  '/:id/basicstats',
+  auth.roles(['facultyStatistics']),
+  async (req, res) => {
+    const { id: facultyId } = req.params
+    const { year_type: yearType, programme_filter: programmeFilter, special_groups: specialGroups } = req.query
 
-  const { year_type: yearType, programme_filter: programmeFilter, special_groups: specialGroups } = req.query
-  const data = await getBasicStats(code, yearType, programmeFilter, specialGroups)
-  if (data) {
-    return res.json(data)
-  }
+    const faculty = await getFacultyCodeById(facultyId)
+    if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
-  const programmes = await getDegreeProgrammesOfFaculty(code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
-  if (!programmes.length) {
-    return res.status(422).end()
-  }
+    const data = await getBasicStats(faculty.code, yearType, programmeFilter, specialGroups)
+    if (data) return res.json(data)
 
-  // TODO: The types could be handled better. Possible null value is causing issues with TS.
-  let updatedStats: any = await combineFacultyBasics(code, programmes, yearType, specialGroups)
-  if (updatedStats) {
-    updatedStats = await setBasicStats(updatedStats, yearType, programmeFilter, specialGroups)
-  }
-  return res.json(updatedStats)
-})
+    const programmes = await getDegreeProgrammesOfFaculty(faculty.code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
+    if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
 
-interface GetCreditStatsRequest extends Request {
-  query: {
-    year_type: YearType
+    const updatedStats = await combineFacultyBasics(faculty.code, programmes, yearType, specialGroups)
+
+    if (updatedStats) await setBasicStats(updatedStats, yearType, programmeFilter, specialGroups)
+    else throw new ApplicationError('No data', 204)
+
+    return res.json(updatedStats)
   }
+)
+
+type GetFacultyStatsResBody = ThesisWriterData
+
+router.get<GetStatsParams, CanError<GetFacultyStatsResBody, GenericApplicationError>, GetStatsReqBody, GetStatsQuery>(
+  '/:id/thesisstats',
+  auth.roles(['facultyStatistics']),
+  async (req, res) => {
+    const { id: facultyId } = req.params
+    const { year_type: yearType, programme_filter: programmeFilter, special_groups: specialGroups } = req.query
+
+    const faculty = await getFacultyCodeById(facultyId)
+    if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+    const data = await getThesisWritersStats(faculty.code, yearType, programmeFilter, specialGroups)
+    if (data) return res.json(data)
+
+    const programmes = await getDegreeProgrammesOfFaculty(faculty.code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
+    if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
+
+    const updatedStats = await combineFacultyThesisWriters(faculty.code, programmes, yearType, specialGroups)
+    if (updatedStats) await setThesisWritersStats(updatedStats, yearType, programmeFilter, specialGroups)
+    else throw new ApplicationError('No data', 204)
+
+    return res.json(updatedStats)
+  }
+)
+
+type GetCreditStatsResBody = {
+  codes: string[]
+  ids: string[]
+  programmeNames: Record<string, NameWithCode>
+}
+type GetCreditStatsQuery = {
+  year_type: YearType
 }
 
-router.get('/:id/creditstats', auth.roles(['facultyStatistics']), async (req: GetCreditStatsRequest, res: Response) => {
-  const facultyId = req.params.id
-  const code = await getFacultyCodeById(facultyId)
+router.get<
+  GetStatsParams,
+  CanError<GetCreditStatsResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetCreditStatsQuery
+>('/:id/creditstats', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
   const { year_type: yearType } = req.query
-  const stats = await getFacultyCredits(code, yearType === 'ACADEMIC_YEAR')
-  return res.json(stats)
+
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+  return res.json(await getFacultyCredits(faculty.code, yearType === 'ACADEMIC_YEAR'))
 })
 
-router.get('/:id/thesisstats', auth.roles(['facultyStatistics']), async (req: GetStatsRequest, res: Response) => {
-  const facultyId = req.params.id
-  const code = await getFacultyCodeById(facultyId)
-  if (!code) {
-    return res.status(422).end()
-  }
+type GetGraduationStatsResBody = GraduationData
+type GetGraduationStatsQuery = {
+  programme_filter: ProgrammeFilter
+}
 
-  const { year_type: yearType, programme_filter: programmeFilter, special_groups: specialGroups } = req.query
-  const data = await getThesisWritersStats(code, yearType, programmeFilter, specialGroups)
-  if (data) {
-    return res.json(data)
-  }
+router.get<
+  GetStatsParams,
+  CanError<GetGraduationStatsResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetGraduationStatsQuery
+>('/:id/graduationtimes', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
+  const { programme_filter: programmeFilter } = req.query
 
-  const programmes = await getDegreeProgrammesOfFaculty(code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
-  if (!programmes.length) {
-    return res.status(422).end()
-  }
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+  const data = await getGraduationStats(faculty.code, programmeFilter)
+  if (data) return res.json(data)
+
+  const programmes = await getDegreeProgrammesOfFaculty(faculty.code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
+  if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
+
+  const updatedStats = await countGraduationTimes(faculty.code, programmes)
+  if (updatedStats) await setGraduationStats(updatedStats, programmeFilter)
 
-  let updatedStats: any = await combineFacultyThesisWriters(code, programmes, yearType, specialGroups)
-  if (updatedStats) {
-    updatedStats = await setThesisWritersStats(updatedStats, yearType, programmeFilter, specialGroups)
-  }
   return res.json(updatedStats)
 })
 
-interface GetGraduationStatsRequest extends Request {
-  query: {
-    programme_filter: ProgrammeFilter
-  }
+type GetProgressStatsResBody = {
+  bachelorsProgStats: Record<string, number[][]>
+  bcMsProgStats: Record<string, number[][]>
+  creditCounts: Record<string, Record<string, number[]>>
+  doctoralProgStats: Record<string, number[][]>
+  id: string
+  mastersProgStats: Record<string, number[][]>
+  programmeNames: Record<string, NameWithCode>
+  yearlyBachelorTitles: string[][]
+  yearlyBcMsTitles: string[][]
+  yearlyMasterTitles: string[][]
+  years: string[]
 }
 
-router.get(
-  '/:id/graduationtimes',
-  auth.roles(['facultyStatistics']),
-  async (req: GetGraduationStatsRequest, res: Response) => {
-    const facultyId = req.params.id
-    const code = await getFacultyCodeById(facultyId)
-    if (!code) {
-      return res.status(422).end()
-    }
-
-    const { programme_filter: programmeFilter } = req.query
-    const data = await getGraduationStats(code, programmeFilter)
-    if (data) {
-      return res.json(data)
-    }
-
-    const programmes = await getDegreeProgrammesOfFaculty(code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
-    if (!programmes) {
-      return res.status(422).end()
-    }
-
-    let updatedStats: any = await countGraduationTimes(code, programmes)
-    if (updatedStats) {
-      updatedStats = await setGraduationStats(updatedStats, programmeFilter)
-    }
-    return res.json(updatedStats)
-  }
-)
-
-interface GetProgressStatsRequest extends Request {
-  query: {
-    graduated: Graduated
-    special_groups: SpecialGroups
-  }
+type GetProgressStatsQuery = {
+  graduated: Graduated
+  special_groups: SpecialGroups
 }
 
-router.get(
-  '/:id/progressstats',
-  auth.roles(['facultyStatistics']),
-  async (req: GetProgressStatsRequest, res: Response) => {
-    const facultyId = req.params.id
-    const code = await getFacultyCodeById(facultyId)
-    if (!code) {
-      return res.status(422).end()
-    }
+router.get<
+  GetStatsParams,
+  CanError<GetProgressStatsResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetProgressStatsQuery
+>('/:id/progressstats', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
+  const { special_groups: specialGroups } = req.query
 
-    const programmes = await getDegreeProgrammesOfFaculty(code, true)
-    if (!programmes) {
-      return res.status(422).end()
-    }
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
-    const { graduated, special_groups: specialGroups } = req.query
-    const data = await getFacultyProgressStats(code, specialGroups, graduated)
-    if (data) {
-      return res.json(data)
-    }
+  const programmes = await getDegreeProgrammesOfFaculty(faculty.code, true)
+  if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
 
-    let updatedStats: any = await combineFacultyStudentProgress(code, programmes, specialGroups, graduated)
-    if (updatedStats) {
-      updatedStats = await setFacultyProgressStats(updatedStats, specialGroups, graduated)
-    }
-    return res.json(updatedStats)
-  }
-)
+  const data = await getFacultyProgressStats(faculty.code, specialGroups)
+  if (data) return res.json(data)
 
-interface GetStudentStatsRequest extends GetProgressStatsRequest {}
+  const updatedStats = await combineFacultyStudentProgress(faculty.code, programmes, specialGroups)
+  if (updatedStats) await setFacultyProgressStats(updatedStats, specialGroups)
 
-router.get(
-  '/:id/studentstats',
-  auth.roles(['facultyStatistics']),
-  async (req: GetStudentStatsRequest, res: Response) => {
-    const facultyId = req.params.id
-    const code = await getFacultyCodeById(facultyId)
-    if (!code) {
-      return res.status(422).end()
-    }
+  return res.json(updatedStats)
+})
 
-    const { graduated, special_groups: specialGroups } = req.query
+export type DegreeProgramme = Pick<
+  Unarray<ProgrammesOfOrganization>,
+  'code' | 'name' | 'degreeProgrammeType' | 'progId'
+>
 
-    /* Fetch cached data if exists */
-    const data = await getFacultyStudentStats(code, specialGroups, graduated)
-    if (data) {
-      return res.json(data)
-    }
-
-    const newProgrammes = await getDegreeProgrammesOfFaculty(code, true)
-    if (!newProgrammes.length) {
-      return res.status(422).end()
-    }
-
-    const updatedStats = await combineFacultyStudents(code, newProgrammes, specialGroups, graduated)
-
-    if (updatedStats) {
-      void setFacultyStudentStats(updatedStats, specialGroups, graduated)
-    }
-    return res.json(updatedStats)
-  }
-)
-
-interface GetUpdateBasicViewRequest extends Request {
-  query: {
-    stats_type: StatsType
-  }
+type GetStudetStatsResBody = {
+  id: string
+  years: string[]
+  facultyTableStats: Record<string, (number | string)[]>
+  facultyTableStatsExtra: Record<string, Record<string, Record<string, number>>>
+  /**
+   * NOTE: programmeStats order is random (not by the year). Make sure to index properly.
+   *
+   * (Order is based on the order in which studyrights were processed in the backend)
+   */
+  programmeStats: Record<string, Record<string, (string | number)[]>>
+  titles: string[]
+  programmeNames: Record<string, DegreeProgramme>
 }
 
-router.get(
-  '/:id/update_basicview',
-  auth.roles(['facultyStatistics']),
-  async (req: GetUpdateBasicViewRequest, res: Response) => {
-    const facultyId = req.params.id
-    const code = await getFacultyCodeById(facultyId)
-    if (!code) {
-      return res.status(422).end()
-    }
-    const { stats_type: statsType } = req.query
-    try {
-      const result = await updateFacultyOverview(code, statsType)
-      return res.json(result)
-    } catch (error) {
-      const message = `Failed to update basic stats for faculty ${code}`
-      logger.error(message, { error })
-      return res.status(500).json({ error: message })
-    }
-  }
-)
+router.get<
+  GetStatsParams,
+  CanError<GetStudetStatsResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetProgressStatsQuery
+>('/:id/studentstats', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
+  const { special_groups: specialGroups } = req.query
+
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+  const data = await getFacultyStudentStats(faculty.code, specialGroups)
+  if (data) return res.json(data)
+
+  const programmes = await getDegreeProgrammesOfFaculty(faculty.code, true)
+  if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
+
+  const updatedStats = await combineFacultyStudents(faculty.code, programmes, specialGroups)
+  if (updatedStats) await setFacultyStudentStats(updatedStats, specialGroups)
+
+  return res.json(updatedStats)
+})
+
+type GetUpdateBasicViewResBody = never
+type GetUpdateBasicViewQuery = {
+  stats_type: StatsType
+}
+
+router.get<
+  GetStatsParams,
+  CanError<GetUpdateBasicViewResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetUpdateBasicViewQuery
+>('/:id/update_basicview', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
+  const { stats_type: statsType } = req.query
+
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
-router.get('/:id/update_progressview', auth.roles(['facultyStatistics']), async (req: Request, res: Response) => {
-  const facultyId = req.params.id
-  const code = await getFacultyCodeById(facultyId)
-  if (!code) {
-    return res.status(422).end()
-  }
   try {
-    const result = await updateFacultyProgressOverview(code)
-    return res.json(result)
+    await updateFacultyOverview(faculty.code, statsType)
+    return res.json()
   } catch (error) {
-    const message = `Failed to update progress tab stats for faculty ${code}`
-    logger.error(message, { error })
-    return res.status(500).json({ error: message })
+    throw new ApplicationError(`Failed to update basic stats for faculty ${faculty.code}`, 500)
   }
 })
 
+router.get<GetStatsParams, CanError<never, GenericApplicationError>, GetStatsReqBody, GetStatsQuery>(
+  '/:id/update_progressview',
+  auth.roles(['facultyStatistics']),
+  async (req, res) => {
+    const { id: facultyId } = req.params
+
+    const faculty = await getFacultyCodeById(facultyId)
+    if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+    try {
+      await updateFacultyProgressOverview(faculty.code)
+      return res.json()
+    } catch (error) {
+      throw new ApplicationError(`Failed to update progress tab stats for faculty ${faculty.code}`, 500)
+    }
+  }
+)
+
 export default router

services/backend/src/routes/feedback.ts

@@ -1,6 +1,6 @@
 import { Request, Response, Router } from 'express'
 
-import { sendFeedbackToToska } from '../services/mailService'
+import { sendFeedback } from '../services/mailService'
 import { ApplicationError } from '../util/customErrors'
 import logger from '../util/logger'
 
@@ -20,7 +20,7 @@ router.post('/email', async (req: EmailRequest, res: Response) => {
     throw new ApplicationError('User not found', 404)
   }
 
-  await sendFeedbackToToska({
+  await sendFeedback({
     feedbackContent: content,
     user,
   })

services/backend/src/routes/login.ts

@@ -17,7 +17,7 @@ router.get<never, CanError<LoginResBody>, LoginReqBody>('/', (req, res) => {
   const { user, logoutUrl } = req
 
   if (!user) {
-    throw new ApplicationError('User not found', 404)
+    throw new ApplicationError('User not found', 401)
   }
 
   res.send({

services/backend/src/routes/population.ts

@@ -1,6 +1,5 @@
-import crypto from 'crypto'
 import { Router } from 'express'
-import { difference, intersection, uniq } from 'lodash'
+import { difference, intersection, uniq } from 'lodash-es'
 
 import type { CanError } from '@oodikone/shared/routes'
 import {
@@ -10,34 +9,107 @@ import {
   type PopulationstatisticsbycourseResBody,
   type PopulationstatisticsbycourseReqBody,
   type PopulationstatisticsbycourseParams,
-  type PostByStudentNumbersResBody,
-  type PostByStudentNumbersReqBody,
+  type GetCustomPopulationResBody,
+  type CustomPopulationQuery,
   type PopulationstatisticsStudyprogrammesResBody,
   PopulationstatisticsMaxYearsToCreatePopulationFormQuery,
   PopulationstatisticsMaxYearsToCreatePopulationFormResBody,
 } from '@oodikone/shared/routes/populations'
-import { GenderCode, Unification, Unarray } from '@oodikone/shared/types'
-import { mapToProviders } from '@oodikone/shared/util'
+import { Unification } from '@oodikone/shared/types'
+import { keyBy, mapToProviders } from '@oodikone/shared/util'
 import { rootOrgId } from '../config'
-import { SISStudyRightModel } from '../models'
-import { maxYearsToCreatePopulationFrom, getCourseProvidersForCourses } from '../services/courses'
+import { getCourseProvidersForCourses, maxYearsToCreatePopulationFrom } from '../services/courses'
 import { encrypt } from '../services/encrypt'
-import { getDegreeProgrammesOfOrganization, ProgrammesOfOrganization } from '../services/faculty/faculty'
-import { getStudentTags } from '../services/populations/getStudentData'
+import { getDegreeProgrammesOfOrganization } from '../services/faculty/faculty'
+import { getStudentTagMap } from '../services/populations/getStudentData'
 import { parseDateRangeFromParams } from '../services/populations/shared'
 import { statisticsOf } from '../services/populations/statisticsOf'
 import { getStudentNumbersWithStudyRights } from '../services/populations/studentNumbersWithStudyRights'
 import { findByCourseAndSemesters } from '../services/students'
-import { ParsedCourse } from '../types'
 import { getFullStudyProgrammeRights, hasFullAccessToStudentData } from '../util'
 
 const router = Router()
 
+const obfuscateStuff = ({
+  result,
+
+  programme,
+  combinedProgramme,
+
+  userRoles,
+  userProgrammeRights,
+  studentsUserCanAccess,
+}: {
+  result: Awaited<ReturnType<typeof statisticsOf>>
+  [key: string]: any
+}): Awaited<ReturnType<typeof statisticsOf>> => {
+  const hasFullAccessToStudents = hasFullAccessToStudentData(userRoles)
+  const userFullProgrammeRights = getFullStudyProgrammeRights(userProgrammeRights)
+  const hasFullRightsToProgramme = userFullProgrammeRights.includes(programme)
+  const hasFullRightsToCombinedProgramme = userFullProgrammeRights.includes(combinedProgramme)
+
+  if (hasFullAccessToStudents || hasFullRightsToProgramme || hasFullRightsToCombinedProgramme) {
+    return result
+  }
+
+  const { criteria, students, coursestatistics } = result
+  const encryptionMap = new Map<string, string>()
+
+  return {
+    criteria,
+    students: students.map(student => {
+      if (studentsUserCanAccess.has(student.studentNumber)) return student
+
+      // correct year for age distribution calculation but the date is always January 1st
+      const obfuscatedBirthDate = new Date(Date.UTC(new Date(student.birthdate).getUTCFullYear(), 0))
+      const { iv, encryptedData: studentNumber } = encrypt(student.studentNumber)
+
+      encryptionMap.set(student.studentNumber, studentNumber)
+
+      return {
+        ...student,
+
+        obfuscated: true,
+        iv,
+        studentNumber,
+        firstnames: '',
+        lastname: '',
+        phoneNumber: '',
+        name: '',
+        email: '',
+        secondaryEmail: '',
+        sis_person_id: '',
+        tags: [],
+        birthdate: obfuscatedBirthDate,
+      }
+    }),
+    coursestatistics: {
+      courses: coursestatistics.courses,
+      enrollments: coursestatistics.enrollments.map(enrollment => ({
+        ...enrollment,
+        studentnumber: encryptionMap.get(enrollment.studentnumber) ?? enrollment.studentnumber,
+      })),
+      credits: coursestatistics.credits.map(credit => ({
+        ...credit,
+        student_studentnumber: encryptionMap.get(credit.student_studentnumber) ?? credit.student_studentnumber,
+      })),
+    },
+  }
+}
+
 router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReqBody, PopulationstatisticsQuery>(
   '/v3/populationstatistics',
   async (req, res) => {
-    const { id: userId, roles: userRoles, programmeRights: userProgrammeRights } = req.user
-    const { years, semesters, programme, combinedProgramme, studentStatuses } = req.query
+    const { id: userId, roles: userRoles, programmeRights: userProgrammeRights, studentsUserCanAccess } = req.user
+    const { programme, combinedProgramme } = req.query
+
+    // NOTE: Make sure we have only arrays, "/...&years=2021" is parsed by express as years: 2021, not [2021]
+    const handleQueryArrays = <T>(requestField: T | T[]): T[] => {
+      return Array.isArray(requestField) ? requestField : [requestField]
+    }
+    const years = handleQueryArrays(req.query.years)
+    const semesters = handleQueryArrays(req.query.semesters)
+    const studentStatuses = handleQueryArrays(req.query.studentStatuses) // NOTE: TRANSFERRED means transferred out of the program
 
     const requiredFields = [years, semesters, programme]
     if (requiredFields.some(field => !field)) {
@@ -48,18 +120,16 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
       return res.status(400).json({ error: 'Semester should be either SPRING OR FALL' })
     }
 
-    const hasCorrectStatus = (studentStatuses: string[]) =>
-      studentStatuses.every(status => ['EXCHANGE', 'NONDEGREE', 'TRANSFERRED'].includes(status))
-    if (studentStatuses && !hasCorrectStatus(studentStatuses)) {
+    const hasCorrectStatus = (studentStatuses: (string | undefined)[]) =>
+      studentStatuses.every(status => status && ['EXCHANGE', 'NONDEGREE', 'TRANSFERRED'].includes(status))
+
+    // NOTE: This has always the length of at least 1, [undefined]
+    if (studentStatuses[0] && !hasCorrectStatus(studentStatuses)) {
       return res.status(400).json({ error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' })
     }
 
     const hasFullAccessToStudents = hasFullAccessToStudentData(userRoles)
 
-    const userFullProgrammeRights = getFullStudyProgrammeRights(userProgrammeRights)
-    const hasFullRightsToProgramme = userFullProgrammeRights.includes(programme)
-    const hasFullRightsToCombinedProgramme = userFullProgrammeRights.includes(combinedProgramme!)
-
     const userProgrammeRightsCodes = userProgrammeRights.map(({ code }) => code)
     const hasAccessToProgramme = userProgrammeRightsCodes.includes(programme)
     const hasAccessToCombinedProgramme = userProgrammeRightsCodes.includes(combinedProgramme!)
@@ -68,10 +138,8 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
       return res.status(403).json({ error: 'Trying to request unauthorized students data' })
     }
 
-    const { startDate, endDate } = parseDateRangeFromParams({
-      ...req.query,
-      years: req.query.years,
-    })
+    // NOTE: Years are academic years
+    const { startDate, endDate } = parseDateRangeFromParams({ semesters, years })
 
     const studentNumbers = await getStudentNumbersWithStudyRights({
       programmeCodes: [programme],
@@ -81,35 +149,21 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
     })
 
     const studyRights = [programme]
-    const tagList = await getStudentTags(studyRights, studentNumbers, userId)
+    const tagMap = await getStudentTagMap(studyRights, studentNumbers, userId)
 
-    const result = await statisticsOf(studentNumbers, studyRights, tagList, startDate)
+    const result = await statisticsOf(studentNumbers, studyRights, tagMap, startDate)
+    const processed = obfuscateStuff({
+      result,
 
-    // Obfuscate if user has only limited degree programme rights and there are any students
-    if (!hasFullAccessToStudents && !hasFullRightsToProgramme && !hasFullRightsToCombinedProgramme) {
-      result.students = result.students.map(student => {
-        const { iv, encryptedData: studentNumber } = encrypt(student.studentNumber)
-        // correct year for age distribution calculation but the date is always January 1st
-        const obfuscatedBirthDate = new Date(Date.UTC(new Date(student.birthdate).getUTCFullYear(), 0))
-        return {
-          ...student,
-          firstnames: '',
-          lastname: '',
-          phoneNumber: '',
-          iv,
-          studentNumber,
-          name: '',
-          email: '',
-          secondaryEmail: '',
-          sis_person_id: '',
-          tags: [],
-          birthdate: obfuscatedBirthDate,
-          obfuscated: true,
-        }
-      })
-    }
+      programme,
+      combinedProgramme,
 
-    return res.json(result)
+      userRoles,
+      userProgrammeRights,
+      studentsUserCanAccess: new Set(studentsUserCanAccess),
+    })
+
+    return res.json(processed)
   }
 )
 
@@ -119,7 +173,12 @@ router.get<
   PopulationstatisticsbycourseReqBody,
   PopulationstatisticsbycourseParams
 >('/v3/populationstatisticsbycourse', async (req, res) => {
-  const { id: userId, roles, studentsUserCanAccess: allStudentsUserCanAccess, programmeRights } = req.user
+  const {
+    id: userId,
+    roles: userRoles,
+    programmeRights: userProgrammeRights,
+    studentsUserCanAccess: allStudentsUserCanAccess,
+  } = req.user
   const { coursecodes: coursecodeJSON, from, to, separate, unifyCourses } = req.query
 
   if (!coursecodeJSON || !from || !to) {
@@ -139,79 +198,51 @@ router.get<
 
   const studentNumbers = await findByCourseAndSemesters(coursecodes, Number(from), Number(to), isSeparate, unifyCourses)
 
-  const studyRights = []
-  const tagList = await getStudentTags(studyRights, studentNumbers, userId)
-  const result: any = await statisticsOf(studentNumbers, studyRights, tagList)
-  const courseProviders: string[] = !hasFullAccessToStudentData(roles)
-    ? await getCourseProvidersForCourses(coursecodes)
-    : []
-  const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
+  const courseProviders: string[] = await getCourseProvidersForCourses(coursecodes)
+  const fullStudyProgrammeRights = getFullStudyProgrammeRights(userProgrammeRights)
   const rightsMappedToProviders = mapToProviders(fullStudyProgrammeRights)
-  const found = courseProviders.some(provider => rightsMappedToProviders.includes(provider))
 
-  const studentsUserCanAccess =
-    hasFullAccessToStudentData(roles) || found
-      ? new Set(studentNumbers)
-      : new Set(intersection(studentNumbers, allStudentsUserCanAccess))
+  const studentsUserCanAccess = courseProviders.some(provider => rightsMappedToProviders.includes(provider))
+    ? new Set(studentNumbers)
+    : new Set(allStudentsUserCanAccess)
 
-  const randomHash = crypto.randomBytes(12).toString('hex')
-  const obfuscateStudent = ({
-    studyRights,
-    studentNumber,
-    courses,
-    gender_code,
-  }: {
-    studyRights: SISStudyRightModel[]
-    studentNumber: string
-    courses: ParsedCourse[]
-    gender_code: GenderCode
-  }) => ({
-    courses,
-    studyRights,
-    gender_code,
-    studentNumber: crypto.createHash('md5').update(`${studentNumber}${randomHash}`).digest('hex'),
-    firstnames: '',
-    lastname: '',
-    phone_number: '',
-    name: '',
-    email: '',
-    started: null,
-    credits: '',
-    tags: [],
-    obfuscated: true,
+  const studyRights = []
+  const tagMap = await getStudentTagMap(studyRights, studentNumbers, userId)
+  const result = await statisticsOf(studentNumbers, studyRights, tagMap)
+  const processed = obfuscateStuff({
+    result,
+
+    userRoles,
+    userProgrammeRights,
+    studentsUserCanAccess,
   })
 
-  if ('students' in result) {
-    result.students = result.students.map(student =>
-      studentsUserCanAccess.has(student.studentNumber) ? student : obfuscateStudent(student)
-    )
-  }
-
-  res.json(result)
+  return res.json(processed)
 })
 
-router.post<never, PostByStudentNumbersResBody, PostByStudentNumbersReqBody>(
+// Used in custom population and single study guidance groups
+router.post<never, GetCustomPopulationResBody, CustomPopulationQuery>(
   '/v3/populationstatisticsbystudentnumbers',
   async (req, res) => {
-    const { studentnumberlist, tags } = req.body
+    const { studentNumbers, tags } = req.body
     const { id: userId, roles, studentsUserCanAccess } = req.user
     const filteredStudentNumbers = hasFullAccessToStudentData(roles)
-      ? studentnumberlist
-      : intersection(studentnumberlist, studentsUserCanAccess)
+      ? studentNumbers
+      : intersection(studentNumbers, studentsUserCanAccess)
 
     const studyProgrammeCode = tags?.studyProgramme?.split('+')[0]
 
     const studyRights = [studyProgrammeCode].filter(value => value !== undefined)
-    const tagList = await getStudentTags(studyRights, filteredStudentNumbers, userId)
+    const tagMap = await getStudentTagMap(studyRights, filteredStudentNumbers, userId)
 
     const { startDate } = tags?.year
       ? parseDateRangeFromParams({ semesters: ['FALL', 'SPRING'], years: [tags?.year] })
       : { startDate: undefined }
 
-    const result = await statisticsOf(filteredStudentNumbers, studyRights, tagList, startDate)
+    const result = await statisticsOf(filteredStudentNumbers, studyRights, tagMap, startDate)
 
     const resultWithStudyProgramme = { ...result, studyProgramme: tags?.studyProgramme }
-    const discardedStudentNumbers = difference(studentnumberlist, filteredStudentNumbers)
+    const discardedStudentNumbers = difference(studentNumbers, filteredStudentNumbers)
 
     res.status(200).json({ ...resultWithStudyProgramme, discardedStudentNumbers })
   }
@@ -229,14 +260,13 @@ router.get<never, PopulationstatisticsStudyprogrammesResBody>(
     const filteredProgrammes = hasFullAccessToStudentData(roles)
       ? programmes
       : programmes.filter(programme => allRights.includes(programme.code))
-    const formattedProgrammes = filteredProgrammes.reduce<Record<string, Unarray<ProgrammesOfOrganization>>>(
-      (acc, curr) => {
-        acc[curr.code] = curr
-        return acc
-      },
-      {}
-    )
-    res.json(formattedProgrammes)
+    const formattedFilteredProgrammes = keyBy(filteredProgrammes, 'code')
+    const formattedAllProgrammes = keyBy(programmes, 'code')
+
+    res.json({
+      allProgrammes: formattedAllProgrammes,
+      filteredProgrammes: formattedFilteredProgrammes,
+    })
   }
 )
 

services/backend/src/routes/students.ts

@@ -1,65 +1,58 @@
-import { Request, Response, Router } from 'express'
+import { Router } from 'express'
 
+import type { CanError } from '@oodikone/shared/routes'
+import type {
+  GetStudentRequestResBody,
+  GetStudentRequestReqBody,
+  GetStudentRequestQuery,
+  GetStudentDetailParams,
+  GetStudentDetailResBody,
+  GetStudentDetailReqBody,
+} from '@oodikone/shared/routes/students'
+import { splitByEmptySpace } from '@oodikone/shared/util'
 import { bySearchTermAndStudentNumbers, withStudentNumber } from '../services/students'
-import { hasFullAccessToStudentData, splitByEmptySpace } from '../util'
-import { ApplicationError } from '../util/customErrors'
+import { hasFullAccessToStudentData } from '../util'
 
 const router = Router()
 
-interface GetStudentsRequest extends Request {
-  query: {
-    searchTerm: string
+router.get<never, CanError<GetStudentRequestResBody>, GetStudentRequestReqBody, GetStudentRequestQuery>(
+  '/',
+  async (req, res) => {
+    const { searchTerm } = req.query
+    const trimmedSearchTerm = searchTerm?.trim()
+    const validSearchTerm =
+      trimmedSearchTerm &&
+      !splitByEmptySpace(trimmedSearchTerm)
+        .slice(0, 2)
+        .find(term => term.length > 2)
+
+    if (!trimmedSearchTerm || validSearchTerm) {
+      return res.status(400).json({ error: 'at least one search term must be longer than 2 characters' })
+    }
+
+    const { roles, studentsUserCanAccess } = req.user
+    const results: ReturnType<typeof bySearchTermAndStudentNumbers> = hasFullAccessToStudentData(roles)
+      ? bySearchTermAndStudentNumbers(trimmedSearchTerm)
+      : bySearchTermAndStudentNumbers(trimmedSearchTerm, studentsUserCanAccess)
+
+    return res.json(await results)
   }
-}
+)
 
-router.get('/', async (req: GetStudentsRequest, res: Response) => {
-  const {
-    user: { roles, studentsUserCanAccess },
-    query: { searchTerm },
-  } = req
+router.get<GetStudentDetailParams, CanError<GetStudentDetailResBody>, GetStudentDetailReqBody>(
+  '/:studentNumber',
+  async (req, res) => {
+    const { studentNumber } = req.params
+    const { id, roles, studentsUserCanAccess } = req.user
 
-  const trimmedSearchTerm = searchTerm ? searchTerm.trim() : undefined
+    if (!hasFullAccessToStudentData(roles) && !studentsUserCanAccess.includes(studentNumber))
+      return res.status(403).json({ error: 'User does not have permission to view current student.' })
 
-  if (
-    trimmedSearchTerm &&
-    !splitByEmptySpace(trimmedSearchTerm)
-      .slice(0, 2)
-      .find(term => term.length > 2)
-  ) {
-    throw new ApplicationError('at least one search term must be longer than 2 characters', 400)
+    const student = await withStudentNumber(studentNumber, id)
+    if (!student) return res.status(404).json({ error: 'Student not found' })
+
+    res.json(student)
   }
-
-  let results: Awaited<ReturnType<typeof bySearchTermAndStudentNumbers>> = []
-  if (trimmedSearchTerm) {
-    results = hasFullAccessToStudentData(roles)
-      ? await bySearchTermAndStudentNumbers(trimmedSearchTerm)
-      : await bySearchTermAndStudentNumbers(trimmedSearchTerm, studentsUserCanAccess)
-  }
-  res.json(results)
-})
-
-interface GetStudentRequest extends Request {
-  params: {
-    studentNumber: string
-  }
-}
-
-router.get('/:studentNumber', async (req: GetStudentRequest, res: Response) => {
-  const { studentNumber } = req.params
-  const {
-    user: { id, roles, studentsUserCanAccess },
-  } = req
-
-  if (!hasFullAccessToStudentData(roles) && !studentsUserCanAccess.includes(studentNumber)) {
-    return res
-      .status(403)
-      .json({ error: `User does not have permission to view data for student number ${studentNumber}.` })
-  }
-  const student = await withStudentNumber(studentNumber, id)
-  if (!student) {
-    return res.status(404).json({ error: `Student not found with student number ${studentNumber}.` })
-  }
-  res.json(student)
-})
+)
 
 export default router

services/backend/src/routes/studyProgramme.ts

@@ -1,6 +1,7 @@
 import { Request, Response, Router } from 'express'
 
 import { Graduated, SpecialGroups, YearType } from '@oodikone/shared/types'
+import { CreditStatsPayload } from '@oodikone/shared/types/studyProgramme'
 import {
   getBasicStats,
   setBasicStats,
@@ -11,10 +12,8 @@ import {
 } from '../services/analyticsService'
 import { getCreditsProduced } from '../services/providerCredits'
 import { getBasicStatsForStudytrack } from '../services/studyProgramme/studyProgrammeBasics'
-import {
-  getStudyProgrammeCoursesForStudyTrack,
-  getStudyProgrammeStatsForColorizedCoursesTable,
-} from '../services/studyProgramme/studyProgrammeCourses'
+import { getStudyProgrammeCoursesForStudyTrack } from '../services/studyProgramme/studyProgrammeCourses'
+import { getStudyProgrammeStatsForColorizedCoursesTable } from '../services/studyProgramme/studyProgrammeCoursesBySemester'
 import { getGraduationStatsForStudyTrack } from '../services/studyProgramme/studyProgrammeGraduations'
 import { updateBasicView, updateStudyTrackView } from '../services/studyProgramme/studyProgrammeUpdates'
 import { getStudyRightsInProgramme, getStudyTracksForProgramme } from '../services/studyProgramme/studyRightFinders'
@@ -34,11 +33,11 @@ interface GetCreditStatsRequest extends Request {
 
 router.get('/creditstats', async (req: GetCreditStatsRequest, res: Response) => {
   const { codes, specialGroups, yearType } = req.query
-  const stats = {}
+  const stats: CreditStatsPayload = {}
   for (const code of JSON.parse(codes)) {
     stats[code] = await getCreditsProduced(code, yearType === 'ACADEMIC_YEAR', specialGroups === 'SPECIAL_INCLUDED')
   }
-  return res.json({ stats })
+  return res.json(stats)
 })
 
 interface GetStatsRequest extends Request {
@@ -134,13 +133,13 @@ interface GetStudyTrackStatsRequest extends Request {
 
 router.get('/:id/studytrackstats', async (req: GetStudyTrackStatsRequest, res: Response) => {
   const code = req.params.id
-  const { graduated, special_groups: specialGroups, combined_programme: combinedProgramme } = req.query
+  const { special_groups: specialGroups, combined_programme: combinedProgramme } = req.query
   if (!code) {
     return res.status(422).end()
   }
 
   void logInfoForGrafana(code, combinedProgramme)
-  const data = await getStudyTrackStats(code, combinedProgramme, graduated, specialGroups)
+  const data = await getStudyTrackStats(code, combinedProgramme, specialGroups)
   if (data) {
     return res.json(data)
   }
@@ -150,13 +149,12 @@ router.get('/:id/studytrackstats', async (req: GetStudyTrackStatsRequest, res: R
     studyProgramme: code,
     combinedProgramme,
     settings: {
-      graduated: graduated === 'GRADUATED_INCLUDED',
       specialGroups: specialGroups === 'SPECIAL_INCLUDED',
     },
     studyRightsOfProgramme,
   })
   if (updated) {
-    await setStudyTrackStats(updated, graduated, specialGroups)
+    await setStudyTrackStats(updated, specialGroups)
   }
   return res.json(updated)
 })

services/backend/src/routes/tags.ts

@@ -1,5 +1,5 @@
 import { Router } from 'express'
-import { difference } from 'lodash'
+import { difference } from 'lodash-es'
 
 import { CanError } from '@oodikone/shared/routes'
 import { NewTag, Role, StudentTag, Tag } from '@oodikone/shared/types'

services/backend/src/routes/teachers.ts

@@ -1,19 +1,17 @@
 import { Request, Response, Router } from 'express'
 
-import { mapToProviders } from '@oodikone/shared/util'
-import { serviceProvider } from '../config'
+import { mapToProviders, splitByEmptySpace } from '@oodikone/shared/util'
 import * as auth from '../middleware/auth'
 import { getProvidersOfFaculty, isFaculty } from '../services/organizations'
 import { getTeachersBySearchTerm, getTeacherStatistics, getYearlyStatistics } from '../services/teachers'
 import { CategoryID, getTeacherStats, findAndSaveTeachers, getCategoriesAndYears } from '../services/teachers/top'
-import { getFullStudyProgrammeRights, splitByEmptySpace } from '../util'
+import { getFullStudyProgrammeRights, validateParamLength } from '../util'
 
 const router = Router()
 
 const iamGroupsGivingFullAccess = ['hy-dekaanit', 'hy-varadekaanit-opetus']
 
-const fullAccessAuth = () =>
-  serviceProvider === 'toska' ? auth.roles([], iamGroupsGivingFullAccess) : auth.roles(['teachers'])
+const fullAccessAuth = () => auth.roles([], iamGroupsGivingFullAccess)
 
 interface GetTeachersRequest extends Request {
   query: {
@@ -28,9 +26,10 @@ router.get('/', fullAccessAuth(), async (req: GetTeachersRequest, res: Response)
   }
 
   const trimmedSearchTerm = searchTerm.trim()
-  const stringSearchTermIsInvalid = !splitByEmptySpace(trimmedSearchTerm).find(searchTerm => searchTerm.length >= 4)
-  const numericSearchTermIsInvalid = !Number.isNaN(Number(trimmedSearchTerm)) && trimmedSearchTerm.length < 6
-  if (stringSearchTermIsInvalid || numericSearchTermIsInvalid) {
+  const searchTermIsInvalid =
+    !validateParamLength(trimmedSearchTerm, 4) &&
+    !splitByEmptySpace(trimmedSearchTerm).find(searchTerm => searchTerm.length >= 4)
+  if (searchTermIsInvalid) {
     return res.status(400).json({ error: 'Invalid search term' })
   }
 
@@ -57,7 +56,7 @@ router.get('/top', fullAccessAuth(), async (req: GetTopTeachersRequest, res: Res
   }
   await findAndSaveTeachers(Number(yearcode), Number(yearcode))
   const updatedStats = await getTeacherStats(category, Number(yearcode))
-  res.json(updatedStats)
+  return res.json(updatedStats)
 })
 
 interface PostTopTeachersRequest extends Request {

services/backend/src/routes/university.ts

@@ -1,8 +1,7 @@
 import { Request, Response, Router } from 'express'
-import { cloneDeep } from 'lodash'
+import { cloneDeep } from 'lodash-es'
 
 import { Graduated, NameWithCode } from '@oodikone/shared/types'
-import { serviceProvider } from '../config'
 import { magicFacultyCode } from '../config/organizationConstants'
 import { OrganizationModel } from '../models'
 import { getDegreeProgrammesOfFaculty } from '../services/faculty/faculty'
@@ -15,6 +14,7 @@ import {
   setFacultyProgressStats,
 } from '../services/faculty/facultyService'
 import { combineFacultyStudentProgress, FacultyProgressData } from '../services/faculty/facultyStudentProgress'
+import { GraduationTarget } from '../services/graduationHelpers'
 import { getMedian } from '../services/studyProgramme/studyProgrammeHelpers'
 import logger from '../util/logger'
 
@@ -39,18 +39,17 @@ interface GetProgressStatsRequest extends Request {
 
 router.get('/allprogressstats', async (req: GetProgressStatsRequest, res: Response) => {
   const specialGroups = req.query?.specialsIncluded === 'true' ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED'
-  const graduated = req.query?.graduated
   const allFaculties = await getSortedFaculties()
   const facultyCodes = allFaculties.map(faculty => faculty.code)
   const codeToData: Record<string, FacultyProgressData> = {}
 
   for (const facultyCode of facultyCodes) {
-    let data: any = await getFacultyProgressStats(facultyCode, specialGroups, graduated)
+    let data = await getFacultyProgressStats(facultyCode, specialGroups)
     if (!data) {
       logger.info(`Data missing from server: Refreshing progress faculty data for faculty ${facultyCode}`)
       const programmes = await getDegreeProgrammesOfFaculty(facultyCode, true)
-      data = await combineFacultyStudentProgress(facultyCode, programmes, specialGroups, graduated)
-      await setFacultyProgressStats(data, specialGroups, graduated)
+      data = await combineFacultyStudentProgress(facultyCode, programmes, specialGroups)
+      await setFacultyProgressStats(data, specialGroups)
     }
     codeToData[facultyCode] = data
   }
@@ -71,6 +70,12 @@ router.get('/allprogressstats', async (req: GetProgressStatsRequest, res: Respon
       master: {} as Record<string, number[]>,
       doctor: {} as Record<string, number[]>,
     },
+    graduatedCount: {
+      bachelor: {} as Record<string, number>,
+      bachelorMaster: {} as Record<string, number>,
+      master: {} as Record<string, number>,
+      doctor: {} as Record<string, number>,
+    },
   }
 
   const unifyProgressStats = (progStats: number[][][]) => {
@@ -96,6 +101,14 @@ router.get('/allprogressstats', async (req: GetProgressStatsRequest, res: Respon
         if (!universityData.creditCounts[fieldName][year]) universityData.creditCounts[fieldName][year] = []
         universityData.creditCounts[fieldName][year].push(...facultyData.creditCounts[fieldName][year])
       }
+
+      for (const fieldName of degreeNames) {
+        if (!facultyData.graduatedCount[fieldName] || Object.keys(facultyData.graduatedCount[fieldName]).length === 0)
+          continue
+        if (!universityData.graduatedCount[fieldName][year]) universityData.graduatedCount[fieldName][year] = 0
+        universityData.graduatedCount[fieldName][year] += facultyData.graduatedCount[fieldName][year]
+      }
+
       const progStats = ['bachelorsProgStats', 'bcMsProgStats', 'mastersProgStats', 'doctoralProgStats'] as const
       for (const fieldName of progStats) {
         if (Object.keys(facultyData[fieldName] || {}).length === 0) continue
@@ -112,7 +125,7 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
   const allFaculties = await getSortedFaculties()
   const facultyCodes = allFaculties.map(faculty => faculty.code)
   const facultyData: Record<string, Awaited<ReturnType<typeof countGraduationTimes>>> = {}
-  const programmeFilter = serviceProvider === 'toska' ? 'NEW_DEGREE_PROGRAMMES' : 'ALL_PROGRAMMES'
+  const programmeFilter = 'NEW_DEGREE_PROGRAMMES'
   for (const facultyCode of facultyCodes) {
     let data: any = await getGraduationStats(facultyCode, programmeFilter, true)
     if (!data) {
@@ -125,7 +138,12 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
   }
 
   const universityData = {
-    goals: { bachelor: 36, bcMsCombo: 60, master: 24, doctor: 48 },
+    goals: {
+      bachelor: GraduationTarget.THREE_YEARS,
+      bcMsCombo: GraduationTarget.FIVE_YEARS,
+      master: GraduationTarget.TWO_YEARS,
+      doctor: GraduationTarget.FOUR_YEARS,
+    },
     programmeNames: getProgrammeNames(allFaculties),
     byGradYear: {
       medians: {} as Record<string, LevelGraduationStats[]>,

services/backend/src/routes/users.ts

@@ -4,11 +4,11 @@ import { LANGUAGE_CODES, Language } from '@oodikone/shared/language'
 import { CanError } from '@oodikone/shared/routes'
 import { Role } from '@oodikone/shared/types'
 import { tryCatch } from '@oodikone/shared/util'
-import { FormattedUser } from 'src/types'
 import { roles } from '../config/roles'
 import * as auth from '../middleware/auth'
 import { sendNotificationAboutAccessToUser, previewNotificationAboutAccessToUser } from '../services/mailService'
 import * as userService from '../services/userService'
+import { FormattedUser } from '../types'
 import logger from '../util/logger'
 
 const router = Router()

services/backend/src/routes/usersFd.ts

@@ -1,68 +0,0 @@
-import { Request, Response, Router } from 'express'
-
-import { isDev } from '../config'
-import * as auth from '../middleware/auth'
-import * as userService from '../services/userService'
-
-const router = Router()
-
-interface NewUserEppnRequest extends Request {
-  params: {
-    newUserEppn: string
-  }
-}
-
-router.get('/from-sisu-by-eppn/:newUserEppn', auth.roles(['admin']), async (req: NewUserEppnRequest, res: Response) => {
-  let { username: requesterEppn } = req.user
-  const { newUserEppn } = req.params
-  // In order to test this feature in the dev environment we need to set an eppn that demo sisu will recognize
-  if (isDev && requesterEppn === 'mluukkai') {
-    requesterEppn = newUserEppn
-  }
-  const person = await userService.getUserFromSisuByEppn(requesterEppn, newUserEppn)
-  res.json(person)
-})
-
-interface User {
-  id: string
-  first_name: string
-  last_name: string
-  call_name: string
-  eppn: string
-  email_address: string
-  student_number: string
-  phone_number: string
-  document_state: string
-}
-
-interface AddUserRequest extends Request {
-  body: {
-    user: User
-  }
-}
-
-interface DeleteUserRequest extends Request {
-  body: {
-    userId: string
-  }
-}
-
-router.post('/add', auth.roles(['admin']), async (req: AddUserRequest, res: Response) => {
-  const { user } = req.body
-  const oldUser = await userService.findByUsername(user.eppn)
-  if (oldUser) return res.status(400).json({ error: 'Trying to add a user already present in the user database.' })
-  const success = await userService.addNewUser(user)
-  res.json(success)
-})
-
-router.delete('/delete', auth.roles(['admin']), async (req: DeleteUserRequest, res: Response) => {
-  const { userId } = req.body
-  try {
-    await userService.deleteUserById(userId)
-    res.json({ deleted: userId })
-  } catch (error) {
-    return res.status(400).json({ error: 'Deletion was unsuccessful with the user id included in the request' })
-  }
-})
-
-export default router

services/backend/src/services/analyticsService.ts

@@ -1,4 +1,5 @@
-import { Graduated, SpecialGroups, StudyTrackStats, YearType } from '@oodikone/shared/types'
+import { SpecialGroups, StudyTrackStats, YearType } from '@oodikone/shared/types'
+import { BasicStats, CreditStats, ProgrammeGraduationStats } from '@oodikone/shared/types/studyProgramme'
 import { facultyCodes, ignoredFacultyCodes } from '../config/organizationConstants'
 import { redisClient } from './redis'
 import { isRelevantProgramme } from './studyProgramme/studyProgrammeHelpers'
@@ -15,11 +16,12 @@ const createRedisKeyForCreditStats = (id: string, yearType: YearType, specialGro
 const createRedisKeyForGraduationStats = (id: string, yearType: YearType, specialGroups: SpecialGroups) => {
   return `GRADUATION_STATS_${id}_${yearType}_${specialGroups}`
 }
-const createRedisKeyForStudyTrackStats = (id: string, graduated: Graduated, specialGroups: SpecialGroups) => {
-  return `STUDYTRACK_STATS_${id}_${graduated}_${specialGroups}`
+const createRedisKeyForStudyTrackStats = (id: string, specialGroups: SpecialGroups) => {
+  return `STUDYTRACK_STATS_${id}_${specialGroups}`
 }
 
-const shouldSaveToRedis = (id: unknown) => {
+// *Should* always be a string (previously unknown)
+const shouldSaveToRedis = (id: string) => {
   if (typeof id !== 'string') return false
   if (isRelevantProgramme(id)) return true
   const splitId = id.split('-')
@@ -31,7 +33,7 @@ export const getBasicStats = async (
   combinedProgramme: string | null,
   yearType: YearType,
   specialGroups: SpecialGroups
-) => {
+): Promise<BasicStats | null> => {
   const searchKey = combinedProgramme ? `${id}-${combinedProgramme}` : id
   const redisKey = createRedisKeyForBasicStats(searchKey, yearType, specialGroups)
   const dataFromRedis = await redisClient.get(redisKey)
@@ -41,22 +43,13 @@ export const getBasicStats = async (
   return JSON.parse(dataFromRedis)
 }
 
-export const setBasicStats = async (data, yearType: YearType, specialGroups: SpecialGroups) => {
+export const setBasicStats = async (data: BasicStats, yearType: YearType, specialGroups: SpecialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
   if (!shouldSaveToRedis(id)) {
-    return dataToRedis
+    return
   }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
 export const getCreditStats = async (providerCode: string, isAcademicYear: boolean, specialGroups: boolean) => {
@@ -69,29 +62,20 @@ export const getCreditStats = async (providerCode: string, isAcademicYear: boole
   if (!dataFromRedis) {
     return null
   }
-  return JSON.parse(dataFromRedis)
+  return JSON.parse(dataFromRedis) as CreditStats
 }
 
-export const setCreditStats = async (data, isAcademicYear: boolean, specialGroups: boolean) => {
+export const setCreditStats = async (data: CreditStats, isAcademicYear: boolean, specialGroups: boolean) => {
   const { id } = data
   const redisKey = createRedisKeyForCreditStats(
     id,
     isAcademicYear ? 'ACADEMIC_YEAR' : 'CALENDAR_YEAR',
     specialGroups ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED'
   )
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
   if (!isRelevantProgramme(id) && !isFaculty(id)) {
-    return dataToRedis
+    return
   }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
 export const getGraduationStats = async (
@@ -106,56 +90,37 @@ export const getGraduationStats = async (
   if (!dataFromRedis) {
     return null
   }
-  return JSON.parse(dataFromRedis)
+  return JSON.parse(dataFromRedis) as ProgrammeGraduationStats
 }
 
 export const setGraduationStats = async (data, yearType: YearType, specialGroups: SpecialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
   if (!shouldSaveToRedis(id)) {
-    return dataToRedis
+    return
   }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
 export const getStudyTrackStats = async (
   id: string,
   combinedProgramme: string | null,
-  graduated: Graduated,
   specialGroups: SpecialGroups
 ) => {
   const searchKey = combinedProgramme ? `${id}-${combinedProgramme}` : id
-  const redisKey = createRedisKeyForStudyTrackStats(searchKey, graduated, specialGroups)
+  const redisKey = createRedisKeyForStudyTrackStats(searchKey, specialGroups)
   const dataFromRedis = await redisClient.get(redisKey)
   if (!dataFromRedis) {
     return null
   }
-  return JSON.parse(dataFromRedis) as StudyTrackStats & { status: string; lastUpdated: string }
+  return JSON.parse(dataFromRedis) as StudyTrackStats
 }
 
-export const setStudyTrackStats = async (data: StudyTrackStats, graduated: Graduated, specialGroups: SpecialGroups) => {
+export const setStudyTrackStats = async (data: StudyTrackStats, specialGroups: SpecialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForStudyTrackStats(id, graduated, specialGroups)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
   if (!shouldSaveToRedis(id)) {
-    return dataToRedis
+    return
   }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  const redisKey = createRedisKeyForStudyTrackStats(id, specialGroups)
+  await redisClient.set(redisKey, JSON.stringify(data))
 }

services/backend/src/services/courses/courseFinders.ts

@@ -1,9 +1,9 @@
-import { orderBy } from 'lodash'
+import { orderBy } from 'lodash-es'
 import { Op } from 'sequelize'
 
+import { CourseWithSubsId } from '@oodikone/shared/types/course'
+import { getSortRank } from '../../../src/util/sortRank'
 import { CourseModel } from '../../models'
-import { CourseWithSubsId } from '../../types'
-import { getSortRank } from '../../util/sortRank'
 
 const nameLikeTerm = (name: string) => {
   if (!name) {
@@ -40,63 +40,50 @@ const codeLikeTerm = (code: string) => {
   }
 }
 
-const getRawCourses = async (name: string, code: string) => {
-  return await CourseModel.findAll({
+const getRawCourses = async (name: string, code: string, includeSpecial: boolean) =>
+  await CourseModel.findAll({
     where: {
+      ...(includeSpecial ? {} : { mainCourseCode: { [Op.ne]: null } }),
       ...nameLikeTerm(name),
       ...codeLikeTerm(code),
     },
-    order: [['id', 'desc']],
+    raw: true,
   })
-}
 
-const getCourses = async (name: string, code: string) => {
-  const rawCourses = await getRawCourses(name, code)
-  const courses: CourseWithSubsId[] = rawCourses
+const getCourses = async (name: string, code: string, includeSpecial: boolean) => {
+  const rawCourses = await getRawCourses(name, code, includeSpecial)
+  return rawCourses
     .map(course => ({
-      ...course.toJSON(),
+      ...course,
       substitutions: orderBy(course.substitutions, [
         substitution => {
-          if (/^A/.exec(substitution)) return 4 // open university codes come last
-          if (/^\d/.exec(substitution)) return 2 // old numeric codes come second
-          if (/^[A-Za-z]/.exec(substitution)) return 1 // new letter based codes come first
+          if (/^A/.exec(substitution.at(0) ?? '')) return 4 // open university codes come last
+          if (/^\d/.exec(substitution.at(0) ?? '')) return 2 // old numeric codes come second
+          if (/^[A-Za-z]/.exec(substitution.at(0) ?? '')) return 1 // new letter based codes come first
           return 3 // unknown, comes before open uni?
         },
       ]),
     }))
-    .sort((a, b) => getSortRank(b.code) - getSortRank(a.code))
-  return courses
+    .sort((a, b) => getSortRank(b.code) - getSortRank(a.code)) as CourseWithSubsId[]
 }
 
-export const getCoursesByNameAndOrCode = async (name: string, code: string) => {
-  const courses = await getCourses(name, code)
-  let substitutionGroupIndex = 0
-  const visitedCourses: string[] = []
+export const getCoursesByNameAndOrCode = async (name: string, code: string, includeSpecial: boolean) => {
+  const courses = await getCourses(name, code, includeSpecial)
 
-  const assignSubstitutionGroup = (course: CourseWithSubsId) => {
-    if (visitedCourses.includes(course.code)) {
-      return
-    }
-    let relatedCourses: CourseWithSubsId[] = []
-    if (course.substitutions !== null) {
-      relatedCourses = courses.filter(currentCourse => course.substitutions.includes(currentCourse.code))
-    }
+  const visitedCourses = new Set<string>()
+
+  const assignSubstitutionGroup = (course: CourseWithSubsId, index: number) => {
+    const relatedCourses = courses.filter(currentCourse => course.substitutions.includes(currentCourse.code))
     relatedCourses.unshift(course)
     relatedCourses.forEach(relatedCourse => {
-      if (visitedCourses.includes(relatedCourse.id)) {
-        return
-      }
-      visitedCourses.push(relatedCourse.id)
-      relatedCourse.subsId = substitutionGroupIndex
+      visitedCourses.add(relatedCourse.id)
+      relatedCourse.subsId = index
     })
   }
 
-  courses.forEach(course => {
-    if (!visitedCourses.includes(course.id)) {
-      substitutionGroupIndex++
-      assignSubstitutionGroup(course)
-    }
-  })
+  courses
+    .filter(course => !visitedCourses.has(course.id) && !visitedCourses.has(course.code))
+    .forEach((course, index) => assignSubstitutionGroup(course, index))
 
   return courses
 }

services/backend/src/services/courses/courseYearlyStatsCounter.ts

@@ -1,108 +1,23 @@
-import { max, min } from 'lodash'
-
-import { Name } from '@oodikone/shared/types'
-import { getSemesterNames } from '../semesters'
-import { FormattedProgramme, OrganizationDetails } from './helpers'
-
-type Programme = {
-  name: Name
-  students: Record<number, string[]>
-  passed: Record<number, string[]>
-  credits: Record<number, number>
-}
-
-type Faculty = {
-  name: Name | null
-  students: string[]
-  passed: string[]
-  credits: number
-}
-
-type FacultyYearStats = {
-  year: string
-  allStudents: string[]
-  allPassed: string[]
-  faculties: Record<string, Faculty>
-  allCredits: number
-}
-
-type Grades = Record<string, string[]>
-
-type GroupAttempts = {
-  grades: Grades
-  categories: {
-    passed: string[]
-    failed: string[]
-  }
-}
-
-type GroupStudents = {
-  grades: Record<string, { grade: string; passed: boolean }>
-  studentNumbers: string[]
-}
-
-type Group = {
-  code: number
-  name: string | Name
-  coursecode: string
-  attempts: GroupAttempts
-  students: GroupStudents
-  enrollments: { studentNumber: string; enrollmentDateTime: Date }[]
-  allEnrollments: { studentNumber: string; enrollmentDateTime: Date }[]
-  yearCode: number
-}
-
-type Student = {
-  earliestAttainment: Date
-  code: number
-}
+import type { Name } from '@oodikone/shared/types'
+import type { Programme, FacultyYearStats, Grades, Group, Student } from '@oodikone/shared/types/courseYearlyStats'
+import { getSemesterNamesByCode } from '../semesters'
+import type { OrganizationDetails } from './helpers'
 
 export class CourseYearlyStatsCounter {
   private groups: Record<number, Group>
   private programmes: Record<string, Programme>
   private facultyStats: Record<number, FacultyYearStats>
-  private obfuscated: boolean
   private students: Map<string, Student>
 
   constructor() {
     this.groups = {}
     this.programmes = {}
     this.facultyStats = {}
-    this.obfuscated = false
     this.students = new Map()
   }
 
-  private initProgramme(code: string, name: Name) {
-    this.programmes[code] = {
-      name,
-      students: {},
-      passed: {},
-      credits: {},
-    }
-  }
-
-  private initFacultyYear(code: number) {
-    const year = `${1949 + Number(code)}-${1950 + Number(code)}`
-    this.facultyStats[code] = {
-      year,
-      allStudents: [],
-      allPassed: [],
-      faculties: {},
-      allCredits: 0,
-    }
-  }
-
-  private initFaculty(yearCode: number, facultyCode: string, organization: OrganizationDetails | null) {
-    this.facultyStats[yearCode].faculties[facultyCode] = {
-      name: organization ? organization.name : null,
-      students: [],
-      passed: [],
-      credits: 0,
-    }
-  }
-
   private initGroup(groupCode: number, name: string | Name, courseCode: string, yearCode: number) {
-    this.groups[groupCode] = {
+    this.groups[groupCode] ??= {
       code: groupCode,
       name,
       coursecode: courseCode,
@@ -124,37 +39,45 @@ export class CourseYearlyStatsCounter {
   }
 
   public markStudyProgramme(
-    code: string,
-    name: Name,
     studentNumber: string,
     yearCode: number,
     passed: boolean,
     credits: number,
-    facultyCode: string | null,
-    organization: OrganizationDetails | null
+    code: string,
+    name: Name,
+    facultyCode: string,
+    organization: OrganizationDetails
   ) {
-    if (!this.programmes[code]) {
-      this.initProgramme(code, name)
-    }
-    if (!this.facultyStats[yearCode]) {
-      this.initFacultyYear(yearCode)
-    }
-    if (
-      facultyCode &&
-      !this.facultyStats[yearCode].faculties[facultyCode] &&
-      !this.facultyStats[yearCode].allPassed.includes(studentNumber)
-    ) {
-      this.initFaculty(yearCode, facultyCode, organization)
+    this.programmes[code] ??= {
+      name,
+      students: {},
+      passed: {},
+      credits: {},
     }
 
-    this.programmes[code].students[yearCode] = this.programmes[code].students[yearCode] || []
+    const year = `${1949 + Number(yearCode)}-${1950 + Number(yearCode)}`
+    this.facultyStats[yearCode] ??= {
+      year,
+      allStudents: [],
+      allPassed: [],
+      faculties: {},
+      allCredits: 0,
+    }
+
+    this.facultyStats[yearCode].faculties[facultyCode] ??= {
+      name: organization?.name ?? null,
+      students: [],
+      passed: [],
+      credits: 0,
+    }
+
+    this.programmes[code].students[yearCode] ??= []
     this.programmes[code].students[yearCode].push(studentNumber)
 
-    if (!this.programmes[code].passed[yearCode]) {
-      this.programmes[code].passed[yearCode] = []
-      this.programmes[code].credits[yearCode] = 0
-    }
-    if (facultyCode && !this.facultyStats[yearCode].allStudents.includes(studentNumber)) {
+    this.programmes[code].passed[yearCode] ??= []
+    this.programmes[code].credits[yearCode] ??= 0
+
+    if (!this.facultyStats[yearCode].allStudents.includes(studentNumber)) {
       this.facultyStats[yearCode].allStudents.push(studentNumber)
       this.facultyStats[yearCode].faculties[facultyCode].students.push(studentNumber)
     }
@@ -162,7 +85,7 @@ export class CourseYearlyStatsCounter {
       this.programmes[code].passed[yearCode].push(studentNumber)
       this.programmes[code].credits[yearCode] += credits
     }
-    if (facultyCode && passed && !this.facultyStats[yearCode].allPassed.includes(studentNumber)) {
+    if (passed && !this.facultyStats[yearCode].allPassed.includes(studentNumber)) {
       this.facultyStats[yearCode].allPassed.push(studentNumber)
       this.facultyStats[yearCode].faculties[facultyCode].passed.push(studentNumber)
       this.facultyStats[yearCode].faculties[facultyCode].credits += credits
@@ -170,18 +93,6 @@ export class CourseYearlyStatsCounter {
     }
   }
 
-  public markStudyProgrammes(
-    studentNumber: string,
-    programmes: FormattedProgramme[],
-    yearCode: number,
-    passed: boolean,
-    credits: number
-  ) {
-    programmes.forEach(({ code, name, facultyCode, organization }) => {
-      this.markStudyProgramme(code, name, studentNumber, yearCode, passed, credits, facultyCode, organization)
-    })
-  }
-
   public markCreditToGroup(
     studentNumber: string,
     passed: boolean,
@@ -191,11 +102,26 @@ export class CourseYearlyStatsCounter {
     courseCode: string,
     yearCode: number
   ) {
-    if (!this.groups[groupCode]) {
-      this.initGroup(groupCode, groupName, courseCode, yearCode)
+    this.initGroup(groupCode, groupName, courseCode, yearCode)
+
+    // mark credit to attempts
+    const { grades, categories } = this.groups[groupCode].attempts
+
+    grades[grade] ??= []
+    grades[grade].push(studentNumber)
+
+    if (passed) {
+      categories.passed.push(studentNumber)
+    } else {
+      categories.failed.push(studentNumber)
     }
-    this.markCreditToAttempts(studentNumber, passed, grade, groupCode)
-    this.markBestEffortGrade(studentNumber, passed, grade, groupCode)
+
+    // mark best effort grade
+    const current = this.groups[groupCode].students.grades[studentNumber]
+
+    if (current?.passed && !passed) return
+    if (current?.passed && Number(grade) <= Number(current?.grade)) return
+    this.groups[groupCode].students.grades[studentNumber] = { grade, passed }
   }
 
   public markEnrollmentToGroup(
@@ -206,16 +132,15 @@ export class CourseYearlyStatsCounter {
     courseCode: string,
     yearCode: number
   ) {
-    if (!this.groups[groupCode]) {
-      this.initGroup(groupCode, groupName, courseCode, yearCode)
-    }
-    const enrollment = { studentNumber, enrollmentDateTime }
-    this.groups[groupCode].allEnrollments.push(enrollment)
-    const oldEnrollment = this.groups[groupCode].enrollments.find(
+    this.initGroup(groupCode, groupName, courseCode, yearCode)
+    const oldEnrollment = this.groups[groupCode].enrollments.some(
       enrollment => enrollment.studentNumber === studentNumber
     )
+
+    const enrollment = { studentNumber, enrollmentDateTime }
+    this.groups[groupCode].allEnrollments.push(enrollment)
     if (!oldEnrollment) {
-      this.groups[groupCode].enrollments.push({ studentNumber, enrollmentDateTime })
+      this.groups[groupCode].enrollments.push(enrollment)
     } else {
       this.groups[groupCode].enrollments = this.groups[groupCode].enrollments
         .filter(enrollment => enrollment.studentNumber !== studentNumber)
@@ -223,35 +148,6 @@ export class CourseYearlyStatsCounter {
     }
   }
 
-  private markCreditToAttempts(studentNumber: string, passed: boolean, grade: string, groupCode: number) {
-    const { attempts } = this.groups[groupCode]
-    const { grades, categories } = attempts
-    if (!grades[grade]) {
-      grades[grade] = []
-    }
-    grades[grade].push(studentNumber)
-    if (passed) {
-      categories.passed.push(studentNumber)
-    } else {
-      categories.failed.push(studentNumber)
-    }
-  }
-
-  private markBestEffortGrade(studentNumber: string, passed: boolean, grade: string, groupCode: number) {
-    const current = this.groups[groupCode].students.grades[studentNumber]
-    if (!current) {
-      this.groups[groupCode].students.grades[studentNumber] = { grade, passed }
-      return
-    }
-    if (current.passed && !passed) {
-      return
-    }
-    if (current.passed && Number(current.grade) >= Number(grade)) {
-      return
-    }
-    this.groups[groupCode].students.grades[studentNumber] = { grade, passed }
-  }
-
   public markCreditToStudentCategories(studentNumber: string, attainmentDate: Date, groupCode: number) {
     if (!this.students.has(studentNumber)) {
       this.addOrUpdateStudent(studentNumber, attainmentDate, groupCode)
@@ -269,14 +165,90 @@ export class CourseYearlyStatsCounter {
 
   private updateExistingStudent(studentNumber: string, attainmentDate: Date, groupCode: number) {
     const student = this.students.get(studentNumber)
-    if (attainmentDate < student!.earliestAttainment) {
-      this.addOrUpdateStudent(studentNumber, attainmentDate, groupCode)
+    if (attainmentDate < student!.earliestAttainment) this.addOrUpdateStudent(studentNumber, attainmentDate, groupCode)
+  }
+
+  private async insertEmptyRows() {
+    const groups = Object.keys(this.groups).map(Number)
+    if (groups.length === 0) return
+
+    const first = Math.min(...groups)
+    const last = Math.max(...groups)
+
+    const a = first
+    const b = last + 1
+    const range = [...Array(b - a).keys()].map(v => v + a)
+    const diff = range.filter(v => !groups.includes(v))
+
+    const semesters = await getSemesterNamesByCode(diff)
+
+    const getNextYear = (name: string) => {
+      const [start, end] = name.split('-')
+      const startYear = parseInt(start, 10) + 1
+      const endYear = parseInt(end, 10) + 1
+      return `${startYear}-${endYear}`
     }
+
+    diff.forEach(i => {
+      const previous = this.groups[i - 1]
+      const name = typeof previous.name === 'string' ? getNextYear(previous.name) : semesters[i]
+
+      this.initGroup(i, name, previous.coursecode, i)
+    })
+  }
+
+  private async parseGroupStatistics(anonymizationSalt: string | null) {
+    await this.insertEmptyRows()
+    for (const [studentNumber, data] of this.students) {
+      this.groups[data.code].students.studentNumbers.push(studentNumber)
+    }
+
+    const groupStatistics = Object.values(this.groups).map(group => {
+      if (anonymizationSalt && group.students.studentNumbers.length < 6) {
+        return {
+          obfuscated: true,
+          code: group.code,
+          name: group.name,
+          coursecode: group.coursecode,
+          attempts: {
+            categories: {
+              failed: [] as string[],
+              passed: [] as string[],
+            },
+            grades: Object.keys(group.attempts.grades).reduce<Grades>((grades, grade) => {
+              grades[grade] = []
+              return grades
+            }, {}),
+          },
+          yearCode: group.yearCode,
+          students: {
+            studentNumbers: [] as string[],
+          },
+        }
+      }
+
+      return {
+        ...group,
+        students: {
+          ...group.students,
+          grades: Object.entries(group.students.grades)
+            .map(([studentNumber, { grade, passed }]) => [studentNumber, passed ? grade : '0'])
+            .reduce<Grades>((grades, [studentNumber, grade]) => {
+              grades[grade] ??= []
+              grades[grade].push(studentNumber)
+
+              return grades
+            }, {}),
+        },
+      }
+    })
+
+    return groupStatistics
   }
 
   private parseProgrammeStatistics(anonymizationSalt: string | null) {
-    if (anonymizationSalt) {
-      this.programmes = {
+    if (anonymizationSalt)
+      return {
         '000000': {
           name: { en: '', fi: '', sv: '' },
           credits: {},
@@ -284,115 +256,34 @@ export class CourseYearlyStatsCounter {
           students: {},
         },
       }
-    }
+
     return this.programmes
   }
 
-  private insertEmptyRows = async () => {
-    const first = min(Object.keys(this.groups).map(Number))
-    const last = max(Object.keys(this.groups).map(Number))
-    if (!(first && last)) {
-      return
-    }
-
-    const getNextYear = (name: string) => {
-      const years = name.split('-')
-      const startYear = parseInt(years[0], 10) + 1
-      const endYear = parseInt(years[1], 10) + 1
-      return `${startYear}-${endYear}`
-    }
-
-    const groups = Object.keys(this.groups)
-    const a = first
-    const b = last + 1
-    const range = [...Array(b - a).keys()].map(v => v + a)
-    const diff = range.filter(v => !groups.includes(v.toString()))
-
-    const semesters = await getSemesterNames(diff)
-
-    diff.forEach(i => {
-      const previous = this.groups[i - 1]
-      const semester = semesters[i]
-      const name = typeof previous.name === 'string' ? getNextYear(previous.name) : semester
-
-      this.initGroup(i, name, previous.coursecode, i)
-    })
-  }
-
-  private async parseGroupStatistics(anonymizationSalt: string | null) {
-    for (const [studentNumber, data] of this.students) {
-      this.groups[data.code].students.studentNumbers.push(studentNumber)
-    }
-
-    await this.insertEmptyRows()
-
-    const groupStatistics = Object.values(this.groups).map(({ ...rest }) => {
-      const { students } = rest
-      const grades = {} as Grades
-      Object.keys(students.grades).forEach(student => {
-        const { grade, passed } = students.grades[student]
-        const parsedGrade = passed ? grade : '0'
-        if (!grades[parsedGrade]) {
-          grades[parsedGrade] = []
-        }
-        grades[parsedGrade].push(student)
-      })
-      const normalStats = {
-        ...rest,
-        students: { ...students, grades },
-      }
-      if (anonymizationSalt && normalStats.students.studentNumbers.length < 6) {
-        this.obfuscated = true
-        const gradeSpread = {} as Grades
-        for (const grade of Object.keys(normalStats.attempts.grades)) {
-          gradeSpread[grade] = []
-        }
-        const obfuscatedStats = {
-          obfuscated: true,
-          code: rest.code,
-          name: rest.name,
-          coursecode: rest.coursecode,
-          attempts: {
-            categories: {
-              failed: [] as string[],
-              passed: [] as string[],
-            },
-            grades: gradeSpread,
-          },
-          yearCode: rest.yearCode,
-          students: {
-            studentNumbers: [] as string[],
-          },
-        }
-        return obfuscatedStats
-      }
-      return normalStats
-    })
-
-    return groupStatistics
-  }
-
   private parseFacultyStatistics(anonymizationSalt: string | null) {
-    if (anonymizationSalt) {
-      this.facultyStats = [
-        {
+    if (anonymizationSalt)
+      return {
+        0: {
           year: 'NA',
           allCredits: 0,
           allPassed: [],
           allStudents: [],
           faculties: {},
         },
-      ]
-    }
+      }
+
     return this.facultyStats
   }
 
   public async getFinalStatistics(anonymizationSalt: string | null) {
+    const statistics = await this.parseGroupStatistics(anonymizationSalt)
+    const obfuscated = statistics.some(row => row.obfuscated)
+
     return {
+      obfuscated,
+      statistics,
       programmes: this.parseProgrammeStatistics(anonymizationSalt),
-      statistics: await this.parseGroupStatistics(anonymizationSalt),
       facultyStats: this.parseFacultyStatistics(anonymizationSalt),
-      obfuscated: this.obfuscated,
     }
   }
 }

services/backend/src/services/courses/creditsAndEnrollmentsOfCourse.ts

@@ -3,7 +3,6 @@ import { Op } from 'sequelize'
 import { EnrollmentState, Unification } from '@oodikone/shared/types'
 import {
   CreditModel,
-  StudentModel,
   SemesterModel,
   OrganizationModel,
   EnrollmentModel,
@@ -12,13 +11,18 @@ import {
 } from '../../models'
 import { getIsOpen } from './helpers'
 
-export const getCreditsForCourses = async (codes: string[], unification: Unification) => {
-  return await CreditModel.findAll({
+export const getCreditsForCourses = async (codes: string[], unification: Unification) =>
+  CreditModel.findAll({
+    attributes: [
+      'grade',
+      'course_code',
+      'credits',
+      'attainment_date',
+      'student_studentnumber',
+      'studyright_id',
+      'credittypecode',
+    ],
     include: [
-      {
-        model: StudentModel,
-        attributes: ['studentnumber'],
-      },
       {
         model: SemesterModel,
         attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
@@ -37,7 +41,6 @@ export const getCreditsForCourses = async (codes: string[], unification: Unifica
     },
     order: [['attainment_date', 'ASC']],
   })
-}
 
 export const getStudentNumberToSrElementsMap = async (studentNumbers: string[]) => {
   const studyRights = await SISStudyRightModel.findAll({
@@ -53,39 +56,37 @@ export const getStudentNumberToSrElementsMap = async (studentNumbers: string[])
     },
   })
 
-  const studyRightMap = studyRights.reduce((obj, cur) => {
-    obj[cur.id] = cur.toJSON()
-    return obj
-  }, {})
-
-  const studyRightIds = Object.keys(studyRightMap)
+  const studyRightIds = studyRights.map(({ id }) => id)
+  const studyRightMap = new Map<
+    string,
+    {
+      studentNumber: string
+      facultyCode: string
+      organization: Pick<OrganizationModel, 'name' | 'code'>
+    }
+  >(studyRights.map(sr => [sr.id, sr.toJSON()]))
 
   const studyRightElements = await SISStudyRightElementModel.findAll({
     attributes: ['code', 'name', 'startDate', 'endDate', 'studyRightId'],
     where: {
-      studyRightId: {
-        [Op.in]: studyRightIds,
-      },
+      studyRightId: { [Op.in]: studyRightIds },
     },
+    raw: true,
   })
 
-  const studentNumberToSrElementsMap = studyRightElements.reduce((obj, cur) => {
-    const studyRight = studyRightMap[cur.studyRightId]
-    const { studentNumber } = studyRight
+  return studyRightElements.reduce((obj, cur) => {
+    const { studentNumber, ...studyRight } = studyRightMap.get(cur.studyRightId)!
     obj[studentNumber] ??= []
-    obj[studentNumber].push({ ...cur.toJSON(), studyRight })
+    obj[studentNumber].push({ ...cur, studyRight })
+
     return obj
   }, {})
-  return studentNumberToSrElementsMap
 }
 
-export const getEnrollmentsForCourses = async (codes: string[], unification: Unification) => {
-  return await EnrollmentModel.findAll({
+export const getEnrollmentsForCourses = async (codes: string[], unification: Unification) =>
+  EnrollmentModel.findAll({
+    attributes: ['studentnumber', 'enrollment_date_time', 'course_code'],
     include: [
-      {
-        model: StudentModel,
-        attributes: ['studentnumber'],
-      },
       {
         model: SemesterModel,
         attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
@@ -105,4 +106,3 @@ export const getEnrollmentsForCourses = async (codes: string[], unification: Uni
       is_open: getIsOpen(unification),
     },
   })
-}

services/backend/src/services/courses/helpers.ts

@@ -6,8 +6,8 @@ export type FormattedProgramme = {
   code: string
   name: Name
   startDate?: Date
-  facultyCode: string | null
-  organization: OrganizationDetails | null
+  facultyCode: string
+  organization: OrganizationDetails
 }
 
 export const getIsOpen = (unification: Unification) => {

services/backend/src/services/courses/index.ts

@@ -4,7 +4,7 @@ import { Op, fn as dbFn, col as dbCol } from 'sequelize'
 import { Name, EnrollmentState, Unification } from '@oodikone/shared/types'
 import { dateIsBetween } from '@oodikone/shared/util/datetime'
 import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, SISStudyRightElementModel } from '../../models'
-import { isOpenUniCourseCode } from '../../util'
+import { getOpenUniCourseCode } from '../../util'
 import { getSortRank } from '../../util/sortRank'
 import { CourseYearlyStatsCounter } from './courseYearlyStatsCounter'
 import {
@@ -25,13 +25,8 @@ const formatStudyRightElement = (studyRightElement: SISStudyRightElementModel):
   code: studyRightElement.code,
   name: studyRightElement.name,
   startDate: studyRightElement.startDate,
-  facultyCode: studyRightElement.studyRight.facultyCode || null,
-  organization: studyRightElement.studyRight.organization
-    ? {
-        name: studyRightElement.studyRight.organization.name,
-        code: studyRightElement.studyRight.organization.code,
-      }
-    : null,
+  facultyCode: studyRightElement.studyRight.facultyCode,
+  organization: studyRightElement.studyRight.organization,
 })
 
 const anonymizeStudentNumber = (studentNumber: string, anonymizationSalt: string) => {
@@ -48,16 +43,15 @@ type FormattedCredit = {
   grade: string
   passed: boolean
   studentNumber: string
-  programmes: FormattedProgramme[]
+  programme: FormattedProgramme
   credits: number
-  obfuscated?: boolean
 }
 
 const parseCredit = (
   credit: CreditModel,
   anonymizationSalt: string | null,
-  studentNumberToSrElementsMap: Record<string, Array<SISStudyRightElementModel>>
-) => {
+  studyRightElements: Array<SISStudyRightElementModel>
+): FormattedCredit => {
   const {
     semester,
     grade,
@@ -69,18 +63,23 @@ const parseCredit = (
   } = credit
   const { yearcode: yearCode, yearname: yearName, semestercode: semesterCode, name: semesterName } = semester
 
-  const studyRightElements = studentNumberToSrElementsMap[studentNumber] || []
   const programmeOfCredit: SISStudyRightElementModel | undefined =
     studyRightElements.find(studyRightElement => studyRightElement.studyRightId === studyRightId) ??
     studyRightElements
-      .filter(studyRightElement =>
-        dateIsBetween(attainmentDate, studyRightElement.startDate, studyRightElement.endDate)
-      )
-      .sort((a, b) => b.startDate.getTime() - a.startDate.getTime())[0] // The newest studyRightElement
+      .filter(({ startDate, endDate }) => dateIsBetween(attainmentDate, startDate, endDate))
+      .sort((a, b) => b.startDate.getTime() - a.startDate.getTime())
+      .at(0) // The newest studyRightElement
 
-  const programmes = programmeOfCredit ? [programmeOfCredit].map(formatStudyRightElement) : []
+  const programme = programmeOfCredit
+    ? formatStudyRightElement(programmeOfCredit)
+    : {
+        code: 'OTHER',
+        name: { en: 'Other', fi: 'Muu', sv: 'Andra' },
+        facultyCode: 'OTHER',
+        organization: { name: { en: 'Other', fi: 'Muu', sv: 'Andra' } },
+      }
 
-  const formattedCredit: FormattedCredit = {
+  return {
     yearCode,
     yearName,
     semesterCode,
@@ -88,18 +87,11 @@ const parseCredit = (
     attainmentDate,
     courseCode,
     grade,
-    passed: !CreditModel.failed(credit) || CreditModel.passed(credit) || CreditModel.improved(credit),
-    studentNumber,
-    programmes,
+    passed: !CreditModel.failed(credit),
+    studentNumber: anonymizationSalt ? anonymizeStudentNumber(studentNumber, anonymizationSalt) : studentNumber,
+    programme,
     credits,
   }
-
-  if (anonymizationSalt) {
-    formattedCredit.obfuscated = true
-    formattedCredit.studentNumber = anonymizeStudentNumber(studentNumber, anonymizationSalt)
-  }
-
-  return formattedCredit
 }
 
 type FormattedEnrollment = {
@@ -110,15 +102,9 @@ type FormattedEnrollment = {
   courseCode: string
   enrollmentDateTime: Date
   studentNumber: string
-  programmes: Array<Record<string, any>>
-  obfuscated?: boolean
 }
 
-const parseEnrollment = (
-  enrollment: EnrollmentModel,
-  anonymizationSalt: string | null,
-  studentNumberToSrElementsMap: Record<string, SISStudyRightElementModel[]>
-) => {
+const parseEnrollment = (enrollment: EnrollmentModel, anonymizationSalt: string | null): FormattedEnrollment => {
   const {
     studentnumber: studentNumber,
     semester,
@@ -127,25 +113,15 @@ const parseEnrollment = (
   } = enrollment
   const { yearcode: yearCode, yearname: yearName, semestercode: semesterCode, name: semesterName } = semester
 
-  const studyRightElements = studentNumberToSrElementsMap[studentNumber] || []
-
-  const formattedEnrollment: FormattedEnrollment = {
+  return {
     yearCode,
     yearName,
     semesterCode,
     semesterName,
     courseCode,
     enrollmentDateTime,
-    studentNumber,
-    programmes: studyRightElements.map(formatStudyRightElement),
+    studentNumber: anonymizationSalt ? anonymizeStudentNumber(studentNumber, anonymizationSalt) : studentNumber,
   }
-
-  if (anonymizationSalt) {
-    formattedEnrollment.obfuscated = true
-    formattedEnrollment.studentNumber = anonymizeStudentNumber(studentNumber, anonymizationSalt)
-  }
-
-  return formattedEnrollment
 }
 
 const getYearlyStatsOfNew = async (
@@ -156,27 +132,22 @@ const getYearlyStatsOfNew = async (
   combineSubstitutions: boolean,
   studentNumberToSrElementsMap: Record<string, SISStudyRightElementModel[]>
 ) => {
-  const courseForSubs = await CourseModel.findOne({
+  const course: Pick<CourseModel, 'code' | 'name' | 'substitutions'> | null = await CourseModel.findOne({
+    attributes: ['code', 'name', 'substitutions'],
     where: { code: courseCode },
   })
 
-  let codes =
-    combineSubstitutions && courseForSubs?.substitutions
-      ? sortMainCode([...courseForSubs.substitutions, courseCode])
-      : [courseCode]
+  const substitutedCodes =
+    combineSubstitutions && course?.substitutions ? sortMainCode([...course.substitutions, courseCode]) : [courseCode]
 
-  if (unification === Unification.REGULAR) {
-    codes = codes.filter(course => !isOpenUniCourseCode(course))
-  }
+  const codes =
+    unification === Unification.REGULAR
+      ? substitutedCodes.filter(course => !getOpenUniCourseCode(course))
+      : substitutedCodes
 
-  const [credits, enrollments, course] = await Promise.all([
+  const [credits, enrollments] = await Promise.all([
     getCreditsForCourses(codes, unification),
     getEnrollmentsForCourses(codes, unification),
-    CourseModel.findOne({
-      where: {
-        code: courseCode,
-      },
-    }),
   ])
 
   const counter = new CourseYearlyStatsCounter()
@@ -191,70 +162,54 @@ const getYearlyStatsOfNew = async (
       yearCode,
       yearName,
       attainmentDate,
-      programmes,
+      programme,
       courseCode,
       credits,
-    } = parseCredit(credit, anonymizationSalt, studentNumberToSrElementsMap)
+    } = parseCredit(credit, anonymizationSalt, studentNumberToSrElementsMap[credit.student_studentnumber] ?? [])
+
+    counter.markStudyProgramme(
+      studentNumber,
+      yearCode,
+      passed,
+      credits,
+      programme.code,
+      programme.name,
+      programme.facultyCode,
+      programme.organization
+    )
 
     const groupCode = separate ? semesterCode : yearCode
     const groupName = separate ? semesterName : yearName
-    const unknownProgramme: FormattedProgramme[] = [
-      {
-        code: 'OTHER',
-        name: {
-          en: 'Other',
-          fi: 'Muu',
-          sv: 'Andra',
-        },
-        facultyCode: 'OTHER',
-        organization: {
-          name: {
-            en: 'Other',
-            fi: 'Muu',
-            sv: 'Andra',
-          },
-        },
-      },
-    ]
-    counter.markStudyProgrammes(
-      studentNumber,
-      programmes.length === 0 ? unknownProgramme : programmes,
-      yearCode,
-      passed,
-      credits
-    )
     counter.markCreditToGroup(studentNumber, passed, grade, groupCode, groupName, courseCode, yearCode)
     counter.markCreditToStudentCategories(studentNumber, attainmentDate, groupCode)
   }
 
-  enrollments.forEach(enrollment => {
+  for (const enrollment of enrollments) {
     const { studentNumber, semesterCode, semesterName, yearCode, yearName, courseCode, enrollmentDateTime } =
-      parseEnrollment(enrollment, anonymizationSalt, studentNumberToSrElementsMap)
+      parseEnrollment(enrollment, anonymizationSalt)
 
     const groupCode = separate ? semesterCode : yearCode
     const groupName = separate ? semesterName : yearName
 
     counter.markEnrollmentToGroup(studentNumber, enrollmentDateTime, groupCode, groupName, courseCode, yearCode)
-  })
+  }
 
   const statistics = await counter.getFinalStatistics(anonymizationSalt)
 
-  let substitutionCourses: CourseModel[] | undefined
-  if (combineSubstitutions && courseForSubs?.substitutions && courseForSubs.substitutions.length > 0) {
-    substitutionCourses = await CourseModel.findAll({
-      where: {
-        code: {
-          [Op.in]: codes,
-        },
-      },
-      attributes: ['code', 'name'],
-    })
-  }
+  const substitutionCourses: Pick<CourseModel, 'code' | 'name'>[] =
+    combineSubstitutions && course?.substitutions?.length
+      ? await CourseModel.findAll({
+          where: {
+            code: { [Op.in]: codes },
+          },
+          attributes: ['code', 'name'],
+        })
+      : [{ code: course!.code, name: course!.name }]
 
   return {
     ...statistics,
     coursecode: courseCode,
-    alternatives: substitutionCourses ?? [{ code: courseForSubs!.code, name: courseForSubs!.name }],
+    alternatives: substitutionCourses,
     name: course?.name,
   }
 }
@@ -274,7 +229,6 @@ export const maxYearsToCreatePopulationFrom = async (courseCodes: string[], unif
   const attainmentDateThreshold = new Date(newestAttainmentDate - 6, 0, 1)
 
   const attainmentsWithinThreshold = await CreditModel.count({
-    attributes: [[dbFn('COUNT', dbCol('id')), 'numAttainments']],
     where: {
       course_code: { [Op.in]: courseCodes },
       attainment_date: { [Op.gt]: attainmentDateThreshold },
@@ -293,57 +247,62 @@ export const getCourseYearlyStats = async (
   anonymizationSalt: string | null,
   combineSubstitutions: boolean
 ) => {
-  const credits = await CreditModel.findAll({
-    attributes: ['student_studentnumber'],
-    where: { course_code: { [Op.in]: courseCodes } },
-  })
-  const enrollments = await EnrollmentModel.findAll({
-    attributes: ['studentnumber'],
-    where: {
-      course_code: {
-        [Op.in]: courseCodes,
+  const [credits, enrollments] = await Promise.all([
+    CreditModel.findAll({
+      attributes: ['student_studentnumber'],
+      where: { course_code: { [Op.in]: courseCodes } },
+    }),
+    EnrollmentModel.findAll({
+      attributes: ['studentnumber'],
+      where: {
+        course_code: {
+          [Op.in]: courseCodes,
+        },
+        state: EnrollmentState.ENROLLED,
       },
-      state: EnrollmentState.ENROLLED,
-    },
-  })
+    }),
+  ])
 
-  const studentNumbers = {}
+  const studentNumbers = new Set<string>()
 
   credits.forEach(credit => {
-    studentNumbers[credit.student_studentnumber] = true
-  })
-  enrollments.forEach(enrollment => {
-    studentNumbers[enrollment.studentnumber] = true
+    studentNumbers.add(credit.student_studentnumber)
   })
 
-  const studentNumberToSrElementsMap = await getStudentNumberToSrElementsMap(Object.keys(studentNumbers))
+  enrollments.forEach(enrollment => {
+    studentNumbers.add(enrollment.studentnumber)
+  })
+
+  const studentNumberToSrElementsMap = await getStudentNumberToSrElementsMap([...studentNumbers])
 
   const statsRegular = await Promise.all(
     courseCodes.map(async courseCode => {
-      const unifyStats = await getYearlyStatsOfNew(
-        courseCode,
-        separate,
-        Unification.UNIFY,
-        anonymizationSalt,
-        combineSubstitutions,
-        studentNumberToSrElementsMap
-      )
-      const regularStats = await getYearlyStatsOfNew(
-        courseCode,
-        separate,
-        Unification.REGULAR,
-        anonymizationSalt,
-        combineSubstitutions,
-        studentNumberToSrElementsMap
-      )
-      const openStats = await getYearlyStatsOfNew(
-        courseCode,
-        separate,
-        Unification.OPEN,
-        anonymizationSalt,
-        combineSubstitutions,
-        studentNumberToSrElementsMap
-      )
+      const [openStats, regularStats, unifyStats] = await Promise.all([
+        getYearlyStatsOfNew(
+          courseCode,
+          separate,
+          Unification.OPEN,
+          anonymizationSalt,
+          combineSubstitutions,
+          studentNumberToSrElementsMap
+        ),
+        getYearlyStatsOfNew(
+          courseCode,
+          separate,
+          Unification.REGULAR,
+          anonymizationSalt,
+          combineSubstitutions,
+          studentNumberToSrElementsMap
+        ),
+        getYearlyStatsOfNew(
+          courseCode,
+          separate,
+          Unification.UNIFY,
+          anonymizationSalt,
+          combineSubstitutions,
+          studentNumberToSrElementsMap
+        ),
+      ])
 
       return { unifyStats, regularStats, openStats }
     })

services/backend/src/services/faculty/faculty.ts

@@ -1,11 +1,12 @@
-import { groupBy, orderBy } from 'lodash'
+import { groupBy, orderBy } from 'lodash-es'
 import { QueryTypes } from 'sequelize'
 
+import { Organization } from '@oodikone/shared/models'
 import { ProgrammeModuleWithRelevantAttributes } from '@oodikone/shared/types'
-import { serviceProvider } from '../../config'
 import { programmeCodes } from '../../config/programmeCodes'
 import { dbConnections } from '../../database/connection'
 import { OrganizationModel } from '../../models'
+import { ApplicationError } from '../../util/customErrors'
 import { CurriculumPeriods, getCurriculumPeriods } from '../curriculumPeriods'
 
 const { sequelize } = dbConnections
@@ -55,9 +56,7 @@ export const getDegreeProgrammesOfOrganization = async (organizationId: string,
   const programmesWithProgIds = programmesOfOrganization.map(programme => ({
     ...programme,
     progId:
-      programme.code in programmeCodes && serviceProvider !== 'fd'
-        ? programmeCodes[programme.code as keyof typeof programmeCodes]
-        : programme.code,
+      programme.code in programmeCodes ? programmeCodes[programme.code as keyof typeof programmeCodes] : programme.code,
   }))
   const programmesGroupedByCode = groupBy(orderBy(programmesWithProgIds, ['valid_from'], ['desc']), prog => prog.code)
   const curriculumPeriods = await getCurriculumPeriods()
@@ -106,20 +105,15 @@ export const getDegreeProgrammesOfFaculty = async (facultyCode: string, onlyCurr
     },
   })
   if (!organization) {
-    throw new Error(`The organization with the code ${facultyCode} was not found.`)
+    throw new ApplicationError(`The organization with the code ${facultyCode} was not found.`)
   }
   return getDegreeProgrammesOfOrganization(organization.id, onlyCurrentProgrammes)
 }
 
-export const getFacultyCodeById = async (facultyId: string) => {
-  const organization = await OrganizationModel.findOne({
+export const getFacultyCodeById = async (facultyId: string): Promise<Pick<Organization, 'code'> | null> =>
+  OrganizationModel.findOne({
     attributes: ['code'],
     where: {
       id: facultyId,
     },
   })
-  if (!organization) {
-    throw new Error(`The organization with the id ${facultyId} was not found.`)
-  }
-  return organization.code
-}

services/backend/src/services/faculty/facultyBasics.ts

@@ -1,22 +1,21 @@
-import { cloneDeep } from 'lodash'
+import { cloneDeep } from 'lodash-es'
 
 import { NameWithCode, DegreeProgrammeType } from '@oodikone/shared/types'
+import { BasicStats } from '@oodikone/shared/types/studyProgramme'
 import { getBasicStats, setBasicStats } from '../analyticsService'
 import { getBasicStatsForStudytrack } from '../studyProgramme/studyProgrammeBasics'
 import type { ProgrammesOfOrganization } from './faculty'
 
-type StudyTrackBasicStats = Awaited<ReturnType<typeof getBasicStatsForStudytrack>>
-
 const calculateCombinedStats = async (
   facultyProgrammes: ProgrammesOfOrganization,
   isAcademicYear: boolean,
   includeAllSpecials: boolean
 ) => {
-  const statsFromAllProgrammes: StudyTrackBasicStats[] = []
+  const statsFromAllProgrammes: BasicStats[] = []
   const studentInfo = {
-    graphStats: [] as StudyTrackBasicStats['graphStats'],
-    programmeTableStats: {} as Record<string, StudyTrackBasicStats['tableStats']>,
-    tableStats: [] as StudyTrackBasicStats['tableStats'],
+    graphStats: [] as BasicStats['graphStats'],
+    programmeTableStats: {} as Record<string, BasicStats['tableStats']>,
+    tableStats: [] as BasicStats['tableStats'],
     titles: ['Code', 'Started\nstudying', 'Accepted', 'Graduated'],
   }
   if (includeAllSpecials) {

services/backend/src/services/faculty/facultyGraduationTimes.ts

@@ -1,7 +1,9 @@
-import { cloneDeep, omit } from 'lodash'
+import { cloneDeep, omit } from 'lodash-es'
 
 import { Name, DegreeProgrammeType, Unarray } from '@oodikone/shared/types'
+import { ProgrammeGraduationStats } from '@oodikone/shared/types/studyProgramme'
 import { getGraduationStats, getStudyTrackStats, setGraduationStats, setStudyTrackStats } from '../analyticsService'
+import { GraduationTarget } from '../graduationHelpers'
 import { getGraduationStatsForStudyTrack, GraduationTimes } from '../studyProgramme/studyProgrammeGraduations'
 import { getMedian } from '../studyProgramme/studyProgrammeHelpers'
 import { getStudyRightsInProgramme } from '../studyProgramme/studyRightFinders'
@@ -22,7 +24,7 @@ export const programmeTypes = {
 } as const
 
 const getStatsByGraduationYear = async (facultyProgrammes: ProgrammesOfOrganization) => {
-  const newStats: Array<Awaited<ReturnType<typeof getGraduationStatsForStudyTrack>>> = []
+  const newStats: ProgrammeGraduationStats[] = []
   const medians: Record<string, LevelGraduationStats[]> = {}
   const programmes: { medians: Record<string, Record<string, ProgrammeStats>> } = { medians: {} }
 
@@ -133,7 +135,7 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
     if (programme.degreeProgrammeType == null || !(programme.degreeProgrammeType in programmeTypes)) {
       continue
     }
-    const statsFromRedis = await getStudyTrackStats(programme.code, '', 'GRADUATED_INCLUDED', 'SPECIAL_EXCLUDED')
+    const statsFromRedis = await getStudyTrackStats(programme.code, '', 'SPECIAL_EXCLUDED')
     if (statsFromRedis) {
       newStats.push(statsFromRedis)
       continue
@@ -144,12 +146,11 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
       studyProgramme: programme.code,
       combinedProgramme: '',
       settings: {
-        graduated: true,
         specialGroups: false,
       },
       studyRightsOfProgramme,
     })
-    await setStudyTrackStats(updatedStats, 'GRADUATED_INCLUDED', 'SPECIAL_EXCLUDED')
+    await setStudyTrackStats(updatedStats, 'SPECIAL_EXCLUDED')
     newStats.push(updatedStats)
   }
 
@@ -161,7 +162,7 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
     const basicStats = statsForProgramme.medians.basic
     const comboStats = statsForProgramme.medians.combo
     const classSizesByYear = basicStats.reduce<Record<string, number>>(
-      (acc, year) => ({ ...acc, [year.name]: year.classSize }),
+      (acc, year) => ({ ...acc, [year.name]: year.classSize ?? 0 }),
       {}
     )
     classSizes.programmes[code] = classSizesByYear
@@ -172,7 +173,8 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
     classSizes[level] ??= {}
     for (const statsForYear of basicStats) {
       classSizes[level][statsForYear.name] ??= 0
-      ;(classSizes[level][statsForYear.name] as number) += statsForYear.classSize
+      ;(classSizes[level][statsForYear.name] as number) += statsForYear.classSize ?? 0
+
       const correctYear = medians[level].find(entry => entry.name === statsForYear.name)
       if (!correctYear) {
         medians[level].push({ ...omit(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
@@ -193,7 +195,7 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
       classSizes.bcMsCombo ??= {}
       for (const statsForYear of comboStats) {
         classSizes.bcMsCombo[statsForYear.name] ??= 0
-        ;(classSizes.bcMsCombo[statsForYear.name] as number) += statsForYear.classSize
+        ;(classSizes.bcMsCombo[statsForYear.name] as number) += statsForYear.classSize ?? 0
         const correctYear = medians.bcMsCombo.find(entry => entry.name === statsForYear.name)
         if (!correctYear) {
           medians.bcMsCombo.push({ ...omit(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
@@ -252,22 +254,23 @@ export const countGraduationTimes = async (faculty: string, programmesOfFaculty:
   } = await getStatsByStartYear(programmesOfFaculty)
 
   const goals = {
-    bachelor: 36,
-    bcMsCombo: faculty === 'H90' ? 72 : 60,
-    master: 24,
-    doctor: 48,
+    bachelor: GraduationTarget.THREE_YEARS,
+    bcMsCombo: faculty === 'H90' ? GraduationTarget.SIX_YEARS : GraduationTarget.FIVE_YEARS,
+    master: GraduationTarget.TWO_YEARS,
+    doctor: GraduationTarget.FOUR_YEARS,
     exceptions: {
-      MH30_004: 6, // months more
-      '420420-ma': 6,
-      '420074-ma': 6,
-      '420119-ma': 6,
-      MH30_001: 48,
-      '320011-ma': 48,
-      '320001-ma': 48,
-      MH30_003: 42,
-      '320002-ma': 42,
-      '320009-ma': 42,
-    },
+      // Exceptions are additions to the base amount
+      MH30_004: GraduationTarget.HALF_YEAR,
+      '420420-ma': GraduationTarget.HALF_YEAR,
+      '420074-ma': GraduationTarget.HALF_YEAR,
+      '420119-ma': GraduationTarget.HALF_YEAR,
+      MH30_001: GraduationTarget.FOUR_YEARS,
+      '320011-ma': GraduationTarget.FOUR_YEARS,
+      '320001-ma': GraduationTarget.FOUR_YEARS,
+      MH30_003: GraduationTarget.THREE_POINT_FIVE_YEARS,
+      '320002-ma': GraduationTarget.THREE_POINT_FIVE_YEARS,
+      '320009-ma': GraduationTarget.THREE_POINT_FIVE_YEARS,
+    } as Record<string, number>,
   }
 
   const programmeNames = programmesOfFaculty.reduce<Record<string, Name>>((acc, { code, name }) => {

services/backend/src/services/faculty/facultyService.ts

@@ -1,4 +1,5 @@
-import { Graduated, Name, ProgrammeFilter, SpecialGroups, YearType, DegreeProgrammeType } from '@oodikone/shared/types'
+import { Name, ProgrammeFilter, SpecialGroups, YearType } from '@oodikone/shared/types'
+import { DegreeProgramme } from '../../routes/faculties'
 import { redisClient } from '../redis'
 import { FacultyProgressData } from './facultyStudentProgress'
 
@@ -21,11 +22,11 @@ const createRedisKeyForThesiswriters = (
 const createRedisKeyForGraduationTimeStats = (id: string, programmeFilter: ProgrammeFilter) => {
   return `FACULTY_GRADUATION_TIME_STATS_${id}_${programmeFilter}`
 }
-const createRedisKeyForFacultyProgress = (id: string, specialGroups: SpecialGroups, graduated: Graduated) => {
-  return `FACULTY_PROGRESS_STATS_${id}_${specialGroups}_${graduated}`
+const createRedisKeyForFacultyProgress = (id: string, specialGroups: SpecialGroups) => {
+  return `FACULTY_PROGRESS_STATS_${id}_${specialGroups}`
 }
-const createRedisKeyForFacultyStudents = (id: string, specialGroups: SpecialGroups, graduated: Graduated) => {
-  return `FACULTY_STUDENTS_STATS_${id}_${specialGroups}_${graduated}`
+const createRedisKeyForFacultyStudents = (id: string, specialGroups: SpecialGroups) => {
+  return `FACULTY_STUDENTS_STATS_${id}_${specialGroups}`
 }
 
 /*
@@ -62,7 +63,7 @@ type StudentInfo = {
   titles: string[]
 }
 
-type BasicData = {
+export type BasicData = {
   graduationInfo: GraduationInfo
   id: string
   programmeNames: ProgrammeNames
@@ -78,16 +79,9 @@ export const setBasicStats = async (
 ) => {
   const { id } = data
   const redisKey = createRedisKeyForBasicStats(id, yearType, programmeFilter, specialGroups)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(data))
+
+  return setOperationStatus === 'OK'
 }
 
 export const getBasicStats = async (
@@ -98,15 +92,13 @@ export const getBasicStats = async (
 ) => {
   const redisKey = createRedisKeyForBasicStats(id, yearType, programmeFilter, specialGroups)
   const dataFromRedis = await redisClient.get(redisKey)
-  if (!dataFromRedis) {
-    return null
-  }
-  return JSON.parse(dataFromRedis) as BasicData & { status: string; lastUpdated: string }
+
+  return dataFromRedis ? (JSON.parse(dataFromRedis) as BasicData) : null
 }
 
 type ProgrammeNames = Record<string, Name & { code: string }>
 
-type ThesisWriterData = {
+export type ThesisWriterData = {
   id: string
   years: (string | number)[]
   tableStats: (string | number)[][]
@@ -117,8 +109,6 @@ type ThesisWriterData = {
   programmeTableStats: Record<string, (string | number)[][]>
   titles: string[]
   programmeNames: ProgrammeNames
-  status: string
-  lastUpdated: string
 }
 
 export const setThesisWritersStats = async (
@@ -129,14 +119,7 @@ export const setThesisWritersStats = async (
 ) => {
   const { id } = data
   const redisKey = createRedisKeyForThesiswriters(id, yearType, programmeFilter, specialGroups)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') return null
-  return dataToRedis
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
 export const getThesisWritersStats = async (
@@ -150,7 +133,7 @@ export const getThesisWritersStats = async (
   if (!dataFromRedis) {
     return null
   }
-  return JSON.parse(dataFromRedis) as ThesisWriterData & { status: string; lastUpdated: string }
+  return JSON.parse(dataFromRedis) as ThesisWriterData
 }
 
 type MediansAndProgrammes = {
@@ -160,7 +143,7 @@ type MediansAndProgrammes = {
   }
 }
 
-type GraduationData = {
+export type GraduationData = {
   id: string
   goals: {
     bachelor: number
@@ -178,16 +161,7 @@ type GraduationData = {
 export const setGraduationStats = async (data: GraduationData, programmeFilter: ProgrammeFilter) => {
   const { id } = data
   const redisKey = createRedisKeyForGraduationTimeStats(id, programmeFilter)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
 export const getGraduationStats = async (id: string, programmeFilter: ProgrammeFilter, keepGraduationTimes = false) => {
@@ -196,39 +170,24 @@ export const getGraduationStats = async (id: string, programmeFilter: ProgrammeF
   if (!dataFromRedis) {
     return null
   }
-  const data = JSON.parse(dataFromRedis) as GraduationData & { status: string; lastUpdated: string }
+  const data = JSON.parse(dataFromRedis) as GraduationData
   if (!keepGraduationTimes) {
     removeGraduationTimes(data)
   }
   return data
 }
 
-export const setFacultyProgressStats = async (
-  data: FacultyProgressData,
-  specialGroups: SpecialGroups,
-  graduated: Graduated
-) => {
+export const setFacultyProgressStats = async (data: FacultyProgressData, specialGroups: SpecialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForFacultyProgress(id, specialGroups, graduated)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  const redisKey = createRedisKeyForFacultyProgress(id, specialGroups)
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
-export const getFacultyProgressStats = async (id: string, specialGroups: SpecialGroups, graduated: Graduated) => {
-  const redisKey = createRedisKeyForFacultyProgress(id, specialGroups, graduated)
+export const getFacultyProgressStats = async (id: string, specialGroups: SpecialGroups) => {
+  const redisKey = createRedisKeyForFacultyProgress(id, specialGroups)
   const dataFromRedis = await redisClient.get(redisKey)
-  if (!dataFromRedis) {
-    return null
-  }
-  return JSON.parse(dataFromRedis) as FacultyProgressData & { status: string; lastUpdated: string }
+
+  return dataFromRedis ? (JSON.parse(dataFromRedis) as FacultyProgressData) : null
 }
 
 type FacultyStudentsData = {
@@ -238,36 +197,18 @@ type FacultyStudentsData = {
   facultyTableStatsExtra: Record<string, Record<string, Record<string, number>>>
   programmeStats: Record<string, Record<string, (string | number)[]>>
   titles: string[]
-  programmeNames: Record<
-    string,
-    { name: Name; code: string; degreeProgrammeType: DegreeProgrammeType | null; progId: string }
-  >
+  programmeNames: Record<string, DegreeProgramme>
 }
 
-export const setFacultyStudentStats = async (
-  data: FacultyStudentsData,
-  specialGroups: SpecialGroups,
-  graduated: Graduated
-) => {
+export const setFacultyStudentStats = async (data: FacultyStudentsData, specialGroups: SpecialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForFacultyStudents(id, specialGroups, graduated)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  const redisKey = createRedisKeyForFacultyStudents(id, specialGroups)
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
-export const getFacultyStudentStats = async (id: string, specialGroups: SpecialGroups, graduated: Graduated) => {
-  const redisKey = createRedisKeyForFacultyStudents(id, specialGroups, graduated)
+export const getFacultyStudentStats = async (id: string, specialGroups: SpecialGroups) => {
+  const redisKey = createRedisKeyForFacultyStudents(id, specialGroups)
   const dataFromRedis = await redisClient.get(redisKey)
-  if (!dataFromRedis) {
-    return null
-  }
-  return JSON.parse(dataFromRedis) as FacultyStudentsData & { status: string; lastUpdated: string }
+
+  return dataFromRedis ? (JSON.parse(dataFromRedis) as FacultyStudentsData) : null
 }

services/backend/src/services/faculty/facultyStudentProgress.ts

@@ -1,7 +1,7 @@
 import dayjs from 'dayjs'
-import { cloneDeep } from 'lodash'
+import { cloneDeep } from 'lodash-es'
 
-import { Graduated, SpecialGroups, Unarray } from '@oodikone/shared/types'
+import { SpecialGroups } from '@oodikone/shared/types'
 import { rootOrgId } from '../../config'
 import { getStudyTrackStats, setStudyTrackStats } from '../analyticsService'
 import { getYearsArray } from '../studyProgramme/studyProgrammeHelpers'
@@ -10,35 +10,38 @@ import { getStudyTrackStatsForStudyProgramme } from '../studyProgramme/studyTrac
 import { getDegreeProgrammesOfOrganization, ProgrammesOfOrganization } from './faculty'
 import { programmeTypes } from './facultyGraduationTimes'
 
+type CreditLimits = Array<[number | null, number | null]>
+
 const createLimits = (
   months: number,
-  creditsToAdd: number,
-  level: 'bachelor' | 'master' | 'doctor'
-): Array<[number, number | null]> => {
+  startingCredits: number,
+  level: 'bachelor' | 'master' | 'bcMs' | 'doctor'
+): CreditLimits => {
   if (level === 'doctor') {
     return [
       [40, null],
       [30, 40],
       [20, 30],
       [10, 20],
-      [0, null],
+      [null, 0],
     ]
   }
+
   return [
-    [Math.ceil(months * (60 / 12)) + creditsToAdd, null],
-    [Math.ceil(months * (45 / 12)) + creditsToAdd, Math.ceil(months * (60 / 12)) + creditsToAdd],
-    [Math.ceil(months * (30 / 12)) + creditsToAdd, Math.ceil(months * (45 / 12)) + creditsToAdd],
-    [Math.ceil(months * (15 / 12)) + creditsToAdd, Math.ceil(months * (30 / 12)) + creditsToAdd],
-    [creditsToAdd + 1, Math.ceil(months * (15 / 12)) + creditsToAdd],
-    [creditsToAdd, null],
+    [startingCredits + Math.ceil(months * (60 / 12)), null],
+    [startingCredits + Math.ceil(months * (45 / 12)), startingCredits + Math.ceil(months * (60 / 12))],
+    [startingCredits + Math.ceil(months * (30 / 12)), startingCredits + Math.ceil(months * (45 / 12))],
+    [startingCredits + Math.ceil(months * (15 / 12)), startingCredits + Math.ceil(months * (30 / 12))],
+    [startingCredits + 1, startingCredits + Math.ceil(months * (15 / 12))],
+    [null, startingCredits],
   ]
 }
 
-const createYearlyTitles = (limitList: ReturnType<typeof createLimits>) => {
+const createYearlyTitles = (limitList: CreditLimits) => {
   const titles: string[] = []
   for (let i = limitList.length - 1; i >= 0; i--) {
-    if (limitList[i][0] === 0) {
-      titles.push('0 Credits')
+    if (limitList[i][0] === null) {
+      titles.push(`${limitList[i][1]} Credits`)
     } else if (limitList[i][1] === null) {
       titles.push(`${limitList[0][0]} ≤ Credits`)
     } else {
@@ -53,18 +56,20 @@ const calculateProgressStats = (
   creditCounts: Record<string, number[]>,
   yearlyTitles: Record<string, string[][]>,
   progressStats: Record<string, Record<string, number[][]>>,
-  programmeInfo: Unarray<ProgrammesOfOrganization>
+  progId: string
 ) => {
-  const goalMonthsForLevels = { bachelor: 36, master: 24, bcMs: 60, doctor: 48 } as const
+  // "bcMs" is the same as masters, as the students have graduated from bachelors.
+  // We need to add the 180cr to the starting point for compensation.
+  const goalMonthsForLevels = { bachelor: 36, master: 24, bcMs: 24, doctor: 48 } as const
 
   const allYears = Object.keys(creditCounts)
-  const limitsForYears = allYears.reduce<Record<string, ReturnType<typeof createLimits>>>((acc, year) => {
+  const limitsForYears = allYears.reduce<Record<string, CreditLimits>>((acc, year) => {
     const startDate = new Date(`${year.slice(0, 4)}-08-01`)
     const lastDayOfMonth = dayjs().endOf('month')
     const months = Math.round(dayjs(lastDayOfMonth).diff(startDate, 'months', true))
     const goalMonths = goalMonthsForLevels[level]
 
-    acc[year] = createLimits(Math.min(months, goalMonths), 0, level === 'bcMs' ? 'master' : level)
+    acc[year] = createLimits(Math.min(months, goalMonths), Number(level === 'bcMs') * 180, level)
     return acc
   }, {})
 
@@ -79,29 +84,29 @@ const calculateProgressStats = (
     progressStats[level] = {}
   }
 
-  progressStats[level][programmeInfo.progId] = []
+  progressStats[level][progId] = []
 
   for (const year of Object.keys(creditCounts)) {
     const limit = limitsForYears[year]
     const statsForYear: number[] = new Array(limit.length).fill(0)
     for (const credits of creditCounts[year]) {
-      const correctIndex = limit.length - 1 - limit.findIndex(element => credits >= element[0])
+      const correctIndex = limit.length - 1 - limit.findIndex(element => credits >= (element[0] ?? 0))
       statsForYear[correctIndex] += 1
     }
-    progressStats[level][programmeInfo.progId].unshift(statsForYear)
+    progressStats[level][progId].unshift(statsForYear)
   }
 }
 
 export const combineFacultyStudentProgress = async (
   faculty: string,
   programmes: ProgrammesOfOrganization,
-  specialGroups: SpecialGroups,
-  graduated: Graduated
+  specialGroups: SpecialGroups
 ) => {
   const since = new Date('2017-08-01')
   const statsOfProgrammes: Array<Awaited<ReturnType<typeof getStudyTrackStatsForStudyProgramme>>> = []
   const allDegreeProgrammes = await getDegreeProgrammesOfOrganization(rootOrgId, false)
   const creditCounts: Record<string, Record<string, number[]>> = {}
+  const graduatedCount: Record<string, Record<string, number>> = {}
   const progressStats: Record<string, Record<string, number[][]>> = {}
   const yearlyTitles: Record<string, string[][]> = {}
 
@@ -114,7 +119,7 @@ export const combineFacultyStudentProgress = async (
     ) {
       continue
     }
-    const statsFromRedis = await getStudyTrackStats(studyProgramme, null, graduated, specialGroups)
+    const statsFromRedis = await getStudyTrackStats(studyProgramme, null, specialGroups)
     if (statsFromRedis) {
       statsOfProgrammes.push(statsFromRedis)
     } else {
@@ -122,22 +127,23 @@ export const combineFacultyStudentProgress = async (
       const updatedStats = await getStudyTrackStatsForStudyProgramme({
         studyProgramme,
         settings: {
-          graduated: graduated === 'GRADUATED_INCLUDED',
           specialGroups: specialGroups === 'SPECIAL_INCLUDED',
         },
         studyRightsOfProgramme,
       })
       statsOfProgrammes.push(updatedStats)
-      await setStudyTrackStats(updatedStats, graduated, specialGroups)
+      await setStudyTrackStats(updatedStats, specialGroups)
     }
   }
 
-  const updateCreditCounts = (
+  const updateCreditAndGraduatedCount = (
     level: 'bachelor' | 'bcMs' | 'master' | 'doctor',
-    creditCountsOfProgramme: Record<string, number[]>
+    creditCountsOfProgramme: Record<string, number[]>,
+    graduatedCountOfProgramme: Record<string, number>
   ) => {
     if (!(level in creditCounts)) {
       creditCounts[level] = cloneDeep(creditCountsOfProgramme)
+      graduatedCount[level] = cloneDeep(graduatedCountOfProgramme)
     } else {
       for (const year of Object.keys(creditCountsOfProgramme)) {
         if (!(year in creditCounts[level])) {
@@ -146,27 +152,28 @@ export const combineFacultyStudentProgress = async (
           creditCounts[level][year].push(...creditCountsOfProgramme[year])
         }
       }
+
+      for (const year of Object.keys(graduatedCountOfProgramme)) {
+        graduatedCount[level][year] += cloneDeep(graduatedCountOfProgramme[year])
+      }
     }
   }
 
   for (const stats of statsOfProgrammes) {
     const programmeInfo = allDegreeProgrammes.find(programme => programme.code === stats.id)
-    if (
-      !programmeInfo ||
-      programmeInfo.degreeProgrammeType == null ||
-      !(programmeInfo.degreeProgrammeType in programmeTypes)
-    ) {
+    if (!programmeInfo?.degreeProgrammeType || !(programmeInfo.degreeProgrammeType in programmeTypes)) {
       continue
     }
 
-    const level = programmeTypes[programmeInfo.degreeProgrammeType as keyof typeof programmeTypes]
+    const { degreeProgrammeType, progId } = programmeInfo
+    const level = programmeTypes[degreeProgrammeType]
 
-    updateCreditCounts(level, stats.creditCounts)
-    calculateProgressStats(level, stats.creditCounts, yearlyTitles, progressStats, programmeInfo)
+    updateCreditAndGraduatedCount(level, stats.creditCounts, stats.graduatedCount)
+    calculateProgressStats(level, stats.creditCounts, yearlyTitles, progressStats, progId)
 
     if (level === 'master' && Object.keys(stats.creditCountsCombo).length > 0) {
-      updateCreditCounts('bcMs', stats.creditCountsCombo)
-      calculateProgressStats('bcMs', stats.creditCountsCombo, yearlyTitles, progressStats, programmeInfo)
+      updateCreditAndGraduatedCount('bcMs', stats.creditCountsCombo, stats.graduatedCount)
+      calculateProgressStats('bcMs', stats.creditCountsCombo, yearlyTitles, progressStats, progId)
     }
   }
 
@@ -178,6 +185,7 @@ export const combineFacultyStudentProgress = async (
     bcMsProgStats: progressStats.bcMs,
     doctoralProgStats: progressStats.doctor,
     creditCounts: {} as Record<string, Record<string, number[]>>,
+    graduatedCount: {} as Record<string, Record<string, number>>,
     yearlyBachelorTitles: yearlyTitles.bachelor,
     yearlyBcMsTitles: yearlyTitles.bcMs,
     yearlyMasterTitles: yearlyTitles.master,
@@ -200,6 +208,14 @@ export const combineFacultyStudentProgress = async (
       }
       result.creditCounts[levelName][year] = creditCounts[level][year]
     }
+
+    for (const year of Object.keys(graduatedCount[level]).toSorted(sortByYear)) {
+      const levelName = level === 'bcMs' ? 'bachelorMaster' : level
+      if (!(levelName in result.graduatedCount)) {
+        result.graduatedCount[levelName] = {}
+      }
+      result.graduatedCount[levelName][year] = graduatedCount[level][year]
+    }
   }
 
   return result

services/backend/src/services/faculty/facultyStudents.ts

@@ -61,15 +61,14 @@ const calculateCombinedStats = (programmeCodes: string[], stats: StudyTrackStats
 export const combineFacultyStudents = async (
   code: string,
   programmes: ProgrammesOfOrganization,
-  specialGroups: 'SPECIAL_INCLUDED' | 'SPECIAL_EXCLUDED',
-  graduated: 'GRADUATED_INCLUDED' | 'GRADUATED_EXCLUDED'
+  specialGroups: 'SPECIAL_INCLUDED' | 'SPECIAL_EXCLUDED'
 ) => {
   let years: string[] = []
   const programmeCodes = programmes.map(programme => programme.code)
   const newStats: StudyTrackStats[] = []
 
   for (const studyProgrammeCode of programmeCodes) {
-    const statsFromRedis = await getStudyTrackStats(studyProgrammeCode, null, graduated, specialGroups)
+    const statsFromRedis = await getStudyTrackStats(studyProgrammeCode, null, specialGroups)
     if (statsFromRedis) {
       newStats.push(statsFromRedis)
       if (!years.length) {
@@ -81,12 +80,11 @@ export const combineFacultyStudents = async (
     const updatedStats = await getStudyTrackStatsForStudyProgramme({
       studyProgramme: studyProgrammeCode,
       settings: {
-        graduated: graduated === 'GRADUATED_INCLUDED',
         specialGroups: specialGroups === 'SPECIAL_INCLUDED',
       },
       studyRightsOfProgramme,
     })
-    await setStudyTrackStats(updatedStats, graduated, specialGroups)
+    await setStudyTrackStats(updatedStats, specialGroups)
     if (!years.length) {
       years = updatedStats.years
     }

services/backend/src/services/faculty/facultyThesisWriters.ts

@@ -1,17 +1,10 @@
-import { cloneDeep } from 'lodash'
+import { cloneDeep } from 'lodash-es'
 
-import { DegreeProgrammeType } from '@oodikone/shared/types'
+import { DegreeProgrammeType, type NameWithCode } from '@oodikone/shared/types'
 import { getGraduationStats, setGraduationStats } from '../analyticsService'
 import { getGraduationStatsForStudyTrack } from '../studyProgramme/studyProgrammeGraduations'
 import { getDegreeProgrammesOfFaculty, ProgrammesOfOrganization } from './faculty'
 
-type ProgrammeName = {
-  code: string
-  en?: string
-  fi?: string
-  sv?: string
-}
-
 const calculateCombinedStats = async (
   isAcademicYear: boolean,
   includeAllSpecials: boolean,
@@ -28,7 +21,7 @@ const calculateCombinedStats = async (
   const newStats: Array<Awaited<ReturnType<typeof getGraduationStatsForStudyTrack>>> = []
   const combinedTableStats: Array<Array<string | number>> = []
   let years: Array<string | number> = []
-  const programmeNames: Record<string, ProgrammeName> = {}
+  const programmeNames: Record<string, NameWithCode> = {}
 
   for (const programme of facultyProgrammes) {
     if (
@@ -90,6 +83,7 @@ const calculateCombinedStats = async (
       isBachelor ? wroteThesis : 0,
       isMaster ? wroteThesis : 0,
     ])
+
     programmeTableStats[progId] = tableStatsForProgramme
 
     if (!combinedTableStats.length) {
@@ -135,7 +129,5 @@ export const combineFacultyThesisWriters = async (
     programmeTableStats,
     titles: ['Code', 'All', 'Bachelors', 'Masters'],
     programmeNames,
-    status: 'Done',
-    lastUpdated: '',
   }
 }

services/backend/src/services/faculty/facultyUpdates.ts

@@ -18,22 +18,15 @@ import { combineFacultyThesisWriters } from './facultyThesisWriters'
 const yearOptions = ['CALENDAR_YEAR', 'ACADEMIC_YEAR'] as const
 const specialGroupOptions = ['SPECIAL_INCLUDED', 'SPECIAL_EXCLUDED'] as const
 const programmeFilterOptions = ['ALL_PROGRAMMES', 'NEW_DEGREE_PROGRAMMES'] as const
-const graduatedOptions = ['GRADUATED_INCLUDED', 'GRADUATED_EXCLUDED'] as const
 
 export const updateFacultyOverview = async (facultyCode: string, statsType: StatsType) => {
-  const all = await getDegreeProgrammesOfFaculty(facultyCode, false)
-  const onlyNew = await getDegreeProgrammesOfFaculty(facultyCode, true)
+  for (const programmeFilter of programmeFilterOptions) {
+    const programmes = await getDegreeProgrammesOfFaculty(facultyCode, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
 
-  for (const yearType of yearOptions) {
-    for (const specialGroups of specialGroupOptions) {
-      for (const programmeFilter of programmeFilterOptions) {
+    for (const yearType of yearOptions) {
+      for (const specialGroups of specialGroupOptions) {
         if (statsType === 'ALL' || statsType === 'STUDENT') {
-          const updatedStudentInfo = await combineFacultyBasics(
-            facultyCode,
-            programmeFilter === 'NEW_DEGREE_PROGRAMMES' ? onlyNew : all,
-            yearType,
-            specialGroups
-          )
+          const updatedStudentInfo = await combineFacultyBasics(facultyCode, programmes, yearType, specialGroups)
           await setBasicStats(updatedStudentInfo, yearType, programmeFilter, specialGroups)
         }
         if ((statsType === 'ALL' || statsType === 'CREDITS') && specialGroups !== 'SPECIAL_EXCLUDED') {
@@ -43,7 +36,7 @@ export const updateFacultyOverview = async (facultyCode: string, statsType: Stat
         if (statsType === 'ALL' || statsType === 'THESIS') {
           const updateThesisWriters = await combineFacultyThesisWriters(
             facultyCode,
-            programmeFilter === 'NEW_DEGREE_PROGRAMMES' ? onlyNew : all,
+            programmes,
             yearType,
             specialGroups
           )
@@ -51,30 +44,22 @@ export const updateFacultyOverview = async (facultyCode: string, statsType: Stat
         }
       }
     }
+
+    const updatedTimes = await countGraduationTimes(facultyCode, programmes)
+    await setGraduationStats(updatedTimes, programmeFilter)
   }
 
-  const updatedTimesAll = await countGraduationTimes(facultyCode, all)
-  await setGraduationStats(updatedTimesAll, 'ALL_PROGRAMMES')
-  const updatedTimesNew = await countGraduationTimes(facultyCode, onlyNew)
-  await setGraduationStats(updatedTimesNew, 'NEW_DEGREE_PROGRAMMES')
   return 'OK'
 }
 
 export const updateFacultyProgressOverview = async (facultyCode: string) => {
   const onlyNew = await getDegreeProgrammesOfFaculty(facultyCode, true)
 
-  for (const graduated of graduatedOptions) {
-    for (const specialGroups of specialGroupOptions) {
-      const updateFacultyStudentStats = await combineFacultyStudents(facultyCode, onlyNew, specialGroups, graduated)
-      await setFacultyStudentStats(updateFacultyStudentStats, specialGroups, graduated)
-      const updateFacultyProgressStats = await combineFacultyStudentProgress(
-        facultyCode,
-        onlyNew,
-        specialGroups,
-        graduated
-      )
-      await setFacultyProgressStats(updateFacultyProgressStats, specialGroups, graduated)
-    }
+  for (const specialGroups of specialGroupOptions) {
+    const updateFacultyStudentStats = await combineFacultyStudents(facultyCode, onlyNew, specialGroups)
+    await setFacultyStudentStats(updateFacultyStudentStats, specialGroups)
+    const updateFacultyProgressStats = await combineFacultyStudentProgress(facultyCode, onlyNew, specialGroups)
+    await setFacultyProgressStats(updateFacultyProgressStats, specialGroups)
   }
 
   return 'OK'

services/backend/src/services/graduationHelpers.ts

@@ -1,13 +1,27 @@
-export const countTimeCategories = (times: number[], goal: number) => {
+/** Map years to semesters */
+export const GraduationTarget = {
+  HALF_YEAR: 1,
+  ONE_YEAR: 2,
+  TWO_YEARS: 4,
+  THREE_YEARS: 6,
+  THREE_POINT_FIVE_YEARS: 7,
+  FOUR_YEARS: 8,
+  FIVE_YEARS: 10,
+  SIX_YEARS: 12,
+} as const
+
+/** Uses semesters, not months */
+export const countTimeCategories = (times: number[], target: number) => {
   const statistics = { onTime: 0, yearOver: 0, wayOver: 0 }
-  times.forEach(time => {
-    if (time <= goal) {
-      statistics.onTime += 1
-    } else if (time <= goal + 12) {
-      statistics.yearOver += 1
+
+  for (const time of times) {
+    if (time <= target) {
+      statistics.onTime++
+    } else if (time <= target + GraduationTarget.ONE_YEAR) {
+      statistics.yearOver++
     } else {
-      statistics.wayOver += 1
+      statistics.wayOver++
     }
-  })
+  }
   return statistics
 }

services/backend/src/services/languageCenterData.js

@@ -1,10 +1,10 @@
-const { orderBy } = require('lodash')
-const { Op } = require('sequelize')
+import { orderBy } from 'lodash-es'
+import { Op } from 'sequelize'
 
-const { CourseModel, CreditModel, EnrollmentModel, SISStudyRightModel } = require('../models')
-const { redisClient } = require('./redis')
+import { CourseModel, CreditModel, EnrollmentModel, SISStudyRightModel } from '../models'
+import { redisClient } from './redis'
 
-const LANGUAGE_CENTER_REDIS_KEY = 'LANGUAGE_CENTER_DATA'
+export const LANGUAGE_CENTER_REDIS_KEY = 'LANGUAGE_CENTER_DATA'
 
 const isBetween = (start, date, end) => {
   return new Date(start).getTime() <= new Date(date).getTime() && new Date(date).getTime() <= new Date(end).getTime()
@@ -33,7 +33,7 @@ const getLanguageCenterCourses = async () => {
   return courses
 }
 
-const createArrayOfCourses = async (attempts, courses) => {
+export const createArrayOfCourses = async (attempts, courses) => {
   const fields = { completions: 0, enrollments: 0, difference: 0, rejected: 0 }
   const semesters = {}
   const faculties = {}
@@ -88,7 +88,7 @@ const createArrayOfCourses = async (attempts, courses) => {
   return courseList
 }
 
-const computeLanguageCenterData = async () => {
+export const computeLanguageCenterData = async () => {
   const courses = await getLanguageCenterCourses()
   const autumnSemester2017 = 135
 
@@ -218,17 +218,10 @@ const computeLanguageCenterData = async () => {
   return { tableData, faculties }
 }
 
-const getLanguageCenterData = async () => {
+export const getLanguageCenterData = async () => {
   const dataOnRedis = await redisClient.get(LANGUAGE_CENTER_REDIS_KEY)
   if (dataOnRedis) return JSON.parse(dataOnRedis)
   const freshData = await computeLanguageCenterData()
   await redisClient.set(LANGUAGE_CENTER_REDIS_KEY, JSON.stringify(freshData))
   return freshData
 }
-
-module.exports = {
-  computeLanguageCenterData,
-  createArrayOfCourses,
-  getLanguageCenterData,
-  LANGUAGE_CENTER_REDIS_KEY,
-}

services/backend/src/services/mailService.ts

@@ -2,7 +2,7 @@ import axios from 'axios'
 
 import { isProduction, pateToken } from '../config'
 import { FormattedUser } from '../types'
-import { ApplicationError } from '../util/customErrors'
+import logger from '../util/logger'
 
 const pateClient = axios.create({
   baseURL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/pate/',
@@ -22,18 +22,12 @@ const baseSettings = {
 }
 
 const sendEmail = async (options = {}) => {
-  if (!isProduction) throw new ApplicationError('Email sending is disabled in development mode.')
-  if (!pateToken) throw new ApplicationError('Email sending failed because pate token is missing.')
+  if (!isProduction) return logger.error('Email sending is disabled in development mode.')
+  if (!pateToken) return logger.error('Email sending failed because pate token is missing.')
   return await pateClient.post('/', options)
 }
 
-export const sendFeedbackToToska = async ({
-  feedbackContent,
-  user,
-}: {
-  feedbackContent: string
-  user: FormattedUser
-}) => {
+export const sendFeedback = async ({ feedbackContent, user }: { feedbackContent: string; user: FormattedUser }) => {
   const { name, userId, email } = user
   const userDetails = `Sent by ${name}, userid: ${userId}, email: ${email}`
   const text = [feedbackContent, userDetails].join('<br />')

services/backend/src/services/populations/closeToGraduation.ts

@@ -286,9 +286,14 @@ export const getCloseToGraduationData = async (studentNumbers?: string[]) => {
     if (dataOnRedis) {
       return JSON.parse(dataOnRedis)
     }
-    const freshData = { ...(await findStudentsCloseToGraduation()), lastUpdated: new Date().toISOString() }
+
+    const students = await findStudentsCloseToGraduation()
+
+    const freshData = { ...students, lastUpdated: new Date().toISOString() }
     await redisClient.set(CLOSE_TO_GRADUATION_REDIS_KEY, JSON.stringify(freshData))
+
     return freshData
   }
+
   return findStudentsCloseToGraduation(studentNumbers)
 }

services/backend/src/services/populations/formatStatisticsForApi.ts

@@ -1,21 +1,10 @@
 import { SISStudyRight } from '@oodikone/shared/models'
-import { FormattedStudent, CriteriaYear, Name, ProgressCriteria } from '@oodikone/shared/types'
+import { FormattedStudent, Name } from '@oodikone/shared/types'
 import { StudentData, StudentTags } from '@oodikone/shared/types/studentData'
-import { dateYearsFromNow, dateDaysFromNow, dayInMilliseconds } from '@oodikone/shared/util/datetime'
-import { CreditModel } from '../../models'
+import { dateDaysFromNow, dayInMilliseconds } from '@oodikone/shared/util/datetime'
 import { hasTransferredFromOrToProgramme } from '../studyProgramme/studyProgrammeHelpers'
-import type { StudentStudyPlan, StudentStudyRight } from './getStudentData'
+import type { StudentStudyRight } from './getStudentData'
 import { getCurriculumVersion } from './shared'
-import type { AnonymousCredit, AnonymousEnrollment } from './statisticsOf'
-
-const yearMap: [string, keyof ProgressCriteria['courses']][] = [
-  ['year1', 'yearOne'],
-  ['year2', 'yearTwo'],
-  ['year3', 'yearThree'],
-  ['year4', 'yearFour'],
-  ['year5', 'yearFive'],
-  ['year6', 'yearSix'],
-]
 
 const getTransferSource = (code: string, studyRights: StudentStudyRight[]): [boolean, string | undefined] => {
   if (code) {
@@ -48,136 +37,18 @@ const getTransferSource = (code: string, studyRights: StudentStudyRight[]): [boo
   return [false, undefined]
 }
 
-const getCriteriaBase = (criteria: ProgressCriteria): [boolean, Record<string, CriteriaYear>] => {
-  const thereAreCriteriaCourses = !!Object.values(criteria.courses).flatMap(val => val).length
-  const thereAreCriteriaCredits = !!Object.values(criteria.credits).reduce((acc, cur) => acc + cur, 0)
-
-  const createEmptyCriteriaYear = (criteria: ProgressCriteria, year: keyof ProgressCriteria['courses']) => ({
-    credits: false,
-    totalSatisfied: 0,
-    coursesSatisfied: Object.fromEntries(criteria.courses[year].map(course => [course, null])),
-  })
-
-  const criteriaChecked: Record<string, CriteriaYear> = {
-    year1: createEmptyCriteriaYear(criteria, 'yearOne'),
-    year2: createEmptyCriteriaYear(criteria, 'yearTwo'),
-    year3: createEmptyCriteriaYear(criteria, 'yearThree'),
-    year4: createEmptyCriteriaYear(criteria, 'yearFour'),
-    year5: createEmptyCriteriaYear(criteria, 'yearFive'),
-    year6: createEmptyCriteriaYear(criteria, 'yearSix'),
-  }
-
-  return [thereAreCriteriaCourses || thereAreCriteriaCredits, criteriaChecked]
-}
-
-const getProgressCriteria = (
-  startDate: string,
-  criteria: ProgressCriteria,
-  credits: AnonymousCredit[],
-  hops: StudentStudyPlan | undefined
-) => {
-  const startDateFromISO = new Date(startDate)
-
-  const criteriaCoursesBySubstitutions: Record<string, string> = Object.fromEntries(
-    Object.entries(criteria.allCourses).flatMap(([courseCode, substitutionCodes]) => [
-      [courseCode, courseCode],
-      ...substitutionCodes.map(code => [code, courseCode]),
-    ])
-  )
-
-  const [thereAreCriteria, criteriaChecked] = getCriteriaBase(criteria)
-  if (!thereAreCriteria) return criteriaChecked
-
-  const academicYears = { year1: 0, year2: 0, year3: 0, year4: 0, year5: 0, year6: 0 }
-
-  const courses = credits.map(({ attainment_date, course_code, credits, credittypecode }) => ({
-    course_code,
-    credits,
-    credittypecode,
-    date:
-      attainment_date < startDateFromISO
-        ? dateDaysFromNow(startDateFromISO, 1).toISOString()
-        : attainment_date.toISOString(),
-  }))
-
-  courses
-    .filter(({ course_code }) => !!criteriaCoursesBySubstitutions[course_code])
-    .filter(({ credittypecode }) => CreditModel.passed({ credittypecode }) || CreditModel.improved({ credittypecode }))
-    .forEach(course => {
-      const courseDate = new Date(course.date)
-      const correctCode = criteriaCoursesBySubstitutions[course.course_code]
-
-      yearMap.forEach(([yearToAdd, criteriaYear]) => {
-        if (criteria.courses[criteriaYear].includes(correctCode)) {
-          const currentDate = criteriaChecked[yearToAdd].coursesSatisfied[correctCode]
-          if (!currentDate || courseDate < new Date(currentDate)) {
-            criteriaChecked[yearToAdd].coursesSatisfied[correctCode] = course.date
-          }
-        }
-      })
-    })
-
-  courses.forEach(course => {
-    const courseDate = new Date(course.date)
-    const correctCode = criteriaCoursesBySubstitutions[course.course_code]
-
-    if (
-      startDateFromISO < courseDate &&
-      !!hops &&
-      (hops.included_courses.includes(course.course_code) || hops.included_courses.includes(correctCode))
-    )
-      Object.keys(academicYears)
-        .filter((_, index) => courseDate < dateYearsFromNow(startDateFromISO, index + 1))
-        .forEach(year => (academicYears[year] += course.credits))
-  })
-
-  yearMap.forEach(([yearToAdd, criteriaYear]) => {
-    criteriaChecked[yearToAdd].totalSatisfied +=
-      Object.values(criteriaChecked[yearToAdd].coursesSatisfied).filter(course => !!course).length ?? 0
-    // UPDATE CREDIT CRITERIA
-    if (!!criteria.credits[criteriaYear] && criteria.credits[criteriaYear] < academicYears[yearToAdd]) {
-      criteriaChecked[yearToAdd].credits = true
-      criteriaChecked[yearToAdd].totalSatisfied += 1
-    }
-  })
-
-  return criteriaChecked
-}
-
 export const formatStudentForAPI = (
   code: string,
   startDate: string,
   student: StudentData,
   tags: StudentTags[],
-  credits: AnonymousCredit[],
-  enrollments: AnonymousEnrollment[],
-  optionData: Name | undefined,
-  criteria: ProgressCriteria
-): FormattedStudent => {
+  optionData: Name | undefined
+): Omit<FormattedStudent, 'criteriaProgress' | 'courses' | 'enrollments'> => {
   const { studentnumber, studyRights, studyplans } = student
 
   const hops = studyplans.find(plan => plan.programme_code === code)
   const [transferredStudyright, transferSource] = getTransferSource(code, studyRights)
 
-  const courses = credits.map(credit => {
-    const attainmentDateNormalized = credit.attainment_date.toISOString()
-    const passed =
-      CreditModel.passed({ credittypecode: credit.credittypecode }) ||
-      CreditModel.improved({ credittypecode: credit.credittypecode })
-
-    return {
-      course_code: credit.course_code,
-      date: attainmentDateNormalized,
-      passed,
-      grade: passed ? credit.grade : 'Hyl.',
-      credits: credit.credits,
-      isStudyModuleCredit: credit.isStudyModule,
-      credittypecode: credit.credittypecode,
-      language: credit.language,
-      studyright_id: credit.studyright_id,
-    }
-  })
-
   return {
     firstnames: student.firstnames,
     lastname: student.lastname,
@@ -197,15 +68,13 @@ export const formatStudentForAPI = (
     birthdate: student.birthdate,
     sis_person_id: student.sis_person_id,
     citizenships: student.citizenships,
-    criteriaProgress: getProgressCriteria(startDate, criteria, credits, hops),
     curriculumVersion: getCurriculumVersion(hops?.curriculum_period_id),
+    hasPersonalIdentityCode: student.hasPersonalIdentityCode,
 
     tags,
     transferredStudyright,
     transferSource,
     studyRights,
     studyplans,
-    courses,
-    enrollments,
   }
 }

services/backend/src/services/populations/getStudentData.ts

@@ -17,7 +17,7 @@ export type StudentTags = TagStudent & {
   tag: Pick<Tag, 'tag_id' | 'tagname' | 'personal_user_id'>
 }
 
-export const getStudentTags = async (studyRights: string[], studentNumbers: string[], userId: string) => {
+export const getStudentTagMap = async (studyRights: string[], studentNumbers: string[], userId: string) => {
   const studentTags = await TagStudentModel.findAll({
     attributes: ['tag_id', 'studentnumber'],
     include: {
@@ -33,19 +33,24 @@ export const getStudentTags = async (studyRights: string[], studentNumbers: stri
     },
   })
 
-  const studentTagList: Record<string, StudentTags[]> = Object.fromEntries(studentNumbers.map(n => [n, []]))
-  studentTags.forEach(studentTag => studentTagList[studentTag.studentnumber].push(studentTag))
+  const studentTagMap = new Map<string, StudentTags[]>()
+  for (const studentTag of studentTags) {
+    const data = studentTagMap.get(studentTag.studentnumber) ?? []
 
-  return studentTagList
+    data.push(studentTag)
+    studentTagMap.set(studentTag.studentnumber, data)
+  }
+
+  return studentTagMap
 }
 
 export type StudentEnrollment = Pick<
   Enrollment,
-  'course_code' | 'state' | 'enrollment_date_time' | 'semestercode' | 'studentnumber'
+  'course_code' | 'state' | 'enrollment_date_time' | 'semestercode' | 'studentnumber' | 'studyright_id'
 >
 export const getEnrollments = (studentNumbers: string[], startDate: string): Promise<Array<StudentEnrollment>> =>
   EnrollmentModel.findAll({
-    attributes: ['course_code', 'state', 'enrollment_date_time', 'semestercode', 'studentnumber'],
+    attributes: ['course_code', 'state', 'enrollment_date_time', 'semestercode', 'studentnumber', 'studyright_id'],
     where: {
       studentnumber: {
         [Op.in]: studentNumbers,
@@ -74,6 +79,7 @@ type StudentPersonalData = Pick<
   | 'gender_code'
   | 'birthdate'
   | 'sis_person_id'
+  | 'hasPersonalIdentityCode'
 >
 
 type StudentStudyRightElement = Pick<
@@ -120,6 +126,7 @@ export const getStudents = (studentNumbers: string[]): Promise<Array<StudentData
       'gender_code',
       'birthdate',
       'sis_person_id',
+      'hasPersonalIdentityCode',
     ],
     include: [
       {
@@ -203,3 +210,22 @@ export const getCredits = async (studentNumbers: string[]): Promise<Array<Studen
     },
     raw: true,
   })
+
+export const getStudyRightElementsForStudyRight = async (studentNumbers: string[], code: string) =>
+  SISStudyRightElementModel.findAll({
+    attributes: [],
+    where: { code },
+    include: {
+      model: SISStudyRightModel,
+      attributes: ['studentNumber'],
+      where: {
+        studentNumber: { [Op.in]: studentNumbers },
+      },
+      include: [
+        {
+          model: SISStudyRightElementModel,
+          attributes: ['code', 'name', 'degreeProgrammeType', 'startDate', 'endDate'],
+        },
+      ],
+    },
+  })

services/backend/src/services/populations/shared.ts

@@ -1,11 +1,10 @@
-import { orderBy } from 'lodash'
+import { orderBy } from 'lodash-es'
 import { Op } from 'sequelize'
 
-import type { CourseStats } from '@oodikone/shared/routes/populations'
-import { Name, DegreeProgrammeType, EnrollmentState } from '@oodikone/shared/types'
-import { SISStudyRightModel, SISStudyRightElementModel, CreditModel, CourseModel } from '../../models'
-import { getPassingSemester, SemesterStart } from '../../util/semester'
-import { StudentCredit, StudentEnrollment } from './getStudentData'
+import { CourseStats } from '@oodikone/shared/routes/populations'
+import { Name, DegreeProgrammeType } from '@oodikone/shared/types'
+import { SISStudyRightElementModel, CourseModel } from '../../models'
+import { SemesterStart } from '../../util/semester'
 
 type QueryParams = {
   semesters: string[]
@@ -47,254 +46,41 @@ export const getCurriculumVersion = (curriculumPeriodId: string | undefined) =>
   return curriculumVersion
 }
 
-export const getOptionsForStudents = async (
-  studentNumbers: string[],
-  code: string,
+export const getOptionsForStudents = (
+  studyRightElementsForStudyRight: SISStudyRightElementModel[],
   degreeProgrammeType: DegreeProgrammeType | null
-): Promise<Record<string, Name>> => {
-  if (!code || !studentNumbers.length) {
-    return {}
-  } else if (
+): Map<string, Name> => {
+  if (
     degreeProgrammeType &&
     ![DegreeProgrammeType.BACHELOR, DegreeProgrammeType.MASTER].includes(degreeProgrammeType)
   ) {
-    return {}
+    return new Map()
   }
 
-  const studyRightElementsForStudyRight = await SISStudyRightElementModel.findAll({
-    attributes: [],
-    where: { code },
-    include: {
-      model: SISStudyRightModel,
-      attributes: ['studentNumber'],
-      where: {
-        studentNumber: { [Op.in]: studentNumbers },
-      },
-      include: [
-        {
-          model: SISStudyRightElementModel,
-          attributes: ['code', 'name', 'degreeProgrammeType', 'startDate', 'endDate'],
-        },
-      ],
-    },
-  })
+  const levelIsMasters = degreeProgrammeType === DegreeProgrammeType.MASTER
+  const filter = levelIsMasters ? DegreeProgrammeType.BACHELOR : DegreeProgrammeType.MASTER
 
-  return Object.fromEntries(
-    studyRightElementsForStudyRight
-      .map(({ studyRight }) => {
-        const levelIsMasters = degreeProgrammeType === DegreeProgrammeType.MASTER
-        const filter = levelIsMasters ? DegreeProgrammeType.BACHELOR : DegreeProgrammeType.MASTER
+  const optionMap = new Map()
+  for (const { studyRight } of studyRightElementsForStudyRight) {
+    const [latestProgramme] = orderBy(
+      studyRight.studyRightElements.filter(element => element.degreeProgrammeType === filter),
+      [levelIsMasters ? 'endDate' : 'startDate'],
+      [levelIsMasters ? 'desc' : 'asc']
+    )
 
-        // NOTE: If in masters, then select latest finished bachlor studyRight otherwise select first started masters studyRight
-        const [latestProgramme] = orderBy(
-          studyRight.studyRightElements.filter(element => element.degreeProgrammeType === filter),
-          [levelIsMasters ? 'endDate' : 'startDate'],
-          [levelIsMasters ? 'desc' : 'asc']
-        )
+    if (latestProgramme?.name === undefined) continue
 
-        return [studyRight.studentNumber, latestProgramme?.name]
-      })
-      .filter(([_, name]) => !!name)
-  )
+    optionMap.set(studyRight.studentNumber, latestProgramme.name)
+  }
+
+  return optionMap
 }
 
-export const getCourses = (courses: string[]): Promise<Array<Pick<CourseModel, 'code' | 'name' | 'substitutions'>>> =>
+export const getCourses = (courses: string[]): Promise<Array<CourseStats>> =>
   CourseModel.findAll({
-    attributes: ['code', 'name', 'substitutions'],
+    attributes: ['code', 'name', 'substitutions', 'substitution_groups', 'is_study_module'],
     where: {
       code: { [Op.in]: courses },
     },
     raw: true,
   })
-
-const defaultCourse = {
-  attempts: 0,
-  enrollments: {
-    [EnrollmentState.ENROLLED]: new Set<string>(),
-    [EnrollmentState.REJECTED]: new Set<string>(),
-    semesters: {
-      BEFORE: {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-      '0-FALL': {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-      '0-SPRING': {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-      '1-FALL': {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-      '1-SPRING': {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-      '2-FALL': {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-      '2-SPRING': {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-      '3-FALL': {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-      '3-SPRING': {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-      '4-FALL': {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-      '4-SPRING': {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-      '5-FALL': {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-      '5-SPRING': {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-      LATER: {
-        [EnrollmentState.ENROLLED]: new Set<string>(),
-        [EnrollmentState.REJECTED]: new Set<string>(),
-      },
-    },
-  },
-  grades: {},
-  students: {
-    all: new Set<string>(),
-    passed: new Set<string>(),
-    failed: new Set<string>(),
-    improvedPassedGrade: new Set<string>(),
-    markedToSemester: new Set<string>(),
-    enrolledNoGrade: new Set<string>(),
-  },
-
-  stats: {
-    passingSemesters: {
-      BEFORE: 0,
-      '0-FALL': 0,
-      '0-SPRING': 0,
-      '1-FALL': 0,
-      '1-SPRING': 0,
-      '2-FALL': 0,
-      '2-SPRING': 0,
-      '3-FALL': 0,
-      '3-SPRING': 0,
-      '4-FALL': 0,
-      '4-SPRING': 0,
-      '5-FALL': 0,
-      '5-SPRING': 0,
-      LATER: 0,
-    },
-  },
-}
-
-export const parseCourseData = async (
-  studentStartingYears: Map<string, number>,
-  enrollments: StudentEnrollment[],
-  credits: StudentCredit[]
-): Promise<CourseStats[]> => {
-  const getYear = (studentnumber: string) => studentStartingYears.get(studentnumber)!
-
-  const coursestats = new Map<string, typeof defaultCourse>()
-  for (const enrollment of enrollments) {
-    const { course_code, studentnumber, state, enrollment_date_time } = enrollment
-
-    // We cannot display these
-    if (course_code === null) continue
-
-    if (!coursestats.has(course_code)) coursestats.set(course_code, structuredClone(defaultCourse))
-    const course = coursestats.get(course_code)!
-
-    const initialDate = new Date(enrollment_date_time)
-    const semester = getPassingSemester(getYear(studentnumber), initialDate)
-
-    course.students.all.add(studentnumber)
-    course.students.enrolledNoGrade.add(studentnumber)
-    course.enrollments[state].add(studentnumber)
-    course.enrollments.semesters[semester][state].add(studentnumber)
-  }
-
-  for (const credit of credits) {
-    const { course_code, student_studentnumber: studentnumber, grade, attainment_date: date } = credit
-
-    // We cannot display these
-    if (course_code === null) continue
-
-    const passingGrade = CreditModel.passed(credit)
-    const failingGrade = CreditModel.failed(credit)
-    const improvedGrade = CreditModel.improved(credit)
-
-    if (!coursestats.has(course_code)) coursestats.set(course_code, structuredClone(defaultCourse))
-    const course = coursestats.get(course_code)!
-
-    course.attempts += 1
-    const gradeCount = course.grades[grade]?.count ?? 0
-    course.grades[grade] = { count: gradeCount + 1, status: { passingGrade, failingGrade, improvedGrade } }
-
-    course.students.all.add(studentnumber)
-    course.students.enrolledNoGrade.delete(studentnumber)
-    if (passingGrade) {
-      if (!course.students.markedToSemester.has(studentnumber)) {
-        course.students.markedToSemester.add(studentnumber)
-
-        const semester = getPassingSemester(getYear(studentnumber), new Date(date))
-        course.stats.passingSemesters[semester]++
-      }
-
-      course.students.passed.add(studentnumber)
-      course.students.failed.delete(studentnumber)
-    } else if (improvedGrade) {
-      course.students.improvedPassedGrade.add(studentnumber)
-      course.students.passed.add(studentnumber)
-      course.students.failed.delete(studentnumber)
-    } else if (failingGrade && !course.students.passed.has(studentnumber)) {
-      course.students.failed.add(studentnumber)
-    }
-  }
-
-  const courses = await getCourses(Array.from(coursestats.keys()))
-  const courseMap = new Map(courses.map(({ code, name }) => [code, name]))
-  const substitutionMap = new Map(courses.map(({ code, substitutions }) => [code, substitutions]))
-
-  return Array.from(coursestats.entries()).map(([code, { attempts, enrollments, grades, students, stats }]) => {
-    return {
-      course: {
-        code,
-        name: courseMap.get(code) ?? { en: code },
-        substitutions: substitutionMap.get(code) ?? [],
-      },
-      attempts,
-      enrollments: {
-        [EnrollmentState.ENROLLED]: Array.from(enrollments[EnrollmentState.ENROLLED]),
-        [EnrollmentState.REJECTED]: Array.from(enrollments[EnrollmentState.REJECTED]),
-        semesters: Object.fromEntries(
-          Object.entries(enrollments.semesters).map(([key, val]) => [
-            key,
-            {
-              [EnrollmentState.ENROLLED]: Array.from(val[EnrollmentState.ENROLLED]),
-              [EnrollmentState.REJECTED]: Array.from(val[EnrollmentState.REJECTED]),
-            },
-          ])
-        ),
-      },
-      grades,
-      students: Object.fromEntries(
-        Object.entries(students).map(([key, val]) => [key, Array.from(val)])
-      ) as CourseStats['students'],
-      stats,
-    }
-  })
-}

services/backend/src/services/populations/statisticsOf.ts

@@ -1,4 +1,3 @@
-import { omitKeys } from '@oodikone/shared/util'
 import { getDegreeProgrammeType } from '../../util'
 import { getCriteria } from '../studyProgramme/studyProgrammeCriteria'
 import { formatStudentForAPI } from './formatStatisticsForApi'
@@ -9,8 +8,9 @@ import {
   getStudents,
   getEnrollments,
   getCredits,
+  getStudyRightElementsForStudyRight,
 } from './getStudentData'
-import { getOptionsForStudents, parseCourseData } from './shared'
+import { getCourses, getOptionsForStudents } from './shared'
 
 export type OptimizedStatisticsQuery = {
   userId: string
@@ -29,58 +29,38 @@ export type StudentCreditObject = Map<StudentCredit['student_studentnumber'], An
 export const statisticsOf = async (
   studentNumbers: string[],
   studyRights: string[],
-  tagList: Record<string, StudentTags[]>,
+  tagMap: Map<string, StudentTags[]>,
   startDate?: string
 ) => {
   const code = studyRights[0] ?? ''
-  const mockedStartDate = startDate ?? new Date(1900).toISOString()
+  const mockedStartDate = startDate ?? new Date(1900, 0, 1).toISOString()
 
-  const [enrollments, credits, students] = await Promise.all([
-    getEnrollments(studentNumbers, mockedStartDate),
-    getCredits(studentNumbers),
-    getStudents(studentNumbers),
-  ])
-
-  const studentStartingYears = new Map(
-    students.map(({ studentnumber, studyRights }) => [
-      studentnumber,
-      studyRights
-        ?.flatMap(({ studyRightElements }) => studyRightElements)
-        .find(element => element.code === code)
-        ?.startDate.getFullYear() ?? +mockedStartDate,
+  const [students, enrollments, credits, degreeProgrammeType, criteria, studyRightElementsForStudyRight] =
+    await Promise.all([
+      getStudents(studentNumbers),
+      getEnrollments(studentNumbers, mockedStartDate),
+      getCredits(studentNumbers),
+      getDegreeProgrammeType(code),
+      getCriteria(code),
+      getStudyRightElementsForStudyRight(studentNumbers, code),
     ])
-  )
 
-  const formattedCoursestats = await parseCourseData(studentStartingYears, enrollments, credits)
+  const courseCodes = new Set<string>()
+  for (const { course_code } of credits) courseCodes.add(course_code)
+  for (const { course_code } of enrollments) courseCodes.add(course_code)
+  const courses = await getCourses(Array.from(courseCodes))
 
-  const creditsByStudent: StudentCreditObject = new Map<string, AnonymousCredit[]>(studentNumbers.map(n => [n, []]))
-  credits.forEach(credit =>
-    creditsByStudent.get(credit.student_studentnumber)!.push(omitKeys(credit, ['student_studentnumber']))
-  )
+  const optionData = getOptionsForStudents(studyRightElementsForStudyRight, degreeProgrammeType)
+  const formattedStudents = students.map(student => {
+    const tags = tagMap.get(student.studentnumber) ?? []
+    const options = optionData.get(student.studentnumber)
 
-  const enrollmentsByStudent: StudentEnrollmentObject = new Map<string, AnonymousEnrollment[]>(
-    studentNumbers.map(n => [n, []])
-  )
-  enrollments.forEach(enrollment =>
-    enrollmentsByStudent.get(enrollment.studentnumber)!.push(omitKeys(enrollment, ['studentnumber']))
-  )
-
-  const criteria = await getCriteria(code)
-  const optionData = await getOptionsForStudents(studentNumbers, code, await getDegreeProgrammeType(code))
+    return formatStudentForAPI(code, mockedStartDate, student, tags, options)
+  })
 
   return {
-    coursestatistics: formattedCoursestats,
-    students: students.map(student =>
-      formatStudentForAPI(
-        code,
-        mockedStartDate,
-        student,
-        tagList[student.studentnumber],
-        creditsByStudent.get(student.studentnumber)!,
-        enrollmentsByStudent.get(student.studentnumber)!,
-        optionData[student.studentnumber],
-        criteria
-      )
-    ),
+    students: formattedStudents,
+    criteria,
+    coursestatistics: { courses, enrollments, credits },
   }
 }

services/backend/src/services/providerCredits.ts

@@ -1,9 +1,11 @@
 import { SISStudyRight } from '@oodikone/shared/models'
 import { EnrollmentType, ExtentCode } from '@oodikone/shared/types'
+import { CreditStats } from '@oodikone/shared/types/studyProgramme'
 import { mapToProviders } from '@oodikone/shared/util'
 import { getCreditStats, setCreditStats } from './analyticsService'
 import { getCourseCodesOfProvider } from './providers'
 import { getCreditsForProvider, getTransferredCredits } from './studyProgramme/creditGetters'
+import { getStudentHetuStateMap } from './studyProgramme/studentGetters'
 import { defineYear } from './studyProgramme/studyProgrammeHelpers'
 import { getSISStudyRightsOfStudents } from './studyProgramme/studyRightFinders'
 
@@ -46,15 +48,17 @@ const basicDegreeExtentCodes = [ExtentCode.BACHELOR, ExtentCode.MASTER, ExtentCo
   This is probably very rare to cause differences, only one such was found in MH60_001
 */
 
-// TODO: How should licentiate and doctorate studies be categorized?
-const getCategory = (extentCode?: ExtentCode, degreeStudyRightExtentCode?: ExtentCode) => {
+const getCategory = (hasHetu: boolean, extentCode?: ExtentCode, degreeStudyRightExtentCode?: ExtentCode) => {
   if (degreeStudyRightExtentCode && basicDegreeExtentCodes.includes(degreeStudyRightExtentCode)) {
     return 'basic'
   }
   switch (extentCode) {
     case ExtentCode.OPEN_UNIVERSITY_STUDIES:
-    case ExtentCode.SUMMER_AND_WINTER_SCHOOL:
-      return 'open-uni'
+      if (hasHetu) {
+        return 'open-uni-hetu'
+      } else {
+        return 'open-uni-no-hetu'
+      }
     case ExtentCode.MASTER:
     case ExtentCode.BACHELOR_AND_MASTER:
     case ExtentCode.BACHELOR:
@@ -62,6 +66,7 @@ const getCategory = (extentCode?: ExtentCode, degreeStudyRightExtentCode?: Exten
     case ExtentCode.NON_DEGREE_STUDIES:
     case ExtentCode.NON_DEGREE_PEGAGOGICAL_STUDIES_FOR_TEACHERS:
     case ExtentCode.NON_DEGREE_PROGRAMME_FOR_SPECIAL_EDUCATION_TEACHERS:
+    case ExtentCode.SPECIALIZATION_STUDIES:
       return 'separate'
     case ExtentCode.EXCHANGE_STUDIES:
     case ExtentCode.EXCHANGE_STUDIES_POSTGRADUATE:
@@ -71,9 +76,9 @@ const getCategory = (extentCode?: ExtentCode, degreeStudyRightExtentCode?: Exten
     case ExtentCode.DOCTOR:
     case ExtentCode.LICENTIATE:
     case ExtentCode.CONTINUING_EDUCATION:
-    case ExtentCode.SPECIALIZATION_STUDIES:
     case ExtentCode.SPECIALIST_TRAINING_IN_MEDICINE_AND_DENTISTRY:
     case ExtentCode.STUDIES_FOR_SECONDARY_SCHOOL_STUDENTS:
+    case ExtentCode.SUMMER_AND_WINTER_SCHOOL:
     case undefined:
       return 'other'
     default:
@@ -81,8 +86,14 @@ const getCategory = (extentCode?: ExtentCode, degreeStudyRightExtentCode?: Exten
   }
 }
 
-const getBasicDegreeStudyRight = (studyRights: Array<SISStudyRight> | undefined, date: Date, semestercode: number) => {
-  return studyRights?.find(studyRight => {
+const getBasicDegreeStudyRight = (
+  studyRights:
+    | Pick<SISStudyRight, 'id' | 'studentNumber' | 'extentCode' | 'semesterEnrollments' | 'startDate' | 'endDate'>[]
+    | undefined,
+  date: Date,
+  semestercode: number
+) =>
+  studyRights?.find(studyRight => {
     const rightDates =
       new Date(studyRight.startDate).getTime() <= new Date(date).getTime() &&
       new Date(date).getTime() <= new Date(studyRight.endDate).getTime()
@@ -91,10 +102,9 @@ const getBasicDegreeStudyRight = (studyRights: Array<SISStudyRight> | undefined,
     )
     return basicDegreeExtentCodes.includes(studyRight.extentCode) && rightDates && enrolledAsPresent
   })
-}
 
 /* Calculates credits produced by provider (programme or faculty) */
-export const computeCreditsProduced = async (providerCode: string, isAcademicYear: boolean) => {
+export const computeCreditsProduced = async (providerCode: string, isAcademicYear: boolean): Promise<CreditStats> => {
   const since = new Date('2017-01-01')
   const rapoFormattedProviderCode = mapToProviders([providerCode])[0]
   const courses = await getCourseCodesOfProvider(rapoFormattedProviderCode)
@@ -116,40 +126,43 @@ export const computeCreditsProduced = async (providerCode: string, isAcademicYea
 
   const students = [...new Set(credits.map(({ studentNumber }) => studentNumber))]
 
-  const studyRights = await getSISStudyRightsOfStudents(students)
+  const [studyRights, studentToHetuStateMap] = await Promise.all([
+    getSISStudyRightsOfStudents(students),
+    getStudentHetuStateMap(students),
+  ])
 
-  const studyRightIdToStudyRightMap = studyRights.reduce<Record<string, SISStudyRight>>((obj, cur) => {
-    obj[cur.id] = cur
-    return obj
-  }, {})
+  const studyRightIdToStudyRightMap = new Map(studyRights.map(sr => [sr.id, sr]))
 
-  const studentNumberToStudyRightsMap = studyRights.reduce<Record<string, Array<SISStudyRight>>>((obj, cur) => {
-    if (!obj[cur.studentNumber]) {
-      obj[cur.studentNumber] = []
+  const studentNumberToStudyRightsMap = studyRights.reduce((map, cur) => {
+    // Node does not support getOrInsert as of writing :-(
+    if (map.has(cur.studentNumber)) {
+      map.get(cur.studentNumber)!.push(cur)
+    } else {
+      map.set(cur.studentNumber, [cur])
     }
-    obj[cur.studentNumber].push(cur)
-    return obj
-  }, {})
+    return map
+  }, new Map<string, [(typeof studyRights)[number]]>())
 
   const stats: Record<string, Record<string, number>> = {}
 
   for (const { attainmentDate, studentNumber, credits: numOfCredits, studyrightId, semestercode } of credits) {
-    const studyRightLinkedToAttainment = studyRightIdToStudyRightMap[studyrightId]
+    const studyRightLinkedToAttainment = studyRightIdToStudyRightMap.get(studyrightId)
     const attainmentYear = defineYear(attainmentDate, isAcademicYear)
     const degreeStudyRight = getBasicDegreeStudyRight(
-      studentNumberToStudyRightsMap[studentNumber],
+      studentNumberToStudyRightsMap.get(studentNumber),
       attainmentDate,
       semestercode
     )
+    const hasHetu = !!studentToHetuStateMap.get(studentNumber)
 
     if (!studyRightLinkedToAttainment && !degreeStudyRight) {
       continue
     }
 
-    const category = getCategory(studyRightLinkedToAttainment?.extentCode, degreeStudyRight?.extentCode)
+    const category = getCategory(hasHetu, studyRightLinkedToAttainment?.extentCode, degreeStudyRight?.extentCode)
     stats[attainmentYear] ??= {}
     stats[attainmentYear][category] ??= 0
-    stats[attainmentYear][category] += numOfCredits || 0
+    stats[attainmentYear][category] += numOfCredits
   }
 
   const transferredCredits = await getTransferredCredits(rapoFormattedProviderCode, since)
@@ -161,18 +174,18 @@ export const computeCreditsProduced = async (providerCode: string, isAcademicYea
     const attainmentYear = defineYear(createdate, isAcademicYear)
     stats[attainmentYear] ??= {}
     stats[attainmentYear].transferred ??= 0
-    stats[attainmentYear].transferred += credits || 0
+    stats[attainmentYear].transferred += credits
   })
 
   return { stats, id: providerCode }
 }
 
 export const getCreditsProduced = async (providerCode: string, isAcademicYear: boolean, specialIncluded = true) => {
-  let data = await getCreditStats(providerCode, isAcademicYear, specialIncluded)
+  const data = await getCreditStats(providerCode, isAcademicYear, specialIncluded)
   if (data) {
     return data
   }
-  data = await computeCreditsProduced(providerCode, isAcademicYear)
-  await setCreditStats(data, isAcademicYear, specialIncluded)
-  return data
+  const newData = await computeCreditsProduced(providerCode, isAcademicYear)
+  await setCreditStats(newData, isAcademicYear, specialIncluded)
+  return newData
 }

services/backend/src/services/semesters.ts

@@ -26,7 +26,7 @@ type SemestersAndYears = {
   >
 }
 
-export const getSemesterNames = async (semesterCodes: number[]) => {
+export const getSemesterNamesByCode = async (semesterCodes: number[]): Promise<Record<number, Name>> => {
   const semesters = await SemesterModel.findAll({
     attributes: ['name', 'semestercode'],
     where: {
@@ -35,7 +35,7 @@ export const getSemesterNames = async (semesterCodes: number[]) => {
     raw: true,
   })
 
-  return Object.fromEntries(semesters?.map(({ name, semestercode }) => [semestercode, name]) ?? [])
+  return Object.fromEntries(semesters.map(({ name, semestercode }) => [semestercode, name]))
 }
 
 export const getCurrentSemester = async () => {

services/backend/src/services/students.ts

@@ -2,7 +2,8 @@ import { Op, QueryTypes } from 'sequelize'
 
 import type { Credit, Student } from '@oodikone/shared/models'
 import { EnrollmentState, UnifyStatus } from '@oodikone/shared/types'
-import { StudentPageStudent } from '@oodikone/shared/types/studentData'
+import { FormattedStudentForSearch, StudentPageStudent } from '@oodikone/shared/types/studentData'
+import { splitByEmptySpace } from '@oodikone/shared/util'
 import { dbConnections } from '../database/connection'
 import {
   StudentModel,
@@ -15,7 +16,6 @@ import {
   SISStudyRightElementModel,
 } from '../models'
 import { TagModel, TagStudentModel } from '../models/kone'
-import { splitByEmptySpace } from '../util'
 
 const { sequelize } = dbConnections
 
@@ -307,16 +307,14 @@ const formatStudentForSearch = ({
   lastname,
   studentnumber,
   studyRights,
-}: Student) => {
-  return {
-    activeStudyRights: studyRights || [],
-    credits: creditcount || 0,
-    firstNames: firstnames,
-    lastName: lastname,
-    studentNumber: studentnumber,
-    started: dateofuniversityenrollment,
-  }
-}
+}: Student): FormattedStudentForSearch => ({
+  activeStudyRights: studyRights || [],
+  credits: creditcount || 0,
+  firstNames: firstnames,
+  lastName: lastname,
+  studentNumber: studentnumber,
+  started: dateofuniversityenrollment?.toString(),
+})
 
 export const bySearchTermAndStudentNumbers = async (searchTerm: string, studentNumbers?: string[]) => {
   const terms = splitByEmptySpace(searchTerm)

services/backend/src/services/studyGuidanceGroups.ts

@@ -1,6 +1,6 @@
 import * as Sentry from '@sentry/node'
 import { AxiosError } from 'axios'
-import { uniq } from 'lodash'
+import { uniq } from 'lodash-es'
 
 import type { StudyGuidanceGroup, Tags, TagsByGroupId, GroupsWithTags } from '@oodikone/shared/types/studyGuidanceGroup'
 import { StudyGuidanceGroupTagModel } from '../models/kone'
@@ -9,30 +9,25 @@ import logger from '../util/logger'
 
 const importerClient = getImporterClient()
 
-const getGroupsFromImporter = async (sisPersonId: string) => {
-  if (!importerClient || !sisPersonId) {
-    return []
-  }
-  const answerTimeout = new Promise(resolve => setTimeout(resolve, 2000))
+const getGroupsFromImporter = async (sisPersonId: string): Promise<StudyGuidanceGroup[]> => {
+  if (!importerClient || !sisPersonId) return []
+
   try {
-    const response = (await Promise.race([
-      importerClient.get(`/person-groups/person/${sisPersonId}`),
-      answerTimeout,
-    ])) as { data: Record<string, StudyGuidanceGroup> }
-    if (!response) {
-      return []
-    }
-    const studyGuidanceGroups: StudyGuidanceGroup[] = Object.values(response.data)
-    return studyGuidanceGroups
+    const response: { data: Record<string, StudyGuidanceGroup> } = await importerClient.get(
+      `/person-groups/person/${sisPersonId}`
+    )
+
+    return response?.data ? Object.values(response.data) : []
   } catch (error) {
-    logger.error("Couldn't fetch users study guidance groups")
+    logger.error('Could not fetch study guidance groups!')
     if (error instanceof AxiosError) {
       logger.error(error.stack)
       logger.error(JSON.stringify(error.response?.data))
       Sentry.captureException(error)
     }
-    return []
   }
+
+  return []
 }
 
 export const getAllGroupsAndStudents = async (sisPersonId: string) => {

services/backend/src/services/studyProgramme/index.ts

@@ -1,4 +1,4 @@
-import { memoize } from 'lodash'
+import { memoize } from 'lodash-es'
 import { Op, QueryTypes } from 'sequelize'
 
 import { Name, CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
@@ -52,15 +52,13 @@ export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, p
       },
     })
 
-    const allEnrollments = {} as Record<string, string[]>
+    const allEnrollments = new Map<string, Set<string>>()
     for (const { studentnumber, course_code: courseCode } of enrollmentsCourses) {
       const code = getCourseCode(courseCode)
-      if (!(code in allEnrollments)) {
-        allEnrollments[code] = []
-      }
-      if (!allEnrollments[code].includes(studentnumber)) {
-        allEnrollments[code].push(studentnumber)
+      if (!allEnrollments.has(code)) {
+        allEnrollments.set(code, new Set())
       }
+      allEnrollments.get(code)!.add(studentnumber)
     }
 
     const credits = await CreditModel.findAll({
@@ -75,7 +73,7 @@ export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, p
       },
     })
 
-    const courseNames = (
+    const courseCodeToName = (
       await CourseModel.findAll({
         attributes: ['code', 'name'],
         where: {
@@ -87,63 +85,50 @@ export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, p
       return acc
     }, new Map())
 
-    const creditCourses = credits.map(credit => {
-      return {
-        code: getCourseCode(credit.course_code),
-        studentNumber: credit.student_studentnumber,
-        creditTypeCode: credit.credittypecode,
-        courseName: courseNames.get(credit.course_code)!,
-        isStudyModule: credit.isStudyModule,
-      }
-    })
+    const creditCourses = credits.map(credit => ({
+      code: getCourseCode(credit.course_code),
+      studentNumber: credit.student_studentnumber,
+      creditTypeCode: credit.credittypecode,
+      courseName: courseCodeToName.get(credit.course_code)!,
+      isStudyModule: credit.isStudyModule,
+    }))
 
-    const passedByCourseCodes = {} as Record<string, string[]>
-    const notCompletedByCourseCodes = {} as Record<string, string[]>
-    const courses: Record<string, { code: string; name: Name; isStudyModule: boolean }> = {}
-    for (const course of creditCourses) {
-      if (!(course.code in courses)) {
-        courses[course.code] = {
-          code: course.code,
-          name: course.courseName,
-          isStudyModule: course.isStudyModule,
-        }
-        passedByCourseCodes[course.code] = []
-        notCompletedByCourseCodes[course.code] = []
+    const passedByCourseCodes = new Map<string, Set<string>>()
+    const notCompletedByCourseCodes = new Map<string, Set<string>>()
+    const courses = new Map<string, { name: Name; isStudyModule: boolean }>()
+
+    for (const { code, courseName, creditTypeCode, isStudyModule, studentNumber } of creditCourses) {
+      if (!courses.has(code)) {
+        courses.set(code, {
+          isStudyModule,
+          name: courseName,
+        })
+        passedByCourseCodes.set(code, new Set())
+        notCompletedByCourseCodes.set(code, new Set())
       }
-      if ([CreditTypeCode.PASSED, CreditTypeCode.IMPROVED, CreditTypeCode.APPROVED].includes(course.creditTypeCode)) {
-        passedByCourseCodes[course.code].push(course.studentNumber)
+      if ([CreditTypeCode.PASSED, CreditTypeCode.IMPROVED, CreditTypeCode.APPROVED].includes(creditTypeCode)) {
+        passedByCourseCodes.get(code)!.add(studentNumber)
       }
-      if (
-        course.creditTypeCode === CreditTypeCode.FAILED &&
-        !passedByCourseCodes[course.code].includes(course.studentNumber)
-      ) {
-        notCompletedByCourseCodes[course.code].push(course.studentNumber)
+      if (creditTypeCode === CreditTypeCode.FAILED && !passedByCourseCodes.get(code)!.has(studentNumber)) {
+        notCompletedByCourseCodes.get(code)!.add(studentNumber)
       }
     }
 
     // If student has enrollments, but no attainment for a particular course, they have no credit info.
     programmeCourses.forEach(courseCode => {
-      if (allEnrollments[courseCode]) {
-        allEnrollments[courseCode].forEach(studentnumber => {
-          if (passedByCourseCodes[courseCode] && !passedByCourseCodes[courseCode].includes(studentnumber)) {
-            notCompletedByCourseCodes[courseCode].push(studentnumber)
-          }
-        })
-      }
+      allEnrollments.get(courseCode)?.forEach(studentnumber => {
+        if (passedByCourseCodes.get(courseCode)?.has(studentnumber)) {
+          notCompletedByCourseCodes.get(courseCode)?.add(studentnumber)
+        }
+      })
     })
 
-    return Object.keys(courses)
-      .reduce<Array<{ code: string; name: Name; isStudyModule: boolean }>>(
-        (acc, val) => [...acc, { ...courses[val] }],
-        []
-      )
-      .map(course => ({
-        code: course.code,
-        name: course.name,
-        type: 'notCompleted',
-        isStudyModule: course.isStudyModule,
-        totalNotCompleted: [...new Set(notCompletedByCourseCodes[course.code])].length,
-      }))
+    return [...courses.entries()].map(([code, { isStudyModule, name }]) => ({
+      code,
+      name,
+      isStudyModule,
+      allNotPassed: notCompletedByCourseCodes.get(code)?.size ?? 0,
+    }))
   } catch (error) {
     logger.error(`getNotCompletedForProgrammeCourses failed ${error}`)
     return []

services/backend/src/services/studyProgramme/studentGetters.ts

@@ -1,348 +1,261 @@
-import { QueryTypes } from 'sequelize'
+import { Op, QueryTypes } from 'sequelize'
 
-import { Name, ExtentCode } from '@oodikone/shared/types'
+import { CreditTypeCode, ExtentCode, Name } from '@oodikone/shared/types'
 import { dbConnections } from '../../database/connection'
+import { StudentModel } from '../../models'
 import logger from '../../util/logger'
 
-const { sequelize } = dbConnections
+// Categories here are as per Linjaukset.md in internal docs.
+// Refer to them before making changes
+const degreeExtentCodes = [ExtentCode.BACHELOR, ExtentCode.MASTER, ExtentCode.BACHELOR_AND_MASTER]
 
-type QueryResult = {
-  totalStudents: string
-  totalCredits: number
-  code: string
+const exchangeStudentExtentCodes = [ExtentCode.EXCHANGE_STUDIES, ExtentCode.EXCHANGE_STUDIES_POSTGRADUATE]
+
+const otherUniversityExtentCodes = [ExtentCode.CONTRACT_TRAINING]
+
+const separateStudyExtentCodes = [
+  ExtentCode.NON_DEGREE_STUDIES,
+  ExtentCode.NON_DEGREE_PEGAGOGICAL_STUDIES_FOR_TEACHERS,
+  ExtentCode.NON_DEGREE_PROGRAMME_FOR_SPECIAL_EDUCATION_TEACHERS,
+  ExtentCode.SPECIALIZATION_STUDIES,
+]
+
+const otherExtentCodes = [
+  ExtentCode.CONTINUING_EDUCATION,
+  ExtentCode.DOCTOR,
+  ExtentCode.LICENTIATE,
+  ExtentCode.SPECIALIST_TRAINING_IN_MEDICINE_AND_DENTISTRY, // Includes veterinary
+  ExtentCode.STUDIES_FOR_SECONDARY_SCHOOL_STUDENTS,
+  ExtentCode.SUMMER_AND_WINTER_SCHOOL,
+]
+
+const openUniversityExtentCodes = [ExtentCode.OPEN_UNIVERSITY_STUDIES]
+
+export type ProgrammeCourseCreditRow = {
+  courseCode: string
   courseName: Name
+  studentNumber: string
+  attainmentDate: string
+  credits: number
+
+  variant: 'degree' | 'exchange' | 'separate' | 'otherUniversity' | 'other' | 'openUni' | 'none'
+
   isStudyModule: boolean
 }
 
-export const getStudentsForProgrammeCourses = async (from: Date, to: Date, programmeCourses: string[]) => {
-  if (!programmeCourses.length) {
-    return []
-  }
-  try {
-    const result: QueryResult[] = await sequelize.query(
-      `
-      WITH Dist AS (
-        SELECT DISTINCT
-          cr.student_studentnumber AS student,
-          cr.credits AS credits,
-          co.code AS code,
-          co.name AS "courseName",
-          cr."isStudyModule"
-        FROM
-          credit cr
-        INNER JOIN
-          course co
-          ON cr.course_code = co.code
-        WHERE
-          cr.attainment_date BETWEEN :from AND :to
-          AND cr.course_code IN (:programmeCourses)
-          AND cr.credittypecode = 4
-      )
-      SELECT
-        COUNT(student) AS "totalStudents",
-        SUM(credits) AS "totalCredits",
-        code,
-        "courseName",
-        "isStudyModule"
-      FROM
-        Dist
-      GROUP BY
-        dist.code,
-        dist."courseName",
-        dist."isStudyModule";
-      `,
-      {
-        type: QueryTypes.SELECT,
-        replacements: { from, to, programmeCourses },
-      }
-    )
-    return result.map(course => ({
-      code: course.code,
-      name: course.courseName,
-      type: 'passed',
-      isStudyModule: course.isStudyModule,
-      totalPassed: parseInt(course.totalStudents, 10),
-      totalAllCredits: course.totalCredits,
-    }))
-  } catch (error) {
-    logger.error(`getStudentsForProgrammeCourses() function failed ${error}`)
-    return []
-  }
+export type TransferCourseCreditRow = {
+  courseCode: string
+  courseName: Name
+  studentNumber: string
+  attainmentDate: string
+  credits: number
+  isStudyModule: boolean
 }
 
-export const getOwnStudentsForProgrammeCourses = async (
-  from: Date,
-  to: Date,
-  programmeCourses: string[],
-  studyProgramme?: string
-) => {
-  if (!programmeCourses.length) {
+const { sequelize } = dbConnections
+
+/**
+NOTE:
+ 1. There are some attainments that are linked to a valid study right, but the attainmentdate
+    is outside the study right's validity period. (maybe human error when the credits were registered)
+ 2. There are some attainments with no attached study right at all.
+ 3. Possibly some other edge cases.
+
+ Such cases get variant "none" and are not included in any category
+
+ Order of cases is somewhat important. refer to linjaukset.md, but TLDR;
+   degree student -> has any degree (bsc / msc) study right in the university
+   openuni -> only if no other study rights
+**/
+export const getProgrammeCourseAggregates = async (params: {
+  courseCodes: string[]
+  from: Date
+  to: Date
+}): Promise<ProgrammeCourseCreditRow[]> => {
+  const { courseCodes, from, to } = params
+
+  if (courseCodes.length === 0) {
     return []
   }
+
   try {
-    const result: QueryResult[] = await sequelize.query(
+    return await sequelize.query<ProgrammeCourseCreditRow>(
       `
-    WITH Dist AS (
-      SELECT DISTINCT
-        cr.student_studentnumber AS student,
-        cr.credits AS credits,
-        co.code AS code,
-        co.name AS "courseName",
-        cr."isStudyModule"
-      FROM
-        credit cr
-      INNER JOIN
-        sis_study_rights sr
-        ON sr.student_number = cr.student_studentnumber
-      INNER JOIN
-        sis_study_right_elements se
-        ON se.study_right_id = sr.id
-      INNER JOIN
-        course co
-        ON cr.course_code = co.code
-      WHERE
-        cr.attainment_date BETWEEN :from AND :to
-        AND cr.course_code IN (:programmeCourses)
-        AND cr.credittypecode = 4
-        AND (
-          se.code = :studyProgramme
-          AND cr.attainment_date BETWEEN se.start_date AND se.end_date
+        WITH filtered_credits AS (
+          SELECT
+            cr.course_code AS "courseCode",
+            co.name AS "courseName",
+            cr.student_studentnumber AS "studentNumber",
+            cr.attainment_date AS "attainmentDate",
+            cr.credits AS credits,
+            cr."isStudyModule" AS "isStudyModule"
+          FROM
+            credit cr
+            INNER JOIN course co ON co.code = cr.course_code
+          WHERE
+            cr.course_code IN (:courseCodes)
+            AND cr.attainment_date BETWEEN :from AND :to
+            AND cr.credittypecode = :creditType
         )
-    )
-    SELECT
-      COUNT(student) AS "totalStudents",
-      SUM(credits) AS "totalCredits",
-      code,
-      "courseName",
-      "isStudyModule"
-    FROM
-      Dist
-    GROUP BY
-      dist.code,
-      "courseName",
-      "isStudyModule";
-    `,
-      {
-        type: QueryTypes.SELECT,
-        replacements: { from, to, programmeCourses, studyProgramme },
-      }
-    )
-    return result.map(course => ({
-      code: course.code,
-      name: course.courseName,
-      type: 'ownProgramme',
-      totalProgrammeStudents: parseInt(course.totalStudents, 10),
-      totalProgrammeCredits: course.totalCredits,
-    }))
-  } catch (error) {
-    logger.error(`getOwnStudentsForProgrammeCourses() function failed ${error}`)
-    return []
-  }
-}
-
-export const getOtherStudentsForProgrammeCourses = async (
-  from: Date,
-  to: Date,
-  programmeCourses: string[],
-  studyProgramme?: string
-) => {
-  if (!programmeCourses.length) {
-    return []
-  }
-  try {
-    const result: QueryResult[] = await sequelize.query(
-      `
-      WITH Dist AS (
-        SELECT DISTINCT
-          cr.student_studentnumber AS student,
-          cr.credits AS credits,
-          co.code AS code,
-          co.name AS "courseName",
-          cr."isStudyModule"
+        SELECT
+          fc."courseCode",
+          fc."courseName",
+          fc."studentNumber",
+          fc."attainmentDate",
+          fc.credits,
+          fc."isStudyModule",
+          CASE
+            WHEN EXISTS (
+              SELECT
+                1
+              FROM
+                sis_study_rights sr
+              WHERE
+                sr.student_number = fc."studentNumber"
+                AND sr.extent_code IN (:degreeExtentCodes)
+                AND fc."attainmentDate" BETWEEN sr.start_date AND COALESCE(sr.end_date, 'infinity'::date)
+            ) THEN 'degree'
+            WHEN EXISTS (
+              SELECT
+                1
+              FROM
+                sis_study_rights sr
+              WHERE
+                sr.student_number = fc."studentNumber"
+                AND sr.extent_code IN (:exchangeStudentExtentCodes)
+                AND fc."attainmentDate" BETWEEN sr.start_date AND COALESCE(sr.end_date, 'infinity'::date)
+            ) THEN 'exchange'
+            WHEN EXISTS (
+              SELECT
+                1
+              FROM
+                sis_study_rights sr
+              WHERE
+                sr.student_number = fc."studentNumber"
+                AND sr.extent_code IN (:separateStudyExtentCodes)
+                AND fc."attainmentDate" BETWEEN sr.start_date AND COALESCE(sr.end_date, 'infinity'::date)
+            ) THEN 'separate'
+            WHEN EXISTS (
+              SELECT
+                1
+              FROM
+                sis_study_rights sr
+              WHERE
+                sr.student_number = fc."studentNumber"
+                AND sr.extent_code IN (:otherUniversityExtentCodes)
+                AND fc."attainmentDate" BETWEEN sr.start_date AND COALESCE(sr.end_date, 'infinity'::date)
+            ) THEN 'otherUniversity'
+            WHEN EXISTS (
+              SELECT
+                1
+              FROM
+                sis_study_rights sr
+              WHERE
+                sr.student_number = fc."studentNumber"
+                AND sr.extent_code IN (:otherExtentCodes)
+                AND fc."attainmentDate" BETWEEN sr.start_date AND COALESCE(sr.end_date, 'infinity'::date)
+            ) THEN 'other'
+            WHEN EXISTS (
+              SELECT
+                1
+              FROM
+                sis_study_rights sr
+              WHERE
+                sr.student_number = fc."studentNumber"
+                AND sr.extent_code IN (:openUniversityExtentCodes)
+                AND fc."attainmentDate" BETWEEN sr.start_date AND COALESCE(sr.end_date, 'infinity'::date)
+            ) THEN 'openUni'
+          ELSE 'none'
+          END AS variant
         FROM
-          credit cr
-        INNER JOIN
-          course co
-          ON cr.course_code = co.code
-        WHERE
-          cr.attainment_date BETWEEN :from AND :to
-          AND cr.course_code IN (:programmeCourses)
-          AND cr.credittypecode = 4
-          AND cr.student_studentnumber NOT IN (
-            SELECT
-              sr.student_number
-            FROM
-              sis_study_rights sr
-            INNER JOIN
-              sis_study_right_elements sre
-              ON sre.study_right_id = sr.id
-              AND sre.code = :studyProgramme
-              AND cr.attainment_date BETWEEN sre.start_date AND sre.end_date
-            WHERE
-              sr.student_number = cr.student_studentnumber
-          )
-      )
-      SELECT
-        COUNT(student) AS "totalStudents",
-        SUM(credits) AS "totalCredits",
-        code,
-        "courseName",
-        "isStudyModule"
-      FROM
-        Dist
-      GROUP BY
-        dist.code,
-        "courseName",
-        "isStudyModule";
-      `,
-      {
-        type: QueryTypes.SELECT,
-        replacements: { from, to, programmeCourses, studyProgramme },
-      }
-    )
-    return result.map(course => ({
-      code: course.code,
-      name: course.courseName,
-      type: 'otherProgramme',
-      isStudyModule: course.isStudyModule,
-      totalOtherProgrammeStudents: parseInt(course.totalStudents, 10),
-      totalOtherProgrammeCredits: course.totalCredits,
-    }))
-  } catch (error) {
-    logger.error(`getOtherStudentsForProgrammeCourses() function failed ${error}`)
-    return []
-  }
-}
-
-export const getTransferStudentsForProgrammeCourses = async (from: Date, to: Date, programmeCourses: string[]) => {
-  if (!programmeCourses.length) {
-    return []
-  }
-  try {
-    const result: QueryResult[] = await sequelize.query(
-      `
-      WITH Dist AS (
-        SELECT DISTINCT
-          cr.student_studentnumber AS student,
-          cr.credits AS credits,
-          co.code AS code,
-          co.name AS "courseName",
-          cr."isStudyModule"
-        FROM
-          credit cr
-        INNER JOIN
-          course co
-          ON cr.course_code = co.code
-        WHERE
-          cr.attainment_date BETWEEN :from AND :to
-          AND cr.course_code IN (:programmeCourses)
-          AND cr.credittypecode = 9
-      )
-      SELECT
-        COUNT(student) AS "totalStudents",
-        SUM(credits) AS "totalCredits",
-        code,
-        "courseName",
-        "isStudyModule"
-      FROM
-        Dist
-      GROUP BY
-        dist.code,
-        "courseName",
-        "isStudyModule";
-      `,
-      {
-        type: QueryTypes.SELECT,
-        replacements: { from, to, programmeCourses },
-      }
-    )
-    return result.map(course => ({
-      code: course.code,
-      name: course.courseName,
-      type: 'transfer',
-      isStudyModule: course.isStudyModule,
-      totalTransferStudents: parseInt(course.totalStudents, 10),
-      totalTransferCredits: course.totalCredits,
-    }))
-  } catch (error) {
-    logger.error(`getTransferStudentsForProgrammeCourses() function failed ${error}`)
-    return []
-  }
-}
-
-export const getStudentsWithoutStudyRightForProgrammeCourses = async (
-  from: Date,
-  to: Date,
-  programmeCourses: string[]
-) => {
-  if (!programmeCourses.length) {
-    return []
-  }
-  try {
-    const result: QueryResult[] = await sequelize.query(
-      `
-      WITH Dist AS (
-        SELECT DISTINCT
-          cr.student_studentnumber AS student,
-          cr.credits AS credits,
-          co.code AS code,
-          co.name AS "courseName",
-          cr."isStudyModule"
-        FROM
-          credit cr
-        INNER JOIN
-          course co
-          ON cr.course_code = co.code
-        LEFT JOIN
-          sis_study_rights sr
-          ON sr.student_number = cr.student_studentnumber
-          AND sr.extent_code IN (:degreeExtentCodes)
-          AND cr.attainment_date BETWEEN sr.start_date AND sr.end_date
-        WHERE
-          cr.attainment_date BETWEEN :from AND :to
-          AND cr.course_code IN (:programmeCourses)
-          AND cr.credittypecode = 4
-          AND sr.student_number IS NULL
-      )
-      SELECT
-        COUNT(student) AS "totalStudents",
-        SUM(credits) AS "totalCredits",
-        code,
-        "courseName",
-        "isStudyModule"
-      FROM
-        Dist
-      GROUP BY
-        dist.code,
-        "courseName",
-        "isStudyModule";
+          filtered_credits fc
       `,
       {
         type: QueryTypes.SELECT,
         replacements: {
+          courseCodes,
           from,
           to,
-          programmeCourses,
-          degreeExtentCodes: [
-            ExtentCode.BACHELOR,
-            ExtentCode.MASTER,
-            ExtentCode.BACHELOR_AND_MASTER,
-            ExtentCode.DOCTOR,
-            ExtentCode.LICENTIATE,
-            ExtentCode.SPECIALIST_TRAINING_IN_MEDICINE_AND_DENTISTRY,
-          ],
+          creditType: CreditTypeCode.PASSED,
+          degreeExtentCodes,
+          otherUniversityExtentCodes,
+          separateStudyExtentCodes,
+          exchangeStudentExtentCodes,
+          openUniversityExtentCodes,
+          otherExtentCodes,
         },
       }
     )
-    return result.map(course => ({
-      code: course.code,
-      name: course.courseName,
-      type: 'noStudyright',
-      totalWithoutStudyRightStudents: parseInt(course.totalStudents, 10),
-      totalWithoutStudyRightCredits: course.totalCredits,
-    }))
   } catch (error) {
-    logger.error(`getStudentsWithoutStudyRightForProgrammeCourses() function failed ${error}`)
+    logger.error('getProgrammeCourseAggregates failed', { error })
     return []
   }
 }
+
+export const getTransferCourseAggregates = async (params: {
+  courseCodes: string[]
+  from: Date
+  to: Date
+}): Promise<TransferCourseCreditRow[]> => {
+  const { courseCodes, from, to } = params
+
+  if (courseCodes.length === 0) {
+    return []
+  }
+
+  try {
+    return await sequelize.query<TransferCourseCreditRow>(
+      `
+        SELECT
+          cr.course_code AS "courseCode",
+          co.name AS "courseName",
+          cr.student_studentnumber AS "studentNumber",
+          cr.attainment_date AS "attainmentDate",
+          cr.credits AS credits,
+          cr."isStudyModule" AS "isStudyModule"
+        FROM
+          credit cr
+          INNER JOIN course co ON co.code = cr.course_code
+        WHERE
+          cr.course_code IN (:courseCodes)
+          AND cr.attainment_date BETWEEN :from AND :to
+          AND cr.credittypecode = :creditType
+      `,
+      {
+        type: QueryTypes.SELECT,
+        replacements: {
+          courseCodes,
+          from,
+          to,
+          creditType: CreditTypeCode.APPROVED,
+        },
+      }
+    )
+  } catch (error) {
+    logger.error('getTransferCourseAggregates failed', { error })
+    return []
+  }
+}
+
+export const getStudentHetuStateMap = async (studentNumbers: string[]): Promise<Map<string, boolean>> => {
+  if (!studentNumbers.length) return new Map()
+
+  try {
+    return new Map(
+      (
+        await StudentModel.findAll({
+          attributes: ['studentnumber', 'hasPersonalIdentityCode'],
+          where: {
+            studentnumber: {
+              [Op.in]: studentNumbers,
+            },
+          },
+          raw: true,
+        })
+      ).map(student => [student.studentnumber, student.hasPersonalIdentityCode])
+    )
+  } catch (error) {
+    logger.error('getOpenUniHetutonSplit failed', { error })
+    return new Map()
+  }
+}

services/backend/src/services/studyProgramme/studyProgrammeBasics.ts

@@ -1,6 +1,7 @@
-import { indexOf } from 'lodash'
+import { indexOf } from 'lodash-es'
 
 import { Name, EnrollmentType, SemesterEnrollment } from '@oodikone/shared/types'
+import { BasicStats } from '@oodikone/shared/types/studyProgramme'
 import { getSemestersAndYears } from '../semesters'
 import {
   defineYear,
@@ -278,7 +279,7 @@ export const getBasicStatsForStudytrack = async ({
     isAcademicYear: boolean
     includeAllSpecials: boolean
   }
-}) => {
+}): Promise<BasicStats> => {
   const { includeAllSpecials, isAcademicYear } = settings
   const since = getStartDate(isAcademicYear)
   const years = getYearsArray(since.getFullYear(), isAcademicYear) as number[]

services/backend/src/services/studyProgramme/studyProgrammeCourses.ts

@@ -1,390 +1,293 @@
-import { orderBy, range } from 'lodash'
-import { Op } from 'sequelize'
+import { range } from 'lodash-es'
 
-import { Name, StudyProgrammeCourse, CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
+import { Name, StudyProgrammeCourse, YearType } from '@oodikone/shared/types'
 import { mapToProviders } from '@oodikone/shared/util'
-import { CreditModel, EnrollmentModel } from '../../models'
-import { isOpenUniCourseCode } from '../../util'
-import { createArrayOfCourses } from '../languageCenterData'
-import { getCurrentStudyYearStartDate, getNotCompletedForProgrammeCourses, getAllProgrammeCourses } from '.'
-import {
-  getOtherStudentsForProgrammeCourses,
-  getOwnStudentsForProgrammeCourses,
-  getStudentsForProgrammeCourses,
-  getStudentsWithoutStudyRightForProgrammeCourses,
-  getTransferStudentsForProgrammeCourses,
-} from './studentGetters'
+import { createEmptyStats, YearStats } from '@oodikone/shared/util/studyProgramme'
 
-const getCurrentYearStartDate = () => {
-  return new Date(new Date().getFullYear(), 0, 1)
-}
+import { getAllProgrammeCourses, getCurrentStudyYearStartDate, getNotCompletedForProgrammeCourses } from '.'
+import { getStudentHetuStateMap, getProgrammeCourseAggregates, getTransferCourseAggregates } from './studentGetters'
+
+const START_YEAR = 2017
+const JULY = 6
+const AUGUST = 7
+
+const getCurrentYearStartDate = () => new Date(new Date().getFullYear(), 0, 1)
+
+const getFrom = (yearType: YearType, year: number) =>
+  yearType === 'ACADEMIC_YEAR' ? new Date(year, AUGUST, 1, 0, 0, 0) : new Date(year, 0, 1, 0, 0, 0)
+
+const getTo = (yearType: YearType, year: number) =>
+  yearType === 'ACADEMIC_YEAR' ? new Date(year + 1, JULY, 31, 23, 59, 59) : new Date(year, 11, 31, 23, 59, 59)
 
 const getAllStudyProgrammeCourses = async (studyProgramme: string) => {
   const providerCode = mapToProviders([studyProgramme])[0]
+  if (!providerCode) return []
+
   const normalCourses = await getAllProgrammeCourses(providerCode)
-  return normalCourses.reduce((acc, curr) => {
-    acc.push(curr.code)
-    if (curr.substitutions?.includes(`AY${curr.code}`)) {
-      acc.push(`AY${curr.code}`)
+  const courseCodes = new Set<string>()
+
+  for (const course of normalCourses) {
+    courseCodes.add(course.code)
+    if (course.substitutions?.includes(`AY${course.code}`)) {
+      courseCodes.add(`AY${course.code}`)
     }
-    return acc
-  }, [] as string[])
+  }
+
+  return [...courseCodes]
 }
 
-const getFrom = (academicYear: string, year: number) => {
-  return academicYear === 'ACADEMIC_YEAR' ? new Date(year, 7, 1, 0, 0, 0) : new Date(year, 0, 1, 0, 0, 0)
+const getYearKey = (date: Date, isAcademicYear: boolean) => {
+  const year = date.getFullYear()
+  if (!isAcademicYear) return year
+  return date.getMonth() >= 7 ? year : year - 1
 }
 
-const getTo = (academicYear: string, year: number) => {
-  return academicYear === 'ACADEMIC_YEAR' ? new Date(year + 1, 6, 31, 23, 59, 59) : new Date(year, 11, 31, 23, 59, 59)
+type YearAccumulator = {
+  stats: YearStats
+  allPassed: Set<string>
+
+  degreeStudents: Set<string>
+  exchangeStudents: Set<string>
+  otherUniversityStudents: Set<string>
+  separateStudents: Set<string>
+  otherStudents: Set<string>
+
+  openStudentsHasHetu: Set<string>
+  openStudentsNoHetu: Set<string>
+  transferStudents: Set<string>
 }
 
-// * Combined return type, should be split into smaller types
-type Students = {
+type CourseAccumulator = {
   code: string
   name: Name
-  type: string
-  isStudyModule?: boolean
-  year?: number
-  totalPassed?: number
-  totalAllCredits?: number
-  totalNotCompleted?: number
-  totalProgrammeStudents?: number
-  totalProgrammeCredits?: number
-  totalWithoutStudyRightStudents?: number
-  totalWithoutStudyRightCredits?: number
-  totalOtherProgrammeStudents?: number
-  totalOtherProgrammeCredits?: number
-  totalTransferStudents?: number
-  totalTransferCredits?: number
+  isStudyModule: boolean
+  years: Map<number, YearAccumulator>
 }
 
-// TODO: It would be a good idea to split this into smaller functions
-// The amount of different return types makes this very messy
-const makeYearlyPromises = (
-  years: number[],
-  academicYear: string,
-  type: 'passed' | 'notCompleted' | 'ownStudents' | 'withoutStudyRight' | 'otherStudents' | 'transfer',
-  programmeCourses: string[],
-  studyProgramme?: string
-) => {
-  return years.map(async year => {
-    const from = getFrom(academicYear, year)
-    const to = getTo(academicYear, year)
-    let result: Students[] | null = null
+const createYearAccumulator = (isStudyModule: boolean): YearAccumulator => ({
+  stats: createEmptyStats(isStudyModule),
+  allPassed: new Set(),
+  degreeStudents: new Set(),
+  exchangeStudents: new Set(),
+  otherUniversityStudents: new Set(),
+  separateStudents: new Set(),
+  otherStudents: new Set(),
 
-    switch (type) {
-      case 'passed':
-        result = await getStudentsForProgrammeCourses(from, to, programmeCourses)
-        break
-      case 'notCompleted':
-        result = await getNotCompletedForProgrammeCourses(from, to, programmeCourses)
-        break
-      case 'ownStudents':
-        result = await getOwnStudentsForProgrammeCourses(from, to, programmeCourses, studyProgramme)
-        break
-      case 'withoutStudyRight':
-        result = await getStudentsWithoutStudyRightForProgrammeCourses(from, to, programmeCourses)
-        break
-      case 'otherStudents':
-        result = await getOtherStudentsForProgrammeCourses(from, to, programmeCourses, studyProgramme)
-        break
-      case 'transfer':
-        result = await getTransferStudentsForProgrammeCourses(from, to, programmeCourses)
-        break
-      default:
-        result = await getStudentsForProgrammeCourses(from, to, programmeCourses)
-    }
+  openStudentsHasHetu: new Set(),
+  openStudentsNoHetu: new Set(),
+  transferStudents: new Set(),
+})
 
-    return result.map(course => {
-      course.year = year
-      return course
+const ensureCourse = (
+  courseMap: Map<string, CourseAccumulator>,
+  code: string,
+  name: Name,
+  isStudyModule: boolean
+): CourseAccumulator => {
+  if (!courseMap.has(code)) {
+    courseMap.set(code, {
+      code,
+      name,
+      isStudyModule,
+      years: new Map(),
     })
-  })
+  }
+
+  const course = courseMap.get(code)!
+  course.isStudyModule = course.isStudyModule || isStudyModule
+  course.name = course.name ?? name
+  return course
 }
 
-type Attempt = {
-  studentNumber: string
-  courseCode: string
-  completed: boolean
-  date: Date
-  semestercode: number
-  enrolled?: boolean
+const ensureYear = (course: CourseAccumulator, year: number, isStudyModule: boolean) => {
+  if (!course.years.has(year)) {
+    course.years.set(year, createYearAccumulator(isStudyModule))
+  }
+  const yearAccumulator = course.years.get(year)!
+  yearAccumulator.stats.isStudyModule ??= isStudyModule
+  return yearAccumulator
 }
 
-export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgramme: string) => {
-  const courses = await getAllProgrammeCourses(mapToProviders([studyProgramme])[0])
-  const autumnSemester2017 = 135
-  const courseCodes = courses.map(course => course.code)
+const finalizeYearStats = (yearAccumulator: YearAccumulator): YearStats => {
+  const { stats } = yearAccumulator
 
-  const credits = await CreditModel.findAll({
-    attributes: ['course_code', 'student_studentnumber', 'semestercode', 'attainment_date'],
-    where: {
-      course_code: { [Op.in]: courseCodes },
-      semestercode: { [Op.gte]: autumnSemester2017 },
-      credittypecode: CreditTypeCode.PASSED,
-    },
-    raw: true,
-  })
+  stats.allPassed = yearAccumulator.allPassed.size
 
-  const enrollments = await EnrollmentModel.findAll({
-    attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'state'],
-    where: {
-      course_code: { [Op.in]: courseCodes },
-      semestercode: { [Op.gte]: autumnSemester2017 },
-      state: { [Op.in]: [EnrollmentState.ENROLLED, EnrollmentState.REJECTED] },
-    },
-    raw: true,
-  })
+  stats.degreeStudents = yearAccumulator.degreeStudents.size
+  stats.exchangeStudents = yearAccumulator.exchangeStudents.size
+  stats.otherUniversityStudents = yearAccumulator.otherUniversityStudents.size
+  stats.separateStudents = yearAccumulator.separateStudents.size
+  stats.otherStudents = yearAccumulator.otherStudents.size
 
-  const studentList = new Set<string>()
-  const attemptsByStudents = {} as Record<string, Attempt[]>
+  stats.openStudentsHasHetu = yearAccumulator.openStudentsHasHetu.size
+  stats.openStudentsNoHetu = yearAccumulator.openStudentsNoHetu.size
+  stats.transferStudents = yearAccumulator.transferStudents.size
 
-  credits.forEach(credit => {
-    const studentNumber = credit.student_studentnumber
-    studentList.add(studentNumber)
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
-    attemptsByStudents[studentNumber].push({
-      studentNumber,
-      courseCode: credit.course_code,
-      completed: true,
-      date: credit.attainment_date,
-      semestercode: credit.semestercode,
-    })
-  })
+  stats.allStudents = stats.allPassed + stats.allNotPassed
 
-  enrollments.forEach(enrollment => {
-    const studentNumber = enrollment.studentnumber
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
-    studentList.add(studentNumber)
-    if (
-      attemptsByStudents[studentNumber].find(
-        attempt =>
-          !attempt.completed &&
-          attempt.semestercode === enrollment.semestercode &&
-          attempt.courseCode === enrollment.course_code
-      )
-    ) {
-      return
-    }
-    attemptsByStudents[studentNumber].push({
-      studentNumber,
-      courseCode: enrollment.course_code,
-      completed: false,
-      date: enrollment.enrollment_date_time,
-      semestercode: enrollment.semestercode,
-      enrolled: enrollment.state === EnrollmentState.ENROLLED,
-    })
-  })
-
-  const attemptsArray = [] as Attempt[]
-  studentList.forEach(studentNumber => attemptsArray.push(...attemptsByStudents[studentNumber]))
-
-  const unorderedTableData = await createArrayOfCourses(attemptsArray, courses)
-  const tableData = orderBy(unorderedTableData, 'code')
-
-  return { tableData }
+  return stats
 }
 
+const getYearRange = async (unixMillis: number, isAcademicYear: boolean) => {
+  const startDate = isAcademicYear ? await getCurrentStudyYearStartDate(unixMillis) : getCurrentYearStartDate()
+  const lastYear = startDate.getFullYear()
+  return range(START_YEAR, lastYear + 1)
+}
+
+/**
+ * @endpoint "programme courses"
+ */
 export const getStudyProgrammeCoursesForStudyTrack = async (
   unixMillis: number,
   studyProgramme: string,
-  academicYear: string,
+  academicYear: YearType,
   combinedProgramme: string
 ) => {
-  const startDate =
-    academicYear === 'ACADEMIC_YEAR' ? await getCurrentStudyYearStartDate(unixMillis) : getCurrentYearStartDate()
-  const startYear = startDate.getFullYear()
-  const yearRange = range(2017, startYear + 1)
+  const isAcademicYear = academicYear === 'ACADEMIC_YEAR'
+  const yearRange = await getYearRange(unixMillis, isAcademicYear)
+  if (yearRange.length === 0) {
+    return []
+  }
+
   const mainProgrammeCourses = await getAllStudyProgrammeCourses(studyProgramme)
   const secondProgrammeCourses = combinedProgramme ? await getAllStudyProgrammeCourses(combinedProgramme) : []
-  const programmeCourses = [...mainProgrammeCourses, ...secondProgrammeCourses]
+  const programmeCourses = [...new Set([...mainProgrammeCourses, ...secondProgrammeCourses])]
 
-  const yearlyPassedStudentByCoursePromises = makeYearlyPromises(yearRange, academicYear, 'passed', programmeCourses)
-  const yearlyNotCompletedStudentByCoursePromises = makeYearlyPromises(
-    yearRange,
-    academicYear,
-    'notCompleted',
-    programmeCourses
-  )
-  const yearlyProgrammeStudentsPromises = makeYearlyPromises(
-    yearRange,
-    academicYear,
-    'ownStudents',
-    programmeCourses,
-    studyProgramme
-  )
-  const yearlyStudentsWithoutStudyRightPromises = makeYearlyPromises(
-    yearRange,
-    academicYear,
-    'withoutStudyRight',
-    programmeCourses
-  )
-  const yearlyOtherProgrammeStudentsPromises = makeYearlyPromises(
-    yearRange,
-    academicYear,
-    'otherStudents',
-    programmeCourses,
-    studyProgramme
-  )
-  const yearlyTransferStudentsPromises = makeYearlyPromises(
-    yearRange,
-    academicYear,
-    'transfer',
-    programmeCourses,
-    studyProgramme
-  )
+  if (programmeCourses.length === 0) {
+    return []
+  }
 
-  const [
-    yearlyPassedStudentByCourse,
-    yearlyNotCompletedStudentByCourse,
-    yearlyProgrammeStudents,
-    yearlyStudentsWithoutStudyRight,
-    yearlyOtherProgrammeStudents,
-    yearlyTransferStudents,
-  ] = await Promise.all([
-    Promise.all(yearlyPassedStudentByCoursePromises),
-    Promise.all(yearlyNotCompletedStudentByCoursePromises),
-    Promise.all(yearlyProgrammeStudentsPromises),
-    Promise.all(yearlyStudentsWithoutStudyRightPromises),
-    Promise.all(yearlyOtherProgrammeStudentsPromises),
-    Promise.all(yearlyTransferStudentsPromises),
+  const firstYear = yearRange[0]
+  const lastYear = yearRange[yearRange.length - 1]
+  const from = getFrom(academicYear, firstYear)
+  const to = getTo(academicYear, lastYear)
+
+  const [programmeCredits, transferCredits] = await Promise.all([
+    getProgrammeCourseAggregates({
+      courseCodes: programmeCourses,
+      from,
+      to,
+    }),
+    getTransferCourseAggregates({
+      courseCodes: programmeCourses,
+      from,
+      to,
+    }),
   ])
 
-  let maxYear = 0
-  const allCourses = [
-    ...yearlyPassedStudentByCourse.flat(),
-    ...yearlyNotCompletedStudentByCourse.flat(),
-    ...yearlyProgrammeStudents.flat(),
-    ...yearlyStudentsWithoutStudyRight.flat(),
-    ...yearlyOtherProgrammeStudents.flat(),
-    ...yearlyTransferStudents.flat(),
-  ].reduce(
-    (acc, curr) => {
-      if (curr.year! > maxYear) {
-        maxYear = curr.year!
-      }
-      if (!acc[curr.code]) {
-        acc[curr.code] = {
-          code: curr.code,
-          name: curr.name,
-          isStudyModule: curr.isStudyModule ?? false,
-          years: {},
-        }
-      }
-
-      if (!acc[curr.code].years[curr.year!]) {
-        acc[curr.code].years[curr.year!] = {
-          isStudyModule: curr.isStudyModule ?? false,
-          totalAllStudents: 0,
-          totalPassed: 0,
-          totalNotCompleted: 0,
-          totalAllCredits: 0,
-          totalProgrammeStudents: 0,
-          totalProgrammeCredits: 0,
-          totalOtherProgrammeStudents: 0,
-          totalOtherProgrammeCredits: 0,
-          totalWithoutStudyRightStudents: 0,
-          totalWithoutStudyRightCredits: 0,
-          totalTransferStudents: 0,
-          totalTransferCredits: 0,
-        }
-      }
-      switch (curr.type) {
-        case 'passed':
-          acc[curr.code].years[curr.year!].totalPassed += curr.totalPassed ?? 0
-          acc[curr.code].years[curr.year!].totalAllStudents += acc[curr.code].years[curr.year!].totalPassed
-          acc[curr.code].years[curr.year!].totalAllCredits += curr.totalAllCredits ?? 0
-          break
-        case 'notCompleted':
-          acc[curr.code].years[curr.year!].totalNotCompleted += curr.totalNotCompleted ?? 0
-          acc[curr.code].years[curr.year!].totalAllStudents += acc[curr.code].years[curr.year!].totalNotCompleted
-          break
-        case 'ownProgramme':
-          acc[curr.code].years[curr.year!].totalProgrammeStudents += curr.totalProgrammeStudents ?? 0
-          acc[curr.code].years[curr.year!].totalProgrammeCredits += curr.totalProgrammeCredits ?? 0
-          break
-        case 'otherProgramme':
-          acc[curr.code].years[curr.year!].totalOtherProgrammeStudents += curr.totalOtherProgrammeStudents ?? 0
-          acc[curr.code].years[curr.year!].totalOtherProgrammeCredits += curr.totalOtherProgrammeCredits ?? 0
-          break
-        case 'noStudyright':
-          acc[curr.code].years[curr.year!].totalWithoutStudyRightStudents += curr.totalWithoutStudyRightStudents ?? 0
-          acc[curr.code].years[curr.year!].totalWithoutStudyRightCredits += curr.totalWithoutStudyRightCredits ?? 0
-          break
-        case 'transfer':
-          acc[curr.code].years[curr.year!].totalTransferStudents += curr.totalTransferStudents ?? 0
-          acc[curr.code].years[curr.year!].totalTransferCredits += curr.totalTransferCredits ?? 0
-          break
-        default:
-          break
-      }
-      return acc
-    },
-    {} as Record<string, StudyProgrammeCourse>
+  const openUniStudentHetuMap = await getStudentHetuStateMap(
+    programmeCredits.filter(c => c.variant === 'openUni').map(c => c.studentNumber)
   )
-  const ayCourses = Object.keys(allCourses).filter(courseCode => courseCode.startsWith('AY'))
-  const properties = [
-    'totalAllStudents',
-    'totalPassed',
-    'totalNotCompleted',
-    'totalAllCredits',
-    'totalProgrammeStudents',
-    'totalProgrammeCredits',
-    'totalOtherProgrammeStudents',
-    'totalOtherProgrammeCredits',
-    'totalWithoutStudyrightStudents',
-    'totalWithoutStudyrightCredits',
-    'totalTransferCredits',
-    'totalTransferStudents',
-  ]
-  ayCourses.forEach(ayCourse => {
-    const openUniCourseCode = isOpenUniCourseCode(ayCourse)
-    if (!openUniCourseCode) {
-      return
+
+  const courseMap = new Map<string, CourseAccumulator>()
+  let maxYear = 0
+
+  for (const row of programmeCredits) {
+    const attainmentDate = new Date(row.attainmentDate)
+    const year = getYearKey(attainmentDate, isAcademicYear)
+
+    if (!yearRange.includes(year)) continue
+    maxYear = Math.max(maxYear, year)
+
+    const course = ensureCourse(courseMap, row.courseCode, row.courseName, row.isStudyModule)
+    const yearAccumulator = ensureYear(course, year, row.isStudyModule)
+    const { stats } = yearAccumulator
+
+    stats.allCredits += row.credits
+    yearAccumulator.allPassed.add(row.studentNumber)
+
+    switch (row.variant) {
+      case 'degree':
+        yearAccumulator.degreeStudents.add(row.studentNumber)
+        stats.degreeStudentsCredits += row.credits
+        break
+      case 'exchange':
+        yearAccumulator.exchangeStudents.add(row.studentNumber)
+        stats.exchangeStudentsCredits += row.credits
+        break
+      case 'separate':
+        yearAccumulator.separateStudents.add(row.studentNumber)
+        stats.separateStudentsCredits += row.credits
+        break
+      case 'otherUniversity':
+        yearAccumulator.otherUniversityStudents.add(row.studentNumber)
+        stats.otherUniversityCredits += row.credits
+        break
+      case 'other':
+        yearAccumulator.otherStudents.add(row.studentNumber)
+        stats.otherStudentsCredits += row.credits
+        break
+      case 'openUni':
+        if (openUniStudentHetuMap.get(row.studentNumber)) {
+          stats.openStudentsHasHetuCredits += row.credits
+          yearAccumulator.openStudentsHasHetu.add(row.studentNumber)
+        } else {
+          stats.openStudentsNoHetuCredits += row.credits
+          yearAccumulator.openStudentsNoHetu.add(row.studentNumber)
+        }
+        break
+      default:
+        break
     }
-    const normCode = openUniCourseCode[1]
+  }
 
-    if (allCourses[normCode]) {
-      const mergedCourse = {} as StudyProgrammeCourse
-      mergedCourse.code = allCourses[normCode].code
-      mergedCourse.name = allCourses[normCode].name
-      mergedCourse.years = {}
+  for (const row of transferCredits) {
+    const attainmentDate = new Date(row.attainmentDate)
+    const year = getYearKey(attainmentDate, isAcademicYear)
+    if (!yearRange.includes(year)) continue
+    maxYear = Math.max(maxYear, year)
 
-      yearRange
-        .filter(year => year <= maxYear)
-        .forEach(year => {
-          if (!allCourses[normCode].years[year]) {
-            mergedCourse.years[year] = {
-              totalAllStudents: 0,
-              totalPassed: 0,
-              totalNotCompleted: 0,
-              totalAllCredits: 0,
-              totalProgrammeStudents: 0,
-              totalProgrammeCredits: 0,
-              totalOtherProgrammeStudents: 0,
-              totalOtherProgrammeCredits: 0,
-              totalWithoutStudyRightStudents: 0,
-              totalWithoutStudyRightCredits: 0,
-              totalTransferCredits: 0,
-              totalTransferStudents: 0,
-            }
-          } else {
-            mergedCourse.years[year] = { ...allCourses[normCode].years[year] }
-          }
-          if (allCourses[ayCourse].years[year]) {
-            properties.forEach(prop => {
-              mergedCourse.years[year][prop] += allCourses[ayCourse].years[year][prop]
-            })
-          }
-        })
-      allCourses[normCode] = mergedCourse
-      delete allCourses[ayCourse]
+    const course = ensureCourse(courseMap, row.courseCode, row.courseName, row.isStudyModule)
+    const yearAccumulator = ensureYear(course, year, row.isStudyModule)
+    yearAccumulator.transferStudents.add(row.studentNumber)
+    yearAccumulator.stats.transferStudentsCredits += row.credits
+  }
+
+  const notCompletedResults = await Promise.all(
+    yearRange.map(year => {
+      const yearFrom = getFrom(academicYear, year)
+      const yearTo = getTo(academicYear, year)
+      return getNotCompletedForProgrammeCourses(yearFrom, yearTo, programmeCourses)
+    })
+  )
+
+  notCompletedResults.forEach((results, index) => {
+    const year = yearRange[index]
+    if (!results) return
+    if (results.length > 0) {
+      maxYear = Math.max(maxYear, year)
+    }
+    for (const result of results) {
+      const course = ensureCourse(courseMap, result.code, result.name, result.isStudyModule ?? false)
+      const yearAccumulator = ensureYear(course, year, result.isStudyModule ?? false)
+      yearAccumulator.stats.allNotPassed += result.allNotPassed
     }
   })
-  return Object.values(allCourses)
+
+  if (maxYear === 0) {
+    maxYear = lastYear
+  }
+
+  const coursesRecord: Record<string, StudyProgrammeCourse> = {}
+
+  for (const [code, course] of courseMap.entries()) {
+    const years: Record<number, YearStats> = {}
+    for (const [year, yearAccumulator] of course.years.entries()) {
+      if (year > maxYear) continue
+      years[year] = finalizeYearStats(yearAccumulator)
+    }
+
+    if (Object.keys(years).length === 0) {
+      continue
+    }
+
+    coursesRecord[code] = {
+      code,
+      name: course.name,
+      isStudyModule: course.isStudyModule,
+      years,
+    }
+  }
+
+  return Object.values(coursesRecord)
 }