[Backend][Updater] Add way to flush redis completely in prod

Vastuuvapauslauseke: käyttö omalla vastuulla

.github/actions/lint/action.yaml

@@ -6,7 +6,7 @@ runs:
   steps:
     - uses: actions/setup-node@v6
       with:
-        node-version: 22
+        node-version: 24
     - name: 'Install dependencies'
       shell: bash
       run: npm ci && npm ci --prefix services/shared && npm ci --prefix services/frontend && npm ci --prefix services/backend

.github/actions/run-tests/action.yaml

@@ -19,7 +19,7 @@ runs:
   using: composite
   steps:
     - name: 'Login to toska Docker registry'
-      uses: docker/login-action@v3
+      uses: docker/login-action@v4
       with:
         registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
         username: ${{ inputs.username }}
@@ -27,10 +27,10 @@ runs:
 
     - name: 'Run services, pull rest of the needed images'
       shell: bash
-      run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull
+      run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull
 
     - name: 'Run cypress'
-      uses: cypress-io/github-action@v6
+      uses: cypress-io/github-action@v7
       with:
         browser: chrome
         spec: ${{ inputs.spec }}
@@ -38,7 +38,7 @@ runs:
         CYPRESS_baseUrl: http://localhost:1337
 
     - name: 'Upload video if tests fail'
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v7
       if: failure()
       with:
         name: 'Cypress videos - ${{ inputs.test_name }}'
@@ -47,4 +47,4 @@ runs:
     - name: 'Print logs on failure'
       shell: bash
       if: failure()
-      run: docker compose --file docker-compose.ci.yml logs
+      run: docker compose --file docker-compose.ci.yml --profile oodikone logs && sleep 15

.github/workflows/production.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: 'Get current time'
         id: get_current_time
@@ -49,7 +49,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Extract release version
         id: extract_version
@@ -85,7 +85,7 @@ jobs:
       - release_backend
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Extract release version
         id: extract_version
@@ -119,7 +119,7 @@ jobs:
       - release_backend
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Extract release version
         id: extract_version
@@ -157,13 +157,13 @@ jobs:
     if: success()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
-          softa-url: https://oodikone.cs.helsinki.fi/
+          softa-url: https://oodikone.helsinki.fi/
           deployment-target: production
 
   report_failure:
@@ -178,7 +178,7 @@ jobs:
     if: failure()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:

.github/workflows/pull_request.yaml

@@ -8,7 +8,6 @@ on:
       - 'cypress/**'
       - 'cypress.config.json'
       - 'services/**'
-      - '!services/backend/src/rapodiff/**'
       - 'docker-compose.ci.yml'
       - 'package*.json'
 
@@ -17,14 +16,43 @@ jobs:
     name: 'Lint packages'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Lint backend, frontend and shared
         uses: ./.github/actions/lint
 
+  test_api:
+    needs:
+      - lint_packages
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install node-24 for cypress
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v4
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+
+      - name: Start oodikone detached
+        run: npm run oodikone:ci:setup
+
+      - name: Run api-tests
+        run: npm run oodikone:ci:api
+
   test_end_to_end:
     runs-on: ubuntu-latest
     needs:
       - lint_packages
+      - test_api
     strategy:
       fail-fast: false
       matrix:
@@ -49,8 +77,6 @@ jobs:
               cypress/e2e/Users.js
           - name: 'Test faculty statistics'
             spec: cypress/e2e/Faculty_statistics.js
-          - name: 'Test open uni search'
-            spec: cypress/e2e/Openuni.js
           - name: 'Test language center view'
             spec: cypress/e2e/Language_center.js
           - name: 'Test study guidance groups'
@@ -59,7 +85,7 @@ jobs:
             spec: cypress/e2e/University.js
     name: ${{ matrix.name }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:

.github/workflows/staging.yaml

@@ -11,7 +11,6 @@ on:
       - 'cypress/**'
       - 'cypress.config.json'
       - 'services/**'
-      - '!services/backend/src/rapodiff/**'
       - 'updater/sis-updater-worker/src/db/migrations/**'
       - 'docker-compose.ci.yml'
       - 'package*.json'
@@ -29,7 +28,7 @@ jobs:
     needs: test
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Get current time
         id: get_current_time
@@ -64,7 +63,7 @@ jobs:
     needs: test
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Build backend image
         id: build-image
@@ -99,7 +98,7 @@ jobs:
     if: success()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
@@ -118,7 +117,7 @@ jobs:
     if: failure()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:

.github/workflows/test.yaml

@@ -16,7 +16,7 @@ jobs:
     name: 'Lint packages'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Lint backend, frontend and shared
         uses: ./.github/actions/lint
 
@@ -25,10 +25,10 @@ jobs:
     name: 'Build backend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: docker/setup-buildx-action@v3
+      - uses: actions/checkout@v6
+      - uses: docker/setup-buildx-action@v4
       - name: 'Build image and export it'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           build-args: |
             SENTRY_RELEASE=${{ github.sha }}
@@ -40,9 +40,11 @@ jobs:
           outputs: type=docker,dest=/tmp/backend.tar
         env:
           DOCKER_BUILD_SUMMARY: false
+
       - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v7
         with:
+          if-no-files-found: error
           name: backend-image
           path: /tmp/backend.tar
 
@@ -50,24 +52,70 @@ jobs:
     name: 'Build frontend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
+      - uses: docker/setup-buildx-action@v4
+
       - name: Get current time
         id: get_current_time
         run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
+
       - name: Build frontend image
-        id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: docker/build-push-action@v7
         with:
-          image: oodikone-frontend
-          tags: staging ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/frontend/Dockerfile
           build-args: |
             BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
             BASE_PATH=/oodikone
             STAGING=true
+          context: services
+          file: services/frontend/Dockerfile
+          push: false
+          tags: oodikone-frontend:staging
+          outputs: type=docker,dest=/tmp/frontend.tar
+        env:
+          DOCKER_BUILD_SUMMARY: false
 
+      # - name: 'Upload image for other jobs'
+      #   uses: actions/upload-artifact@v7
+      #   with:
+      #     if-no-files-found: error
+      #     name: frontend-image
+      #     path: /tmp/frontend.tar
+
+  pull_docker_images:
+    name: 'Pull all needed docker images and cache them'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v4
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+
+      - name: 'Pull required images'
+        run: docker compose --file docker-compose.ci.yml --profile oodikone --profile updater pull --ignore-buildable --quiet
+
+      - name: 'Save Docker images for downstream jobs'
+        run: |
+          docker save \
+            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db \
+            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db \
+            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db \
+            redis:7.2.6 \
+            postgres:15.10 \
+            "quay.io/toska/jami:production" \
+            nginx:latest \
+          | zstd -T0 -3 -o /tmp/docker-images.tar.zst
+
+      - name: 'Upload Docker images artifact'
+        uses: actions/upload-artifact@v7
+        with:
+          name: docker-images
+          path: /tmp/docker-images.tar.zst
+          retention-days: 1
+
+  # === Update databases ===
   check_migration_files:
     name: Check if new migrations have been added
     runs-on: ubuntu-latest
@@ -76,7 +124,7 @@ jobs:
       user_db_migrations: ${{ steps.check_migrations.outputs.user_db_migrations }}
       sis_db_migrations: ${{ steps.check_migrations.outputs.sis_db_migrations }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Check if new migrations have been added
         id: check_migrations
         uses: dorny/paths-filter@v3
@@ -94,45 +142,69 @@ jobs:
 
   build_and_push_anon_sis_db:
     name: 'Build anon sis-db and push to toska hub if new migrations have been added'
-    needs: check_migration_files
+    needs:
+      - pull_docker_images
+      - check_migration_files
     runs-on: ubuntu-latest
     if: needs.check_migration_files.outputs.sis_db_migrations == 'true'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
+
+      - name: 'Download Docker images'
+        uses: actions/download-artifact@v8
+        with:
+          name: docker-images
+          path: /tmp
+
+      - name: 'Load Docker images'
+        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
+
       - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
         run: |
-          docker compose --file docker-compose.ci.yml up --detach sis-updater-worker
+          docker compose --file docker-compose.ci.yml up --detach --quiet-pull sis-updater-worker
           sleep 30 # wait for sis-updater-worker to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
 
   build_and_push_anon_kone_db:
     name: 'Build anon kone-db and push to toska hub if new migrations have been added'
     needs:
+      - pull_docker_images
       - build_backend
       - build_frontend
       - check_migration_files
     runs-on: ubuntu-latest
     if: needs.check_migration_files.outputs.kone_db_migrations == 'true'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
+
+      - name: 'Download Docker images'
+        uses: actions/download-artifact@v8
+        with:
+          name: docker-images
+          path: /tmp
       - name: 'Download backend image'
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v8
         with:
           name: backend-image
+
+      - name: 'Load Docker images'
+        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
       - name: 'Load built backend image'
         run: docker load --input backend.tar
+
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
+
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         run: |
           docker compose --file docker-compose.ci.yml up --detach backend
@@ -142,13 +214,14 @@ jobs:
   build_and_push_anon_user_db:
     name: 'Build anon user-db and push to toska hub if new migrations have been added'
     needs:
+      - pull_docker_images
       - build_backend
       - build_frontend
       - check_migration_files
     runs-on: ubuntu-latest
     if: needs.check_migration_files.outputs.user_db_migrations == 'true'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: dorny/paths-filter@v3
         id: check_migrations
         with:
@@ -156,32 +229,87 @@ jobs:
             migrations:
               - 'services/backend/src/database/migrations_user/**'
               - '.github/workflows/test.yaml'
+
+      - name: 'Download Docker images'
+        uses: actions/download-artifact@v8
+        with:
+          name: docker-images
+          path: /tmp
       - name: 'Download backend image'
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v8
         with:
           name: backend-image
+
+      - name: 'Load Docker images'
+        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
       - name: 'Load built backend image'
         run: docker load --input backend.tar
+
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
+
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         run: |
           docker compose --file docker-compose.ci.yml up --detach backend
           sleep 30 # wait for backend to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
 
-  # === End-to-end tests ===
-  test_end_to_end:
+  # === Backend api-tests ===
+  test_api:
     needs:
+      - pull_docker_images
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
-    # Always run the tests, even if some of the needed jobs failed or were skipped
-    if: always()
+      - lint_packages
+    if: always() && !cancelled() && needs.lint_packages.result == 'success'
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install node-24
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: 'Download Docker images'
+        uses: actions/download-artifact@v8
+        with:
+          name: docker-images
+          path: /tmp
+
+      - name: 'Load Docker images'
+        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
+
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v4
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+
+      - name: Start oodikone (without frontend and backend)
+        run: npm run oodikone:ci:setup -- --quiet-pull
+
+      - name: Run api-tests
+        run: npm run oodikone:ci:api
+
+  # === End-to-end tests ===
+  test_end_to_end:
+    needs:
+      - pull_docker_images
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
+      - lint_packages
+    if: always() && !cancelled() && needs.lint_packages.result == 'success'
     strategy:
       fail-fast: false
       matrix:
@@ -206,8 +334,6 @@ jobs:
               cypress/e2e/Users.js
           - name: 'Test faculty statistics'
             spec: cypress/e2e/Faculty_statistics.js
-          - name: 'Test open uni search'
-            spec: cypress/e2e/Openuni.js
           - name: 'Test language center view'
             spec: cypress/e2e/Language_center.js
           - name: 'Test study guidance groups'
@@ -219,7 +345,33 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
+
+      - name: Install node-24 for cypress
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: 'Download Docker images'
+        uses: actions/download-artifact@v8
+        with:
+          name: docker-images
+          path: /tmp
+      - name: 'Download backend image'
+        uses: actions/download-artifact@v8
+        with:
+          name: backend-image
+      # - name: 'Download frontend image'
+      #   uses: actions/download-artifact@v8
+      #   with:
+      #     name: frontend-image
+
+      - name: 'Load Docker images'
+        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
+      - name: 'Load built backend image'
+        run: docker load --input backend.tar
+      # - name: 'Load built frontend image'
+      #   run: docker load --input frontend.tar
 
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
@@ -233,22 +385,45 @@ jobs:
   test_service_exit_statuses:
     name: 'Check services run correctly'
     needs:
+      - pull_docker_images
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
-    if: always()
+      - lint_packages
+    if: always() && !cancelled() && needs.lint_packages.result == 'success'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: 'Run services, pull rest of the needed images'
-        run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull
+      - name: 'Download Docker images'
+        uses: actions/download-artifact@v8
+        with:
+          name: docker-images
+          path: /tmp
+      - name: 'Download backend image'
+        uses: actions/download-artifact@v8
+        with:
+          name: backend-image
+      # - name: 'Download frontend image'
+      #   uses: actions/download-artifact@v8
+      #   with:
+      #     name: frontend-image
+
+      - name: 'Load Docker images'
+        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
+      - name: 'Load built backend image'
+        run: docker load --input backend.tar
+      # - name: 'Load built frontend image'
+      #   run: docker load --input frontend.tar
+
+      - name: 'Run services, build rest of the needed images'
+        run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull
 
       - name: 'Check that exit statuses are not errors'
         run: |
@@ -260,4 +435,4 @@ jobs:
 
       - name: 'Print logs on failure'
         if: failure()
-        run: docker compose --file docker-compose.ci.yml logs
+        run: docker compose --file docker-compose.ci.yml --profile oodikone logs && sleep 15

.github/workflows/updater.yaml

@@ -1,29 +1,26 @@
 name: 'Test (and release) Updater'
 
 on:
-  push:
-    branches:
-      - master
+  release:
+    types: [published]
     paths:
       - '.github/workflows/updater.yaml'
       - 'updater/sis-updater-worker/**'
       - 'updater/sis-updater-scheduler/**'
-  pull_request:
-    branches:
-      - master
-    paths:
-      - 'updater/sis-updater-worker/**'
-      - 'updater/sis-updater-scheduler/**'
 
 jobs:
   lint_scheduler_and_worker:
     name: 'Lint scheduler and worker'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: actions/setup-node@v6
         with:
-          node-version: 22
+          node-version: 24
+
+      - name: Install safe-chain
+        run: set -o pipefail && curl -fsSL https://github.com/AikidoSec/safe-chain/releases/latest/download/install-safe-chain.sh | sh -s -- --ci
+
       - name: 'Install dependencies'
         run: npm ci && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker
       - name: 'Run linter'
@@ -33,10 +30,10 @@ jobs:
     name: 'Build Updater worker'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: docker/setup-buildx-action@v3
+      - uses: actions/checkout@v6
+      - uses: docker/setup-buildx-action@v4
       - name: 'Build the image'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: updater/sis-updater-worker/
           push: false
@@ -45,7 +42,7 @@ jobs:
         env:
           DOCKER_BUILD_SUMMARY: false
       - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v7
         with:
           name: updater-worker-image
           path: /tmp/worker.tar
@@ -54,10 +51,10 @@ jobs:
     name: 'Build Updater scheduler'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: docker/setup-buildx-action@v3
+      - uses: actions/checkout@v6
+      - uses: docker/setup-buildx-action@v4
       - name: 'Build the image'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: updater/sis-updater-scheduler/
           push: false
@@ -66,7 +63,7 @@ jobs:
         env:
           DOCKER_BUILD_SUMMARY: false
       - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v7
         with:
           name: updater-scheduler-image
           path: /tmp/scheduler.tar
@@ -79,15 +76,15 @@ jobs:
       - build_worker
       - build_scheduler
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
       - name: 'Download built images'
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v8
         with:
           pattern: 'updater-{worker,scheduler}-image'
           merge-multiple: true
@@ -122,9 +119,8 @@ jobs:
     name: 'Build and release Updater worker'
     runs-on: ubuntu-latest
     needs: test_updater
-    if: github.event_name == 'push'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Build worker image
         id: build-image
@@ -152,9 +148,8 @@ jobs:
     name: 'Build and release Updater scheduler'
     runs-on: ubuntu-latest
     needs: test_updater
-    if: github.event_name == 'push'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Build scheduler image
         id: build-image

.gitignore

@@ -65,9 +65,6 @@ instrumented/
 .vscode/launch.json
 .vscode/terminals.json
 
-# Funidata specific
-environments/
-
 # Target of shared symlink
 services/**/src/shared
 

.npmrc

@@ -0,0 +1,3 @@
+ignore-scripts=true  
+min-release-age=7  
+save-prefix=  

.nycrc

@@ -1,5 +0,0 @@
-{
-  "all": true,
-  "include": ["services/frontend/src/**/*.js", "services/frontend/src/**/*.jsx"],
-  "extends": "./nyc-config.js"
-}

README.md

@@ -1,4 +1,5 @@
 # Oodikone
+
 [![Maintainability](https://qlty.sh/gh/UniversityOfHelsinkiCS/projects/oodikone/maintainability.svg)](https://qlty.sh/gh/UniversityOfHelsinkiCS/projects/oodikone)
 
 An application for analyzing university data, running at [https://oodikone.helsinki.fi/](https://oodikone.helsinki.fi/).
@@ -112,8 +113,6 @@ The `documentation` directory contains some notes on the sis-db schema and is me
 
 For internal meeting notes and sensitive information, see Toska GitLab.
 
-[SortableTable documentation](./services/frontend/src/components/SortableTable/README.md)
-
 ### Basics
 
 The development environment is entirely configured in the `docker-compose.yml` file located in this repository. The file defines the services for Oodikone's two main components: **oodikone** and **updater**.
@@ -131,7 +130,6 @@ Some useful commands are defined in `package.json` and can be run with `npm run
 - `npm run docker:down`: stops the whole environment
 - `npm run testupdater`: runs tests for updater
 - `npm run flushredis`: clears Redis, which forces big calculations to be redone in e.g. studyprogramme overview and faculty views
-- `npm run rapodiff`: runs [Rapodiff](./services/backend/src/rapodiff/)
 - `npm run install:local`: installs `node_modules` for each package (backend, frontend, sis-updater-scheduler and sis-updater-worker) locally
 - `npm run install:docker`: installs `node_modules` for each package inside containers
 - `npm run install:both`: runs both `npm run install:local` and `npm run install:docker`
@@ -189,8 +187,6 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 - Updater is deployed to production when pushed if tests pass
 - Test databases can be updated, instructions in [anonyymioodi README](anonyymioodi/README.md)
 
-[Rapodiff](./services/backend/src/rapodiff/) can be used to check differences between Rapo and Oodikone data.
-
 ## ❓FAQ
 
 ### Modules are missing after updating package.json

anonyymioodi/README.md

@@ -6,14 +6,14 @@ The quick guide should be enough for most purposes: updating schema, data, or th
 
 ## Quick guide to updating a test database
 
-1. Ensure you have the latest test-db. Just pulling images might not replace your existing volume, so you may need to remove it manually. This command might work: `docker volume rm oodikone_sis-db-data` (replace `oodikone_sis-db-data` with the correct volume name). The new image, including data, should be pulled the next time you launch Oodikone normally.
+1. Ensure you have the latest test-db. Just pulling images might not replace your existing volume, so you may need to remove it manually. This command might work: `docker volume rm oodikone-sis-db-data` (replace `oodikone-sis-db-data` with the correct volume name). The new image, including data, should be pulled the next time you launch Oodikone normally.
 
 2. Do your dev work as usual with the test database, for example, using `npm run both`. Edit data as needed, (for example, run the updater via the frontend), and then run tests to see if they pass (follow the instructions in the [README](../README.md) to run the tests more quickly).
 
 3. If you're updating `sis-db`:
 
    - Modify the scheduler limit in [config.js](../updater/sis-updater-scheduler/src/config.js) by setting `module.exports.DEV_SCHEDULE_COUNT = null`.
-   - Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone_sis-updater-worker_1`).
+   - Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone-sis-updater-worker`).
 
 4. When you're satisfied with the database, create a dump:
 
@@ -67,6 +67,6 @@ The quick guide should be enough for most purposes: updating schema, data, or th
 
 5. Navigate to <http://localhost:3000/updater> and click the three buttons (Update meta, Update students, Update curriculums) one by one. This will populate `sis-db` with data from the new `sis-importer-db`. You might want to do each update a couple of times, as the database tables affect each other, and not all necessary data might be generated on the first try.
 
-6. Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone_sis-updater-worker_1`). If you see any errors, try clicking the buttons again.
+6. Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone-sis-updater-worker`). If you see any errors, try clicking the buttons again.
 
 7. Run all the tests to ensure everything is working as expected (follow the instructions in the [README](../README.md) to run the tests more quickly). If any tests fail, update the test files accordingly.

anonyymioodi/update_staging_db.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+
+# This script will first make db dumps of the test environment and upload the dumps to staging
+
+
+. ../scripts/utils.sh
+set -euo pipefail
+
+main() {
+  declare -r db_names="sis-db kone-db user-db"
+  declare -A db_url_names
+  db_url_names["sis-db"]="SIS_DB_URL"
+  db_url_names["kone-db"]="DB_URL_KONE"
+  db_url_names["user-db"]="DB_URL_USER"
+
+  declare -r dump_directory="dumps"
+
+  # Takes the first db-tools pod
+  check_oc_logged_staging
+  db_tools_name=$(oc get pods -l app=db-tools --output jsonpath='{.items[0].metadata.name}')
+  readonly db_tools_name
+
+
+  create_dumps
+  upload_and_migrate_dumps
+}
+
+
+
+function create_dumps() {
+  echo "Starting test/anon containers..."
+  (npm run docker:down && npm run both) || die "Failed to start containers"
+  echo
+  echo "Waiting for containers to start..."
+  sleep 10 # wait for containers to start
+
+  # Create dumps for all dbs
+  echo
+  echo "Creating dumps for $db_names"
+  echo
+
+  # Create dir if not exists
+  if [ ! -d $dump_directory ]; then mkdir -p $dump_directory; fi
+
+  for db in $db_names
+  do
+    create_dump_from_db "$db"
+    if [ ! -f "$dump_directory/$db.sqz" ]; then die "Db dump not created"; fi
+  done
+
+  echo "Dumps can be found in $PWD/\*.sqz"
+}
+
+
+# $1 db-name
+function create_dump_from_db() {
+  echo "Dumping db $1..."
+
+  docker exec "$1" pg_dump -Fc -U postgres "$1" > "$dump_directory/$1.sqz" || die "Failed to dump db $1"
+}
+
+
+function check_oc_logged_staging() {
+  echo -n "Checking if oc is logged into toska staging..."
+  [[ $(oc config current-context) == "toska/api-ocp-test-0-k8s-it-helsinki-fi"* ]] || die "\nNot logged in to toska staging environment: $(oc config current-context)"
+  echo " OK"
+}
+
+
+function upload_and_migrate_dumps() {
+
+  local target_dump_directory="$db_tools_name:/tmp/oodikone-staging-dumps"
+  echo -n "Dumping $dump_directory/ to $target_dump_directory..."
+  oc rsync "$dump_directory/" "$target_dump_directory" 1> /dev/null # show errors
+  echo " OK"
+  echo
+
+  for db in $db_names
+  do
+    migrate_dumps "$db"
+  done
+}
+
+
+function migrate_dumps() {
+  local db_url_b64
+  local db_url_raw
+  local db_url
+
+  db_url_b64=$(oc get secrets -l app.kubernetes.io/instance=oodikone --output jsonpath="{.items[0].data.${db_url_names[$1]}}")
+  db_url_raw=$(echo "$db_url_b64" | base64 --decode)
+  db_url="${db_url_raw%\?*}"
+
+  [[ $db_url != *"oodikone_*_test"* ]] || die "Trying to use non-test database: $db_url"
+
+  echo "Migrating $1 to url: $db_url"
+
+  oc exec "$db_tools_name" -- pg_restore --clean --no-owner -d "$db_url" "/tmp/oodikone-staging-dumps/$1.sqz"
+}
+
+
+# Run script
+main

cypress.config.js

@@ -3,7 +3,6 @@
 const { defineConfig } = require('cypress')
 
 module.exports = defineConfig({
-  experimentalStudio: process.env.NODE_ENV === 'development',
   projectId: 'c3jsph',
   defaultCommandTimeout: 30000,
   requestTimeout: 30000,
@@ -14,7 +13,6 @@ module.exports = defineConfig({
     // We've imported your old cypress plugins here.
     // You may want to clean this up later by importing these.
     setupNodeEvents(on, config) {
-      require('@cypress/code-coverage/task')(on, config)
       require('./cypress/plugins/index.js')(on, config)
       return config
     },

cypress/e2e/Custom_population.js

@@ -12,7 +12,7 @@ const save = () => {
 }
 
 const selectSavedPopulation = name => {
-  cy.cs('history-search').click().type(`${name}{enter}`)
+  cy.cs('history-search').type(`${name}{enter}`)
 }
 
 const deleteAllSearches = () => {

cypress/e2e/Degree_programme_overview.js

@@ -93,31 +93,14 @@ describe('Degree programme overview', () => {
     it('Basic information contains correct credits', () => {
       const years = getEmptyYears()
       const tableContents = [
-        ...years.map(year => [year, 0, 0, 0, 0, 0]),
-        [2023, 1519, 1519, 0, 0, 222],
-        [2022, 3235, 3205, 0, 30, 209],
-        [2021, 5133, 5108, 0, 25, 428],
-        [2020, 5801, 5796, 0, 5, 94],
-        [2019, 5305, 5305, 0, 0, 162],
-        [2018, 3442, 3432, 0, 10, 21],
-        [2017, 1211, 1211, 0, 0, 189],
-      ]
-
-      cy.checkTableStats(tableContents, 'credits-produced-by-the-study-programme')
-    })
-
-    it("Toggling 'Show special categories' on displays additional information", () => {
-      cy.cs('special-categories-toggle').click()
-      const years = getEmptyYears()
-      const tableContents = [
-        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0]),
-        [2023, 1519, 1519, 0, 0, 0, 0, 222],
-        [2022, 3235, 3205, 0, 30, 0, 0, 209],
-        [2021, 5133, 5108, 0, 25, 0, 0, 428],
-        [2020, 5801, 5796, 0, 5, 0, 0, 94],
-        [2019, 5305, 5305, 0, 0, 0, 0, 162],
-        [2018, 3442, 3432, 0, 10, 0, 0, 21],
-        [2017, 1211, 1211, 0, 0, 0, 0, 189],
+        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+        [2023, 1519, 1519, 0, 0, 0, 0, 0, 222, 0],
+        [2022, 3235, 3205, 0, 0, 30, 0, 0, 209, 0],
+        [2021, 5133, 5108, 0, 0, 25, 0, 0, 428, 25],
+        [2020, 5801, 5796, 0, 0, 5, 0, 0, 94, 0],
+        [2019, 5305, 5305, 0, 0, 0, 0, 0, 162, 0],
+        [2018, 3442, 3432, 0, 0, 10, 0, 0, 21, 0],
+        [2017, 1211, 1211, 0, 0, 0, 0, 0, 189, 0],
       ]
 
       cy.checkTableStats(tableContents, 'credits-produced-by-the-study-programme')
@@ -190,14 +173,14 @@ describe('Degree programme overview', () => {
       cy.checkTableStats(studentTableContents, 'students-of-the-study-programme')
 
       const creditTableContents = [
-        ...years.map(year => [year, 0, 0, 0, 0, 0]),
-        ['2023 - 2024', 160, 160, 0, 0, 67],
-        ['2022 - 2023', 2725, 2720, 0, 5, 337],
-        ['2021 - 2022', 4092, 4042, 0, 50, 198],
-        ['2020 - 2021', 5420, 5415, 0, 5, 321],
-        ['2019 - 2020', 6043, 6043, 0, 0, 101],
-        ['2018 - 2019', 4856, 4851, 0, 5, 107],
-        ['2017 - 2018', 2350, 2345, 0, 5, 26],
+        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+        ['2023 - 2024', 160, 160, 0, 0, 0, 0, 0, 67, 0],
+        ['2022 - 2023', 2725, 2720, 0, 0, 5, 0, 0, 337, 0],
+        ['2021 - 2022', 4092, 4042, 0, 0, 50, 0, 0, 198, 0],
+        ['2020 - 2021', 5420, 5415, 0, 0, 5, 0, 0, 321, 25],
+        ['2019 - 2020', 6043, 6043, 0, 0, 0, 0, 0, 101, 0],
+        ['2018 - 2019', 4856, 4851, 0, 0, 5, 0, 0, 107, 0],
+        ['2017 - 2018', 2350, 2345, 0, 0, 5, 0, 0, 26, 0],
       ]
 
       cy.checkTableStats(creditTableContents, 'credits-produced-by-the-study-programme')
@@ -224,19 +207,20 @@ describe('Degree programme overview', () => {
         .should('contain', 47)
         .should('contain', 76)
 
-      cy.cs('bachelor-breakdown-bar-chart-section')
+      cy.cs('unset-breakdown-bar-chart')
       cy.cs('graduation-time-toggle').click()
-      cy.cs('bachelor-median-time-bar-chart-section').within(() => {
-        cy.contains('Graduation year')
-        cy.contains('2022')
-        cy.contains('48 graduated').trigger('mouseover')
-        cy.contains('48 students graduated in year 2022')
-        cy.contains('median study time: 44.5 months')
-        cy.contains('22 graduated on time')
-        cy.contains('12 graduated max year overtime')
-        cy.contains('14 graduated over year late')
+      cy.cs('unset-median-bar-chart').within(() => {
+        cy.contains('48 graduated').should('be.visible')
+        cy.contains('48 graduated').hover()
       })
 
+      cy.get('.grad-vals').contains('48 students graduated in year 2022').should('be.visible')
+
+      cy.contains('median study time: 6.5 semesters')
+      cy.contains('24 graduated on time')
+      cy.contains('12 graduated max year overtime')
+      cy.contains('12 graduated over year late')
+
       cy.cs('programmes-before-or-after-stacked-bar-chart-section')
         .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
         .should('contain', 'Datatieteen maisteriohjelma')
@@ -252,33 +236,37 @@ describe('Degree programme overview', () => {
       cy.init('/study-programme')
       cy.contains('a', 'Matematiikan ja tilastotieteen maisteriohjelma').click({ force: true })
 
-      cy.cs('bachelor-breakdown-bar-chart-section')
-      cy.cs('master-breakdown-bar-chart-section')
+      cy.cs('unset-breakdown-bar-chart')
+      cy.cs('unset-breakdown-bar-chart')
 
       cy.cs('graduation-time-toggle').click()
-      cy.cs('master-median-time-bar-chart-section').within(() => {
-        cy.contains('Master study right')
-        cy.contains('Graduation year')
-        cy.contains('2021')
-        cy.contains('2 graduated').trigger('mouseover')
-        cy.contains('2 students graduated in year 2021')
-        cy.contains('median study time: 25 months')
-        cy.contains('1 graduated on time')
-        cy.contains('1 graduated max year overtime')
-        cy.contains('0 graduated over year late')
-      })
+      cy.cs('unset-graduation-times-section')
+        .eq(1)
+        .within(() => {
+          cy.contains('Master study right')
+          cy.contains('2 graduated').should('be.visible')
+          cy.contains('2 graduated').hover()
+        })
 
-      cy.cs('bachelor-median-time-bar-chart-section').within(() => {
-        cy.contains('Bachelor + master study right')
-        cy.contains('Graduation year')
-        cy.contains('2023')
-        cy.contains('11 graduated').trigger('mouseover')
-        cy.contains('11 students graduated in year 2023')
-        cy.contains('median study time: 69 months')
-        cy.contains('3 graduated on time')
-        cy.contains('7 graduated max year overtime')
-        cy.contains('1 graduated over year late')
-      })
+      cy.get('.grad-vals').contains('2 students graduated in year 2021').should('be.visible')
+      cy.contains('median study time: 4.5 semesters')
+      cy.contains('1 graduated on time')
+      cy.contains('1 graduated max year overtime')
+      cy.contains('0 graduated over year late')
+
+      cy.cs('unset-graduation-times-section')
+        .eq(0)
+        .within(() => {
+          cy.contains('Bachelor + master study right')
+          cy.contains('11 graduated').should('be.visible')
+          cy.contains('11 graduated').hover()
+        })
+
+      cy.get('.grad-vals').contains('11 students graduated in year 2023').should('be.visible')
+      cy.contains('median study time: 12 semesters')
+      cy.contains('4 graduated on time')
+      cy.contains('6 graduated max year overtime')
+      cy.contains('1 graduated over year late')
     })
   })
 
@@ -286,7 +274,7 @@ describe('Degree programme overview', () => {
     beforeEach(() => {
       cy.intercept(
         'GET',
-        'api/v2/studyprogrammes/KH50_001/studytrackstats?graduated=GRADUATED_INCLUDED&special_groups=SPECIAL_INCLUDED&combined_programme='
+        'api/studyprogrammes/KH50_001/studytrackstats?special_groups=SPECIAL_INCLUDED&combined_programme='
       ).as('stQuery')
       cy.init('/study-programme')
       cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click()
@@ -319,7 +307,7 @@ describe('Degree programme overview', () => {
 
     it('Students of the degree programme are shown correctly', () => {
       const tableContents = [
-        // [Year, All, Started studying, Present, Absent, Inactive, Graduated, Men, Women, Other/Unknown, Finland, Other]
+        // [Year, All, Started studying, Present, Absent, Passive, Graduated, Men, Women, Other/Unknown, Finland, Other]
         ['2023 - 2024', 8, 8, 0, 0, 8, 0, 5, 3, 0, 8, 1],
         ['2022 - 2023', 26, 25, 0, 0, 24, 2, 19, 7, 0, 25, 1],
         ['2021 - 2022', 38, 29, 0, 0, 33, 5, 29, 9, 0, 36, 4],
@@ -372,14 +360,14 @@ describe('Degree programme overview', () => {
     describe('Population link button works for', () => {
       it('a single year', () => {
         cy.cs('2023-population-link-button').first().click()
-        cy.contains('Matemaattisten tieteiden kandiohjelma 2023 - 2024')
-        cy.contains('Class size 8 students')
+        cy.contains('Matemaattisten tieteiden kandiohjelma')
+        cy.contains('Class of 2023 - 2024, 8 students')
       })
 
       it('total', () => {
         cy.cs('total-population-link-button').click()
         cy.contains('Matemaattisten tieteiden kandiohjelma')
-        cy.contains('Class size 227 students')
+        cy.contains('Class of 2017 - 2026, 227 students')
       })
 
       it('Links to class statistics page with study track info included work', () => {
@@ -388,9 +376,9 @@ describe('Degree programme overview', () => {
           cy.get('a').click()
         })
 
-        cy.contains('Matemaattisten tieteiden kandiohjelma 2022 - 2023')
+        cy.contains('Matemaattisten tieteiden kandiohjelma')
         cy.contains('Studytrack MAT-MAT')
-        cy.contains('Class size 26 students')
+        cy.contains('Class of 2022 - 2023, 26 students')
         cy.contains('Showing 3 out of 26 students')
       })
     })
@@ -398,55 +386,68 @@ describe('Degree programme overview', () => {
     it('Student progress data is shown correctly', () => {
       const years = getEmptyYears(true)
       const tableContents = [
-        // [Year, All, < 30 credits, 30–60 credits, 60–90 credits, 90–120 credits, 120–150 credits, 150–180 credits, ≥ 180 credits]
-        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0]),
-        ['2023 - 2024', 8, 8, 0, 0, 0, 0, 0, 0],
-        ['2022 - 2023', 26, 9, 9, 4, 3, 0, 1, 0],
-        ['2021 - 2022', 38, 8, 6, 11, 8, 5, 0, 0],
-        ['2020 - 2021', 30, 2, 1, 3, 7, 4, 4, 9],
-        ['2019 - 2020', 35, 1, 0, 2, 1, 0, 4, 27],
-        ['2018 - 2019', 45, 0, 1, 1, 2, 0, 3, 38],
-        ['2017 - 2018', 47, 0, 1, 3, 0, 1, 2, 40],
-        ['Total', 229, 28, 18, 24, 21, 10, 14, 114],
+        // [Year, All, < 30 credits, 30–60 credits, 60–90 credits, 90–120 credits, 120–150 credits, 150–180 credits, ≥ 180 credits, Graduated]
+        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+        ['2023 - 2024', 8, 8, 0, 0, 0, 0, 0, 0, 0],
+        ['2022 - 2023', 26, 9, 8, 4, 3, 0, 0, 0, 2],
+        ['2021 - 2022', 38, 8, 4, 9, 8, 4, 0, 0, 5],
+        ['2020 - 2021', 30, 2, 1, 2, 5, 0, 0, 1, 19],
+        ['2019 - 2020', 35, 1, 0, 2, 1, 0, 0, 4, 27],
+        ['2018 - 2019', 45, 0, 1, 1, 2, 0, 0, 2, 39],
+        ['2017 - 2018', 47, 0, 1, 3, 0, 0, 1, 0, 42],
+        ['Total', 229, 28, 15, 21, 19, 4, 1, 7, 134],
       ]
 
       cy.checkTableStats(tableContents, 'study-programme-progress')
     })
 
     it('Progress section', () => {
+      cy.cs('programme-progress-bar-chart-section').scrollIntoView()
+
       cy.cs('programme-progress-bar-chart-section')
         .should('contain', 'Less than 30 credits')
         .should('contain', '30–60 credits')
         .should('contain', 'At least 180 credits')
-        .should('contain', '49.8%') // The percentage for total, at least 180 credits, to check that the graph renders
+        .should('contain', '58.5%') // The percentage for total graduated, to check that the graph renders
 
-      cy.cs('programme-progress-bar-chart-section').contains('49.8%').trigger('mouseover', { force: true })
-      cy.contains('At least 180 credits: 114')
+      cy.cs('programme-progress-bar-chart-section').contains('58.5%').hover()
+      cy.contains('Graduated: 134')
     })
 
-    it('Average graduation times section', () => {
+    // CI contains different data than locally
+    it.skip('Average graduation times section', () => {
+      cy.cs('average-graduation-times-section').scrollIntoView()
+
       cy.cs('average-graduation-times-section').within(() => {
-        cy.contains('2020 - 2021')
-        cy.get('[aria-label="2020 - 2021, 15. On time."]').trigger('mouseover')
-        cy.contains('On time: 15')
-        cy.contains("Click a bar to view that year's study track level breakdown")
-        cy.get('[aria-label="2019 - 2020, 22. On time."]').click()
+        cy.cs('unset-graduation-times-section').within(() => {
+          cy.contains('2020 - 2021')
+        })
+
+        cy.get('text').contains('16').hover()
+        cy.contains('On time: 16')
+
+        cy.cs('unset-graduation-times-section').within(() => {
+          cy.contains("Click a bar to view that year's study track level breakdown")
+          cy.get('text').contains('22').click()
+        })
+
         cy.contains('Year 2019 - 2020 by start year')
-        cy.get('[aria-label="MAT-MAT, 14. On time."]').trigger('mouseover')
+
+        cy.get('text').contains('13').hover()
         cy.contains('Matematiikka')
         cy.contains('MAT-MAT')
-        cy.contains('On time: 14')
+        cy.contains('On time: 13')
       })
 
       cy.cs('graduation-time-toggle').click()
 
       cy.cs('average-graduation-times-section').within(() => {
         cy.contains('2020 - 2021')
-        cy.contains('19 graduated').trigger('mouseover')
+        cy.contains('19 graduated').hover()
         cy.contains('From class of 2020 - 2021, 19/30 students have graduated')
-        cy.contains('median study time: 33 months')
-        cy.contains('15 graduated on time')
-        cy.contains('3 graduated max year overtime')
+        cy.contains('median study time: 6 semesters')
+        cy.contains('16 graduated on time')
+        cy.contains('2 graduated max year overtime')
         cy.contains('1 graduated over year late')
       })
     })
@@ -465,7 +466,7 @@ describe('Degree programme overview', () => {
       it('Students of the study track are shown correctly', () => {
         cy.cs('study-track-overview-section').contains('Students of the study track MAT-MAT by starting year')
         const tableContents = [
-          // [Year, All, Started studying, Present, Absent, Inactive, Graduated, Men, Women, Other/Unknown, Finland, Other]
+          // [Year, All, Started studying, Present, Absent, Passive, Graduated, Men, Women, Other/Unknown, Finland, Other]
           ['2022 - 2023', 3, 3, 0, 0, 1, 2, 2, 1, 0, 3, 0],
           ['2021 - 2022', 5, 1, 0, 0, 1, 4, 3, 2, 0, 5, 0],
           ['2020 - 2021', 13, 10, 0, 0, 1, 12, 7, 6, 0, 12, 3],
@@ -480,13 +481,14 @@ describe('Degree programme overview', () => {
       it('Links to class statistics page with study track info included work', () => {
         cy.cs('2020-population-link-button').first().click()
 
-        cy.contains('Matemaattisten tieteiden kandiohjelma 2020 - 2021')
+        cy.contains('Matemaattisten tieteiden kandiohjelma')
         cy.contains('Studytrack MAT-MAT')
-        cy.contains('Class size 30 students')
+        cy.contains('Class of 2020 - 2021, 30 students')
         cy.contains('Showing 10 out of 30 students')
       })
 
-      it('Info message about missing progress stats is displayed', () => {
+      // FIXME: This feature was implemented. The data is no-good, so maybe reimplement this when it's clean.
+      it.skip('Info message about missing progress stats is displayed', () => {
         cy.contains('Progress of students of the study track MAT-MAT by starting year')
         cy.contains('Progress data is currently only available for all students of the degree programme.')
       })
@@ -496,7 +498,7 @@ describe('Degree programme overview', () => {
         cy.get("[data-cy='graduation-times-graph-breakdownBachelor']").within(() => {
           cy.contains('Start year')
           cy.contains('2020 - 2021')
-          cy.get('[aria-label="2020 - 2021, 9. On time."]').trigger('mouseover')
+          cy.get('[aria-label="2020 - 2021, 9. On time."]').hover()
           cy.contains('Graduated On time: 9 students')
         })
 
@@ -504,7 +506,7 @@ describe('Degree programme overview', () => {
         cy.cs('bachelor-median-time-bar-chart').within(() => {
           cy.contains('Start year')
           cy.contains('2020 - 2021')
-          cy.contains('12 graduated').trigger('mouseover')
+          cy.contains('12 graduated').hover()
           cy.contains('From class of 2020 - 2021, 12/13 students have graduated')
           cy.contains('median study time: 34 months')
           cy.contains('9 graduated on time')
@@ -528,14 +530,21 @@ describe('Degree programme overview', () => {
 
     describe('By credit type tab', () => {
       beforeEach(() => {
-        cy.get('tbody').within(() => {
+        cy.get('thead').within(() => {
           cy.get('tr')
             .eq(0)
+            .within(() => {
+              cy.get('th').eq(3).click()
+            })
+        })
+        cy.get('tbody').within(() => {
+          cy.get('tr')
+            .eq(8)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('341')
+              cy.get('td').eq(1).contains('MAT11004')
+              cy.get('td').eq(2).contains('Differentiaalilaskenta')
+              cy.get('td').eq(3).contains('1170')
             })
         })
       })
@@ -548,12 +557,12 @@ describe('Degree programme overview', () => {
 
         cy.get('tbody').within(() => {
           cy.get('tr')
-            .eq(0)
+            .eq(9)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('121')
+              cy.get('td').eq(1).contains('MAT21009')
+              cy.get('td').eq(2).contains('Kandidaatintutkielma')
+              cy.get('td').eq(3).contains('414')
             })
         })
       })
@@ -568,12 +577,12 @@ describe('Degree programme overview', () => {
 
         cy.get('tbody').within(() => {
           cy.get('tr')
-            .eq(0)
+            .eq(5)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('311')
+              cy.get('td').eq(1).contains('MAT11001')
+              cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
+              cy.get('td').eq(3).contains('1220')
             })
         })
       })
@@ -632,31 +641,67 @@ describe('Degree programme overview', () => {
       })
 
       it("'Show credits/Show students' toggle works", () => {
-        cy.get('thead tr').within(() => {
-          cy.get('th').should('have.length', 8)
-          const headers = ['Type', 'Code', 'Name', 'Total', 'Major', 'Non-major', 'Non-degree', 'Transferred']
-          headers.forEach((header, index) => {
-            cy.get('th').eq(index).contains(header)
+        cy.get('thead tr')
+          .eq(0)
+          .within(() => {
+            cy.get('th').should('have.length', 6)
+            const headers = ['Type', 'Code', 'Name', 'Total credits', 'Credits produced by', 'Transferred credits']
+            headers.forEach((header, index) => {
+              cy.get('th').eq(index).contains(header)
+            })
+          })
+        cy.get('thead tr')
+          .eq(1)
+          .within(() => {
+            cy.get('th').should('have.length', 7)
+            const headers = [
+              'Degree students',
+              'Open university (hetu)',
+              'Open university (no hetu)',
+              'Exchange students',
+              'Other universities',
+              'Separate studies',
+              'Other',
+            ]
+            headers.forEach((header, index) => {
+              cy.get('th').eq(index).contains(header)
+            })
           })
-        })
 
         cy.cs('show-credits-students-toggle').click()
 
+        cy.get('thead tr')
+          .eq(0)
+          .within(() => {
+            cy.get('th').should('have.length', 7)
+            const headers = [
+              'Type',
+              'Code',
+              'Name',
+              'Total students',
+              'Breakdown of total',
+              'Breakdown of passed',
+              'Not included in total or passed',
+            ]
+            headers.forEach((header, index) => {
+              cy.get('th').eq(index).contains(header)
+            })
+          })
         cy.get('thead tr')
           .eq(1)
           .within(() => {
             cy.get('th').should('have.length', 10)
             const headers = [
-              'Type',
-              'Code',
-              'Name',
-              'Total',
               'Passed',
-              'Not',
-              'Major',
-              'Non-major',
-              'Non-degree',
-              'Transferred',
+              'Not completed',
+              'Degree students',
+              'Open university students (with hetu)',
+              'Open university students (without hetu)',
+              'Exchange students',
+              'Other university students',
+              'Separate studies',
+              'Other students',
+              'Students with transferred credits',
             ]
             headers.forEach((header, index) => {
               cy.get('th').eq(index).contains(header)
@@ -665,12 +710,12 @@ describe('Degree programme overview', () => {
 
         cy.get('tbody').within(() => {
           cy.get('tr')
-            .eq(0)
+            .eq(5)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('341')
+              cy.get('td').eq(1).contains('MAT11001')
+              cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
+              cy.get('td').eq(3).contains('375')
             })
         })
       })
@@ -682,7 +727,7 @@ describe('Degree programme overview', () => {
       })
 
       // FIXME: Flaky
-      it('test', () => {
+      it.skip('test', () => {
         // TODO: Implement some real tests for this tab
         cy.contains('From')
       })
@@ -797,7 +842,8 @@ describe('Degree programme overview', () => {
         cy.contains('Students added to tag')
 
         cy.cs('population-link-button').click()
-        cy.contains('Matemaattisten tieteiden kandiohjelma 2022 - 2023')
+        cy.contains('Matemaattisten tieteiden kandiohjelma')
+        cy.contains('Class of 2022 - 2023')
         cy.contains(`Tagged with: ${tagName}`)
         cy.contains('Students (5)')
           .parent()

cypress/e2e/Filters.js

@@ -184,7 +184,7 @@ describe.skip('Population Statistics', { testIsolation: false }, () => {
     cy.selectFromDropdown(`courseFilter-${courses[0].code}`, 1)
     checkFilteredStudentCount(7)
     cy.selectFromAutocomplete('courseFilter', `${courses[1].code} - ${courses[1].name}`)
-    checkFilteredStudentCount(5)
+    checkFilteredStudentCount(6)
     cy.selectFromDropdown(`courseFilter-${courses[1].code}`, 1)
     checkFilteredStudentCount(4)
 
@@ -273,12 +273,12 @@ describe('Population Statistics with Bachelor + Master', { testIsolation: false
     checkFilteredStudentCount(0)
 
     // Non-Bachelor
-    cy.get('input[name="studyRightStatusFilter"][value="Inactive Bachelor study right"]').click()
-    cy.get('input[name="studyRightStatusFilter"][value="Inactive Bachelor study right"]').should('be.checked')
+    cy.get('input[name="studyRightStatusFilter"][value="Passive Bachelor study right"]').click()
+    cy.get('input[name="studyRightStatusFilter"][value="Passive Bachelor study right"]').should('be.checked')
     checkFilteredStudentCount(5)
     // Non-Master (combined)
-    cy.get('input[name="studyRightStatusFilter"][value="Inactive Master study right"]').click()
-    cy.get('input[name="studyRightStatusFilter"][value="Inactive Master study right"]').should('be.checked')
+    cy.get('input[name="studyRightStatusFilter"][value="Passive Master study right"]').click()
+    cy.get('input[name="studyRightStatusFilter"][value="Passive Master study right"]').should('be.checked')
     checkFilteredStudentCount(24)
 
     cy.cs('studyRightStatusFilter-clear').click()
@@ -453,9 +453,9 @@ describe('Custom Population Statistics', { testIsolation: false }, () => {
     cy.selectFromDropdown(`courseFilter-${courses[0].code}`, 1)
     checkFilteredStudentCount(6)
     cy.selectFromAutocomplete('courseFilter', `${courses[1].code} - ${courses[1].name}`)
-    checkFilteredStudentCount(5)
+    checkFilteredStudentCount(6)
     cy.selectFromDropdown(`courseFilter-${courses[1].code}`, 3)
-    checkFilteredStudentCount(2)
+    checkFilteredStudentCount(6)
 
     cy.cs(`courseFilter-${courses[1].code}-clear`).click()
     checkFilteredStudentCount(6)

cypress/e2e/Language_center.js

@@ -44,7 +44,7 @@ describe('When language center is opened', () => {
         cy.get('table > tbody > tr:first').within(() => {
           cy.get('td').should('have.length', 15)
           cy.get('td').eq(0).contains('All courses total')
-          const numbers = [2076, 36, 9, 33, 756, 15, 40, 6, 48, 1, 24, 1059, 49, 0]
+          const numbers = [2076, 36, 9, 33, 757, 15, 40, 6, 48, 1, 24, 1059, 48, 0]
           numbers.forEach((number, index) => {
             cy.get('td')
               .eq(index + 1)
@@ -55,12 +55,12 @@ describe('When language center is opened', () => {
 
       it('Faculties tab "exceeding" button works', () => {
         cy.cs('difference-button').click()
-        checkNumbers([65, 2, 0, 3, 13, 0, 1, 1, 2, 0, 1, 30, 11, 1], 15, 'faculties')
+        checkNumbers([63, 2, 0, 3, 13, 0, 1, 1, 2, 0, 0, 30, 10, 1], 15, 'faculties')
       })
 
       it('Faculties tab semester selector changes numbers', () => {
         chooseSemester('Syksy 2020', 'from')
-        checkNumbers([1184, 28, 6, 27, 496, 4, 10, 2, 33, 0, 9, 535, 34, 0], 15, 'faculties')
+        checkNumbers([1184, 28, 6, 27, 497, 4, 10, 2, 33, 0, 9, 535, 33, 0], 15, 'faculties')
       })
     })
 

cypress/e2e/Openuni.js

@@ -1,126 +0,0 @@
-/// <reference types="cypress" />
-
-const deleteAllSearches = () => {
-  cy.cs('open-uni-search-button').click()
-  cy.contains('Saved populations').parent().parent().parent().parent().get('.search').as('search')
-
-  cy.get('@search').then($el => {
-    const searchItems = $el.find('div[role=option] > span[class=text]')
-    for (let i = searchItems.length - 1; 0 <= i; i--) {
-      if (searchItems[i].textContent.includes('TEST-')) {
-        cy.get('[data-cy="history-search"]').children().eq(0).type(`${searchItems[i].textContent}{enter}`)
-        cy.get('button').contains('Delete').click()
-      }
-    }
-  })
-}
-
-// Skipping the whole test suite as it is unclear whether the feature has any use or will be
-// removed alltogether.
-describe.skip('Open uni population tests', () => {
-  describe('Open uni form can be used', () => {
-    // When openUniUser is created, change it here
-    beforeEach(() => {
-      cy.init('/openunipopulation', 'admin')
-    })
-    after(() => {
-      cy.init('/openunipopulation', 'admin')
-      cy.contains('Open uni student population')
-      deleteAllSearches()
-    })
-
-    // TODO: Rewrite the whole component
-    it.skip('Finds a proper population', () => {
-      cy.fixture('openUniPopulation').then(({ courseCodesSet1 }) => {
-        cy.get('[data-cy="open-uni-search-button"]').click()
-        cy.contains('Insert course code(s)').siblings().get('textarea').as('textarea')
-        cy.get('@textarea').type(courseCodesSet1.join('\n'))
-        cy.cs('begin-of-search').click()
-        cy.cs('begin-of-search').clear()
-        cy.cs('begin-of-search').type('01.01.2015')
-        cy.cs('end-of-search').click()
-        cy.cs('end-of-search').clear()
-        cy.cs('end-of-search').type('01.01.2020')
-        cy.get('button').contains('Search population').click()
-        cy.contains('Beginning of the search for all fields:')
-        cy.contains('01.01.2015')
-        cy.contains('End of the search for enrollments:')
-        cy.contains('01.01.2020')
-        cy.location('pathname').should('eq', '/openunipopulation')
-        cy.location('search').then(search => {
-          const params = new URLSearchParams(search)
-
-          const courseCodes = params.getAll('courseCode')
-          expect(courseCodes).to.include.members(['TKT10001', 'TKT10002'])
-
-          expect(params.get('startdate')).to.eq('01-01-2015')
-          expect(params.get('enddate')).to.eq('01-01-2020')
-        })
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[0])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[1])
-      })
-    })
-
-    it('Textarea can handle different kinds of input', () => {
-      cy.get('[data-cy="open-uni-search-button"]').click()
-      cy.contains('Insert course code(s)')
-        .siblings()
-        .get('textarea')
-        .type('tkt10001    tkt10002,tkt10003\n TKT10004,,TKt10005\nTKT-1234')
-      cy.get('button').contains('Search population').click()
-      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10001')
-      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10002')
-      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10003')
-      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10004')
-      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10005')
-      cy.get('[data-cy="open-uni-table-div"]').should('not.contain', 'TKT-1234')
-    })
-
-    it('Saves a custom population search', () => {
-      cy.fixture('openUniPopulation').then(({ courseCodesSet1 }) => {
-        cy.get('[data-cy="open-uni-search-button"]').click()
-        cy.get('[data-cy="search-name"]').type('TEST-Avoimen-populaatio')
-        cy.contains('Insert course code(s)').siblings().get('textarea').type(courseCodesSet1.join('\n'))
-        cy.get('[data-cy="save-search"]').click()
-        cy.get('[data-cy="history-search"]').children().eq(0).type('TEST-Avoimen-populaatio{enter}')
-        cy.contains('TEST-Avoimen-populaatio')
-        cy.get('button').contains('Search population').as('btn')
-        cy.get('@btn').click()
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[0])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[1])
-      })
-    })
-
-    it('Updates a custom population search', () => {
-      cy.fixture('openUniPopulation').then(({ courseCodesSet1, courseCodesSet2 }) => {
-        cy.get('[data-cy="open-uni-search-button"]').click()
-        cy.get('[data-cy="search-name"]').type('TEST-Avoimen-populaatio')
-        cy.contains('Insert course code(s)').siblings().get('textarea').type(courseCodesSet1.join('\n'))
-        cy.get('[data-cy="save-search"]').click()
-        cy.get('button').contains('Search population').click()
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[0])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[1])
-
-        cy.get('[data-cy="open-uni-search-button"]').click()
-        cy.get('[data-cy="history-search"]').children().eq(0).type('TEST-Avoimen-populaatio{enter}')
-        cy.contains('Insert course code(s)').siblings().get('textarea').should('contain', courseCodesSet1.join(', '))
-        cy.contains('Insert course code(s)').siblings().get('textarea').type('\n')
-        cy.contains('Insert course code(s)').siblings().get('textarea').type(courseCodesSet2.join('\n'))
-        cy.get('[data-cy="save-search"]').click()
-        cy.get('button').contains('Search population').click()
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[0])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[1])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet2[0])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet2[1])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet2[2])
-      })
-    })
-
-    it('View does not crash with non-existing courses', () => {
-      cy.get('[data-cy="open-uni-search-button"]').click()
-      cy.contains('Insert course code(s)').siblings().get('textarea').type('cypress-fun')
-      cy.get('button').contains('Search population').click()
-      cy.get('[data-cy="open-uni-table-div"]').should('not.contain', 'cypress-fun')
-    })
-  })
-})

cypress/e2e/Population_statistics.js

@@ -27,14 +27,14 @@ describe('Population statistics tests', () => {
   describe('When using basic user', () => {
     beforeEach(() => {
       cy.init('/populations')
-      cy.intercept('/api/v3/populationstatistics/studyprogrammes').as('studyprogrammes')
-      cy.intercept('/api/v2/studyprogrammes/**/studytracks').as('studytracks')
+      cy.intercept('/api/populationstatistics/studyprogrammes').as('studyprogrammes')
+      cy.intercept('/api/studyprogrammes/**/studytracks').as('studytracks')
     })
 
     describe('Population search', () => {
       it('Info box works', () => {
         cy.cs('PopulationSearch-info-box-content').should('not.exist')
-        cy.cs('PopulationSearch-info-box-button').trigger('mouseover')
+        cy.cs('PopulationSearch-info-box-button').hover()
         cy.cs('PopulationSearch-info-box-content').should('be.visible')
         cy.cs('PopulationSearch-info-box-content').contains('Tässä osiossa voi tarkastella')
         cy.cs('PopulationSearch-info-box-button').trigger('mouseout')
@@ -74,8 +74,8 @@ describe('Population statistics tests', () => {
           selectStudyProgramme('Matemaattisten tieteiden kandiohjelma')
           cy.wait('@studytracks').its('response.statusCode').should('be.oneOf', [200, 304])
           cy.contains('See class').click()
-          cy.contains('Matemaattisten tieteiden kandiohjelma 2017 - 2018')
-          cy.contains('Class size 47 students')
+          cy.contains('Matemaattisten tieteiden kandiohjelma')
+          cy.contains('Class of 2017 - 2018, 47 students')
         })
 
         // TODO: Delete MUI
@@ -95,7 +95,7 @@ describe('Population statistics tests', () => {
       cy.visit(pathToMathBSc2020)
       cy.cs('filtered-students')
       cy.contains('Matemaattisten tieteiden kandiohjelma')
-      cy.contains('Class size 30 students')
+      cy.contains('Class of 2020 - 2021, 30 students')
       cy.contains('Excludes exchange students')
       cy.contains('Excludes students with non-degree study right')
       cy.contains('Excludes students who have transferred out of this programme')
@@ -251,15 +251,15 @@ describe('Population statistics tests', () => {
       it('Is displayed and link to individual course stats page works', () => {
         cy.visit(pathToMathBSc2020)
         cy.contains('Courses of class').click()
-        cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
+        cy.intercept('/api/courseyearlystats**').as('coursePage')
         cy.cs('toggle-group-module-MAT110').click()
         cy.contains('td', 'MAT11001').siblings().find('[data-testid="NorthEastIcon"]').click()
         cy.wait('@coursePage')
         cy.url().should('include', '/coursestatistics')
-        cy.contains('MAT11001 • Johdatus yliopistomatematiikkaan')
-        cy.contains('AYMAT11001 • Avoin yo: Johdatus yliopistomatematiikkaan')
-        cy.contains('57033 • Johdatus yliopistomatematiikkaan')
-        cy.contains('A57033 • Avoin yo: Johdatus yliopistomatematiikkaan')
+        cy.contains('MAT11001') // Johdatus yliopistomatematiikkaan
+        cy.contains('AYMAT11001') // Avoin yo: Johdatus yliopistomatematiikkaan
+        cy.contains('57033') // Johdatus yliopistomatematiikkaan
+        cy.contains('A57033') // Avoin yo: Johdatus yliopistomatematiikkaan
       })
 
       it('Curriculum selection works', () => {
@@ -383,7 +383,7 @@ describe('Population statistics tests', () => {
       cy.contains(nonExisting).should('not.exist')
       cy.contains('button', 'Check student numbers').click()
       cy.contains('Check for student numbers')
-      cy.cs('check-student-numbers').click().type(`${existing}{enter}${nonExisting}`)
+      cy.cs('check-student-numbers').type(`${existing}{enter}${nonExisting}`)
       cy.contains('button', 'Check students').click()
       cy.get('#checkstudentsresults').within(() => {
         cy.cs('found-title').click()
@@ -401,8 +401,8 @@ describe('Population statistics tests', () => {
   describe('When using IAM user', () => {
     beforeEach(() => {
       cy.init(pathToMathBSc2020, 'onlyiamrights')
-      cy.contains('Matemaattisten tieteiden kandiohjelma 2020 - 2021')
-      cy.contains('Class size 30 students')
+      cy.contains('Matemaattisten tieteiden kandiohjelma')
+      cy.contains('Class of 2020 - 2021, 30 students')
     })
 
     it('Population statistics is visible', () => {

cypress/e2e/Students.js

@@ -114,9 +114,9 @@ describe('Students tests', () => {
           cy.get('a').click()
         })
       cy.url().should('include', '/coursestatistics')
-      cy.contains('MAT12004 • Tilastollinen päättely I')
-      cy.contains('AYMAT12004 • Avoin yo: Tilastollinen päättely I')
-      cy.contains('57046 • Johdatus tilastolliseen päättelyyn')
+      cy.contains('MAT12004') // Tilastollinen päättely I
+      cy.contains('AYMAT12004') // Avoin yo: Tilastollinen päättely I
+      cy.contains('57046') // Johdatus tilastolliseen päättelyyn
     })
 
     it('Has correct Sisu link', () => {
@@ -162,8 +162,8 @@ describe('Students tests', () => {
           .parent()
           .within(() => cy.get('a').click())
       })
-      cy.contains('Matemaattisten tieteiden kandiohjelma 2020 - 2021')
-      cy.contains('Class size 30 students')
+      cy.contains('Matemaattisten tieteiden kandiohjelma')
+      cy.contains('Class of 2020 - 2021, 30 students')
     })
 
     it('Grade graph works in all three different modes', () => {
@@ -174,18 +174,13 @@ describe('Students tests', () => {
         cy.contains('label', 'Group size')
         cy.get('input').should('have.value', '5')
       })
-      cy.contains('.highcharts-container text', "Nov '22")
-      cy.contains('.highcharts-container text', "Jul '22").should('not.exist')
       cy.cs('group-size-input').within(() => {
         cy.contains('label', 'Group size')
         cy.get('input').clear()
         cy.get('input').type('10')
         cy.get('input').should('have.value', '10')
       })
-      cy.contains('.highcharts-container text', "Jul '22")
-      cy.contains('.highcharts-container text', "Nov '22").should('not.exist')
       cy.contains('button', 'Show semester mean').click()
-      cy.contains('.highcharts-container text', "Jan '22")
     })
 
     describe('Bachelor Honours section', () => {
@@ -277,13 +272,9 @@ describe('Students tests', () => {
     it('When a study plan is selected, the time frame of the credit graph is updated', () => {
       cy.clock(new Date('2024-08-30').getTime(), ['Date'])
       cy.visit('/students/550789')
-      cy.contains('.highcharts-container text', '1 Aug 2020')
-      cy.contains('.highcharts-container text', '30 Aug 2024')
       cy.contains('table tbody tr', 'Kulttuurien tutkimuksen kandiohjelma (01.08.2020–30.06.2023)').within(() => {
         cy.get('td').eq(0).click()
       })
-      cy.contains('.highcharts-container text', '1 Aug 2020')
-      cy.contains('.highcharts-container text', '19 Aug 2023')
     })
 
     it("If there's a study plan corresponding to the degree programme, completed credits are displayed", () => {

cypress/e2e/Study_guidance_groups.js

@@ -113,17 +113,17 @@ describe('Study guidance group tests', () => {
         cy.contains('Students (3)')
       })
 
-      it("clicking the 'Show starting from associated year' button activates the 'Date of Course Credits' filter", () => {
-        cy.get('[data-cy="CreditDate-filter-card"][data-active="false"]')
+      it("clicking the 'Show credits starting from the associated academic year' filter", () => {
+        cy.get('[data-cy="ParticipationDate-filter-card"][data-active="false"]')
         cy.cs('Credit accumulation (for 3 students)').click()
-        cy.get('button').contains('Show starting from associated year').click()
-        cy.get('[data-cy="CreditDate-filter-card"][data-active="true"]')
+        cy.get('label').contains('Show credits starting from the associated academic year').click()
+        cy.get('[data-cy="ParticipationDate-filter-card"][data-active="true"]')
         cy.get('.date-picker')
           .eq(0)
           .within(() => cy.get('input').should('have.value', '01.08.2020'))
         cy.contains('Reset All Filters')
-        cy.get('button').contains('Show all credits').click()
-        cy.get('[data-cy="CreditDate-filter-card"][data-active="false"]')
+        cy.get('label').contains('Show credits starting from the associated academic year').click()
+        cy.get('[data-cy="ParticipationDate-filter-card"][data-active="false"]')
       })
 
       it.skip('students table has the correct tabs', () => {

cypress/e2e/Teachers.js

@@ -35,7 +35,7 @@ describe('Teachers page tests', () => {
     cy.cs('Search').click()
     cy.cs('teacher-search').type(teacher1.split(' ')[0])
     cy.get('table tbody tr').should('have.length', 4)
-    cy.get('table tbody tr').eq(3).contains('td', teacher1)
+    cy.get('table tbody tr').eq(1).contains('td', teacher1)
   })
 
   it('Can check teacher page', () => {

cypress/e2e/University.js

@@ -39,12 +39,6 @@ describe('University view', () => {
       checkProgressTables()
     })
 
-    it("'Graduated included / Graduated excluded' toggle works", () => {
-      cy.cs('graduated-toggle').click()
-      checkProgressBarCharts()
-      checkProgressTables()
-    })
-
     it("'All study rights / Special study rights excluded' toggle works", () => {
       cy.cs('study-right-toggle').click()
       checkProgressBarCharts()
@@ -55,7 +49,7 @@ describe('University view', () => {
     it.skip('years in the tables can be clicked to show faculty level breakdown', () => {
       cy.cs('study-right-toggle').click()
       cy.cs('bachelors-faculty-progress-table-show-button3').click()
-      cy.contains('29.5%').trigger('mouseover', { force: true })
+      cy.contains('29.5%').hover()
       cy.contains('Matemaattis-luonnontieteellinen tiedekunta')
       cy.contains('H50')
       cy.contains('0 Credits: 0')

cypress/e2e/Users.js

@@ -72,7 +72,7 @@ describe('Users tests', () => {
       })
 
       it("only the mocked user's programmes are visible", () => {
-        cy.intercept('/api/v3/populationstatistics/studyprogrammes').as('studyprogrammes')
+        cy.intercept('/api/populationstatistics/studyprogrammes').as('studyprogrammes')
         cy.visit('/populations')
         cy.wait('@studyprogrammes').its('response.statusCode').should('be.oneOf', [200, 304])
 

cypress/support/commands.js

@@ -58,17 +58,6 @@ const onlyStudyGuidanceGroupsUser = {
   hypersonsisuid: 'hy-hlo-2222222',
 }
 
-// TODO add user into anon-DB (check the need for changing hypersonid)
-// REMEMBER TO ADD ROLE openUniSearch
-// const openUniUser = {
-//   uid: 'openuniuser',
-//   displayname: 'Open Uni User',
-//   'shib-session-id': 'mock-cypress-session',
-//   hygroupcn: 'grp-oodikone-basics-users;hy-ypa-opa-dojo',
-//   mail: 'grp-toska+mockopenuniuser@helsinki.fi',
-//   hypersonsisuid: 'hy-hlo-9999999',
-// }
-
 const userHeaders = [
   adminUserHeaders,
   basicUserHeaders,
@@ -190,3 +179,20 @@ Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {
     cy.get('tr').filter(':visible').should('have.length', correctStats.length)
   })
 })
+
+/**
+ * Hover on the selected element.
+ */
+Cypress.Commands.add('hover', { prevSubject: 'element' }, (subject, options = {}) => {
+  cy.wrap(subject).trigger('mouseover', {
+    bubbles: true,
+    cancelable: true,
+    ...options,
+  })
+
+  cy.wrap(subject).trigger('mousemove', {
+    bubbles: true,
+    cancelable: true,
+    ...options,
+  })
+})

cypress/support/e2e.js

@@ -13,7 +13,6 @@
 // https://on.cypress.io/configuration
 // ***********************************************************
 
-require('@cypress/code-coverage/support')
 require('./commands')
 
 Cypress.on('uncaught:exception', error => {

docker-compose.ci.yml

@@ -25,6 +25,9 @@ services:
     build:
       context: services
       dockerfile: backend/Dockerfile
+    image: oodikone-backend:staging
+    profiles:
+      - oodikone
 
   frontend:
     <<: *defaults
@@ -32,6 +35,9 @@ services:
     build:
       context: services
       dockerfile: frontend/Dockerfile
+    image: oodikone-frontend:staging
+    profiles:
+      - oodikone
 
   kone-db:
     <<: *defaults
@@ -47,6 +53,7 @@ services:
     command: >
       sh -c 'redis-server --appendonly yes && redis-cli flushall'
     image: redis:7.2.6
+    container_name: redis
     ports:
       - 6379:6379
 
@@ -101,6 +108,8 @@ services:
       - 1337:80
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf
+    profiles:
+      - oodikone
 
   # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
   sis-importer-db:
@@ -114,6 +123,7 @@ services:
     <<: *defaults
     build:
       context: updater/sis-updater-worker
+    container_name: sis-updater-worker
     depends_on:
       - redis
       - sis-db

docker-compose.fd.ci.yml

@@ -1,47 +0,0 @@
-services:
-  oodikone-backend:
-    build:
-      context: ${OODIKONE_REPO:-.}/services
-      dockerfile: backend/Dockerfile
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        SERVICE_PROVIDER: fd
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/oodikone-backend:${DOCKER_IMAGE_TAG:-dev}
-
-  oodikone-frontend:
-    build:
-      context: ${OODIKONE_REPO:-.}/services
-      dockerfile: frontend/Dockerfile
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        SERVICE_PROVIDER: fd
-        LANGUAGE_CENTER_VIEW_ENABLED: 'false' # currently no intention to be enabled
-        SIS_URL: 'https://sis-qa.funidata.fi'
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/oodikone-frontend:${DOCKER_IMAGE_TAG:-dev}
-
-  updater-scheduler:
-    build:
-      context: ${OODIKONE_REPO:-.}/updater/sis-updater-scheduler
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/updater-scheduler:${DOCKER_IMAGE_TAG:-dev}
-
-  updater-worker:
-    build:
-      context: ${OODIKONE_REPO:-.}/updater/sis-updater-worker
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        SERVICE_PROVIDER: fd
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/updater-worker:${DOCKER_IMAGE_TAG:-dev}

docker-compose.fd.yml

@@ -1,190 +0,0 @@
-x-defaults: &defaults
-  restart: unless-stopped
-
-x-build_defaults: &build_defaults
-  dockerfile: dev.Dockerfile
-
-services:
-  adminer:
-    <<: *defaults
-    container_name: adminer
-    environment:
-      ADMINER_DESIGN: pepa-linha
-    image: adminer:4.8.1
-    ports:
-      - 5050:8080
-    volumes:
-      - ./adminer-plugins:/var/www/html/plugins-enabled
-
-  backend:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/backend
-    depends_on:
-      - kone-db
-      - redis
-      - sis-db
-      - user-db
-    env_file:
-      - .env
-    environment:
-      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
-      DB_URL_USER: postgres://postgres@user-db:5432/user-db
-      FRONT_URL: http://localhost:3000
-      REDIS: redis
-      SECRET_TOKEN: dev
-      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
-      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      IMPORTER_DB_API_URL: 'https://localhost:3005'
-      JAMI_URL: 'http://jami:3003/'
-      SERVICE_PROVIDER: fd
-      SIS_HOST: ${SIS_HOST}
-      SIS_GRAPHQL_APPLICATION_ACCOUNT: ${SIS_GRAPHQL_APPLICATION_ACCOUNT}
-      SIS_GRAPHQL_APPLICATION_KEY: ${SIS_GRAPHQL_APPLICATION_KEY}
-      IMPORTER_DB_API_USER: dev_user
-      IMPORTER_DB_API_PASSWORD: dev_password
-      ROOT_ORG_ID: ${ROOT_ORG_ID}
-      LANGUAGE_CENTER_VIEW_ENABLED: 'false'
-    ports:
-      - 8080:8080
-    volumes:
-      - backend-nodemod:/opt/app-root/backend/node_modules
-      - ./environments/testu:/opt/app-root/backend/src/environment
-      - ./services/backend/:/opt/app-root/backend
-      - ./services/shared/:/opt/app-root/shared
-
-  frontend:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/frontend
-    container_name: frontend
-    ports:
-      - 3000:3000
-    environment:
-      REACT_APP_SERVICE_PROVIDER: fd
-      REACT_APP_LANGUAGE_CENTER_VIEW_ENABLED: 'false'
-      REACT_APP_SIS_URL: 'https://sis-qa.funidata.fi'
-    volumes:
-      - frontend-nodemod:/opt/app-root/frontend/node_modules
-      - ./services/frontend/:/opt/app-root/frontend
-      - ./services/shared/:/opt/app-root/shared
-
-  kone-db:
-    <<: *defaults
-    container_name: kone-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: kone-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
-    ports:
-      - 12349:5432
-    volumes:
-      - kone-db-data:/data
-
-  redis:
-    <<: *defaults
-    command: 'redis-server --appendonly yes'
-    container_name: redis
-    image: redis:6.2.16
-    ports:
-      - '6379:6379'
-    volumes:
-      - redis-data:/data
-
-  sis-db:
-    <<: *defaults
-    container_name: sis-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: sis-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
-    volumes:
-      - sis-db-data:/data
-    shm_size: 2gb
-    ports:
-      - 12345:5432
-
-  sis-updater-scheduler:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: updater/sis-updater-scheduler
-    container_name: sis-updater-scheduler
-    extra_hosts:
-      - 'host.docker.internal:host-gateway'
-    depends_on:
-      - redis
-    environment:
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      SECRET_TOKEN: dev
-      SIS_IMPORTER_HOST: ${SIS_IMPORTER_DB_HOST:-host.docker.internal}
-      SIS_IMPORTER_PORT: ${SIS_IMPORTER_DB_PORT:-15234}
-      SIS_IMPORTER_DATABASE: importer-db
-      SIS_IMPORTER_PASSWORD: dev
-      SIS_IMPORTER_USER: dev
-      SERVICE_PROVIDER: fd
-    ports:
-      - 8082:8082
-    volumes:
-      - ./updater/sis-updater-scheduler/:/opt/app-root/src
-
-  sis-updater-worker:
-    <<: *defaults
-    deploy:
-      replicas: 2
-      mode: replicated
-    build:
-      <<: *build_defaults
-      context: updater/sis-updater-worker
-    depends_on:
-      - redis
-      - sis-db
-    extra_hosts:
-      - 'host.docker.internal:host-gateway'
-    environment:
-      DB_URL: postgres://postgres@sis-db:5432/sis-db
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      SIS_IMPORTER_HOST: ${SIS_IMPORTER_DB_HOST:-host.docker.internal}
-      SIS_IMPORTER_PORT: ${SIS_IMPORTER_DB_PORT:-15234}
-      SIS_IMPORTER_DATABASE: importer-db
-      SIS_IMPORTER_PASSWORD: dev
-      SIS_IMPORTER_USER: dev
-      SIS_PASSWORD: postgres
-      ROOT_ORG_ID: ${ROOT_ORG_ID}
-      SERVICE_PROVIDER: fd
-    volumes:
-      - ./updater/sis-updater-worker/:/opt/app-root/src
-
-  user-db:
-    <<: *defaults
-    container_name: user-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: user-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
-    volumes:
-      - user-db-data:/data
-    ports:
-      - 12347:5432
-
-volumes:
-  backend-nodemod:
-  frontend-nodemod:
-  kone-db-data:
-  redis-data:
-  sis-db-data:
-  user-db-data:
-
-networks:
-  default:
-    name: oodikone
-  importer:
-    name: importer_network
-    external: true

docker-compose.test.yml

@@ -1,5 +1,26 @@
 services:
-  frontend:
+  api-tests:
     volumes:
-      - frontend-nodemod:/opt/app-root/frontend/node_modules
-      - ./services/frontend/instrumented:/opt/app-root/frontend/src
+      - backend-tests-nodemod:/opt/app-root/backend/node_modules
+      - ./services/backend/:/opt/app-root/backend
+      - ./services/shared/:/opt/app-root/shared
+    environment:
+      CI: 'true'
+      NODE_ENV: 'test'
+      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
+      FRONT_URL: http://localhost:3000
+      REDIS: redis
+      SECRET_TOKEN: test
+      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
+      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
+      JAMI_URL: 'http://jami:3003/'
+      IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
+      CRYPT_KEY: 'e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'
+    build:
+      context: services/backend
+      dockerfile: dev.Dockerfile
+    command: npm run test
+
+volumes:
+  backend-tests-nodemod:

docker-compose.yml

@@ -21,6 +21,7 @@ services:
     build:
       <<: *build_defaults
       context: services/backend
+    container_name: backend
     depends_on:
       - kone-db
       - redis
@@ -44,9 +45,9 @@ services:
     profiles:
       - oodikone
     volumes:
-      - backend-nodemod:/opt/app-root/backend/node_modules
       - ./services/backend/:/opt/app-root/backend
       - ./services/shared/:/opt/app-root/shared
+      - backend-nodemod:/opt/app-root/backend/node_modules
 
   frontend:
     <<: *defaults
@@ -59,9 +60,9 @@ services:
     profiles:
       - oodikone
     volumes:
-      - frontend-nodemod:/opt/app-root/frontend/node_modules
       - ./services/frontend/:/opt/app-root/frontend
       - ./services/shared/:/opt/app-root/shared
+      - frontend-nodemod:/opt/app-root/frontend/node_modules
 
   kone-db:
     <<: *defaults
@@ -132,6 +133,7 @@ services:
     build:
       <<: *build_defaults
       context: updater/sis-updater-worker
+    container_name: sis-updater-worker
     depends_on:
       - redis
       - sis-db
@@ -168,7 +170,7 @@ services:
       - npm_config_cache=/tmp
       - PORT=3003
       - NODE_ENV=dev
-      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
+      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/jami-db-real
     ports:
       - 3003:3003
     container_name: jami

documentation/access_rights.md

@@ -8,11 +8,10 @@ Roles are defined [here](../services/backend/src/config/roles.ts). Criteria for
 - courseStatistics
 - facultyStatistics
 - fullSisuAccess
-- openUniSearch
 - studyGuidanceGroups
 - teachers
 
-The roles `facultyStatistics`, `openUniSearch`, and `studyGuidanceGroups` simply enable certain views (_Faculties_, _Open uni student population_, and _Guidance groups_ respectively).
+The roles `facultyStatistics` and `studyGuidanceGroups` simply enable certain views (_Faculties_ and _Guidance groups_ respectively).
 
 ### courseStatistics
 
@@ -43,7 +42,6 @@ The roles `facultyStatistics`, `openUniSearch`, and `studyGuidanceGroups` simply
 | Custom population    |             |  ✅   |       ✅       |
 | Completed courses    |     ✅      |  ✅   |       ✅       |
 | Close to graduation  |             |  ✅   |       ✅       |
-| Open uni population  |             |  ✅   |                |
 | Courses              |             |  ✅   |       ✅       |
 
 \* = anyone who can log in to Oodikone, check the [IAM groups](#iam-groups) section for more information
@@ -63,7 +61,6 @@ A user must be a member of either the `grp-oodikone-basic-users` or `grp-oodikon
 | grp-oodikone-basic-users | courseStatistics              |
 | grp-oodikone-users       | facultyStatistics             |
 | grp-toska                | admin                         |
-| hy-ypa-opa-dojo          | openUniSearch                 |
 
 `grp-oodikone-basic-users`: basically the whole staff (hy-\*-allstaff)
 

documentation/combined_programme.md

@@ -36,7 +36,7 @@ All things listed below should work automatically, if new combinations are done.
 
 - _Students of the studyprogramme by starting year_
   - Graduated field separated for both programmes
-  - All = currently enrolled + absent + inactive + graduated licentiate
+  - All = currently enrolled + absent + passive + graduated licentiate
 - _Progress of students of the studyprogramme by starting year_
   - computed for whole bachelor + master/licentiate studyright period
 - _Average graduation times_
@@ -85,12 +85,12 @@ All things listed below should work automatically, if new combinations are done.
 
 - **Graduated from programme** - possibility to filter out students based on graduation either in bachelor's or master's.
 
-- **Studyright status** - active or inactive based on the chosen degree
+- **Studyright status** - active or passive based on the chosen degree
 
 - **Studytrack filter** (needs programme code) is not touched at the moment, thus **may need future manual work**.
 
 ## Study guidance groups
 
-- Backend: student data is fetched by bachelor degree programme code in `/v3/populationstatisticsbystudentnumbers`. Separation from code 'KHxx_xxx+MHxx_xxx' is done in backend.
+- Backend: student data is fetched by bachelor degree programme code in `/populationstatisticsbystudentnumbers`. Separation from code 'KHxx_xxx+MHxx_xxx' is done in backend.
 - Frontend: things are done in the same way as with class stats.
 - **Need manual work**: `./oodikone/services/frontend/src/redux/elementdetails.js` add combined programme codes (both bachelor and master) to `combinedProgrammeCodes` and modify `combinedOptions` list accordingly in `useFilterAndFormattedElementDetails()` function.

documentation/numeric_types.md

@@ -55,11 +55,11 @@ The `semester_enrollments` column is an array of JSON objects. Each object conta
 
 The `type` field represents one of the following enrollment types. The `semester` field refers to a semester code, defined as `semestercode` in the model Semester.
 
-| value | meaning  | notes                            |
-| ----: | :------- | :------------------------------- |
-|     1 | Present  |                                  |
-|     2 | Absent   |                                  |
-|     3 | Inactive | neither 1 or 2 has been declared |
+| value | meaning | notes                            |
+| ----: | :------ | :------------------------------- |
+|     1 | Present |                                  |
+|     2 | Absent  |                                  |
+|     3 | Passive | neither 1 or 2 has been declared |
 
 ## Credit type code
 

documentation/resources.md

@@ -2,26 +2,19 @@
 
 A collection of useful tools, documentation and other resources.
 
-## AI tools
-
-- [ChatGPT](https://chatgpt.com)
-- [CurreChat](https://curre.helsinki.fi/chat)
-- [GitHub Copilot](https://github.com/features/copilot)
-  - Free for students!
-- [Microsoft Copilot](https://copilot.microsoft.com)
-
 ## Code style and maintainability
 
-- [CodeClimate documentation](https://docs.codeclimate.com/docs)
-  - [Advanced configuration](https://docs.codeclimate.com/docs/advanced-configuration)
 - [ESLint documentation](https://eslint.org/docs/v8.x)
 - [Prettier documentation](https://prettier.io/docs/en)
+- [qlty.sh](https://qlty.sh/gh/UniversityOfHelsinkiCS/projects/oodikone)
+  - Qlty.sh replaced Code climate which was introduced by Riku
+  - The usefulness of qlty.sh is sometimes questionable
 
 ## Database
 
 - [pgcli](https://www.pgcli.com)
   - A useful commandline tool to interact with the database
-  - Install: `$ pip install pgcli`
+  - Can be installed from some package managers or optionally via `$ pip install pgcli`
 - [Postgres documentation](https://www.postgresql.org/docs/15/index.html)
 - [Sequelize documentation](https://sequelize.org/docs/v6/)
 
@@ -29,8 +22,6 @@ A collection of useful tools, documentation and other resources.
 
 - [GitHub documentation](https://docs.github.com/en)
   - [GitHub Actions](https://docs.github.com/en/actions)
-- [GitHub Education](https://github.com/education/students)
-  - Free benefits for students!
 - [Mermaid.js documentation](https://mermaid.js.org/intro/syntax-reference.html)
 
 ## TypeScript

eslint.config.mjs

@@ -25,7 +25,7 @@ export default tseslint.config(
       'nyc-config.cjs',
       '.lintstagedrc.mjs',
       'cypress.config.js',
-    ]
+    ],
   },
 
   // Global options (excl. e2e)
@@ -44,8 +44,8 @@ export default tseslint.config(
       parser: tseslint.parser,
       parserOptions: {
         projectService: {
-          allowDefaultProject: ['*.{js,cjs}']
-        }
+          allowDefaultProject: ['*.{js,cjs}'],
+        },
       },
       globals: {
         ...globals.browser,
@@ -53,16 +53,16 @@ export default tseslint.config(
       },
     },
     plugins: {
-      'import-x': eslintPluginImportX
+      'import-x': eslintPluginImportX,
     },
     linterOptions: {
       reportUnusedDisableDirectives: 'error',
     },
     rules: {
-      'camelcase': 'off', // TODO: Enable. old db/backend logic is still using snake_case every now and then and needs a big refactor
+      camelcase: 'off', // TODO: Enable. old db/backend logic is still using snake_case every now and then and needs a big refactor
       'class-methods-use-this': 'off', // Disabled in favor of @typescript-eslint
       'consistent-return': 'off', // Disabled in favor of @typescript-eslint
-      'id-denylist': ['error', 'c', 'd', 'e', 'err', 't'],
+      'id-denylist': ['error', 'err'],
       'no-alert': 'error',
       'no-await-in-loop': 'error',
       'no-console': 'error',
@@ -75,24 +75,42 @@ export default tseslint.config(
       'prefer-const': 'error',
       'prefer-destructuring': 'off', // Disabled in favor of @typescript-eslint
       'dot-notation': 'off', // Disabled in favor of @typescript-eslint
-      'no-restricted-imports': ['error', { 'patterns': [{ 'regex': '^@mui/[^/]+$' }] }],
-      'quotes': ['error', 'single',
-        { avoidEscape: true, allowTemplateLiterals: false }
+      'no-restricted-imports': [
+        'error',
+        {
+          patterns: [
+            {
+              regex: '^@mui/[^/]+$',
+              message:
+                'Import only the needed module. This has no effect in prod due to tree-shake, but speeds up development builds.',
+            },
+            {
+              regex: '^src/.*',
+              message: 'Root imports will not resolve in production build (instead silently crash).',
+            },
+          ],
+        },
       ],
+      quotes: ['error', 'single', { avoidEscape: true, allowTemplateLiterals: false }],
 
       'import-x/no-commonjs': 'error',
       'import-x/no-default-export': 'error',
       'import-x/no-extraneous-dependencies': ['error', { devDependencies: false }],
-      'import-x/no-unused-modules': ['error',
+      'import-x/no-unused-modules': [
+        'error',
         {
           missingExports: true,
           unusedExports: true,
-          src: ['{services,updater}/**/*.{js,jsx,ts,tsx}']
-        }],
-      'import-x/order': ['error', {
-        alphabetize: { order: 'asc', caseInsensitive: true },
-        groups: [['builtin', 'external'], ['internal'], ['parent'], ['sibling', 'index']],
-      }],
+          src: ['{services,updater}/**/*.{js,jsx,ts,tsx}'],
+        },
+      ],
+      'import-x/order': [
+        'error',
+        {
+          alphabetize: { order: 'asc', caseInsensitive: true },
+          groups: [['builtin', 'external'], ['internal'], ['parent'], ['sibling', 'index']],
+        },
+      ],
 
       // TypeScript
       '@typescript-eslint/no-unused-vars': [
@@ -105,9 +123,7 @@ export default tseslint.config(
       '@typescript-eslint/consistent-return': 'error', // NOTE: This is weaker than tsconfig.json noImplicitReturns, and should be replaced once ts migration complete
       '@typescript-eslint/consistent-type-definitions': 'off',
       '@typescript-eslint/dot-notation': 'error',
-      '@typescript-eslint/prefer-destructuring': ['error',
-        { VariableDeclarator: { object: true } }
-      ],
+      '@typescript-eslint/prefer-destructuring': ['error', { VariableDeclarator: { object: true } }],
       '@typescript-eslint/no-empty-object-type': ['error', { allowInterfaces: 'with-single-extends' }],
       '@typescript-eslint/no-explicit-any': 'off', // TODO: enable when feeling brave
       '@typescript-eslint/no-implied-eval': 'error',
@@ -127,12 +143,12 @@ export default tseslint.config(
             'services/frontend/tsconfig.json',
             'services/frontend/tsconfig.node.json',
             'services/backend/tsconfig.json',
-            'services/shared/tsconfig.json'
+            'services/shared/tsconfig.json',
           ],
         }),
         createNodeResolver({
           extensions: ['.js', '.jsx', 'cjs', 'mjs', '.ts', '.tsx'],
-        })
+        }),
       ],
     },
   },
@@ -140,22 +156,22 @@ export default tseslint.config(
   {
     files: ['services/backend/src/routes/*.{ts,js}'],
     rules: {
-      '@typescript-eslint/no-misused-promises': 'off', // Errors come mainly from Express route handlers that are handled correctly by express-async-errors      
+      '@typescript-eslint/no-misused-promises': 'off', // Errors come mainly from Express route handlers that are handled correctly by express-async-errors
       '@typescript-eslint/consistent-return': 'off',
-    }
+    },
   },
 
   // Disable type-aware linting for javascript files
   {
     files: ['{updater,services}/**/*.{js,cjs}'],
-    extends: [tseslint.configs.disableTypeChecked]
+    extends: [tseslint.configs.disableTypeChecked],
   },
 
   // Frontend camelCase and react
   {
     files: ['services/frontend/**/*.{js,jsx,mjs,cjs,ts,tsx}'],
     plugins: {
-      'react': eslintPluginReact,
+      react: eslintPluginReact,
     },
     extends: [
       eslintPluginReact.configs.flat.recommended,
@@ -166,14 +182,15 @@ export default tseslint.config(
       parserOptions: {
         ecmaFeatures: {
           jsx: true,
-        }
-      }
+        },
+      },
     },
     rules: {
-      'camelcase': 'warn',
+      camelcase: 'warn',
       'react/prop-types': 'off', // Typescript but worse
       'react/no-unescaped-entities': 'off',
-      'react/function-component-definition': ['error',
+      'react/function-component-definition': [
+        'error',
         { namedComponents: 'arrow-function', unnamedComponents: 'arrow-function' },
       ],
       'react/no-array-index-key': 'error',
@@ -181,9 +198,7 @@ export default tseslint.config(
       'react/no-this-in-sfc': 'error',
       'react/prefer-stateless-function': 'error',
       'react/jsx-boolean-value': ['error', 'never'],
-      'react/jsx-filename-extension': ['error',
-        { allow: 'as-needed', extensions: ['.jsx', '.tsx'] }
-      ],
+      'react/jsx-filename-extension': ['error', { allow: 'as-needed', extensions: ['.jsx', '.tsx'] }],
       'react/jsx-no-leaked-render': 'warn', // TODO: enable, tho not that useful after full ts migration
       'react/jsx-no-useless-fragment': 'error',
       'react/jsx-props-no-spreading': 'warn', // TODO: enable, improves readability
@@ -191,18 +206,25 @@ export default tseslint.config(
       'react/jsx-tag-spacing': 'warn',
     },
     settings: {
-      'react': {
-        'version': 'detect'
-      }
-    }
+      react: {
+        version: 'detect',
+      },
+    },
   },
 
   // Cypress should only use rules provided by the plugin below
   {
     files: ['cypress/**/*.js'],
-    extends: [
-      eslintPluginCypress.configs.recommended
-    ]
+    extends: [eslintPluginCypress.configs.recommended],
+  },
+
+  // Override for backend api tests
+  {
+    files: ['services/backend/tests/**/*.ts'],
+    rules: {
+      'import-x/no-extraneous-dependencies': 'off',
+      'no-console': 'off',
+    },
   },
 
   // Overrides for CommonJS
@@ -210,7 +232,7 @@ export default tseslint.config(
     files: [
       'services/backend/**/*.js',
       'updater/sis-updater-worker/src/db/migrations/*.cjs',
-      'updater/sis-updater-scheduler/**/*.js'
+      'updater/sis-updater-scheduler/**/*.js',
     ],
     languageOptions: {
       sourceType: 'commonjs',
@@ -221,20 +243,9 @@ export default tseslint.config(
     },
   },
 
-  // Rapodiff
-  {
-    files: ['services/backend/src/rapodiff/**/*'],
-    rules: {
-      'no-console': 'off',
-    },
-  },
-
   // Migration files
   {
-    files: [
-      'updater/sis-updater-worker/src/db/migrations/*.cjs',
-      'services/backend/src/database/**/*.js'
-    ],
+    files: ['updater/sis-updater-worker/src/db/migrations/*.cjs', 'services/backend/src/database/**/*.js'],
     rules: {
       'import-x/no-unused-modules': 'off',
       '@typescript-eslint/no-empty-function': 'off',
@@ -252,5 +263,5 @@ export default tseslint.config(
       'no-promise-executor-return': 'off',
     },
   },
-  eslintPluginPrettierRecommended, // This must be the last plugin so it can override other configs
+  eslintPluginPrettierRecommended // This must be the last plugin so it can override other configs
 )

nyc-config.cjs

@@ -1,6 +0,0 @@
-const { parserPlugins } = require('@istanbuljs/schema').defaults.nyc
-
-module.exports = {
-  cache: false,
-  parserPlugins: parserPlugins.concat('typescript', 'jsx'),
-}

package.json

@@ -2,13 +2,11 @@
   "name": "oodikone",
   "description": "An application for analyzing university data.",
   "engines": {
-    "node": ">=22.0.0"
+    "node": ">=24.0.0"
   },
   "scripts": {
     "cypress": "cypress",
-    "cypress:open": "cypress open",
-    "cypress:run": "cypress run --browser chrome",
-    "instrument": "./scripts/instrument.sh",
+    "cypress:run": "cypress run --browser firefox",
     "eslint": "eslint",
     "lint:all": "eslint services/ updater/",
     "stylelint": "stylelint",
@@ -16,7 +14,8 @@
     "cli": "./cli.sh",
     "oodikone": "./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
-    "oodikone:test": "./run.sh oodikone test up --detach",
+    "oodikone:ci:setup": "docker compose --file docker-compose.ci.yml --profile tests up --detach",
+    "oodikone:ci:api": "docker compose --file docker-compose.test.yml up --exit-code-from api-tests",
     "updater": "./run.sh updater anon up --detach",
     "updater:real": "./run.sh updater real up --detach",
     "both": "./run.sh both anon up --detach",
@@ -25,12 +24,11 @@
     "testupdater": "docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start",
     "flushredis": "docker exec -it redis redis-cli flushall",
     "prepare": "husky",
-    "rapodiff": "docker exec -t oodikone_backend_1 /opt/app-root/backend/node_modules/.bin/tsx /opt/app-root/backend/src/rapodiff/rapodiff.js",
     "install:local": "npm i && npm i --prefix services/shared && npm i --prefix services/frontend && npm i --prefix services/backend && npm i --prefix updater/sis-updater-scheduler && npm i --prefix updater/sis-updater-worker",
-    "ci:local": "npm ci && npm ci --prefix services/shared && npm ci --prefix services/frontend && npm ci --prefix services/backend && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker",
-    "install:docker": "docker exec frontend npm i && docker exec oodikone_backend_1 npm i && docker exec sis-updater-scheduler npm i && docker exec oodikone_sis-updater-worker_1 npm i",
-    "ci:docker": "docker exec frontend npm ci && docker exec oodikone_backend_1 npm ci && docker exec sis-updater-scheduler npm ci && docker exec oodikone_sis-updater-worker_1 npm ci",
+    "install:docker": "docker exec frontend npm i && docker exec backend npm i && docker exec sis-updater-scheduler npm i && docker exec sis-updater-worker npm i",
     "install:both": "npm run install:local && npm run install:docker",
+    "ci:local": "npm ci && npm ci --prefix services/shared && npm ci --prefix services/frontend && npm ci --prefix services/backend && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker",
+    "ci:docker": "docker exec frontend npm ci && docker exec backend npm ci && docker exec sis-updater-scheduler npm ci && docker exec sis-updater-worker npm ci",
     "ci:both": "npm run ci:local && npm run ci:docker"
   },
   "repository": {
@@ -43,28 +41,22 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@cypress/code-coverage": "^3.14.4",
     "@eslint/js": "^9.28.0",
-    "@istanbuljs/schema": "^0.1.3",
-    "cypress": "^15.0.0",
+    "cypress": "^15.13.0",
     "eslint": "^9.28.0",
     "eslint-config-prettier": "^10.1.5",
     "eslint-import-resolver-typescript": "^4.4.3",
-    "eslint-plugin-cypress": "^5.1.1",
+    "eslint-plugin-cypress": "^6.2.3",
     "eslint-plugin-import-x": "^4.16.1",
     "eslint-plugin-prettier": "^5.4.1",
     "eslint-plugin-react": "^7.37.5",
     "eslint-plugin-react-hooks": "^5.2.0",
-    "globals": "^16.2.0",
+    "globals": "^17.4.0",
     "husky": "^9.1.7",
     "lint-staged": "^16.1.0",
-    "nyc": "^17.1.0",
     "prettier": "3.5.3",
-    "stylelint": "^16.20.0",
-    "stylelint-config-standard": "^39.0.0",
+    "stylelint": "^17.6.0",
+    "stylelint-config-standard": "^40.0.0",
     "typescript-eslint": "^8.34.0"
-  },
-  "dependencies": {
-    "lodash": "^4.17.21"
   }
 }

run.sh

@@ -35,7 +35,7 @@ parse_params() {
   option=${args[0]}
 
   #  parse arguments
-  [[ ("$option" != "oodikone" && "$option" != "updater" && "$option" != "both") ]] && \
+  [[ ("$option" != "oodikone" && "$option" != "updater" && "$option" != "both" && "$option" != "tests") ]] && \
     usage && die "Wrong option: $option"
 
   [[ ${#args[@]} -eq 1 ]] && usage && die "Wrong number of arguments"
@@ -47,7 +47,7 @@ parse_params() {
     compose_command=${args[*]:1}
   else
     version=${args[1]}
-    [[ "$version" != "anon" && "$version" != "real" && "$version" != "test" ]] && usage && \
+    [[ "$version" != "anon" && "$version" != "real" && "$version" != "api" ]] && usage && \
       die "Wrong version $version"
     compose_command=${args[*]:2}
   fi
@@ -70,8 +70,8 @@ parse_env() {
   if [[ "$version" == "real" ]]; then
     env="--file docker-compose.yml --file docker-compose.real.yml"
   fi
-  if [[ "$version" == "test" ]]; then
-    env="--file docker-compose.yml --file docker-compose.test.yml"
+  if [[ "$version" == "api" ]]; then
+    env="--file docker-compose.ci.yml"
   fi
   return 0
 }
@@ -85,9 +85,9 @@ parse_params "$@"
 parse_profiles
 parse_env
 if [[ "$compose_command" == "" ]]; then
-  final_command="docker-compose ${env}"
+  final_command="docker compose ${env}"
 else
-  final_command="docker-compose ${env} ${profiles} ${compose_command}"
+  final_command="docker compose ${env} ${profiles} ${compose_command}"
 fi
 
 msg "${BLUE}Running: ${final_command}${NOFORMAT}"

scripts/functions_for_setup.sh

@@ -11,7 +11,6 @@ PROJECT_ROOT="$(git rev-parse --show-toplevel)"
 DUMP_DIR="$PROJECT_ROOT/.databasedumps"
 S3_CONFIG_FILE=~/.s3cfg
 S3_BUCKET="s3://psyduck"
-DOCKER_COMPOSE=$PROJECT_ROOT/docker-compose.yml
 
 ## Following the naming convention in docker-compose, these are names for database
 ## services. The real data databases inside service will have names suffixed by "-real".
@@ -139,29 +138,6 @@ reset_databases() {
   successmsg "Database setup finished"
 }
 
-reset_jami_data() {
-  local database="jami-db"
-  local dump_destination="$DUMP_DIR/$database.sql.gz"
-  local s3_path="jami"
-
-  infomsg "Removing old data"
-
-
-
-  infomsg "Removing database and related volume"
-  docker volume rm oodikone_jami-data || warningmsg "This is okay, continuing"
-  docker-compose -f "$DOCKER_COMPOSE" down "$database" || warningmsg "This is okay, continuing"
-
-  infomsg "Starting postgres in the background"
-  docker-compose -f "$DOCKER_COMPOSE" up -d "$database"
-  retry docker-compose -f "$DOCKER_COMPOSE" exec "$database" pg_isready --dbname="postgres"
-
-  infomsg "Populating Jami"
-  docker exec -i "$database" /bin/bash -c "gunzip | psql -U postgres" < "$dump_destination" 2> /dev/null
-  docker-compose stop "$database"
-  msg ""
-}
-
 reset_all_real_data() {
   infomsg "Downloading real data dumps"
   for i in ${!DATABASES[*]}; do

scripts/instrument.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-
-# This script is used for instrumenting the frontend code
-# Counters are added for calculating the code coverage
-# Instrumented code is used for generating code coverage reports
-
-# Ensure the script exits if any command fails
-set -euo pipefail
-
-# Define the directories
-SOURCE_DIR="services/frontend/src"
-INSTRU_DIR="services/frontend/instrumented"
-
-# Print status
-echo -n "Instrumenting frontend code... "
-
-# Create the instrumented directory if it does not exist
-mkdir -p $INSTRU_DIR
-
-# Instrument the code
-nyc instrument --all --compact=false $SOURCE_DIR $INSTRU_DIR
-
-# Copy remaining files without overwriting instrumented files
-# 
-# Images, CSS and other static files are not instrumended
-# but they are still needed to run the application
-rsync -av --quiet --ignore-existing $SOURCE_DIR/ $INSTRU_DIR/
-
-# Print status
-echo "Done!"

services/.dockerignore

@@ -0,0 +1,7 @@
+*/.dockerignore
+*/.eslintrc.json
+*/.gitignore
+*/*Dockerfile
+*/build
+*/dist
+*/node_modules

services/backend/.dockerignore

@@ -1,3 +0,0 @@
-.dockerignore
-*Dockerfile
-node_modules/

services/backend/.npmrc

@@ -0,0 +1,3 @@
+ignore-scripts=true  
+min-release-age=7  
+save-prefix=  

services/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi9/nodejs-22-minimal AS build
+FROM registry.access.redhat.com/ubi9/nodejs-24-minimal AS build
 
 ENV TZ="Europe/Helsinki"
 
@@ -29,7 +29,7 @@ RUN npm ci --include=dev --prefix shared && npm ci --include=dev --prefix backen
 RUN npm run build --prefix shared && npm run build --prefix backend
 
 
-FROM registry.access.redhat.com/ubi9/nodejs-22-minimal
+FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 
@@ -52,7 +52,8 @@ WORKDIR /opt/app-root/shared
 COPY --from=build /opt/app-root/shared/dist .
 
 WORKDIR /opt/app-root/backend
-COPY --from=build /opt/app-root/backend/package*.json .
+COPY --from=build /opt/app-root/backend/.npmrc .
+COPY --from=build /opt/app-root/backend/package*.json ./
 COPY --from=build /opt/app-root/backend/dist ./dist
 
 RUN npm ci --omit=dev

services/backend/dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM node:22-alpine
+FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 
@@ -6,8 +6,10 @@ ENV NODE_ENV=development
 ENV NODE_OPTIONS=--max-old-space-size=4096
 
 WORKDIR /opt/app-root/backend
-COPY . .
+COPY --chown=1001:1001 . .
+
+USER 1001
 RUN npm ci
 
 EXPOSE 8080
-CMD ["node_modules/.bin/tsx", "watch", "--clear-screen=false", "index.js"]
+CMD ["node_modules/.bin/nodemon", "--exec", "node --import tsx index.ts", "--ext", "ts,js,json", "--watch", "src", "--watch", "index.ts", "--legacy-watch"]

services/backend/package.json

@@ -2,32 +2,30 @@
   "name": "backend",
   "private": true,
   "scripts": {
-    "build": "tsc"
+    "build": "tsc",
+    "test": "node --import tsx --test --test-reporter spec --test-force-exit"
   },
   "dependencies": {
     "@oodikone/shared": "file:../shared",
-    "@sentry/node": "^10.5.0",
-    "@sentry/profiling-node": "^10.5.0",
+    "@sentry/node": "^10.43.0",
+    "@sentry/profiling-node": "^10.43.0",
     "axios": "^1.9.0",
     "bullmq": "^5.53.2",
     "compression": "^1.8.0",
     "cors": "^2.8.5",
     "cron": "^4.3.1",
     "dayjs": "^1.11.13",
-    "express": "^4.21.2",
-    "express-async-errors": "^3.1.1",
-    "lodash": "^4.17.21",
+    "express": "^5.2.1",
+    "lodash-es": "^4.18.1",
     "lru-cache": "^11.1.0",
     "morgan": "^1.10.0",
     "papaparse": "^5.5.3",
-    "pg": "^8.16.0",
+    "pg": "^8.20.0",
     "redis": "^4.7.1",
-    "reflect-metadata": "^0.2.2",
-    "sequelize": "^6.37.7",
+    "sequelize": "^6.37.8",
     "sequelize-typescript": "^2.1.6",
     "umzug": "^3.8.2",
     "unfuck-utf8-headers-middleware": "^1.0.1",
-    "vite": "^7.1.12",
     "winston": "^3.17.0",
     "winston-gelf-transporter": "^1.0.2",
     "winston-loki": "^6.1.3"
@@ -35,11 +33,14 @@
   "devDependencies": {
     "@types/compression": "^1.8.1",
     "@types/cors": "^2.8.19",
-    "@types/express": "^4.17.23",
-    "@types/lodash": "^4.17.17",
+    "@types/express": "^5.0.6",
+    "@types/lodash-es": "^4.17.12",
     "@types/morgan": "^1.9.10",
-    "@types/node": "^22.15.31",
-    "@types/validator": "^13.15.1",
+    "@types/node": "^24.10.1",
+    "@types/pg": "^8.20.0",
+    "@types/supertest": "^6.0.3",
+    "nodemon": "^3.1.11",
+    "supertest": "^7.2.2",
     "tsx": "^4.19.4",
     "typescript": "^5.9.2"
   },

services/backend/src/app.ts

@@ -1,4 +1,3 @@
-import 'express-async-errors'
 import express from 'express'
 
 import { baseUrl, backendPort } from './config'

services/backend/src/config/index.ts

@@ -1,14 +1,13 @@
 // Node env to use
 export const isDev = process.env.NODE_ENV === 'development'
+export const isTest = process.env.NODE_ENV === 'test'
 export const isStaging = process.env.REACT_APP_STAGING === 'true'
 export const isProduction = !isStaging && process.env.NODE_ENV === 'production'
 
 export const runningInCI = process.env.CI === 'true'
 
 // IAM group
-export const requiredGroup = isStaging
-  ? ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
-  : ['grp-oodikone-users', 'grp-oodikone-basic-users']
+export const requiredGroup = ['grp-oodikone-users', 'grp-oodikone-basic-users']
 
 // Pate
 export const pateToken = process.env.PATE_API_TOKEN ?? ''
@@ -32,12 +31,6 @@ export const baseUrl = isDev ? '/api' : ''
 export const frontUrl = process.env.FRONT_URL
 export const backendPort = 8080
 
-// System run in whose environment
-export const serviceProvider = process.env.SERVICE_PROVIDER?.toLowerCase() ?? 'toska'
-
-// Optional logout-url configuration
-export const configLogoutUrl = serviceProvider === 'toska' ? undefined : process.env.LOGOUT_URL
-
 // Other stuff
 export const { DB_URL_KONE, DB_URL_USER, SECRET_TOKEN, SIS_DB_URL, SIS_UPDATER_URL, CRYPT_KEY } = process.env
 

services/backend/src/config/roles.ts

@@ -5,7 +5,6 @@ export const roles = [
   'courseStatistics',
   'facultyStatistics',
   'fullSisuAccess',
-  'openUniSearch',
   'studyGuidanceGroups',
   'teachers',
 ] as const

services/backend/src/events.ts

@@ -13,7 +13,7 @@ import { updateBasicView, updateStudyTrackView } from './services/studyProgramme
 import { findAndSaveTeachers } from './services/teachers/top'
 import { deleteOutdatedUsers } from './services/userService'
 import logger from './util/logger'
-import { jobMaker, addToFlow } from './worker/queue'
+import { jobQueue } from './worker/queue'
 
 const schedule = (cronTime: string, onTick: () => void) => {
   const onComplete = null
@@ -41,7 +41,7 @@ export const refreshFaculties = async () => {
   logger.info('Adding jobs to refresh all faculties')
   const faculties = await getFaculties()
   for (const faculty of faculties) {
-    jobMaker.faculty(faculty.code)
+    jobQueue.refreshFaculty(faculty.code)
   }
 }
 
@@ -53,12 +53,13 @@ export const refreshLanguageCenterData = async () => {
 }
 
 const refreshProgrammesAndFaculties = async () => {
+  logger.info('Refreshing all degree programmes and faculties')
   const facultyCodes = (await getFaculties()).map(faculty => faculty.code)
   for (const faculty of facultyCodes) {
     const programmeCodes = (await getDegreeProgrammesOfFaculty(faculty, true))
       .map(programme => programme.code)
       .filter(code => isRelevantProgramme(code))
-    await addToFlow(faculty, programmeCodes)
+    await jobQueue.addToFlow(faculty, programmeCodes)
   }
 }
 
@@ -82,7 +83,7 @@ export const refreshProgrammes = async () => {
   }
   for (const code of programmeCodes) {
     // If combined programme is given, this updates only the bachelor programme
-    jobMaker.programme(code)
+    jobQueue.refreshProgramme(code)
   }
 }
 
@@ -96,8 +97,12 @@ const dailyJobs = async () => {
   try {
     await refreshTeacherLeaderboard()
     await refreshProgrammesAndFaculties()
-    if (languageCenterViewEnabled) jobMaker.languagecenter()
-    jobMaker.closeToGraduation()
+
+    if (languageCenterViewEnabled) {
+      jobQueue.refreshLanguagecenter()
+    }
+
+    jobQueue.refreshCloseToGraduation()
   } catch (error) {
     logger.error('Daily jobs failed', error)
   }
@@ -118,11 +123,11 @@ export const startCron = () => {
     })
     schedule('0 19 * * 1', () => {
       logger.info('Updating students whose studyplans have not been updated recently')
-      jobMaker.studyplansUpdate(4)
+      jobQueue.refreshStudyplans(4)
     })
     schedule('0 10 * * 2', () => {
       logger.info('Updating students whose studyplans have not been updated recently')
-      jobMaker.studyplansUpdate(5)
+      jobQueue.refreshStudyplans(5)
     })
   }
 }

services/backend/src/middleware/accessLogger.ts

@@ -1,8 +1,8 @@
 import { Request, Response } from 'express'
-import { omit } from 'lodash'
 import morgan from 'morgan'
 
-import { FormattedUser } from '../types'
+import { FormattedUser } from '@oodikone/shared/types'
+import { omitKeys } from '@oodikone/shared/util'
 import { getFullStudyProgrammeRights } from '../util'
 import logger from '../util/logger'
 
@@ -52,7 +52,7 @@ const accessLogger = morgan((tokens, req: Request, res: Response): undefined =>
   const onlyIamRights = !user.isAdmin && fullStudyProgrammeRights.length === 0
 
   logger.info(message, {
-    ...omit(meta, ['studentsUserCanAccess']), // Don't log student list which might be huge
+    ...omitKeys(meta, ['studentsUserCanAccess']), // Don't log student list which might be huge
     isUsageStats: true, // Pass this as a custom field so we can filter by it in Graylog
     isIamRights: onlyIamRights || usingIamRights, // Needed for Grafana IAM users panel
   })

services/backend/src/middleware/currentUser.ts

@@ -1,10 +1,9 @@
 import * as Sentry from '@sentry/node'
 import { NextFunction, Request, Response } from 'express'
-import { intersection } from 'lodash'
+import { intersection } from 'lodash-es'
 
-import { configLogoutUrl, isDev, requiredGroup, serviceProvider } from '../config'
-import { getMockedUser, getMockedUserFd, getOrganizationAccess, getUserFd, getUserToska } from '../services/userService'
-import { FormattedUser, IamAccess } from '../types'
+import { requiredGroup } from '../config'
+import { getMockedUser, getOrganizationAccess, getUser } from '../services/userService'
 import { ApplicationError } from '../util/customErrors'
 import logger from '../util/logger'
 
@@ -14,30 +13,6 @@ const hasRequiredIamGroup = (iamGroups: string[], iamRights: string[]) => {
   return intersection(iamGroups, requiredGroup).length > 0 || iamRights.length > 0
 }
 
-const getUser = async (
-  showAsUser: string | undefined,
-  username: string,
-  name: string,
-  email: string,
-  iamGroups: string[],
-  specialGroup: Record<string, boolean>,
-  sisId: string,
-  iamAccess: IamAccess
-) => {
-  if (showAsUser && specialGroup?.superAdmin) {
-    return await getMockedUser({ userToMock: showAsUser, mockedBy: username })
-  }
-  return await getUserToska({
-    username,
-    name,
-    email,
-    iamGroups,
-    specialGroup,
-    sisId,
-    iamAccess,
-  })
-}
-
 type Headers = {
   displayname?: string
   hygroupcn?: string
@@ -49,7 +24,7 @@ type Headers = {
   'x-show-as-user'?: string
 }
 
-const toskaUserMiddleware = async (req: Request, _res: Response, next: NextFunction) => {
+export default async (req: Request, _: Response, next: NextFunction) => {
   const {
     'shib-session-id': sessionId,
     'x-show-as-user': showAsUser,
@@ -61,20 +36,15 @@ const toskaUserMiddleware = async (req: Request, _res: Response, next: NextFunct
     uid: username,
   } = req.headers as Headers
 
-  const missingHeaders: string[] = []
-  if (!sessionId) missingHeaders.push('shib-session-id')
-  if (!username) missingHeaders.push('uid')
-
-  if (missingHeaders.length > 0) {
-    const reqUser = username ?? 'Anonymous user'
+  if (!username || !sessionId)
     throw new ApplicationError(
-      `${reqUser} requested ${req.url} without valid request headers. Missing: ${missingHeaders.join(', ')}`,
+      `${username ?? 'Anonymous user'} requested ${req.url} without authorization headers.`,
       403,
       { logoutUrl }
     )
-  }
 
   const iamGroups = parseIamGroups(hygroupcn!)
+
   const { iamAccess = {}, specialGroup = {} } = await getOrganizationAccess(sisId!, iamGroups)
   const iamRights = Object.keys(iamAccess)
 
@@ -83,49 +53,29 @@ const toskaUserMiddleware = async (req: Request, _res: Response, next: NextFunct
       message: 'User does not have required iam group',
       meta: { username, name, email, iamGroups, iamRights },
     })
+
     throw new ApplicationError(`User '${username}' does not have required iam group`, 403, { logoutUrl })
   }
 
-  const user = await getUser(showAsUser, username!, name!, email!, iamGroups, specialGroup, sisId!, iamAccess)
-  if (!user) {
-    throw new ApplicationError(`Username ${username} not found.`, 403, { logoutUrl })
-  }
+  const user =
+    showAsUser && specialGroup?.superAdmin
+      ? await getMockedUser({ userToMock: showAsUser, mockedBy: username })
+      : await getUser({
+          username,
+          name: name!,
+          email: email!,
+          iamGroups,
+          specialGroup,
+          sisId: sisId!,
+          iamAccess,
+        })
 
-  Sentry.setUser({ username: user.mockedBy ?? username! })
+  // Set username just-in-case there is error for invalid user
+  Sentry.setUser({ username: user?.mockedBy ?? username })
+  if (!user) throw new ApplicationError(`Username ${username} not found.`, 403, { logoutUrl })
 
   req.user = user
   req.logoutUrl = logoutUrl!
 
   next()
 }
-
-const fdUserMiddleware = async (req: Request, _res: Response, next: NextFunction) => {
-  const { remote_user: remoteUser, 'x-show-as-user': showAsUser } = req.headers
-
-  if (!remoteUser) {
-    throw new ApplicationError('Not enough data in request headers, remote_user was missing', 403, { configLogoutUrl })
-  }
-
-  let user: FormattedUser | null
-
-  // getMockedUser in production requires the superAdmin role, which is only available
-  // via iamGroups, so it's now only implemented for the dev environment
-  if (showAsUser && isDev) {
-    user = await getMockedUserFd({ userToMock: showAsUser as string, mockedBy: remoteUser as string })
-  } else {
-    user = await getUserFd({ username: remoteUser as string })
-  }
-
-  if (!user) {
-    throw new ApplicationError(`Could not grant access with the eppn ${remoteUser}.`, 403, { configLogoutUrl })
-  }
-
-  req.user = user
-  req.logoutUrl = configLogoutUrl
-
-  next()
-}
-
-const currentUserMiddleware = serviceProvider === 'toska' ? toskaUserMiddleware : fdUserMiddleware
-
-export default currentUserMiddleware

services/backend/src/middleware/headers.ts

@@ -1,10 +1,4 @@
 import headersMiddleware from 'unfuck-utf8-headers-middleware'
 
-import { serviceProvider } from '../config'
-
-const headersToska = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
-const headersOther = ['remote_user']
-
-const headers = serviceProvider === 'toska' ? headersToska : headersOther
-
+const headers = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
 export default headersMiddleware(headers)

services/backend/src/models/SISStudyRight.ts

@@ -71,6 +71,9 @@ export class SISStudyRightModel extends Model implements SISStudyRight {
   @Column(DataType.JSONB)
   declare semesterEnrollments: SISStudyRight['semesterEnrollments']
 
+  @Column(DataType.JSONB)
+  declare transferInfo: SISStudyRight['transferInfo']
+
   @HasMany(() => CreditModel, { foreignKey: 'studyright_id', sourceKey: 'id' })
   declare credits: SISStudyRight['credits']
 

services/backend/src/models/course.ts

@@ -73,6 +73,9 @@ export class CourseModel extends Model implements Course {
   @Column(DataType.JSONB)
   declare substitutions: Course['substitutions']
 
+  @Column(DataType.JSONB)
+  declare substitution_groups: Course['substitution_groups']
+
   @Column(DataType.STRING)
   declare course_unit_type: Course['course_unit_type']
 

services/backend/src/rapodiff/.gitignore

@@ -1,2 +0,0 @@
-*.csv
-*.csv#

services/backend/src/rapodiff/README.md

@@ -1,22 +0,0 @@
-# Rapodiff
-
-Use this to check differences between oodikone and rapo.
-
-## How to load the CSV file from Rapo
-
-1. Go to Rapo and open ”Koulutusohjelmien vuosiseuranta”
-2. Navigate to ”Opintopisteet ja opetus / Credits and teaching”
-3. From the faculty dropdown menu, select all the faculties
-4. From the bottom of the page, click ”Vie -> Tiedot -> CSV”
-5. Rename the file to `data.csv` and copy it to this directory
-
-## Usage
-
-1. Have real oodikone running. Then run rapodiff **in the root directory**: `npm run rapodiff -- args`
-2. place args with a mode. omit args to print possible modes.
-
-## Known differences in APIs
-
-- rapo does not show students that have graduated (it however takes some days for rapo to notice a graduation)
-- rapo does not handle the starting year of transfers properly (shows in old staring population)
-- rapo show some students incorrectly passive, see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3521

services/backend/src/rapodiff/creditsDiff.js

@@ -1,188 +0,0 @@
-const { orderBy } = require('lodash')
-
-// dbConnections must be imported to avoid Sequelize errors even if it's not used
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-const { dbConnections } = require('../database/connection')
-const { computeCreditsProduced } = require('../services/providerCredits')
-const { parseCsv, printProgressBar } = require('./helpers')
-
-const diffs = []
-let noDiffCounter = 0
-
-// Change weird rapo programmecodes to oodikone format, example: 300-M003 => MH30_003
-const transformProgrammeCode = oldCode => `${oldCode[4]}H${oldCode.slice(0, 2)}_${oldCode.slice(5, 8)}`
-
-const formatData = data =>
-  data.reduce((obj, cur) => {
-    const parseNum = str => (str === '' ? 0 : parseFloat(str.replace(',', '.')))
-    const [
-      year,
-      facultyCode,
-      programmeInfo,
-      basic,
-      exchange,
-      otherUni,
-      openUni,
-      special,
-      total,
-      transferredFromAbroad,
-      otherTransferred,
-    ] = cur
-    const programmeCode = transformProgrammeCode(programmeInfo.split(' ')[0])
-    if (!obj[programmeCode]) obj[programmeCode] = {}
-    obj[programmeCode][year] = {
-      year: parseNum(year),
-      facultyCode,
-      programmeInfo,
-      basic: parseNum(basic),
-      'incoming-exchange': parseNum(exchange),
-      agreement: parseNum(otherUni),
-      'open-uni': parseNum(openUni),
-      separate: parseNum(special),
-      total: parseNum(total),
-      transferred: parseNum(transferredFromAbroad) + parseNum(otherTransferred),
-    }
-    return obj
-  }, {})
-
-const diff = (rapoData, okData, code) => {
-  for (const year of Object.keys(okData)) {
-    if (year !== '2022' && year !== '2023') continue
-    const rapo = rapoData[year]
-    const ok = okData[year]
-    if (!rapo) {
-      if (!ok) console.log(`Year ${year} not in rapo for code ${code}, skipping`)
-      continue
-    }
-    for (const field of ['basic', 'incoming-exchange', 'open-uni', 'agreement', 'separate', 'transferred']) {
-      const okValue = Math.round(ok[field] || 0)
-      const rapoValue = Math.round(rapo[field] || 0)
-      const diff = Math.abs(okValue - rapoValue)
-      if (diff === 0) {
-        noDiffCounter++
-        continue
-      }
-      const percentage = rapoValue !== 0 ? Number(((diff / rapoValue) * 100).toFixed(2)) : 100
-      diffs.push({ field, year, code, okValue, rapoValue, diff, 'diff %': percentage })
-    }
-  }
-}
-
-const processProgrammes = async data => {
-  let programmeCounter = 0
-  const rapoProgrammeData = formatData(data.slice(1))
-  const allProgrammeCodes = [...new Set(Object.keys(rapoProgrammeData))]
-  for (const programmeCode of allProgrammeCodes) {
-    const okProgrammeData = await computeCreditsProduced(programmeCode, false, true)
-    diff(rapoProgrammeData[programmeCode], okProgrammeData.stats, programmeCode)
-    printProgressBar(allProgrammeCodes.length, ++programmeCounter)
-  }
-  const fieldToSortBy = 'diff' // 'diff %'
-  const orderedDiffs = orderBy(diffs, fieldToSortBy, 'asc')
-  console.table(orderedDiffs)
-  console.log(`${diffs.length} diffs found with current settings. Numbers with no diff: ${noDiffCounter}`)
-  const totalDiff = orderedDiffs.reduce((sum, cur) => cur.diff + sum, 0)
-  console.log('Total difference: ', totalDiff)
-}
-
-const programmeCreditsDiff = async fileName => {
-  console.log('Running studyprogramme credits diff')
-  await parseCsv(fileName, processProgrammes)
-  console.log('Diff completed.')
-}
-
-/* 
-separate
-Suoritus on erillisellä opiskeluoikeudella opettajankoulutuksen opintoja suorittavan opiskelijan suorittama
-Suoritus on erillisellä opiskeluoikeudella opiskelevan suorittama
-
-agreement
-Suoritus on korkeakoulujen välisillä yhteistyösopimuksilla opiskelevan suorittama
-
-
-open-uni
-Suoritus on avoimessa korkeakouluopetuksessa suoritettu
-
-incoming-exchange
-Suoritus on tulevan kansainvälisen vaihto-opiskelijan suorittama
-
-basic
-perustutkinto-opiskelija = 1
-ja
-Tuntematon
-tai (ehkä):
-Suoritus on kotimainen harjoittelujakso
-?? Opintosuoritus on täydennyskoulutuksessa suoritettu ??
-
-other
-perustutkinto-opiskelija = 0 && tuntematon
-
-*/
-
-const getCategory = (typeName, isBasic, hyvaksiluettu) => {
-  if (hyvaksiluettu) return 'transferred' // ?? unsure
-  if (isBasic || ['Suoritus on kotimainen harjoittelujakso', 'Tuntematon'].includes(typeName)) return 'basic'
-  if (typeName === 'Suoritus on avoimessa korkeakouluopetuksessa suoritettu') return 'open-uni'
-  if (typeName === 'Suoritus on tulevan kansainvälisen vaihto-opiskelijan suorittama') return 'incoming-exchange'
-  if (typeName.startsWith('Suoritus on erillisellä opiskeluoikeudella')) return 'separate'
-  if (typeName === 'Suoritus on korkeakoulujen välisillä yhteistyösopimuksilla opiskelevan suorittama')
-    return 'agreement'
-  return 'other'
-}
-
-const processIds = async (rawData, code, field) => {
-  const data = rawData.slice(0, 21628) // 11476 = KTM viimeinen
-  const rapoData = data
-    .map(row => ({
-      id: row[0].slice(11),
-      type: getCategory(row[1], row[3] === '1', row[6] === '1'),
-      typeCode: row[8],
-      isModule: row[9] !== '0',
-      credits: parseFloat(row[10]),
-      included: row[4] === '1',
-    }))
-    .filter(course => !course.isModule && (field === 'transferred' || course.included))
-  const rapoIds = rapoData.filter(course => course.credits > 0 && course.type === field).map(row => row.id)
-  const rapoCredits = rapoData.reduce(
-    (stats, cur) => {
-      stats[cur.type] += cur.credits
-      return stats
-    },
-    {
-      transferred: 0,
-      total: '_',
-      basic: 0,
-      'open-uni': 0,
-      separate: 0,
-      'incoming-exchange': 0,
-      agreement: 0,
-      other: 0,
-    }
-  )
-  const stats = await computeCreditsProduced(code, false, true, field)
-  if (!stats.stats.ids) {
-    console.log({ rapoCredits })
-    console.log('Edit getCreditsProduced code so that it saves a list of the relevant ids you want to compare.')
-    return
-  }
-  const okIdMap = stats.stats.ids
-  const rapoIdMap = rapoIds.reduce((obj, cur) => {
-    obj[cur] = true
-    return obj
-  }, {})
-  const onlyInRapo = Object.keys(rapoIdMap).filter(id => !okIdMap[id])
-  const onlyInOk = Object.keys(okIdMap).filter(id => !rapoIdMap[id])
-  console.log('\n onlyInRapo: ', onlyInRapo.length)
-  console.dir(onlyInRapo.slice(0, 30))
-  console.log('\n onlyInOk: ', onlyInOk.length)
-  console.dir(onlyInOk.slice(0, 30))
-  delete stats.ids
-  console.log(stats.stats['2022'])
-  console.log('rapoCredits: ', rapoCredits)
-}
-
-const testNewCalc = async (fileName, code, field) => {
-  await parseCsv(fileName, async data => processIds(data, code, field))
-}
-
-module.exports = { programmeCreditsDiff, testNewCalc }

services/backend/src/rapodiff/helpers.js

@@ -1,36 +0,0 @@
-const fs = require('fs/promises')
-const Papa = require('papaparse')
-
-const parseCsv = async (fileName, fun) => {
-  try {
-    const file = await fs.readFile(`${__dirname}/${fileName}`, 'utf8')
-    Papa.parsePromise = file => {
-      return new Promise((complete, error) => {
-        Papa.parse(file, { complete, error })
-      })
-    }
-    const data = await Papa.parsePromise(file)
-    await fun(data?.data)
-  } catch (error) {
-    console.log(error)
-  }
-}
-
-const printProgressBar = (total, current) => {
-  const barLength = 30
-  const progress = Math.floor((current / total) * barLength)
-  const empty = barLength - progress
-
-  const progressBar = `[${'#'.repeat(progress)}${'-'.repeat(empty)}]`
-  const percentage = Math.floor((current / total) * 100)
-
-  process.stdout.clearLine()
-  process.stdout.cursorTo(0)
-  process.stdout.write(`${progressBar} ${percentage} %`)
-
-  if (current >= total) {
-    console.log()
-  }
-}
-
-module.exports = { parseCsv, printProgressBar }

services/backend/src/rapodiff/rapodiff.js

@@ -1,31 +0,0 @@
-const { programmeCreditsDiff, testNewCalc } = require('./creditsDiff')
-
-// ** To add new mode, just type key and function here **
-const modes = {
-  // Check credits with current local oodikone data against provided data.csv in rapodiff folder.
-  credits: () => programmeCreditsDiff('data.csv'),
-  // Provide programme code as argument. Parses credits from credits.csv in rapodiff folder. This must be obtained from Rapo people. This tells which credit ids are found in oodikone but not rapo, and vice versa.
-  // notice: check mode requires writing ids in to the stats object in credits calculating function.
-  // Add something like this there:
-  // if (category === 'open-uni' && credits > 0 && new Date(attainmentDate).getFullYear() === 2022) stats.ids[id] = true
-  check: (fileName, code, field) => testNewCalc(fileName, code, field),
-}
-
-const main = async () => {
-  console.log('**RAPODIFF**\n')
-  const mode = process.argv[2]
-  const modeFunction = modes[mode]
-  const arg = process.argv[3]
-  const arg2 = process.argv[4]
-  const arg3 = process.argv[5]
-  if (!modeFunction) {
-    console.log(
-      'Unknown or missing mode. Supply the mode as an argument (e.g. `npm run rapodiff -- credits`). Supported modes:\n'
-    )
-    console.log(Object.keys(modes).join('\n'))
-    return
-  }
-  await modeFunction(arg, arg2, arg3)
-}
-
-main().then(() => process.exit(0))

services/backend/src/routes.ts

@@ -3,7 +3,7 @@ import compression from 'compression'
 import cors from 'cors'
 import { Express, json as jsonExpress } from 'express'
 
-import { frontUrl, languageCenterViewEnabled, serviceProvider } from './config'
+import { frontUrl, languageCenterViewEnabled } from './config'
 import accessLogger from './middleware/accessLogger'
 import * as auth from './middleware/auth'
 import currentUserMiddleware from './middleware/currentUser'
@@ -15,7 +15,6 @@ import completedCoursesSearch from './routes/completedCoursesSearch'
 import courseExclusions from './routes/courseExclusions'
 import courses from './routes/courses'
 import curriculumPeriods from './routes/curriculumPeriods'
-import customOpenUniSearch from './routes/customOpenUniSearch'
 import customPopulationSearch from './routes/customPopulationSearch'
 import faculties from './routes/faculties'
 import feedback from './routes/feedback'
@@ -34,8 +33,7 @@ import tags from './routes/tags'
 import teachers from './routes/teachers'
 import university from './routes/university'
 import updater from './routes/updater'
-import usersToska from './routes/users'
-import usersFd from './routes/usersFd'
+import users from './routes/users'
 
 const routes = (app: Express, url: string) => {
   app.use(cors({ credentials: true, origin: frontUrl }))
@@ -43,7 +41,7 @@ const routes = (app: Express, url: string) => {
   app.use(compression())
 
   app.use(headersMiddleware)
-  app.use(currentUserMiddleware) // eslint-disable-line @typescript-eslint/no-misused-promises
+  app.use(currentUserMiddleware)
   app.use(accessLogger)
   app.use(`${url}/login`, login)
   app.use(url, courses)
@@ -53,10 +51,9 @@ const routes = (app: Express, url: string) => {
   app.use(`${url}/semesters/codes`, semesters)
   app.use(url, tags)
   app.use(url, programmeModules)
-  app.use(`${url}/v3/course-exclusions`, courseExclusions)
-  app.use(`${url}/v2/studyprogrammes`, studyProgramme)
+  app.use(`${url}/course-exclusions`, courseExclusions)
+  app.use(`${url}/studyprogrammes`, studyProgramme)
   app.use(`${url}/programme-criteria`, studyProgrammeCriteria)
-  app.use(`${url}/openunisearch`, auth.roles(['openUniSearch']), customOpenUniSearch)
   app.use(`${url}/changelog`, changelog)
   app.use(`${url}/completedcoursessearch`, completedCoursesSearch)
   if (languageCenterViewEnabled) {
@@ -66,19 +63,14 @@ const routes = (app: Express, url: string) => {
   app.use(`${url}/university`, university)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, teachers)
-  if (serviceProvider === 'toska') {
-    app.use(`${url}/users`, usersToska)
-  } else {
-    app.use(`${url}/users`, usersToska)
-    app.use(`${url}/users`, usersFd)
-  }
+  app.use(`${url}/users`, users)
   app.use(`${url}/feedback`, feedback)
   app.use(`${url}/custom-population-search`, customPopulationSearch)
   app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
   app.use(`${url}/close-to-graduation`, auth.roles(['fullSisuAccess', 'studyGuidanceGroups']), closeToGraduation)
   app.use(`${url}/study-programme-pins`, studyProgrammePins)
   app.use(`${url}/curriculum-periods`, curriculumPeriods)
-  app.get('*', (_, res) => {
+  app.get('', (_, res) => {
     res.status(404).json({ error: 'unknown endpoint' })
   })
   Sentry.setupExpressErrorHandler(app)

services/backend/src/routes/changelog.ts

@@ -1,20 +1,19 @@
 import axios from 'axios'
-import { Request, Response, Router } from 'express'
+import { Router } from 'express'
 
 import { Release } from '@oodikone/shared/types'
-import { isDev } from '../config'
+import { isDev, isTest } from '../config'
 
 const router = Router()
 
 const changelog: { data?: Release[] } = {}
 
-export type ChangelogResBody = Release[]
-
-router.get<never, ChangelogResBody>('/', async (_req: Request, res: Response) => {
+router.get<never, Release[]>('/', async (_, res) => {
   if (changelog.data) {
     return res.status(200).send(changelog.data)
   }
-  if (isDev) {
+
+  if (isDev || isTest) {
     const fakeRelease: Release[] = [
       {
         description: '**Feature 1**\n- Added a fancy new feature \n\n**Feature 2**\n- Fixed a bug\n- Fixed another bug',
@@ -35,8 +34,11 @@ router.get<never, ChangelogResBody>('/', async (_req: Request, res: Response) =>
         version: '0.0.1',
       },
     ]
+
+    changelog.data = fakeRelease
     return res.status(200).json(fakeRelease)
   }
+
   const response = await axios.get('https://api.github.com/repos/UniversityOfHelsinkiCS/oodikone/releases')
   const releasesFromAPI: Release[] = response.data.map((release: Record<string, any>) => ({
     description: release.body,
@@ -44,6 +46,7 @@ router.get<never, ChangelogResBody>('/', async (_req: Request, res: Response) =>
     title: release.name,
     version: release.tag_name,
   }))
+
   changelog.data = releasesFromAPI
   res.status(200).json(releasesFromAPI)
 })

services/backend/src/routes/closeToGraduation.ts

@@ -1,22 +1,24 @@
 import { Router } from 'express'
 
-import { findStudentsCloseToGraduation, getCloseToGraduationData } from '../services/populations/closeToGraduation'
+import type { CloseToGraduationResBody } from '@oodikone/shared/routes/closeToGraduation'
+import { getCloseToGraduationData } from '../services/populations/closeToGraduation'
 import { getAllStudentsUserHasInGroups } from '../services/studyGuidanceGroups'
 import { hasFullAccessToStudentData } from '../util'
 
 const router = Router()
 
-type CloseToGraduationResBody = Awaited<ReturnType<typeof findStudentsCloseToGraduation>>
-
 router.get<never, CloseToGraduationResBody>('/', async (req, res) => {
   const { user } = req
-  if (hasFullAccessToStudentData(user.roles)) {
-    const result = await getCloseToGraduationData()
+
+  if (!hasFullAccessToStudentData(user.roles)) {
+    const studentNumbers = await getAllStudentsUserHasInGroups(user.sisPersonId)
+    const result = await getCloseToGraduationData(studentNumbers)
+
     return res.json(result)
   }
-  const studentsInUsersGuidanceGroups = await getAllStudentsUserHasInGroups(user.sisPersonId)
-  const result = await getCloseToGraduationData(studentsInUsersGuidanceGroups)
-  res.json(result)
+
+  const result = await getCloseToGraduationData()
+  return res.json(result)
 })
 
 export default router

services/backend/src/routes/completedCoursesSearch.ts

@@ -1,14 +1,14 @@
 import { Router } from 'express'
-import { difference, intersection } from 'lodash'
+import { difference, intersection } from 'lodash-es'
 
 import { CanError } from '@oodikone/shared/routes'
 import { tryCatch } from '@oodikone/shared/util'
 import { Courses, FormattedStudent, getCompletedCourses } from '../services/completedCoursesSearch'
 import {
-  getOpenUniSearches,
-  createNewSearch,
-  deleteSearch,
-  updateSearch,
+  getUserSearches,
+  createNewUserSearch,
+  deleteUserSearch,
+  updateUserSearch,
   FoundSearch,
 } from '../services/openUni/openUniManageSearches'
 import { hasFullAccessToStudentData, safeJSONParse } from '../util'
@@ -42,18 +42,11 @@ router.get<never, SearchResBody, SearchReqBody, SearchQuery>('/', async (req, re
     return res.status(400).json({ error: 'Courses must be of type array' })
   }
 
-  const answerTimeout = new Promise((_, reject) => setTimeout(() => reject(new Error('Request timeout')), 6000))
-
   // Teachers can also get rights to students via Importer if
   // the students have enrolled to their courses in last 8 months
   // (acual logic a bit different, see Importer)
   const { data: teacherRightsToStudents, error } = importerClient
-    ? await tryCatch<{ data: string[] }>(
-        Promise.race([
-          importerClient.post('/teacher-rights/', { teacherId, studentNumbers }),
-          answerTimeout as Promise<never>, // This will always reject with an Error
-        ])
-      )
+    ? await tryCatch<{ data: string[] }>(importerClient.post('/teacher-rights/', { teacherId, studentNumbers }))
     : { data: null, error: null }
 
   if (error) {
@@ -79,7 +72,7 @@ export type GetSearchResBody = FoundSearch[]
 
 router.get<never, GetSearchResBody>('/searches', async (req, res) => {
   const userId = req.user.id
-  const foundSearches = await getOpenUniSearches(userId)
+  const foundSearches = await getUserSearches(userId)
   return res.json(foundSearches)
 })
 
@@ -105,7 +98,7 @@ router.post<never, CreateSearchResBody, CreateSearchReqBody>('/searches', async
   if (courseCodes && !Array.isArray(courseCodes)) {
     return res.status(400).json({ error: 'Courselist must be type of array' })
   }
-  const createdSearch = await createNewSearch(userId, name, courseCodes)
+  const createdSearch = await createNewUserSearch(userId, name, courseCodes)
   if (!createdSearch) {
     return res.status(400).json({ error: 'Failed to create courselist' })
   }
@@ -137,7 +130,7 @@ router.put<UpdateSearchParams, UpdateSearchResBody, UpdateSearchReqBody>('/searc
     return res.status(422).end()
   }
 
-  const updatedSearch = await updateSearch(userId, id, courseCodes)
+  const updatedSearch = await updateUserSearch(userId, id, courseCodes)
   if (!updatedSearch) {
     return res.status(404).json({ error: 'Courselist could not be found' })
   }
@@ -162,7 +155,7 @@ router.delete<DeleteSearchParams, DeleteSearchResBody>('/searches/:id', async (r
     return res.status(422).end()
   }
 
-  const deletedSearch = await deleteSearch(userId, id)
+  const deletedSearch = await deleteUserSearch(userId, id)
   if (!deletedSearch) {
     return res.status(404).json({ error: 'Courselist could not be found' })
   }

services/backend/src/routes/courseExclusions.ts

@@ -1,19 +1,20 @@
 import { Router } from 'express'
 
+import type { ExcludedCoursesResBody, ExcludedCoursesReqBody } from '@oodikone/shared/routes/courseExclusions'
 import { addExcludedCourses, removeExcludedCourses } from '../services/excludedCourses'
+import { hasFullAccessToStudentData } from '../util'
 
 const router = Router()
 
-export type ExcludedCoursesParams = { code: string }
-export type ExcludedCoursesResBody = void
-export type ExcludedCoursesReqBody = {
-  courseCodes: string[]
-  curriculumVersion: string
-}
+router.post<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async (req, res) => {
+  const { programmeCode, courseCodes, curriculumVersion } = req.body
+  const { roles, programmeRights } = req.user
+
+  const hasFullAccess = hasFullAccessToStudentData(roles)
+  const hasAccessToProgramme = programmeRights.map(({ code }) => code).includes(programmeCode)
+
+  if (!hasFullAccess && !hasAccessToProgramme) return res.status(403).end()
 
-router.post<ExcludedCoursesParams, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/:code', async (req, res) => {
-  const { code: programmeCode } = req.params
-  const { courseCodes, curriculumVersion } = req.body
   const result = await addExcludedCourses(courseCodes, curriculumVersion, programmeCode)
   if (!result) {
     res.status(400).end()
@@ -22,9 +23,15 @@ router.post<ExcludedCoursesParams, ExcludedCoursesResBody, ExcludedCoursesReqBod
   res.status(201).end()
 })
 
-router.delete<ExcludedCoursesParams, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/:code', async (req, res) => {
-  const { code: programmeCode } = req.params
-  const { courseCodes, curriculumVersion } = req.body
+router.delete<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async (req, res) => {
+  const { programmeCode, courseCodes, curriculumVersion } = req.body
+  const { roles, programmeRights } = req.user
+
+  const hasFullAccess = hasFullAccessToStudentData(roles)
+  const hasAccessToProgramme = programmeRights.map(({ code }) => code).includes(programmeCode)
+
+  if (!hasFullAccess && !hasAccessToProgramme) return res.status(403).end()
+
   const result = await removeExcludedCourses(courseCodes, curriculumVersion, programmeCode)
   if (!result) {
     res.status(400).end()

services/backend/src/routes/courses.ts

@@ -2,108 +2,99 @@ import crypto from 'crypto'
 import { Router } from 'express'
 
 import { CanError } from '@oodikone/shared/routes'
-import { getCourseYearlyStats } from '../services/courses'
-import { getCoursesByNameAndOrCode, getCoursesByCodes } from '../services/courses/courseFinders'
-import { CourseWithSubsId } from '../types/course'
-import { getFullStudyProgrammeRights, hasFullAccessToStudentData, validateParamLength } from '../util'
+import {
+  CoursesMultiResBody,
+  CoursesMultiReqBody,
+  CoursesMultiQuery,
+  CourseYearlyStatsReqBody,
+  CourseYearlyStatsQuery,
+} from '@oodikone/shared/routes/courses'
+import type { CourseWithSubsDetails } from '@oodikone/shared/types/course'
+import { getCourseDetails, getCourseYearlyStats } from '../services/courses'
+import { getCoursesByNameAndOrCode } from '../services/courses/courseFinders'
+import {
+  getFullStudyProgrammeRights,
+  handleQueryArrays,
+  hasFullAccessToStudentData,
+  validateParamLength,
+} from '../util'
 
 const router = Router()
 
-type CoursesMultiReqBody = never
-type CoursesMultiResBody = { courses: CourseWithSubsId[] }
-type CoursesMultiQuery = {
-  name: string
-  code: string
-  combineSubstitutions: string
-}
-
 router.get<never, CanError<CoursesMultiResBody>, CoursesMultiReqBody, CoursesMultiQuery>(
-  '/v2/coursesmulti',
+  '/coursesmulti',
   async (req, res) => {
-    const { name, code, combineSubstitutions } = req.query
+    const { name, code, includeSpecial } = req.query
     if (!(validateParamLength(name, 5) || validateParamLength(code, 2))) {
       return res.status(400).json({ error: 'Query parameter name or code is invalid' })
     }
 
-    const courses: CourseWithSubsId[] = await getCoursesByNameAndOrCode(name, code)
+    const courses = await getCoursesByNameAndOrCode(name, code, includeSpecial === 'true')
 
-    if (combineSubstitutions === 'false') {
-      const courseCodes = courses.map(course => course.code)
-      const substitutions = [
-        ...new Set(
-          courses.flatMap(course => course.substitutions).filter(code => code !== null && !courseCodes.includes(code))
-        ),
-      ]
-      const substitutionDetails = await getCoursesByCodes(substitutions)
-      for (const substitution of substitutionDetails) {
-        courses.push(substitution.toJSON())
-      }
-    }
-
-    const mergedCourses: Record<string, CourseWithSubsId> = {}
+    const mergedCourses = new Map<string, CourseWithSubsDetails>()
 
     for (const course of courses) {
-      const groupId = combineSubstitutions === 'true' ? (course.subsId?.toString() ?? '') : course.code
-      if (!(course.max_attainment_date && course.min_attainment_date)) {
-        continue
+      if (!(course.max_attainment_date && course.min_attainment_date)) continue
+
+      if (!mergedCourses.has(course.code)) mergedCourses.set(course.code, structuredClone(course))
+      const mergedCourse = mergedCourses.get(course.code)!
+
+      if (mergedCourse.max_attainment_date < course.max_attainment_date) {
+        mergedCourse.max_attainment_date = course.max_attainment_date
       }
-      if (!mergedCourses[groupId]) {
-        mergedCourses[groupId] = {
-          ...course,
-          substitutions: combineSubstitutions === 'true' ? course.substitutions : [],
-        }
-      } else {
-        const mergedCourse = mergedCourses[groupId]
-        if (mergedCourse.max_attainment_date < course.max_attainment_date) {
-          mergedCourse.max_attainment_date = course.max_attainment_date
-        }
-        if (mergedCourse.min_attainment_date > course.min_attainment_date) {
-          mergedCourse.min_attainment_date = course.min_attainment_date
-        }
+      if (mergedCourse.min_attainment_date > course.min_attainment_date) {
+        mergedCourse.min_attainment_date = course.min_attainment_date
       }
     }
 
-    res.json({ courses: Object.values(mergedCourses) })
+    res.json({ courses: Array.from(mergedCourses.values()) })
   }
 )
 
 type CourseYearlyStatsResBody = Awaited<ReturnType<typeof getCourseYearlyStats>>
-type CourseYearlyStatsReqBody = never
-type CourseYearlyStatsQuery = {
-  codes: string[]
-  separate: string
-  combineSubstitutions: string
-}
 
 router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody, CourseYearlyStatsQuery>(
-  '/v3/courseyearlystats',
+  '/courseyearlystats',
   async (req, res) => {
-    const { roles, programmeRights } = req.user
+    const { codes, combineSubstitutions, separate } = req.query
 
-    const userHasFullAccessToStudentData = hasFullAccessToStudentData(roles)
-    const userHasCorrectRole = userHasFullAccessToStudentData === true || roles.includes('courseStatistics')
-    const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
+    const courseCodes = handleQueryArrays(codes)
 
-    // If user has rights to see at least one programme, then they are allowed to see all of them
-    if (!userHasCorrectRole && fullStudyProgrammeRights.length === 0) {
-      return res.status(403).json({ error: 'No programmes so no access to course stats' })
-    }
-
-    const { codes } = req.query
-    if (!codes) {
+    if (!courseCodes?.length) {
       return res.status(422).send({ error: 'Missing required query parameters' })
     }
 
-    const combineSubstitutions = req.query.combineSubstitutions !== 'false'
-    const separate = req.query.separate === 'true'
+    const { roles, programmeRights } = req.user
+    const userHasFullAccessToStudentData = hasFullAccessToStudentData(roles)
+    const userHasAccessToCourseStats = userHasFullAccessToStudentData || roles.includes('courseStatistics')
+    const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
+
+    // If user has rights to see at least one programme, then they are allowed to see all of them
+    if (!userHasAccessToCourseStats && !fullStudyProgrammeRights.length) {
+      return res.status(403).json({ error: 'No valid rights provided' })
+    }
 
     // Student numbers should be obfuscated to all other users except admins,
     // fullSisuAccess users, and users with rights to any specific degree programmes
     const anonymize = !userHasFullAccessToStudentData && fullStudyProgrammeRights.length === 0
     const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
-    const results = await getCourseYearlyStats(codes, separate, anonymizationSalt, combineSubstitutions)
+
+    const useCombineSubstitutions = combineSubstitutions !== 'false'
+    const useSeparate = separate === 'true'
+
+    const results = await getCourseYearlyStats(courseCodes, useSeparate, anonymizationSalt, useCombineSubstitutions)
     res.json(results)
   }
 )
 
+router.get<never, CanError<unknown>, never, { codes: string | string[] }>('/coursedetails', async (req, res) => {
+  const { codes: coursecodes } = req.query
+  const codes = handleQueryArrays(coursecodes)
+
+  if (!codes?.length) return res.status(500).end()
+  const details = await getCourseDetails(codes)
+
+  return res.json(details)
+})
+
 export default router

services/backend/src/routes/customOpenUniSearch.ts

@@ -1,148 +0,0 @@
-import { Router } from 'express'
-
-import { CanError } from '@oodikone/shared/routes'
-import {
-  getOpenUniSearches,
-  createNewSearch,
-  deleteSearch,
-  updateSearch,
-} from '../services/openUni/openUniManageSearches'
-import { getCustomOpenUniCourses } from '../services/openUni/openUniStats'
-import { safeJSONParse } from '../util'
-
-const router = Router()
-
-type SearchResBody = CanError<Awaited<ReturnType<typeof getCustomOpenUniCourses>>>
-type SearchReqBody = never
-type SearchQuery = {
-  courselist: string
-  startdate: string
-  enddate: string
-}
-
-router.get<never, SearchResBody, SearchReqBody, SearchQuery>('/', async (req, res) => {
-  const { courselist, startdate, enddate } = req.query
-
-  const courseCodes = (await safeJSONParse(courselist)) ?? []
-  if (!Array.isArray(courseCodes)) {
-    return res.status(400).json({ error: 'Courses must be of type array' })
-  }
-
-  const startDate = startdate !== 'null' ? new Date(startdate) : new Date(2017, 7, 1, 0, 0, 0)
-  const endDate = enddate !== 'null' ? new Date(enddate) : new Date()
-  if (enddate === 'null') {
-    endDate.setHours(23, 59, 59, 999)
-  }
-
-  const customOpenUniSearches = await getCustomOpenUniCourses(courseCodes, startDate, endDate)
-  return res.json(customOpenUniSearches)
-})
-
-type GetSearchResBody = Awaited<ReturnType<typeof getOpenUniSearches>>
-router.get<never, GetSearchResBody>('/searches', async (req, res) => {
-  const { id: userId } = req.user
-  const foundSearches = await getOpenUniSearches(userId)
-  return res.json(foundSearches)
-})
-
-type CreateSearchResBody = CanError<{
-  id: string
-  userId: string
-  name: string
-  courseList: string[]
-  updatedAt: string
-}>
-type CreateSearchReqBody = {
-  courselist: string[]
-  name: string
-}
-
-router.post<never, CreateSearchResBody, CreateSearchReqBody>('/searches', async (req, res) => {
-  const { courselist, name } = req.body
-  const { id: userId } = req.user
-
-  const courseCodes = courselist || []
-
-  if (!name) {
-    return res.status(400).json({ error: 'Name missing' })
-  }
-
-  if (courseCodes && !Array.isArray(courseCodes)) {
-    return res.status(400).json({ error: 'Course codes must be type of array' })
-  }
-
-  const createdSearch = await createNewSearch(userId, name, courseCodes)
-  if (!createdSearch) {
-    return res.status(400).json({ error: 'Failed to create search' })
-  }
-
-  return res.status(201).json({
-    id: createdSearch.id,
-    userId: createdSearch.userId,
-    name: createdSearch.name,
-    courseList: createdSearch.courseCodes,
-    updatedAt: createdSearch.updatedAt,
-  })
-})
-
-type UpdateSearchParams = {
-  id: string
-}
-type UpdateSearchResBody = CanError<{
-  id: string
-  userId: string
-  name: string
-  courseList: string[]
-  updatedAt: string
-}>
-type UpdateSearchReqBody = {
-  courselist: string[]
-}
-
-router.put<UpdateSearchParams, UpdateSearchResBody, UpdateSearchReqBody>('/searches/:id', async (req, res) => {
-  const { id } = req.params
-  const { courselist } = req.body
-  const { id: userId } = req.user
-
-  const courseCodes = courselist || []
-
-  if (!id || !userId) {
-    return res.status(422).end()
-  }
-
-  const updatedSearch = await updateSearch(userId, id, courseCodes)
-  if (!updatedSearch) {
-    return res.status(404).json({ error: 'Open uni search could not be found' })
-  }
-
-  return res.json({
-    id: updatedSearch.id,
-    userId: updatedSearch.userId,
-    name: updatedSearch.name,
-    courseList: updatedSearch.courseCodes,
-    updatedAt: updatedSearch.updatedAt,
-  })
-})
-
-type DeleteSearchParams = {
-  id: string
-}
-type DeleteSearchResBody = CanError<string>
-
-router.delete<DeleteSearchParams, DeleteSearchResBody>('/searches/:id', async (req, res) => {
-  const { id } = req.params
-  const { id: userId } = req.user
-
-  if (!id || !userId) {
-    return res.status(422).end()
-  }
-
-  const deletedSearch = await deleteSearch(userId, id)
-  if (!deletedSearch) {
-    return res.status(404).json({ error: 'Open uni search could not be found' })
-  }
-
-  return res.json(id)
-})
-
-export default router

services/backend/src/routes/faculties.ts

@@ -1,8 +1,18 @@
-import { Request, Response, Router } from 'express'
+import { Router } from 'express'
 
-import { Graduated, ProgrammeFilter, SpecialGroups, StatsType, YearType } from '@oodikone/shared/types'
+import type { CanError, GenericApplicationError } from '@oodikone/shared/routes'
+import type {
+  Graduated,
+  Name,
+  NameWithCode,
+  ProgrammeFilter,
+  SpecialGroups,
+  StatsType,
+  Unarray,
+  YearType,
+} from '@oodikone/shared/types'
 import * as auth from '../middleware/auth'
-import { getDegreeProgrammesOfFaculty, getFacultyCodeById } from '../services/faculty/faculty'
+import { getDegreeProgrammesOfFaculty, getFacultyCodeById, ProgrammesOfOrganization } from '../services/faculty/faculty'
 import { combineFacultyBasics } from '../services/faculty/facultyBasics'
 import { getFacultyCredits } from '../services/faculty/facultyCredits'
 import { countGraduationTimes } from '../services/faculty/facultyGraduationTimes'
@@ -18,242 +28,270 @@ import {
   setFacultyStudentStats,
   getFacultyProgressStats,
   setFacultyProgressStats,
+  type BasicData,
+  type ThesisWriterData,
+  GraduationData,
 } from '../services/faculty/facultyService'
 import { combineFacultyStudentProgress } from '../services/faculty/facultyStudentProgress'
 import { combineFacultyStudents } from '../services/faculty/facultyStudents'
 import { combineFacultyThesisWriters } from '../services/faculty/facultyThesisWriters'
 import { updateFacultyOverview, updateFacultyProgressOverview } from '../services/faculty/facultyUpdates'
-import logger from '../util/logger'
+import { ApplicationError } from '../util/customErrors'
 
 // Faculty uses a lot of tools designed for Degree programme.
 // Some of them have been copied here and slightly edited for faculty purpose.
 
 const router = Router()
 
-router.get('/', async (_req: Request, res: Response) => {
-  const faculties = await getFacultiesForFacultyList()
-  res.json(faculties)
-})
+type GetFacultyListResBody = {
+  id: string
+  code: string
+  name: Name
+}[]
 
-interface GetStatsRequest extends Request {
-  query: {
-    programme_filter: ProgrammeFilter
-    special_groups: SpecialGroups
-    year_type: YearType
-  }
+router.get<never, GetFacultyListResBody>('/', async (_req, res) => res.json(await getFacultiesForFacultyList()))
+
+type GetStatsParams = { id: string }
+type GetStatsReqBody = never
+type GetStatsResBody = BasicData
+type GetStatsQuery = {
+  programme_filter: ProgrammeFilter
+  special_groups: SpecialGroups
+  year_type: YearType
 }
 
-router.get('/:id/basicstats', auth.roles(['facultyStatistics']), async (req: GetStatsRequest, res: Response) => {
-  const facultyId = req.params.id
-  const code = await getFacultyCodeById(facultyId)
-  if (!code) {
-    return res.status(422).end()
-  }
+router.get<GetStatsParams, CanError<GetStatsResBody, GenericApplicationError>, GetStatsReqBody, GetStatsQuery>(
+  '/:id/basicstats',
+  auth.roles(['facultyStatistics']),
+  async (req, res) => {
+    const { id: facultyId } = req.params
+    const { year_type: yearType, programme_filter: programmeFilter, special_groups: specialGroups } = req.query
 
-  const { year_type: yearType, programme_filter: programmeFilter, special_groups: specialGroups } = req.query
-  const data = await getBasicStats(code, yearType, programmeFilter, specialGroups)
-  if (data) {
-    return res.json(data)
-  }
+    const faculty = await getFacultyCodeById(facultyId)
+    if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
-  const programmes = await getDegreeProgrammesOfFaculty(code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
-  if (!programmes.length) {
-    return res.status(422).end()
-  }
+    const data = await getBasicStats(faculty.code, yearType, programmeFilter, specialGroups)
+    if (data) return res.json(data)
 
-  // TODO: The types could be handled better. Possible null value is causing issues with TS.
-  let updatedStats: any = await combineFacultyBasics(code, programmes, yearType, specialGroups)
-  if (updatedStats) {
-    updatedStats = await setBasicStats(updatedStats, yearType, programmeFilter, specialGroups)
-  }
-  return res.json(updatedStats)
-})
+    const programmes = await getDegreeProgrammesOfFaculty(faculty.code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
+    if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
 
-interface GetCreditStatsRequest extends Request {
-  query: {
-    year_type: YearType
+    const updatedStats = await combineFacultyBasics(faculty.code, programmes, yearType, specialGroups)
+
+    if (updatedStats) await setBasicStats(updatedStats, yearType, programmeFilter, specialGroups)
+    else throw new ApplicationError('No data', 204)
+
+    return res.json(updatedStats)
   }
+)
+
+type GetFacultyStatsResBody = ThesisWriterData
+
+router.get<GetStatsParams, CanError<GetFacultyStatsResBody, GenericApplicationError>, GetStatsReqBody, GetStatsQuery>(
+  '/:id/thesisstats',
+  auth.roles(['facultyStatistics']),
+  async (req, res) => {
+    const { id: facultyId } = req.params
+    const { year_type: yearType, programme_filter: programmeFilter, special_groups: specialGroups } = req.query
+
+    const faculty = await getFacultyCodeById(facultyId)
+    if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+    const data = await getThesisWritersStats(faculty.code, yearType, programmeFilter, specialGroups)
+    if (data) return res.json(data)
+
+    const programmes = await getDegreeProgrammesOfFaculty(faculty.code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
+    if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
+
+    const updatedStats = await combineFacultyThesisWriters(faculty.code, programmes, yearType, specialGroups)
+    if (updatedStats) await setThesisWritersStats(updatedStats, yearType, programmeFilter, specialGroups)
+    else throw new ApplicationError('No data', 204)
+
+    return res.json(updatedStats)
+  }
+)
+
+type GetCreditStatsResBody = {
+  codes: string[]
+  ids: string[]
+  programmeNames: Record<string, NameWithCode>
+}
+type GetCreditStatsQuery = {
+  year_type: YearType
 }
 
-router.get('/:id/creditstats', auth.roles(['facultyStatistics']), async (req: GetCreditStatsRequest, res: Response) => {
-  const facultyId = req.params.id
-  const code = await getFacultyCodeById(facultyId)
+router.get<
+  GetStatsParams,
+  CanError<GetCreditStatsResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetCreditStatsQuery
+>('/:id/creditstats', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
   const { year_type: yearType } = req.query
-  const stats = await getFacultyCredits(code, yearType === 'ACADEMIC_YEAR')
-  return res.json(stats)
+
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+  return res.json(await getFacultyCredits(faculty.code, yearType === 'ACADEMIC_YEAR'))
 })
 
-router.get('/:id/thesisstats', auth.roles(['facultyStatistics']), async (req: GetStatsRequest, res: Response) => {
-  const facultyId = req.params.id
-  const code = await getFacultyCodeById(facultyId)
-  if (!code) {
-    return res.status(422).end()
-  }
+type GetGraduationStatsResBody = GraduationData
+type GetGraduationStatsQuery = {
+  programme_filter: ProgrammeFilter
+}
 
-  const { year_type: yearType, programme_filter: programmeFilter, special_groups: specialGroups } = req.query
-  const data = await getThesisWritersStats(code, yearType, programmeFilter, specialGroups)
-  if (data) {
-    return res.json(data)
-  }
+router.get<
+  GetStatsParams,
+  CanError<GetGraduationStatsResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetGraduationStatsQuery
+>('/:id/graduationtimes', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
+  const { programme_filter: programmeFilter } = req.query
 
-  const programmes = await getDegreeProgrammesOfFaculty(code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
-  if (!programmes.length) {
-    return res.status(422).end()
-  }
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+  const data = await getGraduationStats(faculty.code, programmeFilter)
+  if (data) return res.json(data)
+
+  const programmes = await getDegreeProgrammesOfFaculty(faculty.code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
+  if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
+
+  const updatedStats = await countGraduationTimes(faculty.code, programmes)
+  if (updatedStats) await setGraduationStats(updatedStats, programmeFilter)
 
-  let updatedStats: any = await combineFacultyThesisWriters(code, programmes, yearType, specialGroups)
-  if (updatedStats) {
-    updatedStats = await setThesisWritersStats(updatedStats, yearType, programmeFilter, specialGroups)
-  }
   return res.json(updatedStats)
 })
 
-interface GetGraduationStatsRequest extends Request {
-  query: {
-    programme_filter: ProgrammeFilter
-  }
+type GetProgressStatsResBody = {
+  bachelorsProgStats: Record<string, number[][]>
+  bcMsProgStats: Record<string, number[][]>
+  creditCounts: Record<string, Record<string, number[]>>
+  doctoralProgStats: Record<string, number[][]>
+  id: string
+  mastersProgStats: Record<string, number[][]>
+  programmeNames: Record<string, NameWithCode>
+  yearlyBachelorTitles: string[][]
+  yearlyBcMsTitles: string[][]
+  yearlyMasterTitles: string[][]
+  years: string[]
 }
 
-router.get(
-  '/:id/graduationtimes',
-  auth.roles(['facultyStatistics']),
-  async (req: GetGraduationStatsRequest, res: Response) => {
-    const facultyId = req.params.id
-    const code = await getFacultyCodeById(facultyId)
-    if (!code) {
-      return res.status(422).end()
-    }
-
-    const { programme_filter: programmeFilter } = req.query
-    const data = await getGraduationStats(code, programmeFilter)
-    if (data) {
-      return res.json(data)
-    }
-
-    const programmes = await getDegreeProgrammesOfFaculty(code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
-    if (!programmes) {
-      return res.status(422).end()
-    }
-
-    let updatedStats: any = await countGraduationTimes(code, programmes)
-    if (updatedStats) {
-      updatedStats = await setGraduationStats(updatedStats, programmeFilter)
-    }
-    return res.json(updatedStats)
-  }
-)
-
-interface GetProgressStatsRequest extends Request {
-  query: {
-    graduated: Graduated
-    special_groups: SpecialGroups
-  }
+type GetProgressStatsQuery = {
+  graduated: Graduated
+  special_groups: SpecialGroups
 }
 
-router.get(
-  '/:id/progressstats',
-  auth.roles(['facultyStatistics']),
-  async (req: GetProgressStatsRequest, res: Response) => {
-    const facultyId = req.params.id
-    const code = await getFacultyCodeById(facultyId)
-    if (!code) {
-      return res.status(422).end()
-    }
+router.get<
+  GetStatsParams,
+  CanError<GetProgressStatsResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetProgressStatsQuery
+>('/:id/progressstats', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
+  const { special_groups: specialGroups } = req.query
 
-    const programmes = await getDegreeProgrammesOfFaculty(code, true)
-    if (!programmes) {
-      return res.status(422).end()
-    }
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
-    const { graduated, special_groups: specialGroups } = req.query
-    const data = await getFacultyProgressStats(code, specialGroups, graduated)
-    if (data) {
-      return res.json(data)
-    }
+  const programmes = await getDegreeProgrammesOfFaculty(faculty.code, true)
+  if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
 
-    let updatedStats: any = await combineFacultyStudentProgress(code, programmes, specialGroups, graduated)
-    if (updatedStats) {
-      updatedStats = await setFacultyProgressStats(updatedStats, specialGroups, graduated)
-    }
-    return res.json(updatedStats)
-  }
-)
+  const data = await getFacultyProgressStats(faculty.code, specialGroups)
+  if (data) return res.json(data)
 
-interface GetStudentStatsRequest extends GetProgressStatsRequest {}
+  const updatedStats = await combineFacultyStudentProgress(faculty.code, programmes, specialGroups)
+  if (updatedStats) await setFacultyProgressStats(updatedStats, specialGroups)
 
-router.get(
-  '/:id/studentstats',
-  auth.roles(['facultyStatistics']),
-  async (req: GetStudentStatsRequest, res: Response) => {
-    const facultyId = req.params.id
-    const code = await getFacultyCodeById(facultyId)
-    if (!code) {
-      return res.status(422).end()
-    }
+  return res.json(updatedStats)
+})
 
-    const { graduated, special_groups: specialGroups } = req.query
+export type DegreeProgramme = Pick<
+  Unarray<ProgrammesOfOrganization>,
+  'code' | 'name' | 'degreeProgrammeType' | 'progId'
+>
 
-    /* Fetch cached data if exists */
-    const data = await getFacultyStudentStats(code, specialGroups, graduated)
-    if (data) {
-      return res.json(data)
-    }
-
-    const newProgrammes = await getDegreeProgrammesOfFaculty(code, true)
-    if (!newProgrammes.length) {
-      return res.status(422).end()
-    }
-
-    const updatedStats = await combineFacultyStudents(code, newProgrammes, specialGroups, graduated)
-
-    if (updatedStats) {
-      void setFacultyStudentStats(updatedStats, specialGroups, graduated)
-    }
-    return res.json(updatedStats)
-  }
-)
-
-interface GetUpdateBasicViewRequest extends Request {
-  query: {
-    stats_type: StatsType
-  }
+type GetStudetStatsResBody = {
+  id: string
+  years: string[]
+  facultyTableStats: Record<string, (number | string)[]>
+  facultyTableStatsExtra: Record<string, Record<string, Record<string, number>>>
+  /**
+   * NOTE: programmeStats order is random (not by the year). Make sure to index properly.
+   *
+   * (Order is based on the order in which studyrights were processed in the backend)
+   */
+  programmeStats: Record<string, Record<string, (string | number)[]>>
+  titles: string[]
+  programmeNames: Record<string, DegreeProgramme>
 }
 
-router.get(
-  '/:id/update_basicview',
-  auth.roles(['facultyStatistics']),
-  async (req: GetUpdateBasicViewRequest, res: Response) => {
-    const facultyId = req.params.id
-    const code = await getFacultyCodeById(facultyId)
-    if (!code) {
-      return res.status(422).end()
-    }
-    const { stats_type: statsType } = req.query
-    try {
-      const result = await updateFacultyOverview(code, statsType)
-      return res.json(result)
-    } catch (error) {
-      const message = `Failed to update basic stats for faculty ${code}`
-      logger.error(message, { error })
-      return res.status(500).json({ error: message })
-    }
-  }
-)
+router.get<
+  GetStatsParams,
+  CanError<GetStudetStatsResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetProgressStatsQuery
+>('/:id/studentstats', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
+  const { special_groups: specialGroups } = req.query
+
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+  const data = await getFacultyStudentStats(faculty.code, specialGroups)
+  if (data) return res.json(data)
+
+  const programmes = await getDegreeProgrammesOfFaculty(faculty.code, true)
+  if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
+
+  const updatedStats = await combineFacultyStudents(faculty.code, programmes, specialGroups)
+  if (updatedStats) await setFacultyStudentStats(updatedStats, specialGroups)
+
+  return res.json(updatedStats)
+})
+
+type GetUpdateBasicViewResBody = never
+type GetUpdateBasicViewQuery = {
+  stats_type: StatsType
+}
+
+router.get<
+  GetStatsParams,
+  CanError<GetUpdateBasicViewResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetUpdateBasicViewQuery
+>('/:id/update_basicview', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
+  const { stats_type: statsType } = req.query
+
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
-router.get('/:id/update_progressview', auth.roles(['facultyStatistics']), async (req: Request, res: Response) => {
-  const facultyId = req.params.id
-  const code = await getFacultyCodeById(facultyId)
-  if (!code) {
-    return res.status(422).end()
-  }
   try {
-    const result = await updateFacultyProgressOverview(code)
-    return res.json(result)
+    await updateFacultyOverview(faculty.code, statsType)
+    return res.json()
   } catch (error) {
-    const message = `Failed to update progress tab stats for faculty ${code}`
-    logger.error(message, { error })
-    return res.status(500).json({ error: message })
+    throw new ApplicationError(`Failed to update basic stats for faculty ${faculty.code}`, 500)
   }
 })
 
+router.get<GetStatsParams, CanError<never, GenericApplicationError>, GetStatsReqBody, GetStatsQuery>(
+  '/:id/update_progressview',
+  auth.roles(['facultyStatistics']),
+  async (req, res) => {
+    const { id: facultyId } = req.params
+
+    const faculty = await getFacultyCodeById(facultyId)
+    if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+    try {
+      await updateFacultyProgressOverview(faculty.code)
+      return res.json()
+    } catch (error) {
+      throw new ApplicationError(`Failed to update progress tab stats for faculty ${faculty.code}`, 500)
+    }
+  }
+)
+
 export default router

services/backend/src/routes/feedback.ts

@@ -1,6 +1,6 @@
 import { Request, Response, Router } from 'express'
 
-import { sendFeedbackToToska } from '../services/mailService'
+import { sendFeedback } from '../services/mailService'
 import { ApplicationError } from '../util/customErrors'
 import logger from '../util/logger'
 
@@ -20,7 +20,7 @@ router.post('/email', async (req: EmailRequest, res: Response) => {
     throw new ApplicationError('User not found', 404)
   }
 
-  await sendFeedbackToToska({
+  await sendFeedback({
     feedbackContent: content,
     user,
   })

services/backend/src/routes/login.ts

@@ -1,8 +1,8 @@
 import { Router } from 'express'
 
 import { CanError } from '@oodikone/shared/routes'
+import { FormattedUser } from '@oodikone/shared/types'
 import { omitKeys } from '@oodikone/shared/util'
-import { FormattedUser } from '../types/user'
 import { ApplicationError } from '../util/customErrors'
 
 const router = Router()
@@ -17,7 +17,7 @@ router.get<never, CanError<LoginResBody>, LoginReqBody>('/', (req, res) => {
   const { user, logoutUrl } = req
 
   if (!user) {
-    throw new ApplicationError('User not found', 404)
+    throw new ApplicationError('User not found', 401)
   }
 
   res.send({

services/backend/src/routes/population.ts

@@ -1,6 +1,5 @@
-import crypto from 'crypto'
 import { Router } from 'express'
-import { difference, intersection, uniq } from 'lodash'
+import { difference, intersection, uniq } from 'lodash-es'
 
 import type { CanError } from '@oodikone/shared/routes'
 import {
@@ -16,28 +15,101 @@ import {
   PopulationstatisticsMaxYearsToCreatePopulationFormQuery,
   PopulationstatisticsMaxYearsToCreatePopulationFormResBody,
 } from '@oodikone/shared/routes/populations'
-import { GenderCode, Unification, Unarray } from '@oodikone/shared/types'
-import { mapToProviders } from '@oodikone/shared/util'
+import { Unification } from '@oodikone/shared/types'
+import { keyBy, mapToProviders } from '@oodikone/shared/util'
 import { rootOrgId } from '../config'
-import { SISStudyRightModel } from '../models'
-import { maxYearsToCreatePopulationFrom, getCourseProvidersForCourses } from '../services/courses'
+import {
+  getCourseProvidersForCourses,
+  maxYearsToCreatePopulationFrom,
+  searchAndCombineSubstitutionGroupsToCodes,
+} from '../services/courses'
 import { encrypt } from '../services/encrypt'
-import { getDegreeProgrammesOfOrganization, ProgrammesOfOrganization } from '../services/faculty/faculty'
-import { getStudentTags } from '../services/populations/getStudentData'
+import { getDegreeProgrammesOfOrganization } from '../services/faculty/faculty'
+import { getStudentTagMap } from '../services/populations/getStudentData'
 import { parseDateRangeFromParams } from '../services/populations/shared'
 import { statisticsOf } from '../services/populations/statisticsOf'
 import { getStudentNumbersWithStudyRights } from '../services/populations/studentNumbersWithStudyRights'
 import { findByCourseAndSemesters } from '../services/students'
-import { ParsedCourse } from '../types'
-import { getFullStudyProgrammeRights, hasFullAccessToStudentData } from '../util'
+import { getFullStudyProgrammeRights, handleQueryArrays, hasFullAccessToStudentData } from '../util'
 
 const router = Router()
 
+const obfuscateStuff = ({
+  result,
+
+  programme,
+  combinedProgramme,
+
+  userRoles,
+  userProgrammeRights,
+  studentsUserCanAccess,
+}: {
+  result: Awaited<ReturnType<typeof statisticsOf>>
+  [key: string]: any
+}): Awaited<ReturnType<typeof statisticsOf>> => {
+  const hasFullAccessToStudents = hasFullAccessToStudentData(userRoles)
+  const userFullProgrammeRights = getFullStudyProgrammeRights(userProgrammeRights)
+  const hasFullRightsToProgramme = userFullProgrammeRights.includes(programme)
+  const hasFullRightsToCombinedProgramme = userFullProgrammeRights.includes(combinedProgramme)
+
+  if (hasFullAccessToStudents || hasFullRightsToProgramme || hasFullRightsToCombinedProgramme) {
+    return result
+  }
+
+  const { criteria, students, coursestatistics } = result
+  const encryptionMap = new Map<string, string>()
+
+  return {
+    criteria,
+    students: students.map(student => {
+      if (studentsUserCanAccess.has(student.studentNumber)) return student
+
+      // correct year for age distribution calculation but the date is always January 1st
+      const obfuscatedBirthDate = new Date(Date.UTC(new Date(student.birthdate).getUTCFullYear(), 0))
+      const { iv, encryptedData: studentNumber } = encrypt(student.studentNumber)
+
+      encryptionMap.set(student.studentNumber, studentNumber)
+
+      return {
+        ...student,
+
+        obfuscated: true,
+        iv,
+        studentNumber,
+        firstnames: '',
+        lastname: '',
+        phoneNumber: '',
+        name: '',
+        email: '',
+        secondaryEmail: '',
+        sis_person_id: '',
+        tags: [],
+        birthdate: obfuscatedBirthDate,
+      }
+    }),
+    coursestatistics: {
+      courses: coursestatistics.courses,
+      enrollments: coursestatistics.enrollments.map(enrollment => ({
+        ...enrollment,
+        studentnumber: encryptionMap.get(enrollment.studentnumber) ?? enrollment.studentnumber,
+      })),
+      credits: coursestatistics.credits.map(credit => ({
+        ...credit,
+        student_studentnumber: encryptionMap.get(credit.student_studentnumber) ?? credit.student_studentnumber,
+      })),
+    },
+  }
+}
+
 router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReqBody, PopulationstatisticsQuery>(
-  '/v3/populationstatistics',
+  '/populationstatistics',
   async (req, res) => {
-    const { id: userId, roles: userRoles, programmeRights: userProgrammeRights } = req.user
-    const { years, semesters, programme, combinedProgramme, studentStatuses } = req.query
+    const { id: userId, roles: userRoles, programmeRights: userProgrammeRights, studentsUserCanAccess } = req.user
+    const { programme, combinedProgramme } = req.query
+
+    const years = handleQueryArrays(req.query.years)
+    const semesters = handleQueryArrays(req.query.semesters)
+    const studentStatuses = handleQueryArrays(req.query.studentStatuses) // NOTE: TRANSFERRED means transferred out of the program
 
     const requiredFields = [years, semesters, programme]
     if (requiredFields.some(field => !field)) {
@@ -48,18 +120,16 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
       return res.status(400).json({ error: 'Semester should be either SPRING OR FALL' })
     }
 
-    const hasCorrectStatus = (studentStatuses: string[]) =>
-      studentStatuses.every(status => ['EXCHANGE', 'NONDEGREE', 'TRANSFERRED'].includes(status))
-    if (studentStatuses && !hasCorrectStatus(studentStatuses)) {
+    const hasCorrectStatus = (studentStatuses: (string | undefined)[]) =>
+      studentStatuses.every(status => status && ['EXCHANGE', 'NONDEGREE', 'TRANSFERRED'].includes(status))
+
+    // NOTE: This has always the length of at least 1, [undefined]
+    if (studentStatuses[0] && !hasCorrectStatus(studentStatuses)) {
       return res.status(400).json({ error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' })
     }
 
     const hasFullAccessToStudents = hasFullAccessToStudentData(userRoles)
 
-    const userFullProgrammeRights = getFullStudyProgrammeRights(userProgrammeRights)
-    const hasFullRightsToProgramme = userFullProgrammeRights.includes(programme)
-    const hasFullRightsToCombinedProgramme = userFullProgrammeRights.includes(combinedProgramme!)
-
     const userProgrammeRightsCodes = userProgrammeRights.map(({ code }) => code)
     const hasAccessToProgramme = userProgrammeRightsCodes.includes(programme)
     const hasAccessToCombinedProgramme = userProgrammeRightsCodes.includes(combinedProgramme!)
@@ -68,10 +138,8 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
       return res.status(403).json({ error: 'Trying to request unauthorized students data' })
     }
 
-    const { startDate, endDate } = parseDateRangeFromParams({
-      ...req.query,
-      years: req.query.years,
-    })
+    // NOTE: Years are academic years
+    const { startDate, endDate } = parseDateRangeFromParams({ semesters, years })
 
     const studentNumbers = await getStudentNumbersWithStudyRights({
       programmeCodes: [programme],
@@ -81,35 +149,21 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
     })
 
     const studyRights = [programme]
-    const tagList = await getStudentTags(studyRights, studentNumbers, userId)
+    const tagMap = await getStudentTagMap(studyRights, studentNumbers, userId)
 
-    const result = await statisticsOf(studentNumbers, studyRights, tagList, startDate)
+    const result = await statisticsOf(studentNumbers, studyRights, tagMap, startDate)
+    const processed = obfuscateStuff({
+      result,
 
-    // Obfuscate if user has only limited degree programme rights and there are any students
-    if (!hasFullAccessToStudents && !hasFullRightsToProgramme && !hasFullRightsToCombinedProgramme) {
-      result.students = result.students.map(student => {
-        const { iv, encryptedData: studentNumber } = encrypt(student.studentNumber)
-        // correct year for age distribution calculation but the date is always January 1st
-        const obfuscatedBirthDate = new Date(Date.UTC(new Date(student.birthdate).getUTCFullYear(), 0))
-        return {
-          ...student,
-          firstnames: '',
-          lastname: '',
-          phoneNumber: '',
-          iv,
-          studentNumber,
-          name: '',
-          email: '',
-          secondaryEmail: '',
-          sis_person_id: '',
-          tags: [],
-          birthdate: obfuscatedBirthDate,
-          obfuscated: true,
-        }
-      })
-    }
+      programme,
+      combinedProgramme,
 
-    return res.json(result)
+      userRoles,
+      userProgrammeRights,
+      studentsUserCanAccess: new Set(studentsUserCanAccess),
+    })
+
+    return res.json(processed)
   }
 )
 
@@ -118,9 +172,15 @@ router.get<
   CanError<PopulationstatisticsbycourseResBody>,
   PopulationstatisticsbycourseReqBody,
   PopulationstatisticsbycourseParams
->('/v3/populationstatisticsbycourse', async (req, res) => {
-  const { id: userId, roles, studentsUserCanAccess: allStudentsUserCanAccess, programmeRights } = req.user
-  const { coursecodes: coursecodeJSON, from, to, separate, unifyCourses } = req.query
+>('/populationstatisticsbycourse', async (req, res) => {
+  const {
+    id: userId,
+    roles: userRoles,
+    programmeRights: userProgrammeRights,
+    studentsUserCanAccess: allStudentsUserCanAccess,
+  } = req.user
+  const { coursecodes: coursecodeJSON, from, to, separate, unifyCourses, ...restParams } = req.query
+  const includeSubstitutions = restParams.includeSubstitutions === 'true'
 
   if (!coursecodeJSON || !from || !to) {
     return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
@@ -128,98 +188,73 @@ router.get<
 
   const isSeparate = separate === 'true'
   const coursecodes = JSON.parse(coursecodeJSON)
+  const codes = includeSubstitutions ? await searchAndCombineSubstitutionGroupsToCodes(coursecodes) : coursecodes
 
   const toFromDiff = Math.abs(Number(to) - Number(from) + 1)
   const requestedYearsToCreatePopulationFrom = Math.ceil(isSeparate ? toFromDiff / 2 : toFromDiff) // 2 semesters = 1 year
 
-  const maxYearsForPopulation = await maxYearsToCreatePopulationFrom(coursecodes, Unification.REGULAR)
+  const maxYearsForPopulation = await maxYearsToCreatePopulationFrom(codes, Unification.REGULAR)
   if (requestedYearsToCreatePopulationFrom > maxYearsForPopulation) {
     return res.status(400).json({ error: `Max years to create population from is ${maxYearsForPopulation}` })
   }
 
-  const studentNumbers = await findByCourseAndSemesters(coursecodes, Number(from), Number(to), isSeparate, unifyCourses)
+  const studentNumbers = await findByCourseAndSemesters(codes, Number(from), Number(to), isSeparate, unifyCourses)
+
+  const courseProviders: string[] = await getCourseProvidersForCourses(coursecodes)
+  const fullStudyProgrammeRights = getFullStudyProgrammeRights(userProgrammeRights)
+  const rightsMappedToProviders = mapToProviders(fullStudyProgrammeRights)
+
+  const studentsUserCanAccess = courseProviders.some(provider => rightsMappedToProviders.includes(provider))
+    ? new Set(studentNumbers)
+    : new Set(allStudentsUserCanAccess)
 
   const studyRights = []
-  const tagList = await getStudentTags(studyRights, studentNumbers, userId)
-  const result: any = await statisticsOf(studentNumbers, studyRights, tagList)
-  const courseProviders: string[] = !hasFullAccessToStudentData(roles)
-    ? await getCourseProvidersForCourses(coursecodes)
-    : []
-  const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
-  const rightsMappedToProviders = mapToProviders(fullStudyProgrammeRights)
-  const found = courseProviders.some(provider => rightsMappedToProviders.includes(provider))
+  const tagMap = await getStudentTagMap(studyRights, studentNumbers, userId)
+  const result = await statisticsOf(studentNumbers, studyRights, tagMap)
+  const processed = obfuscateStuff({
+    result,
 
-  const studentsUserCanAccess =
-    hasFullAccessToStudentData(roles) || found
-      ? new Set(studentNumbers)
-      : new Set(intersection(studentNumbers, allStudentsUserCanAccess))
-
-  const randomHash = crypto.randomBytes(12).toString('hex')
-  const obfuscateStudent = ({
-    studyRights,
-    studentNumber,
-    courses,
-    gender_code,
-  }: {
-    studyRights: SISStudyRightModel[]
-    studentNumber: string
-    courses: ParsedCourse[]
-    gender_code: GenderCode
-  }) => ({
-    courses,
-    studyRights,
-    gender_code,
-    studentNumber: crypto.createHash('md5').update(`${studentNumber}${randomHash}`).digest('hex'),
-    firstnames: '',
-    lastname: '',
-    phone_number: '',
-    name: '',
-    email: '',
-    started: null,
-    credits: '',
-    tags: [],
-    obfuscated: true,
+    userRoles,
+    userProgrammeRights,
+    studentsUserCanAccess,
   })
 
-  if ('students' in result) {
-    result.students = result.students.map(student =>
-      studentsUserCanAccess.has(student.studentNumber) ? student : obfuscateStudent(student)
-    )
-  }
-
-  res.json(result)
+  return res.json({ ...processed, mainCourseCodes: coursecodes, allCourseCodes: codes })
 })
 
 // Used in custom population and single study guidance groups
 router.post<never, GetCustomPopulationResBody, CustomPopulationQuery>(
-  '/v3/populationstatisticsbystudentnumbers',
+  '/populationstatisticsbystudentnumbers',
   async (req, res) => {
     const { studentNumbers, tags } = req.body
     const { id: userId, roles, studentsUserCanAccess } = req.user
+
+    const students = handleQueryArrays(studentNumbers)
+
     const filteredStudentNumbers = hasFullAccessToStudentData(roles)
-      ? studentNumbers
-      : intersection(studentNumbers, studentsUserCanAccess)
+      ? students
+      : intersection(students, studentsUserCanAccess)
 
     const studyProgrammeCode = tags?.studyProgramme?.split('+')[0]
 
     const studyRights = [studyProgrammeCode].filter(value => value !== undefined)
-    const tagList = await getStudentTags(studyRights, filteredStudentNumbers, userId)
+    const tagMap = await getStudentTagMap(studyRights, filteredStudentNumbers, userId)
 
     const { startDate } = tags?.year
       ? parseDateRangeFromParams({ semesters: ['FALL', 'SPRING'], years: [tags?.year] })
       : { startDate: undefined }
 
-    const result = await statisticsOf(filteredStudentNumbers, studyRights, tagList, startDate)
+    const result = await statisticsOf(filteredStudentNumbers, studyRights, tagMap, startDate)
 
     const resultWithStudyProgramme = { ...result, studyProgramme: tags?.studyProgramme }
-    const discardedStudentNumbers = difference(studentNumbers, filteredStudentNumbers)
+    const discardedStudentNumbers = difference(students, filteredStudentNumbers)
 
     res.status(200).json({ ...resultWithStudyProgramme, discardedStudentNumbers })
   }
 )
 
 router.get<never, PopulationstatisticsStudyprogrammesResBody>(
-  '/v3/populationstatistics/studyprogrammes',
+  '/populationstatistics/studyprogrammes',
   async (req, res) => {
     const { roles, programmeRights } = req.user
     const programmes = await getDegreeProgrammesOfOrganization(rootOrgId, false)
@@ -230,14 +265,13 @@ router.get<never, PopulationstatisticsStudyprogrammesResBody>(
     const filteredProgrammes = hasFullAccessToStudentData(roles)
       ? programmes
       : programmes.filter(programme => allRights.includes(programme.code))
-    const formattedProgrammes = filteredProgrammes.reduce<Record<string, Unarray<ProgrammesOfOrganization>>>(
-      (acc, curr) => {
-        acc[curr.code] = curr
-        return acc
-      },
-      {}
-    )
-    res.json(formattedProgrammes)
+    const formattedFilteredProgrammes = keyBy(filteredProgrammes, 'code')
+    const formattedAllProgrammes = keyBy(programmes, 'code')
+
+    res.json({
+      allProgrammes: formattedAllProgrammes,
+      filteredProgrammes: formattedFilteredProgrammes,
+    })
   }
 )
 
@@ -246,7 +280,7 @@ router.get<
   PopulationstatisticsMaxYearsToCreatePopulationFormResBody,
   never,
   PopulationstatisticsMaxYearsToCreatePopulationFormQuery
->('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
+>('/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
   const courseCodes = JSON.parse(req.query.courseCodes) as string[]
   const maxYearsToCreatePopulationFromOpen = await maxYearsToCreatePopulationFrom(courseCodes, Unification.OPEN)
   const maxYearsToCreatePopulationFromUni = await maxYearsToCreatePopulationFrom(courseCodes, Unification.REGULAR)

services/backend/src/routes/programmeModules.ts

@@ -1,16 +1,21 @@
-import { Request, Response, Router } from 'express'
+import { Router } from 'express'
 
+import { CurriculumOption } from '@oodikone/shared/types'
 import { getCoursesAndModules, getCurriculumOptions } from '../services/programmeModules'
 
 const router = Router()
 
-router.get('/v3/curriculum-options/:code', async (req: Request, res: Response) => {
+type CurriculumOptionParams = { code: string }
+
+router.get<CurriculumOptionParams, CurriculumOption[] | null>('/curriculum-options/:code', async (req, res) => {
   const { code } = req.params
   const result = await getCurriculumOptions(code)
   res.json(result)
 })
 
-router.get('/v3/curriculum/:code/:periodIds', async (req: Request, res: Response) => {
+type CurriculumPeriodParams = { code: string; periodIds: string }
+
+router.get<CurriculumPeriodParams>('/curriculum/:code/:periodIds', async (req, res) => {
   const { code, periodIds } = req.params
   const { defaultProgrammeCourses, secondProgrammeCourses } = await getCoursesAndModules(code, periodIds)
 

services/backend/src/routes/students.ts

@@ -1,66 +1,58 @@
-import { Request, Response, Router } from 'express'
+import { Router } from 'express'
 
+import type { CanError } from '@oodikone/shared/routes'
+import type {
+  GetStudentRequestResBody,
+  GetStudentRequestReqBody,
+  GetStudentRequestQuery,
+  GetStudentDetailParams,
+  GetStudentDetailResBody,
+  GetStudentDetailReqBody,
+} from '@oodikone/shared/routes/students'
 import { splitByEmptySpace } from '@oodikone/shared/util'
 import { bySearchTermAndStudentNumbers, withStudentNumber } from '../services/students'
 import { hasFullAccessToStudentData } from '../util'
-import { ApplicationError } from '../util/customErrors'
 
 const router = Router()
 
-interface GetStudentsRequest extends Request {
-  query: {
-    searchTerm: string
+router.get<never, CanError<GetStudentRequestResBody>, GetStudentRequestReqBody, GetStudentRequestQuery>(
+  '/',
+  async (req, res) => {
+    const { searchTerm } = req.query
+    const trimmedSearchTerm = searchTerm?.trim()
+    const validSearchTerm =
+      trimmedSearchTerm &&
+      !splitByEmptySpace(trimmedSearchTerm)
+        .slice(0, 2)
+        .find(term => term.length > 2)
+
+    if (!trimmedSearchTerm || validSearchTerm) {
+      return res.status(400).json({ error: 'at least one search term must be longer than 2 characters' })
+    }
+
+    const { roles, studentsUserCanAccess } = req.user
+    const results: ReturnType<typeof bySearchTermAndStudentNumbers> = hasFullAccessToStudentData(roles)
+      ? bySearchTermAndStudentNumbers(trimmedSearchTerm)
+      : bySearchTermAndStudentNumbers(trimmedSearchTerm, studentsUserCanAccess)
+
+    return res.json(await results)
   }
-}
+)
 
-router.get('/', async (req: GetStudentsRequest, res: Response) => {
-  const {
-    user: { roles, studentsUserCanAccess },
-    query: { searchTerm },
-  } = req
+router.get<GetStudentDetailParams, CanError<GetStudentDetailResBody>, GetStudentDetailReqBody>(
+  '/:studentNumber',
+  async (req, res) => {
+    const { studentNumber } = req.params
+    const { id, roles, studentsUserCanAccess } = req.user
 
-  const trimmedSearchTerm = searchTerm ? searchTerm.trim() : undefined
+    if (!hasFullAccessToStudentData(roles) && !studentsUserCanAccess.includes(studentNumber))
+      return res.status(403).json({ error: 'User does not have permission to view current student.' })
 
-  if (
-    trimmedSearchTerm &&
-    !splitByEmptySpace(trimmedSearchTerm)
-      .slice(0, 2)
-      .find(term => term.length > 2)
-  ) {
-    throw new ApplicationError('at least one search term must be longer than 2 characters', 400)
+    const student = await withStudentNumber(studentNumber, id)
+    if (!student) return res.status(404).json({ error: 'Student not found' })
+
+    res.json(student)
   }
-
-  let results: Awaited<ReturnType<typeof bySearchTermAndStudentNumbers>> = []
-  if (trimmedSearchTerm) {
-    results = hasFullAccessToStudentData(roles)
-      ? await bySearchTermAndStudentNumbers(trimmedSearchTerm)
-      : await bySearchTermAndStudentNumbers(trimmedSearchTerm, studentsUserCanAccess)
-  }
-  res.json(results)
-})
-
-interface GetStudentRequest extends Request {
-  params: {
-    studentNumber: string
-  }
-}
-
-router.get('/:studentNumber', async (req: GetStudentRequest, res: Response) => {
-  const { studentNumber } = req.params
-  const {
-    user: { id, roles, studentsUserCanAccess },
-  } = req
-
-  if (!hasFullAccessToStudentData(roles) && !studentsUserCanAccess.includes(studentNumber)) {
-    return res
-      .status(403)
-      .json({ error: `User does not have permission to view data for student number ${studentNumber}.` })
-  }
-  const student = await withStudentNumber(studentNumber, id)
-  if (!student) {
-    return res.status(404).json({ error: `Student not found with student number ${studentNumber}.` })
-  }
-  res.json(student)
-})
+)
 
 export default router

services/backend/src/routes/studyProgramme.ts

@@ -1,6 +1,7 @@
-import { Request, Response, Router } from 'express'
+import { Router } from 'express'
 
 import { Graduated, SpecialGroups, YearType } from '@oodikone/shared/types'
+import { CreditStatsPayload } from '@oodikone/shared/types/studyProgramme'
 import {
   getBasicStats,
   setBasicStats,
@@ -11,10 +12,8 @@ import {
 } from '../services/analyticsService'
 import { getCreditsProduced } from '../services/providerCredits'
 import { getBasicStatsForStudytrack } from '../services/studyProgramme/studyProgrammeBasics'
-import {
-  getStudyProgrammeCoursesForStudyTrack,
-  getStudyProgrammeStatsForColorizedCoursesTable,
-} from '../services/studyProgramme/studyProgrammeCourses'
+import { getStudyProgrammeCoursesForStudyTrack } from '../services/studyProgramme/studyProgrammeCourses'
+import { getStudyProgrammeStatsForColorizedCoursesTable } from '../services/studyProgramme/studyProgrammeCoursesBySemester'
 import { getGraduationStatsForStudyTrack } from '../services/studyProgramme/studyProgrammeGraduations'
 import { updateBasicView, updateStudyTrackView } from '../services/studyProgramme/studyProgrammeUpdates'
 import { getStudyRightsInProgramme, getStudyTracksForProgramme } from '../services/studyProgramme/studyRightFinders'
@@ -24,32 +23,30 @@ import { logInfoForGrafana } from '../util/logInfoForGrafana'
 
 const router = Router()
 
-interface GetCreditStatsRequest extends Request {
-  query: {
-    codes: string
-    specialGroups: SpecialGroups
-    yearType: YearType
-  }
+type StudyProgrammeParams = { id: string }
+type UpdateStudyProgrammeQuery = { combined_programme: string }
+type CreditStatsQuery = {
+  codes: string
+  specialGroups: SpecialGroups
+  yearType: YearType
 }
 
-router.get('/creditstats', async (req: GetCreditStatsRequest, res: Response) => {
+router.get<StudyProgrammeParams, unknown, never, CreditStatsQuery>('/creditstats', async (req, res) => {
   const { codes, specialGroups, yearType } = req.query
-  const stats = {}
+  const stats: CreditStatsPayload = {}
   for (const code of JSON.parse(codes)) {
     stats[code] = await getCreditsProduced(code, yearType === 'ACADEMIC_YEAR', specialGroups === 'SPECIAL_INCLUDED')
   }
-  return res.json({ stats })
+  return res.json(stats)
 })
 
-interface GetStatsRequest extends Request {
-  query: {
-    year_type: YearType
-    special_groups: SpecialGroups
-    combined_programme: string
-  }
+type StatsRequestQuery = {
+  year_type: YearType
+  special_groups: SpecialGroups
+  combined_programme: string
 }
 
-router.get('/:id/basicstats', async (req: GetStatsRequest, res: Response) => {
+router.get<StudyProgrammeParams, unknown, never, StatsRequestQuery>('/:id/basicstats', async (req, res) => {
   const code = req.params.id
   const { year_type: yearType, special_groups: specialGroups, combined_programme: combinedProgramme } = req.query
   if (!code) {
@@ -76,7 +73,7 @@ router.get('/:id/basicstats', async (req: GetStatsRequest, res: Response) => {
   return res.json(updatedStats)
 })
 
-router.get('/:id/graduationstats', async (req: GetStatsRequest, res: Response) => {
+router.get<StudyProgrammeParams, unknown, never, StatsRequestQuery>('/:id/graduationstats', async (req, res) => {
   const code = req.params.id
   const { year_type: yearType, special_groups: specialGroups, combined_programme: combinedProgramme } = req.query
   if (!code) {
@@ -103,14 +100,12 @@ router.get('/:id/graduationstats', async (req: GetStatsRequest, res: Response) =
   return res.json(updatedStats)
 })
 
-interface GetCourseStatsRequest extends Request {
-  query: {
-    combinedProgramme: string
-    yearType: YearType
-  }
+type CourseStatsQuery = {
+  combinedProgramme: string
+  yearType: YearType
 }
 
-router.get('/:id/coursestats', async (req: GetCourseStatsRequest, res: Response) => {
+router.get<StudyProgrammeParams, unknown, never, CourseStatsQuery>('/:id/coursestats', async (req, res) => {
   const code = req.params.id
   const { combinedProgramme, yearType } = req.query
   const date = new Date()
@@ -124,23 +119,21 @@ router.get('/:id/coursestats', async (req: GetCourseStatsRequest, res: Response)
   }
 })
 
-interface GetStudyTrackStatsRequest extends Request {
-  query: {
-    graduated: Graduated
-    special_groups: SpecialGroups
-    combined_programme: string
-  }
+type StudyTrackStatsQuery = {
+  graduated: Graduated
+  special_groups: SpecialGroups
+  combined_programme: string
 }
 
-router.get('/:id/studytrackstats', async (req: GetStudyTrackStatsRequest, res: Response) => {
+router.get<StudyProgrammeParams, unknown, never, StudyTrackStatsQuery>('/:id/studytrackstats', async (req, res) => {
   const code = req.params.id
-  const { graduated, special_groups: specialGroups, combined_programme: combinedProgramme } = req.query
+  const { special_groups: specialGroups, combined_programme: combinedProgramme } = req.query
   if (!code) {
     return res.status(422).end()
   }
 
   void logInfoForGrafana(code, combinedProgramme)
-  const data = await getStudyTrackStats(code, combinedProgramme, graduated, specialGroups)
+  const data = await getStudyTrackStats(code, combinedProgramme, specialGroups)
   if (data) {
     return res.json(data)
   }
@@ -150,18 +143,20 @@ router.get('/:id/studytrackstats', async (req: GetStudyTrackStatsRequest, res: R
     studyProgramme: code,
     combinedProgramme,
     settings: {
-      graduated: graduated === 'GRADUATED_INCLUDED',
       specialGroups: specialGroups === 'SPECIAL_INCLUDED',
     },
     studyRightsOfProgramme,
   })
+
   if (updated) {
-    await setStudyTrackStats(updated, graduated, specialGroups)
+    await setStudyTrackStats(updated, specialGroups)
+    return res.json(updated)
   }
-  return res.json(updated)
+
+  return res.status(500).end()
 })
 
-router.get('/:id/colorizedtablecoursestats', async (req: Request, res: Response) => {
+router.get<StudyProgrammeParams>('/:id/colorizedtablecoursestats', async (req, res) => {
   const code = req.params.id
   try {
     const data = await getStudyProgrammeStatsForColorizedCoursesTable(code)
@@ -171,7 +166,7 @@ router.get('/:id/colorizedtablecoursestats', async (req: Request, res: Response)
   }
 })
 
-router.get('/:id/studytracks', async (req: Request, res: Response) => {
+router.get<StudyProgrammeParams>('/:id/studytracks', async (req, res) => {
   const code = req.params.id
   if (!code) {
     return res.status(422).end()
@@ -184,42 +179,42 @@ router.get('/:id/studytracks', async (req: Request, res: Response) => {
   }
 })
 
-interface GetUpdateViewRequest extends Request {
-  query: {
-    combined_programme: string
+router.get<StudyProgrammeParams, unknown, never, UpdateStudyProgrammeQuery>(
+  '/:id/update_basicview',
+  async (req, res) => {
+    const code = req.params.id
+    const combinedProgramme = req.query?.combined_programme
+    if (!code) {
+      return res.status(400).json({ error: 'Missing code' })
+    }
+    try {
+      const result = await updateBasicView(code, combinedProgramme)
+      return res.json(result)
+    } catch (error) {
+      const message = `Failed to update basic stats for programme ${code}${combinedProgramme ? `+${combinedProgramme}` : ''}`
+      logger.error(message, { error })
+      return res.status(500).json({ error: message })
+    }
   }
-}
+)
 
-router.get('/:id/update_basicview', async (req: GetUpdateViewRequest, res: Response) => {
-  const code = req.params.id
-  const combinedProgramme = req.query?.combined_programme
-  if (!code) {
-    return res.status(400).json({ error: 'Missing code' })
+router.get<StudyProgrammeParams, unknown, never, UpdateStudyProgrammeQuery>(
+  '/:id/update_studytrackview',
+  async (req, res) => {
+    const code = req.params.id
+    const combinedProgramme = req.query?.combined_programme
+    if (!code) {
+      return res.status(400).json({ error: 'Missing code' })
+    }
+    try {
+      const result = await updateStudyTrackView(code, combinedProgramme)
+      return res.json(result)
+    } catch (error) {
+      const message = `Failed to update study track stats for programme ${code}${combinedProgramme ? `+${combinedProgramme}` : ''}`
+      logger.error(message, { error })
+      return res.status(500).json({ error: message })
+    }
   }
-  try {
-    const result = await updateBasicView(code, combinedProgramme)
-    return res.json(result)
-  } catch (error) {
-    const message = `Failed to update basic stats for programme ${code}${combinedProgramme ? `+${combinedProgramme}` : ''}`
-    logger.error(message, { error })
-    return res.status(500).json({ error: message })
-  }
-})
-
-router.get('/:id/update_studytrackview', async (req: GetUpdateViewRequest, res: Response) => {
-  const code = req.params.id
-  const combinedProgramme = req.query?.combined_programme
-  if (!code) {
-    return res.status(400).json({ error: 'Missing code' })
-  }
-  try {
-    const result = await updateStudyTrackView(code, combinedProgramme)
-    return res.json(result)
-  } catch (error) {
-    const message = `Failed to update study track stats for programme ${code}${combinedProgramme ? `+${combinedProgramme}` : ''}`
-    logger.error(message, { error })
-    return res.status(500).json({ error: message })
-  }
-})
+)
 
 export default router

services/backend/src/routes/tags.ts

@@ -1,5 +1,5 @@
 import { Router } from 'express'
-import { difference } from 'lodash'
+import { difference } from 'lodash-es'
 
 import { CanError } from '@oodikone/shared/routes'
 import { NewTag, Role, StudentTag, Tag } from '@oodikone/shared/types'

services/backend/src/routes/teachers.ts

@@ -1,19 +1,17 @@
 import { Request, Response, Router } from 'express'
 
 import { mapToProviders, splitByEmptySpace } from '@oodikone/shared/util'
-import { serviceProvider } from '../config'
 import * as auth from '../middleware/auth'
 import { getProvidersOfFaculty, isFaculty } from '../services/organizations'
 import { getTeachersBySearchTerm, getTeacherStatistics, getYearlyStatistics } from '../services/teachers'
 import { CategoryID, getTeacherStats, findAndSaveTeachers, getCategoriesAndYears } from '../services/teachers/top'
-import { getFullStudyProgrammeRights, validateParamLength } from '../util'
+import { getFullStudyProgrammeRights, handleQueryArrays, validateParamLength } from '../util'
 
 const router = Router()
 
 const iamGroupsGivingFullAccess = ['hy-dekaanit', 'hy-varadekaanit-opetus']
 
-const fullAccessAuth = () =>
-  serviceProvider === 'toska' ? auth.roles([], iamGroupsGivingFullAccess) : auth.roles(['teachers'])
+const fullAccessAuth = () => auth.roles([], iamGroupsGivingFullAccess)
 
 interface GetTeachersRequest extends Request {
   query: {
@@ -58,7 +56,7 @@ router.get('/top', fullAccessAuth(), async (req: GetTopTeachersRequest, res: Res
   }
   await findAndSaveTeachers(Number(yearcode), Number(yearcode))
   const updatedStats = await getTeacherStats(category, Number(yearcode))
-  res.json(updatedStats)
+  return res.json(updatedStats)
 })
 
 interface PostTopTeachersRequest extends Request {
@@ -93,11 +91,12 @@ router.get('/stats', async (req: GetTeacherStatsRequest, res: Response) => {
   const { roles, programmeRights, iamGroups } = req.user
   const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
 
-  const { providers, semesterStart, semesterEnd } = req.query
-  if (!providers || !semesterStart) {
+  const { providers: providerStack, semesterStart, semesterEnd } = req.query
+  if (!providerStack || !semesterStart) {
     return res.status(422).send('Missing required query parameters')
   }
 
+  const providers = handleQueryArrays(providerStack)
   const providerRights = mapToProviders(fullStudyProgrammeRights)
   if (
     !(

services/backend/src/routes/university.ts

@@ -1,8 +1,7 @@
 import { Request, Response, Router } from 'express'
-import { cloneDeep } from 'lodash'
+import { cloneDeep } from 'lodash-es'
 
 import { Graduated, NameWithCode } from '@oodikone/shared/types'
-import { serviceProvider } from '../config'
 import { magicFacultyCode } from '../config/organizationConstants'
 import { OrganizationModel } from '../models'
 import { getDegreeProgrammesOfFaculty } from '../services/faculty/faculty'
@@ -15,6 +14,7 @@ import {
   setFacultyProgressStats,
 } from '../services/faculty/facultyService'
 import { combineFacultyStudentProgress, FacultyProgressData } from '../services/faculty/facultyStudentProgress'
+import { GraduationTarget } from '../services/graduationHelpers'
 import { getMedian } from '../services/studyProgramme/studyProgrammeHelpers'
 import logger from '../util/logger'
 
@@ -39,18 +39,17 @@ interface GetProgressStatsRequest extends Request {
 
 router.get('/allprogressstats', async (req: GetProgressStatsRequest, res: Response) => {
   const specialGroups = req.query?.specialsIncluded === 'true' ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED'
-  const graduated = req.query?.graduated
   const allFaculties = await getSortedFaculties()
   const facultyCodes = allFaculties.map(faculty => faculty.code)
   const codeToData: Record<string, FacultyProgressData> = {}
 
   for (const facultyCode of facultyCodes) {
-    let data: any = await getFacultyProgressStats(facultyCode, specialGroups, graduated)
+    let data = await getFacultyProgressStats(facultyCode, specialGroups)
     if (!data) {
       logger.info(`Data missing from server: Refreshing progress faculty data for faculty ${facultyCode}`)
       const programmes = await getDegreeProgrammesOfFaculty(facultyCode, true)
-      data = await combineFacultyStudentProgress(facultyCode, programmes, specialGroups, graduated)
-      await setFacultyProgressStats(data, specialGroups, graduated)
+      data = await combineFacultyStudentProgress(facultyCode, programmes, specialGroups)
+      await setFacultyProgressStats(data, specialGroups)
     }
     codeToData[facultyCode] = data
   }
@@ -71,6 +70,12 @@ router.get('/allprogressstats', async (req: GetProgressStatsRequest, res: Respon
       master: {} as Record<string, number[]>,
       doctor: {} as Record<string, number[]>,
     },
+    graduatedCount: {
+      bachelor: {} as Record<string, number>,
+      bachelorMaster: {} as Record<string, number>,
+      master: {} as Record<string, number>,
+      doctor: {} as Record<string, number>,
+    },
   }
 
   const unifyProgressStats = (progStats: number[][][]) => {
@@ -96,6 +101,14 @@ router.get('/allprogressstats', async (req: GetProgressStatsRequest, res: Respon
         if (!universityData.creditCounts[fieldName][year]) universityData.creditCounts[fieldName][year] = []
         universityData.creditCounts[fieldName][year].push(...facultyData.creditCounts[fieldName][year])
       }
+
+      for (const fieldName of degreeNames) {
+        if (!facultyData.graduatedCount[fieldName] || Object.keys(facultyData.graduatedCount[fieldName]).length === 0)
+          continue
+        if (!universityData.graduatedCount[fieldName][year]) universityData.graduatedCount[fieldName][year] = 0
+        universityData.graduatedCount[fieldName][year] += facultyData.graduatedCount[fieldName][year]
+      }
+
       const progStats = ['bachelorsProgStats', 'bcMsProgStats', 'mastersProgStats', 'doctoralProgStats'] as const
       for (const fieldName of progStats) {
         if (Object.keys(facultyData[fieldName] || {}).length === 0) continue
@@ -112,7 +125,7 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
   const allFaculties = await getSortedFaculties()
   const facultyCodes = allFaculties.map(faculty => faculty.code)
   const facultyData: Record<string, Awaited<ReturnType<typeof countGraduationTimes>>> = {}
-  const programmeFilter = serviceProvider === 'toska' ? 'NEW_DEGREE_PROGRAMMES' : 'ALL_PROGRAMMES'
+  const programmeFilter = 'NEW_DEGREE_PROGRAMMES'
   for (const facultyCode of facultyCodes) {
     let data: any = await getGraduationStats(facultyCode, programmeFilter, true)
     if (!data) {
@@ -125,7 +138,12 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
   }
 
   const universityData = {
-    goals: { bachelor: 36, bcMsCombo: 60, master: 24, doctor: 48 },
+    goals: {
+      bachelor: GraduationTarget.THREE_YEARS,
+      bcMsCombo: GraduationTarget.FIVE_YEARS,
+      master: GraduationTarget.TWO_YEARS,
+      doctor: GraduationTarget.FOUR_YEARS,
+    },
     programmeNames: getProgrammeNames(allFaculties),
     byGradYear: {
       medians: {} as Record<string, LevelGraduationStats[]>,

services/backend/src/routes/updater.ts

@@ -4,6 +4,7 @@ import { languageCenterViewEnabled } from '../config'
 import { refreshFaculties, refreshProgrammes } from '../events'
 import {
   abortUpdate,
+  nukeRedis,
   updateSISCoursesByCourseCode,
   updateSISMetadata,
   updateSISProgrammes,
@@ -12,28 +13,28 @@ import {
   updateSISStudentsByStudentNumber,
 } from '../services/sisUpdaterService'
 import logger from '../util/logger'
-import { getJobs, jobMaker, removeWaitingJobs } from '../worker/queue'
+import { jobQueue } from '../worker/queue'
 
 const router = Router()
 
 const refreshFacultiesByList = (facultyCodes: string[]) => {
-  facultyCodes.forEach(code => jobMaker.faculty(code))
+  facultyCodes.forEach(code => jobQueue.refreshFaculty(code))
   return 'Added jobs for refreshing faculties'
 }
 
 const refreshProgrammesByList = (programmeCodes: string[]) => {
-  programmeCodes.forEach(code => jobMaker.programme(code))
+  programmeCodes.forEach(code => jobQueue.refreshProgramme(code))
   return 'Added jobs for refreshing programme'
 }
 
-router.get('/update/v2/meta', async (_req: Request, res: Response) => {
+router.get('/update/meta', async (_req: Request, res: Response) => {
   const response = await updateSISMetadata()
   if (response) {
     res.status(200).json('Update SIS meta scheduled')
   }
 })
 
-router.get('/update/v2/students', async (_req: Request, res: Response) => {
+router.get('/update/students', async (_req: Request, res: Response) => {
   const response = await updateSISStudents()
   if (response) {
     res.status(200).json('Update SIS students scheduled')
@@ -47,7 +48,7 @@ interface PostCustomListRequest extends Request {
   }
 }
 
-router.post('/update/v2/customlist/:type', async (req: PostCustomListRequest, res: Response) => {
+router.post('/update/customlist/:type', async (req: PostCustomListRequest, res: Response) => {
   const { type } = req.params
   const list = req.body
   const typeToJob = {
@@ -65,7 +66,7 @@ router.post('/update/v2/customlist/:type', async (req: PostCustomListRequest, re
   }
 })
 
-router.get('/update/v2/programmes', async (_req: Request, res: Response) => {
+router.get('/update/programmes', async (_req: Request, res: Response) => {
   const response = await updateSISProgrammes()
   if (response) {
     res.status(200).json('Update SIS programmes scheduled')
@@ -80,11 +81,17 @@ router.get('/refresh_redis_cache', async (req: Request, res: Response) => {
   }
 })
 
+router.get('/nuke_redis', async (req: Request, res: Response) => {
+  logger.info(`${req.user.username} requested complete wipe of redis`)
+  const response = await nukeRedis()
+  if (response) res.status(200).json('Complete flush of redis scheduled')
+})
+
 router.post('/refresh-teacher-leaderboard', (req: Request, res: Response) => {
   logger.info(
     `${req.user.username} requested refresh of teacher leaderboard for the current and previous academic year`
   )
-  jobMaker.teacherLeaderboard()
+  jobQueue.refreshTeacherLeaderboard()
   res.status(200).json('Teacher leaderboard for the current and previous academic year refreshed')
 })
 
@@ -104,13 +111,13 @@ router.post('/refresh_language_center_data', (req: Request, res: Response) => {
   logger.info(`${req.user.username} requested refresh of language center data`)
   if (!languageCenterViewEnabled)
     res.status(418).json({ error: 'The language center functionality is not activated in your environment.' })
-  jobMaker.languagecenter()
+  jobQueue.refreshLanguagecenter()
   res.status(200).json('Added job for refreshing language center data')
 })
 
 router.post('/refresh-close-to-graduation', (req: Request, res: Response) => {
   logger.info(`${req.user.username} requested refresh of close to graduation data`)
-  jobMaker.closeToGraduation()
+  jobQueue.refreshCloseToGraduation()
   res.status(200).json('Added job for refreshing close to graduation data')
 })
 
@@ -120,13 +127,12 @@ router.get('/abort', async (_req: Request, res: Response) => {
 })
 
 router.get('/jobs', async (_req: Request, res: Response) => {
-  const waiting = await getJobs('waiting')
-  const active = await getJobs('active')
-  res.status(200).json({ waiting, active })
+  const jobs = await jobQueue.getJobs()
+  res.status(200).json(jobs)
 })
 
 router.delete('/jobs', async (_req: Request, res: Response) => {
-  await removeWaitingJobs()
+  await jobQueue.clearWaitingJobs()
   res.status(200).end()
 })
 

services/backend/src/routes/users.ts

@@ -1,10 +1,20 @@
 import { Router } from 'express'
 
-import { LANGUAGE_CODES, Language } from '@oodikone/shared/language'
+import { LANGUAGE_CODES } from '@oodikone/shared/language'
 import { CanError } from '@oodikone/shared/routes'
-import { Role } from '@oodikone/shared/types'
+import {
+  ChangeLanguageReqBody,
+  GetUserByIdParams,
+  GetUserByIdResBody,
+  GetUserRolesResBody,
+  GetUsersResBody,
+  ModifyUserRolesReqBody,
+  PostUserEmailReqBody,
+  UserEmailPreviewResBody,
+  UserIDElementsParams,
+  UserIDElementsReqBody,
+} from '@oodikone/shared/routes/users'
 import { tryCatch } from '@oodikone/shared/util'
-import { FormattedUser } from 'src/types'
 import { roles } from '../config/roles'
 import * as auth from '../middleware/auth'
 import { sendNotificationAboutAccessToUser, previewNotificationAboutAccessToUser } from '../services/mailService'
@@ -13,21 +23,12 @@ import logger from '../util/logger'
 
 const router = Router()
 
-type GetUsersResBody = Omit<FormattedUser, 'studentsUserCanAccess' | 'isAdmin' | 'mockedBy' | 'userId'>[]
-
 router.get<never, CanError<GetUsersResBody>>('/', auth.roles(['admin']), async (_, res) => {
   const results = await userService.findAll()
   res.json(results)
 })
 
-type GetUserRolesResBody = readonly string[]
-
-router.get<never, CanError<GetUserRolesResBody>>('/roles', auth.roles(['admin']), (_, res) => {
-  res.json(roles)
-})
-
-type GetUserByIdParams = { uid: string }
-type GetUserByIdResBody = Omit<FormattedUser, 'studentsUserCanAccess' | 'isAdmin' | 'mockedBy' | 'userId'>
+router.get<never, GetUserRolesResBody>('/roles', auth.roles(['admin']), (_, res) => res.json(roles))
 
 router.get<never, CanError<GetUserByIdResBody>, never, GetUserByIdParams>(
   '/:uid',
@@ -41,61 +42,35 @@ router.get<never, CanError<GetUserByIdResBody>, never, GetUserByIdParams>(
   }
 )
 
-type ModifyUserRolesReqBody = {
-  username: string
-  roles: Record<Role, boolean>
-}
-type ModifyUserRolesResBody = never
-
-router.post<never, CanError<ModifyUserRolesResBody, Error>, ModifyUserRolesReqBody>(
+router.post<never, CanError<void, Error>, ModifyUserRolesReqBody>(
   '/modify-roles',
   auth.roles(['admin']),
   async (req, res) => {
     const { username, roles } = req.body
     const { error } = await tryCatch(userService.modifyAccess(username, roles))
-
-    if (error) {
-      return res.status(400).json(error)
-    }
+    if (error) return res.status(400).json(error)
 
     return res.status(204).end()
   }
 )
 
-type UserEmailPreviewResBody = {
-  subject: string
-  html: string
-}
-
 router.get<never, CanError<UserEmailPreviewResBody>>('/email/preview', auth.roles(['admin']), (_, res) => {
   const { accessMessageSubject, accessMessageText } = previewNotificationAboutAccessToUser()
   res.json({ subject: accessMessageSubject, html: accessMessageText })
 })
 
-type PostUserEmailReqBody = { email: string }
-type PostUserEmailResBody = never
+router.post<never, CanError<void>, PostUserEmailReqBody>('/email', auth.roles(['admin']), async (req, res) => {
+  const { email } = req.body
+  if (!email) return res.status(400).json({ error: 'Email address is missing' })
 
-router.post<never, CanError<PostUserEmailResBody>, PostUserEmailReqBody>(
-  '/email',
-  auth.roles(['admin']),
-  async (req, res) => {
-    const { email } = req.body
-    if (!email) return res.status(400).json({ error: 'Email address is missing' })
+  const { error } = await tryCatch(sendNotificationAboutAccessToUser(email))
+  if (error) return res.status(500).json({ error: error.message })
 
-    const { error } = await tryCatch(sendNotificationAboutAccessToUser(email))
-    if (error) return res.status(500).json({ error: error.message })
+  logger.info('Succesfully sent message about Oodikone access to user')
+  return res.status(200).end()
+})
 
-    logger.info('Succesfully sent message about Oodikone access to user')
-    return res.status(200).end()
-  }
-)
-
-type ElementsParams = { uid: string }
-type ElementsReqBody = { codes: string[] }
-
-type PostUserUIDElementsResBody = void
-
-router.post<never, CanError<PostUserUIDElementsResBody>, ElementsReqBody, ElementsParams>(
+router.post<never, CanError<void>, UserIDElementsReqBody, UserIDElementsParams>(
   '/:uid/elements',
   auth.roles(['admin']),
   async (req, res) => {
@@ -106,9 +81,7 @@ router.post<never, CanError<PostUserUIDElementsResBody>, ElementsReqBody, Elemen
   }
 )
 
-type DeleteUserUIDElementsResBody = void
-
-router.delete<never, CanError<DeleteUserUIDElementsResBody>, ElementsReqBody, ElementsParams>(
+router.delete<never, CanError<void>, UserIDElementsReqBody, UserIDElementsParams>(
   '/:uid/elements',
   auth.roles(['admin']),
   async (req, res) => {
@@ -119,10 +92,7 @@ router.delete<never, CanError<DeleteUserUIDElementsResBody>, ElementsReqBody, El
   }
 )
 
-type ChangeLanguageReqBody = { language: Language }
-type PostChangeLanguageResBody = void
-
-router.post<never, CanError<PostChangeLanguageResBody>, ChangeLanguageReqBody>('/language', async (req, res) => {
+router.post<never, CanError<void>, ChangeLanguageReqBody>('/language', async (req, res) => {
   const { language } = req.body
 
   if (!LANGUAGE_CODES.includes(language)) {

services/backend/src/routes/usersFd.ts

@@ -1,68 +0,0 @@
-import { Request, Response, Router } from 'express'
-
-import { isDev } from '../config'
-import * as auth from '../middleware/auth'
-import * as userService from '../services/userService'
-
-const router = Router()
-
-interface NewUserEppnRequest extends Request {
-  params: {
-    newUserEppn: string
-  }
-}
-
-router.get('/from-sisu-by-eppn/:newUserEppn', auth.roles(['admin']), async (req: NewUserEppnRequest, res: Response) => {
-  let { username: requesterEppn } = req.user
-  const { newUserEppn } = req.params
-  // In order to test this feature in the dev environment we need to set an eppn that demo sisu will recognize
-  if (isDev && requesterEppn === 'mluukkai') {
-    requesterEppn = newUserEppn
-  }
-  const person = await userService.getUserFromSisuByEppn(requesterEppn, newUserEppn)
-  res.json(person)
-})
-
-interface User {
-  id: string
-  first_name: string
-  last_name: string
-  call_name: string
-  eppn: string
-  email_address: string
-  student_number: string
-  phone_number: string
-  document_state: string
-}
-
-interface AddUserRequest extends Request {
-  body: {
-    user: User
-  }
-}
-
-interface DeleteUserRequest extends Request {
-  body: {
-    userId: string
-  }
-}
-
-router.post('/add', auth.roles(['admin']), async (req: AddUserRequest, res: Response) => {
-  const { user } = req.body
-  const oldUser = await userService.findByUsername(user.eppn)
-  if (oldUser) return res.status(400).json({ error: 'Trying to add a user already present in the user database.' })
-  const success = await userService.addNewUser(user)
-  res.json(success)
-})
-
-router.delete('/delete', auth.roles(['admin']), async (req: DeleteUserRequest, res: Response) => {
-  const { userId } = req.body
-  try {
-    await userService.deleteUserById(userId)
-    res.json({ deleted: userId })
-  } catch (error) {
-    return res.status(400).json({ error: 'Deletion was unsuccessful with the user id included in the request' })
-  }
-})
-
-export default router

services/backend/src/services/analyticsService.ts

@@ -1,4 +1,5 @@
-import { Graduated, SpecialGroups, StudyTrackStats, YearType } from '@oodikone/shared/types'
+import { SpecialGroups, StudyTrackStats, YearType } from '@oodikone/shared/types'
+import { BasicStats, CreditStats, ProgrammeGraduationStats } from '@oodikone/shared/types/studyProgramme'
 import { facultyCodes, ignoredFacultyCodes } from '../config/organizationConstants'
 import { redisClient } from './redis'
 import { isRelevantProgramme } from './studyProgramme/studyProgrammeHelpers'
@@ -15,11 +16,12 @@ const createRedisKeyForCreditStats = (id: string, yearType: YearType, specialGro
 const createRedisKeyForGraduationStats = (id: string, yearType: YearType, specialGroups: SpecialGroups) => {
   return `GRADUATION_STATS_${id}_${yearType}_${specialGroups}`
 }
-const createRedisKeyForStudyTrackStats = (id: string, graduated: Graduated, specialGroups: SpecialGroups) => {
-  return `STUDYTRACK_STATS_${id}_${graduated}_${specialGroups}`
+const createRedisKeyForStudyTrackStats = (id: string, specialGroups: SpecialGroups) => {
+  return `STUDYTRACK_STATS_${id}_${specialGroups}`
 }
 
-const shouldSaveToRedis = (id: unknown) => {
+// *Should* always be a string (previously unknown)
+const shouldSaveToRedis = (id: string) => {
   if (typeof id !== 'string') return false
   if (isRelevantProgramme(id)) return true
   const splitId = id.split('-')
@@ -31,7 +33,7 @@ export const getBasicStats = async (
   combinedProgramme: string | null,
   yearType: YearType,
   specialGroups: SpecialGroups
-) => {
+): Promise<BasicStats | null> => {
   const searchKey = combinedProgramme ? `${id}-${combinedProgramme}` : id
   const redisKey = createRedisKeyForBasicStats(searchKey, yearType, specialGroups)
   const dataFromRedis = await redisClient.get(redisKey)
@@ -41,22 +43,13 @@ export const getBasicStats = async (
   return JSON.parse(dataFromRedis)
 }
 
-export const setBasicStats = async (data, yearType: YearType, specialGroups: SpecialGroups) => {
+export const setBasicStats = async (data: BasicStats, yearType: YearType, specialGroups: SpecialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
   if (!shouldSaveToRedis(id)) {
-    return dataToRedis
+    return
   }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
 export const getCreditStats = async (providerCode: string, isAcademicYear: boolean, specialGroups: boolean) => {
@@ -69,29 +62,20 @@ export const getCreditStats = async (providerCode: string, isAcademicYear: boole
   if (!dataFromRedis) {
     return null
   }
-  return JSON.parse(dataFromRedis)
+  return JSON.parse(dataFromRedis) as CreditStats
 }
 
-export const setCreditStats = async (data, isAcademicYear: boolean, specialGroups: boolean) => {
+export const setCreditStats = async (data: CreditStats, isAcademicYear: boolean, specialGroups: boolean) => {
   const { id } = data
   const redisKey = createRedisKeyForCreditStats(
     id,
     isAcademicYear ? 'ACADEMIC_YEAR' : 'CALENDAR_YEAR',
     specialGroups ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED'
   )
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
   if (!isRelevantProgramme(id) && !isFaculty(id)) {
-    return dataToRedis
+    return
   }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
 export const getGraduationStats = async (
@@ -106,56 +90,37 @@ export const getGraduationStats = async (
   if (!dataFromRedis) {
     return null
   }
-  return JSON.parse(dataFromRedis)
+  return JSON.parse(dataFromRedis) as ProgrammeGraduationStats
 }
 
 export const setGraduationStats = async (data, yearType: YearType, specialGroups: SpecialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
   if (!shouldSaveToRedis(id)) {
-    return dataToRedis
+    return
   }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
 export const getStudyTrackStats = async (
   id: string,
   combinedProgramme: string | null,
-  graduated: Graduated,
   specialGroups: SpecialGroups
 ) => {
   const searchKey = combinedProgramme ? `${id}-${combinedProgramme}` : id
-  const redisKey = createRedisKeyForStudyTrackStats(searchKey, graduated, specialGroups)
+  const redisKey = createRedisKeyForStudyTrackStats(searchKey, specialGroups)
   const dataFromRedis = await redisClient.get(redisKey)
   if (!dataFromRedis) {
     return null
   }
-  return JSON.parse(dataFromRedis) as StudyTrackStats & { status: string; lastUpdated: string }
+  return JSON.parse(dataFromRedis) as StudyTrackStats
 }
 
-export const setStudyTrackStats = async (data: StudyTrackStats, graduated: Graduated, specialGroups: SpecialGroups) => {
+export const setStudyTrackStats = async (data: StudyTrackStats, specialGroups: SpecialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForStudyTrackStats(id, graduated, specialGroups)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
   if (!shouldSaveToRedis(id)) {
-    return dataToRedis
+    return
   }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  const redisKey = createRedisKeyForStudyTrackStats(id, specialGroups)
+  await redisClient.set(redisKey, JSON.stringify(data))
 }

services/backend/src/services/completedCoursesSearch.ts

@@ -4,19 +4,19 @@ import { CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
 import { omitKeys } from '@oodikone/shared/util'
 import { CourseModel, CreditModel, EnrollmentModel, StudentModel, StudyplanModel } from '../models'
 
-export type Courses = Array<Pick<CourseModel, 'code' | 'name' | 'substitutions'>>
+export type Courses = Array<Pick<CourseModel, 'code' | 'name' | 'substitution_groups'>>
 
 interface StudentCredit {
   date: Date
   courseCode: string
   creditType: CreditTypeCode
-  substitution: string | null
+  substitution: string[] | null
 }
 
 interface StudentEnrollment {
   date: Date
   courseCode: string
-  substitution: string | null
+  substitution: string[] | null
 }
 
 interface StudentInfo {
@@ -58,7 +58,7 @@ interface StudentWithCourses extends Omit<StudentWithStudyplanNested, 'studyplan
 const getCourses = async (courseCodes: string[]) => {
   const courses: Courses = (
     await CourseModel.findAll({
-      attributes: ['code', 'name', 'substitutions'],
+      attributes: ['code', 'name', 'substitution_groups'],
       where: where(fn('LOWER', col('code')), {
         [Op.in]: courseCodes.map(code => code.toLowerCase()),
       }),
@@ -67,16 +67,13 @@ const getCourses = async (courseCodes: string[]) => {
   return courses
 }
 
-const getCredits = async (
-  courses: Courses,
-  fullCourseCodes: string[],
-  courseCodes: string[],
-  studentNumbers: string[]
-) => {
+const getPassedCredits = async (courses: Courses, fullCourseCodes: string[], studentNumbers: string[]) => {
   const credits: Array<
     Pick<CreditModel, 'course_code' | 'student_studentnumber' | 'credittypecode' | 'attainment_date'>
   > = await CreditModel.findAll({
+    raw: true,
     attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'attainment_date'],
+    order: [['attainment_date', 'DESC']],
     where: {
       course_code: {
         [Op.in]: fullCourseCodes,
@@ -84,20 +81,47 @@ const getCredits = async (
       student_studentnumber: {
         [Op.in]: studentNumbers,
       },
+      credittypecode: {
+        [Op.not]: CreditTypeCode.FAILED,
+      },
     },
   })
-  const formattedCredits = credits.map(credit => {
-    const originalCode = courseCodes.includes(credit.course_code)
-      ? null
-      : courses.find(course => course.substitutions.includes(credit.course_code))?.code
-    return {
-      courseCode: originalCode ?? credit.course_code,
-      substitution: originalCode ? credit.course_code : null,
-      studentNumber: credit.student_studentnumber,
-      creditType: credit.credittypecode,
-      date: credit.attainment_date,
+
+  const creditsByStudentNumber = Object.groupBy(credits, ({ student_studentnumber }) => student_studentnumber)
+
+  const formattedCredits: {
+    courseCode: string
+    substitution: string[] | null
+    studentNumber: string
+    creditType: CreditTypeCode
+    date: Date
+  }[] = []
+
+  Object.entries(creditsByStudentNumber).map(([studentNumber, credits]) => {
+    // We know that credits will exist, because of the way we created the previous object
+    const studentCreditCourseCodes = credits!.map(({ course_code }) => course_code)
+    for (const course of courses) {
+      // Also handle the main course code by adding it as a group
+      for (const group of [[course.code]].concat(course.substitution_groups)) {
+        if (group.every(code => studentCreditCourseCodes.includes(code))) {
+          // We just checked that (group) course code exists in credits so .find(...)! is ok
+          const groupCredits = group.map(code => credits!.find(credit => credit.course_code === code)!)
+          if (groupCredits.length) {
+            const groupCreditCodes = groupCredits.map(({ course_code }) => course_code)
+            formattedCredits.push({
+              courseCode: course.code,
+              substitution:
+                groupCreditCodes.includes(course.code) && groupCreditCodes.length === 1 ? null : groupCreditCodes,
+              studentNumber,
+              creditType: groupCredits.length > 1 ? CreditTypeCode.PASSED : groupCredits.at(0)!.credittypecode,
+              date: groupCredits?.at(0)?.attainment_date ?? new Date(), // Credits are sorted by date in desc. order
+            })
+          }
+        }
+      }
     }
   })
+
   return formattedCredits
 }
 
@@ -105,6 +129,7 @@ const getEnrollments = async (courses: Courses, fullCourseCodes: string[], stude
   const enrollments: Array<Pick<EnrollmentModel, 'course_code' | 'enrollment_date_time' | 'studentnumber'>> =
     await EnrollmentModel.findAll({
       attributes: ['course_code', 'enrollment_date_time', 'studentnumber'],
+      order: [['enrollment_date_time', 'DESC']],
       where: {
         course_code: {
           [Op.in]: fullCourseCodes,
@@ -117,15 +142,42 @@ const getEnrollments = async (courses: Courses, fullCourseCodes: string[], stude
         },
       },
     })
-  const formattedEnrollments = enrollments.map(enrollment => {
-    const originalCode = courses.find(course => course.substitutions.includes(enrollment.course_code))?.code
-    return {
-      courseCode: originalCode ?? enrollment.course_code,
-      substitution: originalCode ? enrollment.course_code : null,
-      studentNumber: enrollment.studentnumber,
-      date: enrollment.enrollment_date_time,
+
+  const enrollmentsByStudents = Object.groupBy(enrollments, ({ studentnumber }) => studentnumber)
+
+  const formattedEnrollments: {
+    courseCode: string
+    substitution: string[] | null
+    studentNumber: string
+    date: Date
+  }[] = []
+
+  Object.entries(enrollmentsByStudents).map(([studentNumber, enrollments]) => {
+    // We know that enrollments will exist, because of the way we created the previous object
+    const studentEnrollmentCourseCodes = enrollments!.map(({ course_code }) => course_code)
+    for (const course of courses) {
+      // Also handle the main course code by adding it as a group
+      for (const group of [[course.code]].concat(course.substitution_groups)) {
+        if (group.every(code => studentEnrollmentCourseCodes.includes(code))) {
+          // We just checked that (group) course code exists in enrollments so .find(...)! is ok
+          const groupEnrollments = group.map(code => enrollments!.find(enrollment => enrollment.course_code === code)!)
+          if (groupEnrollments.length) {
+            const groupEnrollmentCodes = groupEnrollments.map(({ course_code }) => course_code)
+            formattedEnrollments.push({
+              courseCode: course.code,
+              substitution:
+                groupEnrollmentCodes.includes(course.code) && groupEnrollmentCodes.length === 1
+                  ? null
+                  : groupEnrollmentCodes,
+              studentNumber,
+              date: groupEnrollments?.at(0)?.enrollment_date_time ?? new Date(), // Enrollments are sorted by date in desc. order
+            })
+          }
+        }
+      }
     }
   })
+
   return formattedEnrollments
 }
 
@@ -159,15 +211,20 @@ export const getCompletedCourses = async (
 ): Promise<{ students: Omit<FormattedStudent, 'allEnrollments'>[]; courses: Courses }> => {
   const courses = await getCourses(courseCodes)
   const courseCodesSet = new Set(courseCodes)
-  courses.forEach(course => {
+
+  // Get *ALL* courses including any courses in any substitution groups
+  for (const course of courses) {
     courseCodesSet.add(course.code)
-    course.substitutions.forEach(substitution => {
-      courseCodesSet.add(substitution)
-    })
-  })
+    for (const group of course.substitution_groups) {
+      for (const code of group) {
+        courseCodesSet.add(code)
+      }
+    }
+  }
+
   const fullCourseCodes = Array.from(courseCodesSet)
 
-  const credits = await getCredits(courses, fullCourseCodes, courseCodes, studentNumbers)
+  const credits = await getPassedCredits(courses, fullCourseCodes, studentNumbers)
   const enrollments = await getEnrollments(courses, fullCourseCodes, studentNumbers)
   const studentInfo = await getStudents(studentNumbers)
 
@@ -243,7 +300,7 @@ export const getCompletedCourses = async (
   students.forEach(student => {
     courseCodes.forEach(courseCode => {
       const [latestEnrollment] = student.allEnrollments
-        .filter(enrollment => enrollment.courseCode === courseCode || enrollment.substitution === courseCode)
+        .filter(enrollment => enrollment.courseCode === courseCode)
         .sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime())
       student.enrollments[courseCode] = latestEnrollment
     })

services/backend/src/services/courses/courseFinders.ts

@@ -1,9 +1,7 @@
-import { orderBy } from 'lodash'
 import { Op } from 'sequelize'
 
+import { CourseWithSubsDetails } from '@oodikone/shared/types/course'
 import { CourseModel } from '../../models'
-import { CourseWithSubsId } from '../../types'
-import { getSortRank } from '../../util/sortRank'
 
 const nameLikeTerm = (name: string) => {
   if (!name) {
@@ -40,65 +38,33 @@ const codeLikeTerm = (code: string) => {
   }
 }
 
-const getRawCourses = async (name: string, code: string) => {
-  return await CourseModel.findAll({
+export const getCoursesByNameAndOrCode = async (name: string, code: string, includeSpecial: boolean) => {
+  const courses = await CourseModel.findAll({
     where: {
+      ...(includeSpecial ? {} : { mainCourseCode: { [Op.ne]: null } }),
       ...nameLikeTerm(name),
       ...codeLikeTerm(code),
     },
-    order: [['id', 'desc']],
+    raw: true,
   })
-}
 
-const getCourses = async (name: string, code: string) => {
-  const rawCourses = await getRawCourses(name, code)
-  const courses: CourseWithSubsId[] = rawCourses
-    .map(course => ({
-      ...course.toJSON(),
-      substitutions: orderBy(course.substitutions, [
-        substitution => {
-          if (/^A/.exec(substitution)) return 4 // open university codes come last
-          if (/^\d/.exec(substitution)) return 2 // old numeric codes come second
-          if (/^[A-Za-z]/.exec(substitution)) return 1 // new letter based codes come first
-          return 3 // unknown, comes before open uni?
-        },
-      ]),
-    }))
-    .sort((a, b) => getSortRank(b.code) - getSortRank(a.code))
-  return courses
-}
+  const coursesWithSubstitutionGroupDetails = await Promise.all(
+    courses.map<Promise<CourseWithSubsDetails>>(async course => {
+      const groupDetails = await Promise.all(
+        course?.substitution_groups?.map(async group => {
+          return await CourseModel.findAll({
+            attributes: ['name', 'code'],
+            where: { code: { [Op.in]: group } },
+            raw: true,
+          })
+        })
+      )
 
-export const getCoursesByNameAndOrCode = async (name: string, code: string) => {
-  const courses = await getCourses(name, code)
-  let substitutionGroupIndex = 0
-  const visitedCourses: string[] = []
-
-  const assignSubstitutionGroup = (course: CourseWithSubsId) => {
-    if (visitedCourses.includes(course.code)) {
-      return
-    }
-    let relatedCourses: CourseWithSubsId[] = []
-    if (course.substitutions !== null) {
-      relatedCourses = courses.filter(currentCourse => course.substitutions.includes(currentCourse.code))
-    }
-    relatedCourses.unshift(course)
-    relatedCourses.forEach(relatedCourse => {
-      if (visitedCourses.includes(relatedCourse.id)) {
-        return
-      }
-      visitedCourses.push(relatedCourse.id)
-      relatedCourse.subsId = substitutionGroupIndex
+      return { ...course, substitution_groups: groupDetails }
     })
-  }
+  )
 
-  courses.forEach(course => {
-    if (!visitedCourses.includes(course.id)) {
-      substitutionGroupIndex++
-      assignSubstitutionGroup(course)
-    }
-  })
-
-  return courses
+  return coursesWithSubstitutionGroupDetails
 }
 
 export const getCoursesByCodes = (codes: string[]) => {

services/backend/src/services/courses/courseYearlyStatsCounter.ts

@@ -1,108 +1,23 @@
-import { max, min } from 'lodash'
-
-import { Name } from '@oodikone/shared/types'
-import { getSemesterNames } from '../semesters'
-import { FormattedProgramme, OrganizationDetails } from './helpers'
-
-type Programme = {
-  name: Name
-  students: Record<number, string[]>
-  passed: Record<number, string[]>
-  credits: Record<number, number>
-}
-
-type Faculty = {
-  name: Name | null
-  students: string[]
-  passed: string[]
-  credits: number
-}
-
-type FacultyYearStats = {
-  year: string
-  allStudents: string[]
-  allPassed: string[]
-  faculties: Record<string, Faculty>
-  allCredits: number
-}
-
-type Grades = Record<string, string[]>
-
-type GroupAttempts = {
-  grades: Grades
-  categories: {
-    passed: string[]
-    failed: string[]
-  }
-}
-
-type GroupStudents = {
-  grades: Record<string, { grade: string; passed: boolean }>
-  studentNumbers: string[]
-}
-
-type Group = {
-  code: number
-  name: string | Name
-  coursecode: string
-  attempts: GroupAttempts
-  students: GroupStudents
-  enrollments: { studentNumber: string; enrollmentDateTime: Date }[]
-  allEnrollments: { studentNumber: string; enrollmentDateTime: Date }[]
-  yearCode: number
-}
-
-type Student = {
-  earliestAttainment: Date
-  code: number
-}
+import type { Name } from '@oodikone/shared/types'
+import type { Programme, FacultyYearStats, Grades, Group, Student } from '@oodikone/shared/types/courseYearlyStats'
+import { getSemesterNamesByCode } from '../semesters'
+import type { OrganizationDetails } from './helpers'
 
 export class CourseYearlyStatsCounter {
   private groups: Record<number, Group>
   private programmes: Record<string, Programme>
   private facultyStats: Record<number, FacultyYearStats>
-  private obfuscated: boolean
   private students: Map<string, Student>
 
   constructor() {
     this.groups = {}
     this.programmes = {}
     this.facultyStats = {}
-    this.obfuscated = false
     this.students = new Map()
   }
 
-  private initProgramme(code: string, name: Name) {
-    this.programmes[code] = {
-      name,
-      students: {},
-      passed: {},
-      credits: {},
-    }
-  }
-
-  private initFacultyYear(code: number) {
-    const year = `${1949 + Number(code)}-${1950 + Number(code)}`
-    this.facultyStats[code] = {
-      year,
-      allStudents: [],
-      allPassed: [],
-      faculties: {},
-      allCredits: 0,
-    }
-  }
-
-  private initFaculty(yearCode: number, facultyCode: string, organization: OrganizationDetails | null) {
-    this.facultyStats[yearCode].faculties[facultyCode] = {
-      name: organization ? organization.name : null,
-      students: [],
-      passed: [],
-      credits: 0,
-    }
-  }
-
   private initGroup(groupCode: number, name: string | Name, courseCode: string, yearCode: number) {
-    this.groups[groupCode] = {
+    this.groups[groupCode] ??= {
       code: groupCode,
       name,
       coursecode: courseCode,
@@ -124,37 +39,45 @@ export class CourseYearlyStatsCounter {
   }
 
   public markStudyProgramme(
-    code: string,
-    name: Name,
     studentNumber: string,
     yearCode: number,
     passed: boolean,
     credits: number,
-    facultyCode: string | null,
-    organization: OrganizationDetails | null
+    code: string,
+    name: Name,
+    facultyCode: string,
+    organization: OrganizationDetails
   ) {
-    if (!this.programmes[code]) {
-      this.initProgramme(code, name)
-    }
-    if (!this.facultyStats[yearCode]) {
-      this.initFacultyYear(yearCode)
-    }
-    if (
-      facultyCode &&
-      !this.facultyStats[yearCode].faculties[facultyCode] &&
-      !this.facultyStats[yearCode].allPassed.includes(studentNumber)
-    ) {
-      this.initFaculty(yearCode, facultyCode, organization)
+    this.programmes[code] ??= {
+      name,
+      students: {},
+      passed: {},
+      credits: {},
     }
 
-    this.programmes[code].students[yearCode] = this.programmes[code].students[yearCode] || []
+    const year = `${1949 + Number(yearCode)}-${1950 + Number(yearCode)}`
+    this.facultyStats[yearCode] ??= {
+      year,
+      allStudents: [],
+      allPassed: [],
+      faculties: {},
+      allCredits: 0,
+    }
+
+    this.facultyStats[yearCode].faculties[facultyCode] ??= {
+      name: organization?.name ?? null,
+      students: [],
+      passed: [],
+      credits: 0,
+    }
+
+    this.programmes[code].students[yearCode] ??= []
     this.programmes[code].students[yearCode].push(studentNumber)
 
-    if (!this.programmes[code].passed[yearCode]) {
-      this.programmes[code].passed[yearCode] = []
-      this.programmes[code].credits[yearCode] = 0
-    }
-    if (facultyCode && !this.facultyStats[yearCode].allStudents.includes(studentNumber)) {
+    this.programmes[code].passed[yearCode] ??= []
+    this.programmes[code].credits[yearCode] ??= 0
+
+    if (!this.facultyStats[yearCode].allStudents.includes(studentNumber)) {
       this.facultyStats[yearCode].allStudents.push(studentNumber)
       this.facultyStats[yearCode].faculties[facultyCode].students.push(studentNumber)
     }
@@ -162,7 +85,7 @@ export class CourseYearlyStatsCounter {
       this.programmes[code].passed[yearCode].push(studentNumber)
       this.programmes[code].credits[yearCode] += credits
     }
-    if (facultyCode && passed && !this.facultyStats[yearCode].allPassed.includes(studentNumber)) {
+    if (passed && !this.facultyStats[yearCode].allPassed.includes(studentNumber)) {
       this.facultyStats[yearCode].allPassed.push(studentNumber)
       this.facultyStats[yearCode].faculties[facultyCode].passed.push(studentNumber)
       this.facultyStats[yearCode].faculties[facultyCode].credits += credits
@@ -170,18 +93,6 @@ export class CourseYearlyStatsCounter {
     }
   }
 
-  public markStudyProgrammes(
-    studentNumber: string,
-    programmes: FormattedProgramme[],
-    yearCode: number,
-    passed: boolean,
-    credits: number
-  ) {
-    programmes.forEach(({ code, name, facultyCode, organization }) => {
-      this.markStudyProgramme(code, name, studentNumber, yearCode, passed, credits, facultyCode, organization)
-    })
-  }
-
   public markCreditToGroup(
     studentNumber: string,
     passed: boolean,
@@ -191,11 +102,26 @@ export class CourseYearlyStatsCounter {
     courseCode: string,
     yearCode: number
   ) {
-    if (!this.groups[groupCode]) {
-      this.initGroup(groupCode, groupName, courseCode, yearCode)
+    this.initGroup(groupCode, groupName, courseCode, yearCode)
+
+    // mark credit to attempts
+    const { grades, categories } = this.groups[groupCode].attempts
+
+    grades[grade] ??= []
+    grades[grade].push(studentNumber)
+
+    if (passed) {
+      categories.passed.push(studentNumber)
+    } else {
+      categories.failed.push(studentNumber)
     }
-    this.markCreditToAttempts(studentNumber, passed, grade, groupCode)
-    this.markBestEffortGrade(studentNumber, passed, grade, groupCode)
+
+    // mark best effort grade
+    const current = this.groups[groupCode].students.grades[studentNumber]
+
+    if (current?.passed && !passed) return
+    if (current?.passed && Number(grade) <= Number(current?.grade)) return
+    this.groups[groupCode].students.grades[studentNumber] = { grade, passed }
   }
 
   public markEnrollmentToGroup(
@@ -206,16 +132,15 @@ export class CourseYearlyStatsCounter {
     courseCode: string,
     yearCode: number
   ) {
-    if (!this.groups[groupCode]) {
-      this.initGroup(groupCode, groupName, courseCode, yearCode)
-    }
-    const enrollment = { studentNumber, enrollmentDateTime }
-    this.groups[groupCode].allEnrollments.push(enrollment)
-    const oldEnrollment = this.groups[groupCode].enrollments.find(
+    this.initGroup(groupCode, groupName, courseCode, yearCode)
+    const oldEnrollment = this.groups[groupCode].enrollments.some(
       enrollment => enrollment.studentNumber === studentNumber
     )
+
+    const enrollment = { studentNumber, enrollmentDateTime }
+    this.groups[groupCode].allEnrollments.push(enrollment)
     if (!oldEnrollment) {
-      this.groups[groupCode].enrollments.push({ studentNumber, enrollmentDateTime })
+      this.groups[groupCode].enrollments.push(enrollment)
     } else {
       this.groups[groupCode].enrollments = this.groups[groupCode].enrollments
         .filter(enrollment => enrollment.studentNumber !== studentNumber)
@@ -223,35 +148,6 @@ export class CourseYearlyStatsCounter {
     }
   }
 
-  private markCreditToAttempts(studentNumber: string, passed: boolean, grade: string, groupCode: number) {
-    const { attempts } = this.groups[groupCode]
-    const { grades, categories } = attempts
-    if (!grades[grade]) {
-      grades[grade] = []
-    }
-    grades[grade].push(studentNumber)
-    if (passed) {
-      categories.passed.push(studentNumber)
-    } else {
-      categories.failed.push(studentNumber)
-    }
-  }
-
-  private markBestEffortGrade(studentNumber: string, passed: boolean, grade: string, groupCode: number) {
-    const current = this.groups[groupCode].students.grades[studentNumber]
-    if (!current) {
-      this.groups[groupCode].students.grades[studentNumber] = { grade, passed }
-      return
-    }
-    if (current.passed && !passed) {
-      return
-    }
-    if (current.passed && Number(current.grade) >= Number(grade)) {
-      return
-    }
-    this.groups[groupCode].students.grades[studentNumber] = { grade, passed }
-  }
-
   public markCreditToStudentCategories(studentNumber: string, attainmentDate: Date, groupCode: number) {
     if (!this.students.has(studentNumber)) {
       this.addOrUpdateStudent(studentNumber, attainmentDate, groupCode)
@@ -269,14 +165,90 @@ export class CourseYearlyStatsCounter {
 
   private updateExistingStudent(studentNumber: string, attainmentDate: Date, groupCode: number) {
     const student = this.students.get(studentNumber)
-    if (attainmentDate < student!.earliestAttainment) {
-      this.addOrUpdateStudent(studentNumber, attainmentDate, groupCode)
+    if (attainmentDate < student!.earliestAttainment) this.addOrUpdateStudent(studentNumber, attainmentDate, groupCode)
+  }
+
+  private async insertEmptyRows() {
+    const groups = Object.keys(this.groups).map(Number)
+    if (groups.length === 0) return
+
+    const first = Math.min(...groups)
+    const last = Math.max(...groups)
+
+    const a = first
+    const b = last + 1
+    const range = [...Array(b - a).keys()].map(v => v + a)
+    const diff = range.filter(v => !groups.includes(v))
+
+    const semesters = await getSemesterNamesByCode(diff)
+
+    const getNextYear = (name: string) => {
+      const [start, end] = name.split('-')
+      const startYear = parseInt(start, 10) + 1
+      const endYear = parseInt(end, 10) + 1
+      return `${startYear}-${endYear}`
     }
+
+    diff.forEach(i => {
+      const previous = this.groups[i - 1]
+      const name = typeof previous.name === 'string' ? getNextYear(previous.name) : semesters[i]
+
+      this.initGroup(i, name, previous.coursecode, i)
+    })
+  }
+
+  private async parseGroupStatistics(anonymizationSalt: string | null) {
+    await this.insertEmptyRows()
+    for (const [studentNumber, data] of this.students) {
+      this.groups[data.code].students.studentNumbers.push(studentNumber)
+    }
+
+    const groupStatistics = Object.values(this.groups).map(group => {
+      if (anonymizationSalt && group.students.studentNumbers.length < 6) {
+        return {
+          obfuscated: true,
+          code: group.code,
+          name: group.name,
+          coursecode: group.coursecode,
+          attempts: {
+            categories: {
+              failed: [] as string[],
+              passed: [] as string[],
+            },
+            grades: Object.keys(group.attempts.grades).reduce<Grades>((grades, grade) => {
+              grades[grade] = []
+              return grades
+            }, {}),
+          },
+          yearCode: group.yearCode,
+          students: {
+            studentNumbers: [] as string[],
+          },
+        }
+      }
+
+      return {
+        ...group,
+        students: {
+          ...group.students,
+          grades: Object.entries(group.students.grades)
+            .map(([studentNumber, { grade, passed }]) => [studentNumber, passed ? grade : '0'])
+            .reduce<Grades>((grades, [studentNumber, grade]) => {
+              grades[grade] ??= []
+              grades[grade].push(studentNumber)
+
+              return grades
+            }, {}),
+        },
+      }
+    })
+
+    return groupStatistics
   }
 
   private parseProgrammeStatistics(anonymizationSalt: string | null) {
-    if (anonymizationSalt) {
-      this.programmes = {
+    if (anonymizationSalt)
+      return {
         '000000': {
           name: { en: '', fi: '', sv: '' },
           credits: {},
@@ -284,115 +256,34 @@ export class CourseYearlyStatsCounter {
           students: {},
         },
       }
-    }
+
     return this.programmes
   }
 
-  private insertEmptyRows = async () => {
-    const first = min(Object.keys(this.groups).map(Number))
-    const last = max(Object.keys(this.groups).map(Number))
-    if (!(first && last)) {
-      return
-    }
-
-    const getNextYear = (name: string) => {
-      const years = name.split('-')
-      const startYear = parseInt(years[0], 10) + 1
-      const endYear = parseInt(years[1], 10) + 1
-      return `${startYear}-${endYear}`
-    }
-
-    const groups = Object.keys(this.groups)
-    const a = first
-    const b = last + 1
-    const range = [...Array(b - a).keys()].map(v => v + a)
-    const diff = range.filter(v => !groups.includes(v.toString()))
-
-    const semesters = await getSemesterNames(diff)
-
-    diff.forEach(i => {
-      const previous = this.groups[i - 1]
-      const semester = semesters[i]
-      const name = typeof previous.name === 'string' ? getNextYear(previous.name) : semester
-
-      this.initGroup(i, name, previous.coursecode, i)
-    })
-  }
-
-  private async parseGroupStatistics(anonymizationSalt: string | null) {
-    for (const [studentNumber, data] of this.students) {
-      this.groups[data.code].students.studentNumbers.push(studentNumber)
-    }
-
-    await this.insertEmptyRows()
-
-    const groupStatistics = Object.values(this.groups).map(({ ...rest }) => {
-      const { students } = rest
-      const grades = {} as Grades
-      Object.keys(students.grades).forEach(student => {
-        const { grade, passed } = students.grades[student]
-        const parsedGrade = passed ? grade : '0'
-        if (!grades[parsedGrade]) {
-          grades[parsedGrade] = []
-        }
-        grades[parsedGrade].push(student)
-      })
-      const normalStats = {
-        ...rest,
-        students: { ...students, grades },
-      }
-      if (anonymizationSalt && normalStats.students.studentNumbers.length < 6) {
-        this.obfuscated = true
-        const gradeSpread = {} as Grades
-        for (const grade of Object.keys(normalStats.attempts.grades)) {
-          gradeSpread[grade] = []
-        }
-        const obfuscatedStats = {
-          obfuscated: true,
-          code: rest.code,
-          name: rest.name,
-          coursecode: rest.coursecode,
-          attempts: {
-            categories: {
-              failed: [] as string[],
-              passed: [] as string[],
-            },
-            grades: gradeSpread,
-          },
-          yearCode: rest.yearCode,
-          students: {
-            studentNumbers: [] as string[],
-          },
-        }
-        return obfuscatedStats
-      }
-      return normalStats
-    })
-
-    return groupStatistics
-  }
-
   private parseFacultyStatistics(anonymizationSalt: string | null) {
-    if (anonymizationSalt) {
-      this.facultyStats = [
-        {
+    if (anonymizationSalt)
+      return {
+        0: {
           year: 'NA',
           allCredits: 0,
           allPassed: [],
           allStudents: [],
           faculties: {},
         },
-      ]
-    }
+      }
+
     return this.facultyStats
   }
 
   public async getFinalStatistics(anonymizationSalt: string | null) {
+    const statistics = await this.parseGroupStatistics(anonymizationSalt)
+    const obfuscated = statistics.some(row => row.obfuscated)
+
     return {
+      obfuscated,
+      statistics,
       programmes: this.parseProgrammeStatistics(anonymizationSalt),
-      statistics: await this.parseGroupStatistics(anonymizationSalt),
       facultyStats: this.parseFacultyStatistics(anonymizationSalt),
-      obfuscated: this.obfuscated,
     }
   }
 }

services/backend/src/services/courses/creditsAndEnrollmentsOfCourse.ts

@@ -1,9 +1,10 @@
 import { Op } from 'sequelize'
 
+import { Credit, Enrollment } from '@oodikone/shared/models'
 import { EnrollmentState, Unification } from '@oodikone/shared/types'
 import {
-  CreditModel,
   StudentModel,
+  CreditModel,
   SemesterModel,
   OrganizationModel,
   EnrollmentModel,
@@ -12,31 +13,82 @@ import {
 } from '../../models'
 import { getIsOpen } from './helpers'
 
-export const getCreditsForCourses = async (codes: string[], unification: Unification) => {
-  return await CreditModel.findAll({
+export const getCreditsForCourses = async (codeGroups: string[][], unification: Unification) => {
+  const allCourseCodes = codeGroups.flatMap(group => group)
+
+  // We need the credits grouped by student numbers so that we can check if a student has
+  // completed a substitution group
+  const students = await StudentModel.findAll({
+    raw: true,
+    nest: true,
+    attributes: ['studentnumber'],
     include: [
       {
-        model: StudentModel,
-        attributes: ['studentnumber'],
-      },
-      {
-        model: SemesterModel,
-        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+        model: CreditModel,
+        attributes: [
+          'grade',
+          'course_code',
+          'credits',
+          'attainment_date',
+          'student_studentnumber',
+          'studyright_id',
+          'credittypecode',
+        ],
         where: {
-          startdate: {
-            [Op.lte]: new Date(),
+          course_code: {
+            [Op.in]: allCourseCodes,
           },
+          is_open: getIsOpen(unification),
         },
+        order: [['attainment_date', 'ASC']],
+        include: [
+          {
+            model: SemesterModel,
+            attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+            where: {
+              startdate: {
+                [Op.lte]: new Date(),
+              },
+            },
+          },
+        ],
       },
     ],
-    where: {
-      course_code: {
-        [Op.in]: codes,
-      },
-      is_open: getIsOpen(unification),
-    },
-    order: [['attainment_date', 'ASC']],
   })
+
+  const studentNumberToCredits = students.reduce<Record<string, Credit[]>>((acc, student) => {
+    acc[student.studentnumber] ??= []
+    acc[student.studentnumber] = acc[student.studentnumber].concat(student.credits)
+
+    return acc
+  }, {})
+
+  const completedGroups: Credit[][] = []
+  for (const studentCredits of Object.values(studentNumberToCredits)) {
+    const codesOfPassedCredits = studentCredits
+      .filter(credit => CreditModel.passed(credit))
+      .map(credit => credit.course_code)
+    for (const group of codeGroups) {
+      if (group.length === 1) {
+        // Failed courses are only calculated for the original course and 1-to-1 substitutions
+        const credits = studentCredits.filter(credit => credit.course_code === group.at(0))
+        if (credits.length) {
+          completedGroups.push(credits)
+        }
+      }
+      // For substitution groups we must only get passed groups
+      else if (group.every(code => codesOfPassedCredits.includes(code))) {
+        // The credit in question should always exist because we just checked that it does
+        completedGroups.push(
+          studentCredits.filter(credit => group.includes(credit.course_code) && CreditModel.passed(credit))
+        )
+      } else {
+        // "Partially completed substitution, do not count towards anything"
+      }
+    }
+  }
+
+  return completedGroups.filter(group => group.length)
 }
 
 export const getStudentNumberToSrElementsMap = async (studentNumbers: string[]) => {
@@ -53,56 +105,91 @@ export const getStudentNumberToSrElementsMap = async (studentNumbers: string[])
     },
   })
 
-  const studyRightMap = studyRights.reduce((obj, cur) => {
-    obj[cur.id] = cur.toJSON()
-    return obj
-  }, {})
-
-  const studyRightIds = Object.keys(studyRightMap)
+  const studyRightIds = studyRights.map(({ id }) => id)
+  const studyRightMap = new Map<
+    string,
+    {
+      studentNumber: string
+      facultyCode: string
+      organization: Pick<OrganizationModel, 'name' | 'code'>
+    }
+  >(studyRights.map(sr => [sr.id, sr.toJSON()]))
 
   const studyRightElements = await SISStudyRightElementModel.findAll({
     attributes: ['code', 'name', 'startDate', 'endDate', 'studyRightId'],
     where: {
-      studyRightId: {
-        [Op.in]: studyRightIds,
-      },
+      studyRightId: { [Op.in]: studyRightIds },
     },
+    raw: true,
   })
 
-  const studentNumberToSrElementsMap = studyRightElements.reduce((obj, cur) => {
-    const studyRight = studyRightMap[cur.studyRightId]
-    const { studentNumber } = studyRight
+  return studyRightElements.reduce((obj, cur) => {
+    const { studentNumber, ...studyRight } = studyRightMap.get(cur.studyRightId)!
     obj[studentNumber] ??= []
-    obj[studentNumber].push({ ...cur.toJSON(), studyRight })
+    obj[studentNumber].push({ ...cur, studyRight })
+
     return obj
   }, {})
-  return studentNumberToSrElementsMap
 }
 
-export const getEnrollmentsForCourses = async (codes: string[], unification: Unification) => {
-  return await EnrollmentModel.findAll({
+export const getEnrollmentsForCourses = async (codeGroups: string[][], unification: Unification) => {
+  const allCourseCodes = codeGroups.flatMap(group => group)
+
+  const students = await StudentModel.findAll({
+    raw: true,
+    nest: true,
+    attributes: ['studentnumber'],
     include: [
       {
-        model: StudentModel,
-        attributes: ['studentnumber'],
-      },
-      {
-        model: SemesterModel,
-        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
-        where: {
-          startdate: {
-            [Op.lte]: new Date(),
+        model: EnrollmentModel,
+        attributes: ['studentnumber', 'enrollment_date_time', 'course_code'],
+        include: [
+          {
+            model: SemesterModel,
+            attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+            where: {
+              startdate: {
+                [Op.lte]: new Date(),
+              },
+            },
           },
+        ],
+        where: {
+          course_code: {
+            [Op.in]: allCourseCodes,
+          },
+          enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
+          state: EnrollmentState.ENROLLED,
+          is_open: getIsOpen(unification),
         },
       },
     ],
-    where: {
-      course_code: {
-        [Op.in]: codes,
-      },
-      enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
-      state: EnrollmentState.ENROLLED,
-      is_open: getIsOpen(unification),
-    },
   })
+
+  const studentNumberToEnrollments = students.reduce<Record<string, Enrollment[]>>((acc, student) => {
+    acc[student.studentnumber] ??= []
+    acc[student.studentnumber] = acc[student.studentnumber].concat(student.enrollments)
+
+    return acc
+  }, {})
+
+  // Calculate all code groups that have enrollments for all of the courses in the group
+  const completedEnrollments: Enrollment[][] = []
+  for (const studentEnrollments of Object.values(studentNumberToEnrollments)) {
+    const codesOfEnrollments = studentEnrollments.map(enrollment => enrollment.course_code)
+
+    for (const group of codeGroups) {
+      if (group.every(code => codesOfEnrollments.includes(code))) {
+        completedEnrollments.push(
+          studentEnrollments
+            .filter(enrollment => group.includes(enrollment.course_code))
+            .sort((a, b) => b.enrollment_date_time.getTime() - a.enrollment_date_time.getTime())
+        )
+      } else {
+        // Partially completed enrollments for a group, skipping...
+      }
+    }
+  }
+
+  return completedEnrollments
 }

services/backend/src/services/courses/helpers.ts

@@ -6,8 +6,8 @@ export type FormattedProgramme = {
   code: string
   name: Name
   startDate?: Date
-  facultyCode: string | null
-  organization: OrganizationDetails | null
+  facultyCode: string
+  organization: OrganizationDetails
 }
 
 export const getIsOpen = (unification: Unification) => {

services/backend/src/services/courses/index.ts

@@ -1,11 +1,12 @@
 import crypto from 'crypto'
 import { Op, fn as dbFn, col as dbCol } from 'sequelize'
 
+import { Credit, Enrollment } from '@oodikone/shared/models'
 import { Name, EnrollmentState, Unification } from '@oodikone/shared/types'
 import { dateIsBetween } from '@oodikone/shared/util/datetime'
+import logger from '../../../src/util/logger'
 import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, SISStudyRightElementModel } from '../../models'
 import { isOpenUniCourseCode } from '../../util'
-import { getSortRank } from '../../util/sortRank'
 import { CourseYearlyStatsCounter } from './courseYearlyStatsCounter'
 import {
   getCreditsForCourses,
@@ -14,24 +15,12 @@ import {
 } from './creditsAndEnrollmentsOfCourse'
 import { FormattedProgramme, getIsOpen } from './helpers'
 
-const sortMainCode = (codes: string[]) => {
-  if (!codes) {
-    return []
-  }
-  return codes.sort((a, b) => getSortRank(b) - getSortRank(a))
-}
-
 const formatStudyRightElement = (studyRightElement: SISStudyRightElementModel): FormattedProgramme => ({
   code: studyRightElement.code,
   name: studyRightElement.name,
   startDate: studyRightElement.startDate,
-  facultyCode: studyRightElement.studyRight.facultyCode || null,
-  organization: studyRightElement.studyRight.organization
-    ? {
-        name: studyRightElement.studyRight.organization.name,
-        code: studyRightElement.studyRight.organization.code,
-      }
-    : null,
+  facultyCode: studyRightElement.studyRight.facultyCode,
+  organization: studyRightElement.studyRight.organization,
 })
 
 const anonymizeStudentNumber = (studentNumber: string, anonymizationSalt: string) => {
@@ -48,39 +37,61 @@ type FormattedCredit = {
   grade: string
   passed: boolean
   studentNumber: string
-  programmes: FormattedProgramme[]
+  programme: FormattedProgramme
   credits: number
-  obfuscated?: boolean
 }
 
+// Is group in question for a single course (original or 1-to-1 substitution) or a substitution group with multiple courses
+const isSingleCourse = (group: Credit[] | Enrollment[]): boolean =>
+  group?.length === 1 || [...new Set(group?.map((course: Credit | Enrollment) => course.course_code))].length === 1
+
 const parseCredit = (
-  credit: CreditModel,
+  creditGroup: Credit[],
   anonymizationSalt: string | null,
-  studentNumberToSrElementsMap: Record<string, Array<SISStudyRightElementModel>>
-) => {
+  mainCourseCode: string,
+  studyRightElements: Array<SISStudyRightElementModel>
+): FormattedCredit => {
+  const singleCourse = isSingleCourse(creditGroup)
+  // Latest attainment for a course
+  const credit =
+    creditGroup
+      .sort((a, b) => b.attainment_date.getTime() - a.attainment_date.getTime())
+      .find(credit => !CreditModel.failed(credit)) ?? creditGroup.at(0)!
+
+  const courseCode = singleCourse ? credit.course_code : mainCourseCode
+  const grade = singleCourse ? credit.grade : 'substituted'
+  const credits = singleCourse ? credit.credits : creditGroup.reduce((acc, credit) => acc + credit.credits, 0)
+
+  // Take the first attainment for the course (see Student statistics info-box)
+  // TODO: Substitution groups should be marked for the semester when
+  // the group was first finished eg. when was the first time the last missing
+  // course of the group was passed
   const {
     semester,
-    grade,
-    course_code: courseCode,
-    credits,
     attainment_date: attainmentDate,
     student_studentnumber: studentNumber,
     studyright_id: studyRightId,
-  } = credit
+  } = creditGroup.at(-1)!
+
   const { yearcode: yearCode, yearname: yearName, semestercode: semesterCode, name: semesterName } = semester
 
-  const studyRightElements = studentNumberToSrElementsMap[studentNumber] || []
   const programmeOfCredit: SISStudyRightElementModel | undefined =
     studyRightElements.find(studyRightElement => studyRightElement.studyRightId === studyRightId) ??
     studyRightElements
-      .filter(studyRightElement =>
-        dateIsBetween(attainmentDate, studyRightElement.startDate, studyRightElement.endDate)
-      )
-      .sort((a, b) => b.startDate.getTime() - a.startDate.getTime())[0] // The newest studyRightElement
+      .filter(({ startDate, endDate }) => dateIsBetween(attainmentDate, startDate, endDate))
+      .sort((a, b) => b.startDate.getTime() - a.startDate.getTime())
+      .at(0) // The newest studyRightElement
 
-  const programmes = programmeOfCredit ? [programmeOfCredit].map(formatStudyRightElement) : []
+  const programme = programmeOfCredit
+    ? formatStudyRightElement(programmeOfCredit)
+    : {
+        code: 'OTHER',
+        name: { en: 'Other', fi: 'Muu', sv: 'Andra' },
+        facultyCode: 'OTHER',
+        organization: { name: { en: 'Other', fi: 'Muu', sv: 'Andra' } },
+      }
 
-  const formattedCredit: FormattedCredit = {
+  return {
     yearCode,
     yearName,
     semesterCode,
@@ -88,18 +99,11 @@ const parseCredit = (
     attainmentDate,
     courseCode,
     grade,
-    passed: !CreditModel.failed(credit) || CreditModel.passed(credit) || CreditModel.improved(credit),
-    studentNumber,
-    programmes,
+    passed: singleCourse ? !CreditModel.failed(credit) : true,
+    studentNumber: anonymizationSalt ? anonymizeStudentNumber(studentNumber, anonymizationSalt) : studentNumber,
+    programme,
     credits,
   }
-
-  if (anonymizationSalt) {
-    formattedCredit.obfuscated = true
-    formattedCredit.studentNumber = anonymizeStudentNumber(studentNumber, anonymizationSalt)
-  }
-
-  return formattedCredit
 }
 
 type FormattedEnrollment = {
@@ -110,15 +114,9 @@ type FormattedEnrollment = {
   courseCode: string
   enrollmentDateTime: Date
   studentNumber: string
-  programmes: Array<Record<string, any>>
-  obfuscated?: boolean
 }
 
-const parseEnrollment = (
-  enrollment: EnrollmentModel,
-  anonymizationSalt: string | null,
-  studentNumberToSrElementsMap: Record<string, SISStudyRightElementModel[]>
-) => {
+const parseEnrollment = (enrollment: Enrollment, anonymizationSalt: string | null): FormattedEnrollment => {
   const {
     studentnumber: studentNumber,
     semester,
@@ -127,28 +125,33 @@ const parseEnrollment = (
   } = enrollment
   const { yearcode: yearCode, yearname: yearName, semestercode: semesterCode, name: semesterName } = semester
 
-  const studyRightElements = studentNumberToSrElementsMap[studentNumber] || []
-
-  const formattedEnrollment: FormattedEnrollment = {
+  return {
     yearCode,
     yearName,
     semesterCode,
     semesterName,
     courseCode,
     enrollmentDateTime,
-    studentNumber,
-    programmes: studyRightElements.map(formatStudyRightElement),
+    studentNumber: anonymizationSalt ? anonymizeStudentNumber(studentNumber, anonymizationSalt) : studentNumber,
   }
+}
 
-  if (anonymizationSalt) {
-    formattedEnrollment.obfuscated = true
-    formattedEnrollment.studentNumber = anonymizeStudentNumber(studentNumber, anonymizationSalt)
-  }
+const getSubstitutionGroupDetails = async (codeGroups: string[][]) => {
+  const substitutionGroupDetails = await CourseModel.findAll({
+    attributes: ['code', 'name'],
+    where: {
+      code: { [Op.in]: codeGroups.flatMap(group => group) },
+    },
+    raw: true,
+  })
 
-  return formattedEnrollment
+  return codeGroups.map(group =>
+    group.map(code => substitutionGroupDetails.find(subCourse => subCourse.code === code)!)
+  )
 }
 
 const getYearlyStatsOfNew = async (
+  course: Pick<CourseModel, 'code' | 'name' | 'substitution_groups'> | null,
   courseCode: string,
   separate: boolean,
   unification: Unification,
@@ -156,32 +159,36 @@ const getYearlyStatsOfNew = async (
   combineSubstitutions: boolean,
   studentNumberToSrElementsMap: Record<string, SISStudyRightElementModel[]>
 ) => {
-  const courseForSubs = await CourseModel.findOne({
-    where: { code: courseCode },
-  })
+  // Includes main course code and substitutions (if enabled)
+  const creditGroupCodes =
+    combineSubstitutions && course?.substitution_groups
+      ? [[courseCode]].concat(course.substitution_groups)
+      : [[courseCode]]
 
-  let codes =
-    combineSubstitutions && courseForSubs?.substitutions
-      ? sortMainCode([...courseForSubs.substitutions, courseCode])
-      : [courseCode]
-
-  if (unification === Unification.REGULAR) {
-    codes = codes.filter(course => !isOpenUniCourseCode(course))
+  let filteredCreditGroupCodes: string[][]
+  switch (unification) {
+    // Include only course codes / group of course codes WITHOUT ANY AY prefix
+    case Unification.REGULAR:
+      filteredCreditGroupCodes = creditGroupCodes.filter(group => group.every(course => !isOpenUniCourseCode(course)))
+      break
+    // Include all courses / group of courses
+    case Unification.OPEN:
+    case Unification.UNIFY:
+    default:
+      filteredCreditGroupCodes = creditGroupCodes
+      break
   }
 
-  const [credits, enrollments, course] = await Promise.all([
-    getCreditsForCourses(codes, unification),
-    getEnrollmentsForCourses(codes, unification),
-    CourseModel.findOne({
-      where: {
-        code: courseCode,
-      },
-    }),
+  const [creditGroups, enrollmentGroups] = await Promise.all([
+    getCreditsForCourses(filteredCreditGroupCodes, unification),
+    getEnrollmentsForCourses(filteredCreditGroupCodes, unification),
   ])
 
   const counter = new CourseYearlyStatsCounter()
 
-  for (const credit of credits) {
+  for (const creditGroup of creditGroups) {
+    if (!creditGroup?.length) continue
+
     const {
       studentNumber,
       grade,
@@ -191,70 +198,72 @@ const getYearlyStatsOfNew = async (
       yearCode,
       yearName,
       attainmentDate,
-      programmes,
-      courseCode,
+      programme,
+      courseCode: creditCourseCode,
       credits,
-    } = parseCredit(credit, anonymizationSalt, studentNumberToSrElementsMap)
+    } = parseCredit(
+      creditGroup,
+      anonymizationSalt,
+      courseCode,
+      studentNumberToSrElementsMap[creditGroup.at(0)?.student_studentnumber ?? 0] ?? []
+    )
+
+    counter.markStudyProgramme(
+      studentNumber,
+      yearCode,
+      passed,
+      credits,
+      programme.code,
+      programme.name,
+      programme.facultyCode,
+      programme.organization
+    )
 
     const groupCode = separate ? semesterCode : yearCode
     const groupName = separate ? semesterName : yearName
-    const unknownProgramme: FormattedProgramme[] = [
-      {
-        code: 'OTHER',
-        name: {
-          en: 'Other',
-          fi: 'Muu',
-          sv: 'Andra',
-        },
-        facultyCode: 'OTHER',
-        organization: {
-          name: {
-            en: 'Other',
-            fi: 'Muu',
-            sv: 'Andra',
-          },
-        },
-      },
-    ]
-    counter.markStudyProgrammes(
-      studentNumber,
-      programmes.length === 0 ? unknownProgramme : programmes,
-      yearCode,
-      passed,
-      credits
-    )
-    counter.markCreditToGroup(studentNumber, passed, grade, groupCode, groupName, courseCode, yearCode)
+    counter.markCreditToGroup(studentNumber, passed, grade, groupCode, groupName, creditCourseCode, yearCode)
     counter.markCreditToStudentCategories(studentNumber, attainmentDate, groupCode)
   }
 
-  enrollments.forEach(enrollment => {
-    const { studentNumber, semesterCode, semesterName, yearCode, yearName, courseCode, enrollmentDateTime } =
-      parseEnrollment(enrollment, anonymizationSalt, studentNumberToSrElementsMap)
+  for (const enrollments of enrollmentGroups) {
+    if (!enrollments?.length) continue
 
-    const groupCode = separate ? semesterCode : yearCode
-    const groupName = separate ? semesterName : yearName
+    for (const enrollment of enrollments) {
+      const {
+        studentNumber,
+        semesterCode,
+        semesterName,
+        yearCode,
+        yearName,
+        courseCode: enrollmentCourseCode,
+        enrollmentDateTime,
+      } = parseEnrollment(enrollment, anonymizationSalt)
 
-    counter.markEnrollmentToGroup(studentNumber, enrollmentDateTime, groupCode, groupName, courseCode, yearCode)
-  })
+      const groupCode = separate ? semesterCode : yearCode
+      const groupName = separate ? semesterName : yearName
+
+      counter.markEnrollmentToGroup(
+        studentNumber,
+        enrollmentDateTime,
+        groupCode,
+        groupName,
+        enrollmentCourseCode,
+        yearCode
+      )
+    }
+  }
 
   const statistics = await counter.getFinalStatistics(anonymizationSalt)
 
-  let substitutionCourses: CourseModel[] | undefined
-  if (combineSubstitutions && courseForSubs?.substitutions && courseForSubs.substitutions.length > 0) {
-    substitutionCourses = await CourseModel.findAll({
-      where: {
-        code: {
-          [Op.in]: codes,
-        },
-      },
-      attributes: ['code', 'name'],
-    })
-  }
+  const substitutionGroups: Pick<CourseModel, 'code' | 'name'>[][] =
+    combineSubstitutions && course?.substitution_groups?.length
+      ? await getSubstitutionGroupDetails(course.substitution_groups)
+      : [[{ code: course!.code, name: course!.name }]]
 
   return {
     ...statistics,
     coursecode: courseCode,
-    alternatives: substitutionCourses ?? [{ code: courseForSubs!.code, name: courseForSubs!.name }],
+    alternatives: substitutionGroups,
     name: course?.name,
   }
 }
@@ -268,13 +277,12 @@ export const maxYearsToCreatePopulationFrom = async (courseCodes: string[], unif
     raw: true,
   })) as { date: Date } | null
 
-  if (lastAttainmentDate === null) return 0
+  if (lastAttainmentDate?.date == null) return 0
 
   const newestAttainmentDate = lastAttainmentDate?.date.getFullYear()
   const attainmentDateThreshold = new Date(newestAttainmentDate - 6, 0, 1)
 
   const attainmentsWithinThreshold = await CreditModel.count({
-    attributes: [[dbFn('COUNT', dbCol('id')), 'numAttainments']],
     where: {
       course_code: { [Op.in]: courseCodes },
       attainment_date: { [Op.gt]: attainmentDateThreshold },
@@ -293,57 +301,75 @@ export const getCourseYearlyStats = async (
   anonymizationSalt: string | null,
   combineSubstitutions: boolean
 ) => {
-  const credits = await CreditModel.findAll({
-    attributes: ['student_studentnumber'],
-    where: { course_code: { [Op.in]: courseCodes } },
-  })
-  const enrollments = await EnrollmentModel.findAll({
-    attributes: ['studentnumber'],
-    where: {
-      course_code: {
-        [Op.in]: courseCodes,
+  const [credits, enrollments] = await Promise.all([
+    CreditModel.findAll({
+      attributes: ['student_studentnumber'],
+      where: { course_code: { [Op.in]: courseCodes } },
+    }),
+    EnrollmentModel.findAll({
+      attributes: ['studentnumber'],
+      where: {
+        course_code: {
+          [Op.in]: courseCodes,
+        },
+        state: EnrollmentState.ENROLLED,
       },
-      state: EnrollmentState.ENROLLED,
-    },
-  })
+    }),
+  ])
 
-  const studentNumbers = {}
+  const studentNumbers = new Set<string>()
 
   credits.forEach(credit => {
-    studentNumbers[credit.student_studentnumber] = true
-  })
-  enrollments.forEach(enrollment => {
-    studentNumbers[enrollment.studentnumber] = true
+    studentNumbers.add(credit.student_studentnumber)
   })
 
-  const studentNumberToSrElementsMap = await getStudentNumberToSrElementsMap(Object.keys(studentNumbers))
+  enrollments.forEach(enrollment => {
+    studentNumbers.add(enrollment.studentnumber)
+  })
+
+  const studentNumberToSrElementsMap = await getStudentNumberToSrElementsMap([...studentNumbers])
 
   const statsRegular = await Promise.all(
     courseCodes.map(async courseCode => {
-      const unifyStats = await getYearlyStatsOfNew(
-        courseCode,
-        separate,
-        Unification.UNIFY,
-        anonymizationSalt,
-        combineSubstitutions,
-        studentNumberToSrElementsMap
-      )
-      const regularStats = await getYearlyStatsOfNew(
-        courseCode,
-        separate,
-        Unification.REGULAR,
-        anonymizationSalt,
-        combineSubstitutions,
-        studentNumberToSrElementsMap
-      )
-      const openStats = await getYearlyStatsOfNew(
-        courseCode,
-        separate,
-        Unification.OPEN,
-        anonymizationSalt,
-        combineSubstitutions,
-        studentNumberToSrElementsMap
-      )
+      const course: Pick<CourseModel, 'code' | 'name' | 'substitution_groups'> | null = await CourseModel.findOne({
+        attributes: ['code', 'name', 'substitution_groups'],
+        where: { code: courseCode },
+      })
+
+      if (!course) {
+        logger.error('Course for course stats not found with code' + courseCode)
+        return []
+      }
+
+      const [openStats, regularStats, unifyStats] = await Promise.all([
+        getYearlyStatsOfNew(
+          course,
+          courseCode,
+          separate,
+          Unification.OPEN,
+          anonymizationSalt,
+          combineSubstitutions,
+          studentNumberToSrElementsMap
+        ),
+        getYearlyStatsOfNew(
+          course,
+          courseCode,
+          separate,
+          Unification.REGULAR,
+          anonymizationSalt,
+          combineSubstitutions,
+          studentNumberToSrElementsMap
+        ),
+        getYearlyStatsOfNew(
+          course,
+          courseCode,
+          separate,
+          Unification.UNIFY,
+          anonymizationSalt,
+          combineSubstitutions,
+          studentNumberToSrElementsMap
+        ),
+      ])
 
       return { unifyStats, regularStats, openStats }
     })
@@ -368,3 +394,29 @@ export const getCourseProvidersForCourses = async (codes: string[]) => {
     })
   ).map(({ code }) => code)
 }
+
+export const getCourseDetails = async (codes: string[]) =>
+  CourseModel.findAll({
+    attributes: ['code', 'name', 'substitution_groups'],
+    where: { code: { [Op.in]: codes } },
+    raw: true,
+  })
+
+// Add all substitution_groups and coursecodes together
+export const searchAndCombineSubstitutionGroupsToCodes = async (coursecodes: string[]) => {
+  const substitutionGroups = await CourseModel.findAll({
+    raw: true,
+    attributes: ['substitution_groups'],
+    where: {
+      code: { [Op.in]: coursecodes },
+    },
+  })
+
+  return [
+    ...new Set(
+      coursecodes.concat(
+        substitutionGroups.flatMap(({ substitution_groups }) => substitution_groups.flatMap(code => code))
+      )
+    ),
+  ]
+}

services/backend/src/services/faculty/faculty.ts

@@ -1,11 +1,12 @@
-import { groupBy, orderBy } from 'lodash'
+import { groupBy, orderBy } from 'lodash-es'
 import { QueryTypes } from 'sequelize'
 
+import { Organization } from '@oodikone/shared/models'
 import { ProgrammeModuleWithRelevantAttributes } from '@oodikone/shared/types'
-import { serviceProvider } from '../../config'
 import { programmeCodes } from '../../config/programmeCodes'
 import { dbConnections } from '../../database/connection'
 import { OrganizationModel } from '../../models'
+import { ApplicationError } from '../../util/customErrors'
 import { CurriculumPeriods, getCurriculumPeriods } from '../curriculumPeriods'
 
 const { sequelize } = dbConnections
@@ -55,9 +56,7 @@ export const getDegreeProgrammesOfOrganization = async (organizationId: string,
   const programmesWithProgIds = programmesOfOrganization.map(programme => ({
     ...programme,
     progId:
-      programme.code in programmeCodes && serviceProvider !== 'fd'
-        ? programmeCodes[programme.code as keyof typeof programmeCodes]
-        : programme.code,
+      programme.code in programmeCodes ? programmeCodes[programme.code as keyof typeof programmeCodes] : programme.code,
   }))
   const programmesGroupedByCode = groupBy(orderBy(programmesWithProgIds, ['valid_from'], ['desc']), prog => prog.code)
   const curriculumPeriods = await getCurriculumPeriods()
@@ -106,20 +105,15 @@ export const getDegreeProgrammesOfFaculty = async (facultyCode: string, onlyCurr
     },
   })
   if (!organization) {
-    throw new Error(`The organization with the code ${facultyCode} was not found.`)
+    throw new ApplicationError(`The organization with the code ${facultyCode} was not found.`)
   }
   return getDegreeProgrammesOfOrganization(organization.id, onlyCurrentProgrammes)
 }
 
-export const getFacultyCodeById = async (facultyId: string) => {
-  const organization = await OrganizationModel.findOne({
+export const getFacultyCodeById = async (facultyId: string): Promise<Pick<Organization, 'code'> | null> =>
+  OrganizationModel.findOne({
     attributes: ['code'],
     where: {
       id: facultyId,
     },
   })
-  if (!organization) {
-    throw new Error(`The organization with the id ${facultyId} was not found.`)
-  }
-  return organization.code
-}

services/backend/src/services/faculty/facultyBasics.ts

@@ -1,22 +1,21 @@
-import { cloneDeep } from 'lodash'
+import { cloneDeep } from 'lodash-es'
 
 import { NameWithCode, DegreeProgrammeType } from '@oodikone/shared/types'
+import { BasicStats } from '@oodikone/shared/types/studyProgramme'
 import { getBasicStats, setBasicStats } from '../analyticsService'
 import { getBasicStatsForStudytrack } from '../studyProgramme/studyProgrammeBasics'
 import type { ProgrammesOfOrganization } from './faculty'
 
-type StudyTrackBasicStats = Awaited<ReturnType<typeof getBasicStatsForStudytrack>>
-
 const calculateCombinedStats = async (
   facultyProgrammes: ProgrammesOfOrganization,
   isAcademicYear: boolean,
   includeAllSpecials: boolean
 ) => {
-  const statsFromAllProgrammes: StudyTrackBasicStats[] = []
+  const statsFromAllProgrammes: BasicStats[] = []
   const studentInfo = {
-    graphStats: [] as StudyTrackBasicStats['graphStats'],
-    programmeTableStats: {} as Record<string, StudyTrackBasicStats['tableStats']>,
-    tableStats: [] as StudyTrackBasicStats['tableStats'],
+    graphStats: [] as BasicStats['graphStats'],
+    programmeTableStats: {} as Record<string, BasicStats['tableStats']>,
+    tableStats: [] as BasicStats['tableStats'],
     titles: ['Code', 'Started\nstudying', 'Accepted', 'Graduated'],
   }
   if (includeAllSpecials) {

services/backend/src/services/faculty/facultyGraduationTimes.ts

@@ -1,7 +1,10 @@
-import { cloneDeep, omit } from 'lodash'
+import { cloneDeep } from 'lodash-es'
 
 import { Name, DegreeProgrammeType, Unarray } from '@oodikone/shared/types'
+import { ProgrammeGraduationStats } from '@oodikone/shared/types/studyProgramme'
+import { omitKeys } from '@oodikone/shared/util'
 import { getGraduationStats, getStudyTrackStats, setGraduationStats, setStudyTrackStats } from '../analyticsService'
+import { GraduationTarget } from '../graduationHelpers'
 import { getGraduationStatsForStudyTrack, GraduationTimes } from '../studyProgramme/studyProgrammeGraduations'
 import { getMedian } from '../studyProgramme/studyProgrammeHelpers'
 import { getStudyRightsInProgramme } from '../studyProgramme/studyRightFinders'
@@ -22,7 +25,7 @@ export const programmeTypes = {
 } as const
 
 const getStatsByGraduationYear = async (facultyProgrammes: ProgrammesOfOrganization) => {
-  const newStats: Array<Awaited<ReturnType<typeof getGraduationStatsForStudyTrack>>> = []
+  const newStats: ProgrammeGraduationStats[] = []
   const medians: Record<string, LevelGraduationStats[]> = {}
   const programmes: { medians: Record<string, Record<string, ProgrammeStats>> } = { medians: {} }
 
@@ -52,12 +55,12 @@ const getStatsByGraduationYear = async (facultyProgrammes: ProgrammesOfOrganizat
     const level = programmeTypes[degreeProgrammeType as keyof typeof programmeTypes]
 
     if (!medians[level]) {
-      medians[level] = cloneDeep(graduationTimes.medians).map(year => ({ ...omit(year, ['y']), median: year.y }))
+      medians[level] = cloneDeep(graduationTimes.medians).map(year => ({ ...omitKeys(year, ['y']), median: year.y }))
     } else {
       for (const statsForYear of graduationTimes.medians) {
         const correctYear = medians[level].find(entry => entry.name === statsForYear.name)
         if (!correctYear) {
-          medians[level].push({ ...omit(cloneDeep(statsForYear), ['y']), median: statsForYear.y })
+          medians[level].push({ ...omitKeys(cloneDeep(statsForYear), ['y']), median: statsForYear.y })
           continue
         }
         correctYear.times.push(...statsForYear.times)
@@ -71,12 +74,12 @@ const getStatsByGraduationYear = async (facultyProgrammes: ProgrammesOfOrganizat
 
     if (level === 'master' && comboTimes.medians.length) {
       if (!medians.bcMsCombo) {
-        medians.bcMsCombo = cloneDeep(comboTimes.medians).map(year => ({ ...omit(year, ['y']), median: year.y }))
+        medians.bcMsCombo = cloneDeep(comboTimes.medians).map(year => ({ ...omitKeys(year, ['y']), median: year.y }))
       } else {
         for (const statsForYear of comboTimes.medians) {
           const correctYear = medians.bcMsCombo.find(entry => entry.name === statsForYear.name)
           if (!correctYear) {
-            medians.bcMsCombo.push({ ...omit(cloneDeep(statsForYear), ['y']), median: statsForYear.y })
+            medians.bcMsCombo.push({ ...omitKeys(cloneDeep(statsForYear), ['y']), median: statsForYear.y })
             continue
           }
           correctYear.times.push(...statsForYear.times)
@@ -99,7 +102,7 @@ const getStatsByGraduationYear = async (facultyProgrammes: ProgrammesOfOrganizat
         programmes.medians[level][year.name] = { data: [], programmes: [] }
       }
       programmes.medians[level][year.name].data.push(
-        omit({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y'])
+        omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y'])
       )
       programmes.medians[level][year.name].programmes.push(code)
     }
@@ -114,7 +117,7 @@ const getStatsByGraduationYear = async (facultyProgrammes: ProgrammesOfOrganizat
           programmes.medians.bcMsCombo[year.name] = { data: [], programmes: [] }
         }
         programmes.medians.bcMsCombo[year.name].data.push(
-          omit({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y'])
+          omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y'])
         )
         programmes.medians.bcMsCombo[year.name].programmes.push(code)
       }
@@ -133,7 +136,7 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
     if (programme.degreeProgrammeType == null || !(programme.degreeProgrammeType in programmeTypes)) {
       continue
     }
-    const statsFromRedis = await getStudyTrackStats(programme.code, '', 'GRADUATED_INCLUDED', 'SPECIAL_EXCLUDED')
+    const statsFromRedis = await getStudyTrackStats(programme.code, '', 'SPECIAL_EXCLUDED')
     if (statsFromRedis) {
       newStats.push(statsFromRedis)
       continue
@@ -144,12 +147,11 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
       studyProgramme: programme.code,
       combinedProgramme: '',
       settings: {
-        graduated: true,
         specialGroups: false,
       },
       studyRightsOfProgramme,
     })
-    await setStudyTrackStats(updatedStats, 'GRADUATED_INCLUDED', 'SPECIAL_EXCLUDED')
+    await setStudyTrackStats(updatedStats, 'SPECIAL_EXCLUDED')
     newStats.push(updatedStats)
   }
 
@@ -161,21 +163,22 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
     const basicStats = statsForProgramme.medians.basic
     const comboStats = statsForProgramme.medians.combo
     const classSizesByYear = basicStats.reduce<Record<string, number>>(
-      (acc, year) => ({ ...acc, [year.name]: year.classSize }),
+      (acc, year) => ({ ...acc, [year.name]: year.classSize ?? 0 }),
       {}
     )
     classSizes.programmes[code] = classSizesByYear
 
     if (!medians[level]) {
-      medians[level] = cloneDeep(basicStats).map(year => ({ ...omit(year, ['y', 'classSize']), median: year.y }))
+      medians[level] = cloneDeep(basicStats).map(year => ({ ...omitKeys(year, ['y', 'classSize']), median: year.y }))
     }
     classSizes[level] ??= {}
     for (const statsForYear of basicStats) {
       classSizes[level][statsForYear.name] ??= 0
-      ;(classSizes[level][statsForYear.name] as number) += statsForYear.classSize
+      ;(classSizes[level][statsForYear.name] as number) += statsForYear.classSize ?? 0
+
       const correctYear = medians[level].find(entry => entry.name === statsForYear.name)
       if (!correctYear) {
-        medians[level].push({ ...omit(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
+        medians[level].push({ ...omitKeys(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
         continue
       }
       correctYear.times.push(...statsForYear.times)
@@ -188,15 +191,18 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
 
     if (level === 'master' && comboStats.length) {
       if (!medians.bcMsCombo) {
-        medians.bcMsCombo = cloneDeep(comboStats).map(year => ({ ...omit(year, ['y', 'classSize']), median: year.y }))
+        medians.bcMsCombo = cloneDeep(comboStats).map(year => ({
+          ...omitKeys(year, ['y', 'classSize']),
+          median: year.y,
+        }))
       }
       classSizes.bcMsCombo ??= {}
       for (const statsForYear of comboStats) {
         classSizes.bcMsCombo[statsForYear.name] ??= 0
-        ;(classSizes.bcMsCombo[statsForYear.name] as number) += statsForYear.classSize
+        ;(classSizes.bcMsCombo[statsForYear.name] as number) += statsForYear.classSize ?? 0
         const correctYear = medians.bcMsCombo.find(entry => entry.name === statsForYear.name)
         if (!correctYear) {
-          medians.bcMsCombo.push({ ...omit(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
+          medians.bcMsCombo.push({ ...omitKeys(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
           continue
         }
         correctYear.times.push(...statsForYear.times)
@@ -218,7 +224,7 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
         programmes.medians[level][year.name] = { data: [], programmes: [] }
       }
       programmes.medians[level][year.name].data.push(
-        omit({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y', 'classSize'])
+        omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y', 'classSize'])
       )
       programmes.medians[level][year.name].programmes.push(code)
     }
@@ -233,7 +239,7 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
           programmes.medians.bcMsCombo[year.name] = { data: [], programmes: [] }
         }
         programmes.medians.bcMsCombo[year.name].data.push(
-          omit({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y', 'classSize'])
+          omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y', 'classSize'])
         )
         programmes.medians.bcMsCombo[year.name].programmes.push(code)
       }
@@ -252,22 +258,23 @@ export const countGraduationTimes = async (faculty: string, programmesOfFaculty:
   } = await getStatsByStartYear(programmesOfFaculty)
 
   const goals = {
-    bachelor: 36,
-    bcMsCombo: faculty === 'H90' ? 72 : 60,
-    master: 24,
-    doctor: 48,
+    bachelor: GraduationTarget.THREE_YEARS,
+    bcMsCombo: faculty === 'H90' ? GraduationTarget.SIX_YEARS : GraduationTarget.FIVE_YEARS,
+    master: GraduationTarget.TWO_YEARS,
+    doctor: GraduationTarget.FOUR_YEARS,
     exceptions: {
-      MH30_004: 6, // months more
-      '420420-ma': 6,
-      '420074-ma': 6,
-      '420119-ma': 6,
-      MH30_001: 48,
-      '320011-ma': 48,
-      '320001-ma': 48,
-      MH30_003: 42,
-      '320002-ma': 42,
-      '320009-ma': 42,
-    },
+      // Exceptions are additions to the base amount
+      MH30_004: GraduationTarget.HALF_YEAR,
+      '420420-ma': GraduationTarget.HALF_YEAR,
+      '420074-ma': GraduationTarget.HALF_YEAR,
+      '420119-ma': GraduationTarget.HALF_YEAR,
+      MH30_001: GraduationTarget.FOUR_YEARS,
+      '320011-ma': GraduationTarget.FOUR_YEARS,
+      '320001-ma': GraduationTarget.FOUR_YEARS,
+      MH30_003: GraduationTarget.THREE_POINT_FIVE_YEARS,
+      '320002-ma': GraduationTarget.THREE_POINT_FIVE_YEARS,
+      '320009-ma': GraduationTarget.THREE_POINT_FIVE_YEARS,
+    } as Record<string, number>,
   }
 
   const programmeNames = programmesOfFaculty.reduce<Record<string, Name>>((acc, { code, name }) => {

services/backend/src/services/faculty/facultyService.ts

@@ -1,4 +1,5 @@
-import { Graduated, Name, ProgrammeFilter, SpecialGroups, YearType, DegreeProgrammeType } from '@oodikone/shared/types'
+import { Name, ProgrammeFilter, SpecialGroups, YearType } from '@oodikone/shared/types'
+import { DegreeProgramme } from '../../routes/faculties'
 import { redisClient } from '../redis'
 import { FacultyProgressData } from './facultyStudentProgress'
 
@@ -21,11 +22,11 @@ const createRedisKeyForThesiswriters = (
 const createRedisKeyForGraduationTimeStats = (id: string, programmeFilter: ProgrammeFilter) => {
   return `FACULTY_GRADUATION_TIME_STATS_${id}_${programmeFilter}`
 }
-const createRedisKeyForFacultyProgress = (id: string, specialGroups: SpecialGroups, graduated: Graduated) => {
-  return `FACULTY_PROGRESS_STATS_${id}_${specialGroups}_${graduated}`
+const createRedisKeyForFacultyProgress = (id: string, specialGroups: SpecialGroups) => {
+  return `FACULTY_PROGRESS_STATS_${id}_${specialGroups}`
 }
-const createRedisKeyForFacultyStudents = (id: string, specialGroups: SpecialGroups, graduated: Graduated) => {
-  return `FACULTY_STUDENTS_STATS_${id}_${specialGroups}_${graduated}`
+const createRedisKeyForFacultyStudents = (id: string, specialGroups: SpecialGroups) => {
+  return `FACULTY_STUDENTS_STATS_${id}_${specialGroups}`
 }
 
 /*
@@ -62,7 +63,7 @@ type StudentInfo = {
   titles: string[]
 }
 
-type BasicData = {
+export type BasicData = {
   graduationInfo: GraduationInfo
   id: string
   programmeNames: ProgrammeNames
@@ -78,16 +79,9 @@ export const setBasicStats = async (
 ) => {
   const { id } = data
   const redisKey = createRedisKeyForBasicStats(id, yearType, programmeFilter, specialGroups)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(data))
+
+  return setOperationStatus === 'OK'
 }
 
 export const getBasicStats = async (
@@ -98,15 +92,13 @@ export const getBasicStats = async (
 ) => {
   const redisKey = createRedisKeyForBasicStats(id, yearType, programmeFilter, specialGroups)
   const dataFromRedis = await redisClient.get(redisKey)
-  if (!dataFromRedis) {
-    return null
-  }
-  return JSON.parse(dataFromRedis) as BasicData & { status: string; lastUpdated: string }
+
+  return dataFromRedis ? (JSON.parse(dataFromRedis) as BasicData) : null
 }
 
 type ProgrammeNames = Record<string, Name & { code: string }>
 
-type ThesisWriterData = {
+export type ThesisWriterData = {
   id: string
   years: (string | number)[]
   tableStats: (string | number)[][]
@@ -117,8 +109,6 @@ type ThesisWriterData = {
   programmeTableStats: Record<string, (string | number)[][]>
   titles: string[]
   programmeNames: ProgrammeNames
-  status: string
-  lastUpdated: string
 }
 
 export const setThesisWritersStats = async (
@@ -129,14 +119,7 @@ export const setThesisWritersStats = async (
 ) => {
   const { id } = data
   const redisKey = createRedisKeyForThesiswriters(id, yearType, programmeFilter, specialGroups)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') return null
-  return dataToRedis
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
 export const getThesisWritersStats = async (
@@ -150,7 +133,7 @@ export const getThesisWritersStats = async (
   if (!dataFromRedis) {
     return null
   }
-  return JSON.parse(dataFromRedis) as ThesisWriterData & { status: string; lastUpdated: string }
+  return JSON.parse(dataFromRedis) as ThesisWriterData
 }
 
 type MediansAndProgrammes = {
@@ -160,7 +143,7 @@ type MediansAndProgrammes = {
   }
 }
 
-type GraduationData = {
+export type GraduationData = {
   id: string
   goals: {
     bachelor: number
@@ -178,16 +161,7 @@ type GraduationData = {
 export const setGraduationStats = async (data: GraduationData, programmeFilter: ProgrammeFilter) => {
   const { id } = data
   const redisKey = createRedisKeyForGraduationTimeStats(id, programmeFilter)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
 export const getGraduationStats = async (id: string, programmeFilter: ProgrammeFilter, keepGraduationTimes = false) => {
@@ -196,39 +170,24 @@ export const getGraduationStats = async (id: string, programmeFilter: ProgrammeF
   if (!dataFromRedis) {
     return null
   }
-  const data = JSON.parse(dataFromRedis) as GraduationData & { status: string; lastUpdated: string }
+  const data = JSON.parse(dataFromRedis) as GraduationData
   if (!keepGraduationTimes) {
     removeGraduationTimes(data)
   }
   return data
 }
 
-export const setFacultyProgressStats = async (
-  data: FacultyProgressData,
-  specialGroups: SpecialGroups,
-  graduated: Graduated
-) => {
+export const setFacultyProgressStats = async (data: FacultyProgressData, specialGroups: SpecialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForFacultyProgress(id, specialGroups, graduated)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  const redisKey = createRedisKeyForFacultyProgress(id, specialGroups)
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
-export const getFacultyProgressStats = async (id: string, specialGroups: SpecialGroups, graduated: Graduated) => {
-  const redisKey = createRedisKeyForFacultyProgress(id, specialGroups, graduated)
+export const getFacultyProgressStats = async (id: string, specialGroups: SpecialGroups) => {
+  const redisKey = createRedisKeyForFacultyProgress(id, specialGroups)
   const dataFromRedis = await redisClient.get(redisKey)
-  if (!dataFromRedis) {
-    return null
-  }
-  return JSON.parse(dataFromRedis) as FacultyProgressData & { status: string; lastUpdated: string }
+
+  return dataFromRedis ? (JSON.parse(dataFromRedis) as FacultyProgressData) : null
 }
 
 type FacultyStudentsData = {
@@ -238,36 +197,18 @@ type FacultyStudentsData = {
   facultyTableStatsExtra: Record<string, Record<string, Record<string, number>>>
   programmeStats: Record<string, Record<string, (string | number)[]>>
   titles: string[]
-  programmeNames: Record<
-    string,
-    { name: Name; code: string; degreeProgrammeType: DegreeProgrammeType | null; progId: string }
-  >
+  programmeNames: Record<string, DegreeProgramme>
 }
 
-export const setFacultyStudentStats = async (
-  data: FacultyStudentsData,
-  specialGroups: SpecialGroups,
-  graduated: Graduated
-) => {
+export const setFacultyStudentStats = async (data: FacultyStudentsData, specialGroups: SpecialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForFacultyStudents(id, specialGroups, graduated)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: new Date().toISOString(),
-  }
-  const setOperationStatus = await redisClient.set(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') {
-    return null
-  }
-  return dataToRedis
+  const redisKey = createRedisKeyForFacultyStudents(id, specialGroups)
+  await redisClient.set(redisKey, JSON.stringify(data))
 }
 
-export const getFacultyStudentStats = async (id: string, specialGroups: SpecialGroups, graduated: Graduated) => {
-  const redisKey = createRedisKeyForFacultyStudents(id, specialGroups, graduated)
+export const getFacultyStudentStats = async (id: string, specialGroups: SpecialGroups) => {
+  const redisKey = createRedisKeyForFacultyStudents(id, specialGroups)
   const dataFromRedis = await redisClient.get(redisKey)
-  if (!dataFromRedis) {
-    return null
-  }
-  return JSON.parse(dataFromRedis) as FacultyStudentsData & { status: string; lastUpdated: string }
+
+  return dataFromRedis ? (JSON.parse(dataFromRedis) as FacultyStudentsData) : null
 }

services/backend/src/services/faculty/facultyStudentProgress.ts

@@ -1,7 +1,7 @@
 import dayjs from 'dayjs'
-import { cloneDeep } from 'lodash'
+import { cloneDeep } from 'lodash-es'
 
-import { Graduated, SpecialGroups, Unarray } from '@oodikone/shared/types'
+import { SpecialGroups } from '@oodikone/shared/types'
 import { rootOrgId } from '../../config'
 import { getStudyTrackStats, setStudyTrackStats } from '../analyticsService'
 import { getYearsArray } from '../studyProgramme/studyProgrammeHelpers'
@@ -10,35 +10,38 @@ import { getStudyTrackStatsForStudyProgramme } from '../studyProgramme/studyTrac
 import { getDegreeProgrammesOfOrganization, ProgrammesOfOrganization } from './faculty'
 import { programmeTypes } from './facultyGraduationTimes'
 
+type CreditLimits = Array<[number | null, number | null]>
+
 const createLimits = (
   months: number,
-  creditsToAdd: number,
-  level: 'bachelor' | 'master' | 'doctor'
-): Array<[number, number | null]> => {
+  startingCredits: number,
+  level: 'bachelor' | 'master' | 'bcMs' | 'doctor'
+): CreditLimits => {
   if (level === 'doctor') {
     return [
       [40, null],
       [30, 40],
       [20, 30],
       [10, 20],
-      [0, null],
+      [null, 0],
     ]
   }
+
   return [
-    [Math.ceil(months * (60 / 12)) + creditsToAdd, null],
-    [Math.ceil(months * (45 / 12)) + creditsToAdd, Math.ceil(months * (60 / 12)) + creditsToAdd],
-    [Math.ceil(months * (30 / 12)) + creditsToAdd, Math.ceil(months * (45 / 12)) + creditsToAdd],
-    [Math.ceil(months * (15 / 12)) + creditsToAdd, Math.ceil(months * (30 / 12)) + creditsToAdd],
-    [creditsToAdd + 1, Math.ceil(months * (15 / 12)) + creditsToAdd],
-    [creditsToAdd, null],
+    [startingCredits + Math.ceil(months * (60 / 12)), null],
+    [startingCredits + Math.ceil(months * (45 / 12)), startingCredits + Math.ceil(months * (60 / 12))],
+    [startingCredits + Math.ceil(months * (30 / 12)), startingCredits + Math.ceil(months * (45 / 12))],
+    [startingCredits + Math.ceil(months * (15 / 12)), startingCredits + Math.ceil(months * (30 / 12))],
+    [startingCredits + 1, startingCredits + Math.ceil(months * (15 / 12))],
+    [null, startingCredits],
   ]
 }
 
-const createYearlyTitles = (limitList: ReturnType<typeof createLimits>) => {
+const createYearlyTitles = (limitList: CreditLimits) => {
   const titles: string[] = []
   for (let i = limitList.length - 1; i >= 0; i--) {
-    if (limitList[i][0] === 0) {
-      titles.push('0 Credits')
+    if (limitList[i][0] === null) {
+      titles.push(`${limitList[i][1]} Credits`)
     } else if (limitList[i][1] === null) {
       titles.push(`${limitList[0][0]} ≤ Credits`)
     } else {
@@ -53,18 +56,20 @@ const calculateProgressStats = (
   creditCounts: Record<string, number[]>,
   yearlyTitles: Record<string, string[][]>,
   progressStats: Record<string, Record<string, number[][]>>,
-  programmeInfo: Unarray<ProgrammesOfOrganization>
+  progId: string
 ) => {
-  const goalMonthsForLevels = { bachelor: 36, master: 24, bcMs: 60, doctor: 48 } as const
+  // "bcMs" is the same as masters, as the students have graduated from bachelors.
+  // We need to add the 180cr to the starting point for compensation.
+  const goalMonthsForLevels = { bachelor: 36, master: 24, bcMs: 24, doctor: 48 } as const
 
   const allYears = Object.keys(creditCounts)
-  const limitsForYears = allYears.reduce<Record<string, ReturnType<typeof createLimits>>>((acc, year) => {
+  const limitsForYears = allYears.reduce<Record<string, CreditLimits>>((acc, year) => {
     const startDate = new Date(`${year.slice(0, 4)}-08-01`)
     const lastDayOfMonth = dayjs().endOf('month')
     const months = Math.round(dayjs(lastDayOfMonth).diff(startDate, 'months', true))
     const goalMonths = goalMonthsForLevels[level]
 
-    acc[year] = createLimits(Math.min(months, goalMonths), 0, level === 'bcMs' ? 'master' : level)
+    acc[year] = createLimits(Math.min(months, goalMonths), Number(level === 'bcMs') * 180, level)
     return acc
   }, {})
 
@@ -79,29 +84,29 @@ const calculateProgressStats = (
     progressStats[level] = {}
   }
 
-  progressStats[level][programmeInfo.progId] = []
+  progressStats[level][progId] = []
 
   for (const year of Object.keys(creditCounts)) {
     const limit = limitsForYears[year]
     const statsForYear: number[] = new Array(limit.length).fill(0)
     for (const credits of creditCounts[year]) {
-      const correctIndex = limit.length - 1 - limit.findIndex(element => credits >= element[0])
+      const correctIndex = limit.length - 1 - limit.findIndex(element => credits >= (element[0] ?? 0))
       statsForYear[correctIndex] += 1
     }
-    progressStats[level][programmeInfo.progId].unshift(statsForYear)
+    progressStats[level][progId].unshift(statsForYear)
   }
 }
 
 export const combineFacultyStudentProgress = async (
   faculty: string,
   programmes: ProgrammesOfOrganization,
-  specialGroups: SpecialGroups,
-  graduated: Graduated
+  specialGroups: SpecialGroups
 ) => {
   const since = new Date('2017-08-01')
   const statsOfProgrammes: Array<Awaited<ReturnType<typeof getStudyTrackStatsForStudyProgramme>>> = []
   const allDegreeProgrammes = await getDegreeProgrammesOfOrganization(rootOrgId, false)
   const creditCounts: Record<string, Record<string, number[]>> = {}
+  const graduatedCount: Record<string, Record<string, number>> = {}
   const progressStats: Record<string, Record<string, number[][]>> = {}
   const yearlyTitles: Record<string, string[][]> = {}
 
@@ -114,7 +119,7 @@ export const combineFacultyStudentProgress = async (
     ) {
       continue
     }
-    const statsFromRedis = await getStudyTrackStats(studyProgramme, null, graduated, specialGroups)
+    const statsFromRedis = await getStudyTrackStats(studyProgramme, null, specialGroups)
     if (statsFromRedis) {
       statsOfProgrammes.push(statsFromRedis)
     } else {
@@ -122,22 +127,23 @@ export const combineFacultyStudentProgress = async (
       const updatedStats = await getStudyTrackStatsForStudyProgramme({
         studyProgramme,
         settings: {
-          graduated: graduated === 'GRADUATED_INCLUDED',
           specialGroups: specialGroups === 'SPECIAL_INCLUDED',
         },
         studyRightsOfProgramme,
       })
       statsOfProgrammes.push(updatedStats)
-      await setStudyTrackStats(updatedStats, graduated, specialGroups)
+      await setStudyTrackStats(updatedStats, specialGroups)
     }
   }
 
-  const updateCreditCounts = (
+  const updateCreditAndGraduatedCount = (
     level: 'bachelor' | 'bcMs' | 'master' | 'doctor',
-    creditCountsOfProgramme: Record<string, number[]>
+    creditCountsOfProgramme: Record<string, number[]>,
+    graduatedCountOfProgramme: Record<string, number>
   ) => {
     if (!(level in creditCounts)) {
       creditCounts[level] = cloneDeep(creditCountsOfProgramme)
+      graduatedCount[level] = cloneDeep(graduatedCountOfProgramme)
     } else {
       for (const year of Object.keys(creditCountsOfProgramme)) {
         if (!(year in creditCounts[level])) {
@@ -146,27 +152,28 @@ export const combineFacultyStudentProgress = async (
           creditCounts[level][year].push(...creditCountsOfProgramme[year])
         }
       }
+
+      for (const year of Object.keys(graduatedCountOfProgramme)) {
+        graduatedCount[level][year] += cloneDeep(graduatedCountOfProgramme[year])
+      }
     }
   }
 
   for (const stats of statsOfProgrammes) {
     const programmeInfo = allDegreeProgrammes.find(programme => programme.code === stats.id)
-    if (
-      !programmeInfo ||
-      programmeInfo.degreeProgrammeType == null ||
-      !(programmeInfo.degreeProgrammeType in programmeTypes)
-    ) {
+    if (!programmeInfo?.degreeProgrammeType || !(programmeInfo.degreeProgrammeType in programmeTypes)) {
       continue
     }
 
-    const level = programmeTypes[programmeInfo.degreeProgrammeType as keyof typeof programmeTypes]
+    const { degreeProgrammeType, progId } = programmeInfo
+    const level = programmeTypes[degreeProgrammeType]
 
-    updateCreditCounts(level, stats.creditCounts)
-    calculateProgressStats(level, stats.creditCounts, yearlyTitles, progressStats, programmeInfo)
+    updateCreditAndGraduatedCount(level, stats.creditCounts, stats.graduatedCount)
+    calculateProgressStats(level, stats.creditCounts, yearlyTitles, progressStats, progId)
 
     if (level === 'master' && Object.keys(stats.creditCountsCombo).length > 0) {
-      updateCreditCounts('bcMs', stats.creditCountsCombo)
-      calculateProgressStats('bcMs', stats.creditCountsCombo, yearlyTitles, progressStats, programmeInfo)
+      updateCreditAndGraduatedCount('bcMs', stats.creditCountsCombo, stats.graduatedCount)
+      calculateProgressStats('bcMs', stats.creditCountsCombo, yearlyTitles, progressStats, progId)
     }
   }
 
@@ -178,6 +185,7 @@ export const combineFacultyStudentProgress = async (
     bcMsProgStats: progressStats.bcMs,
     doctoralProgStats: progressStats.doctor,
     creditCounts: {} as Record<string, Record<string, number[]>>,
+    graduatedCount: {} as Record<string, Record<string, number>>,
     yearlyBachelorTitles: yearlyTitles.bachelor,
     yearlyBcMsTitles: yearlyTitles.bcMs,
     yearlyMasterTitles: yearlyTitles.master,
@@ -200,6 +208,14 @@ export const combineFacultyStudentProgress = async (
       }
       result.creditCounts[levelName][year] = creditCounts[level][year]
     }
+
+    for (const year of Object.keys(graduatedCount[level]).toSorted(sortByYear)) {
+      const levelName = level === 'bcMs' ? 'bachelorMaster' : level
+      if (!(levelName in result.graduatedCount)) {
+        result.graduatedCount[levelName] = {}
+      }
+      result.graduatedCount[levelName][year] = graduatedCount[level][year]
+    }
   }
 
   return result

services/backend/src/services/faculty/facultyStudents.ts

@@ -61,15 +61,14 @@ const calculateCombinedStats = (programmeCodes: string[], stats: StudyTrackStats
 export const combineFacultyStudents = async (
   code: string,
   programmes: ProgrammesOfOrganization,
-  specialGroups: 'SPECIAL_INCLUDED' | 'SPECIAL_EXCLUDED',
-  graduated: 'GRADUATED_INCLUDED' | 'GRADUATED_EXCLUDED'
+  specialGroups: 'SPECIAL_INCLUDED' | 'SPECIAL_EXCLUDED'
 ) => {
   let years: string[] = []
   const programmeCodes = programmes.map(programme => programme.code)
   const newStats: StudyTrackStats[] = []
 
   for (const studyProgrammeCode of programmeCodes) {
-    const statsFromRedis = await getStudyTrackStats(studyProgrammeCode, null, graduated, specialGroups)
+    const statsFromRedis = await getStudyTrackStats(studyProgrammeCode, null, specialGroups)
     if (statsFromRedis) {
       newStats.push(statsFromRedis)
       if (!years.length) {
@@ -81,12 +80,11 @@ export const combineFacultyStudents = async (
     const updatedStats = await getStudyTrackStatsForStudyProgramme({
       studyProgramme: studyProgrammeCode,
       settings: {
-        graduated: graduated === 'GRADUATED_INCLUDED',
         specialGroups: specialGroups === 'SPECIAL_INCLUDED',
       },
       studyRightsOfProgramme,
     })
-    await setStudyTrackStats(updatedStats, graduated, specialGroups)
+    await setStudyTrackStats(updatedStats, specialGroups)
     if (!years.length) {
       years = updatedStats.years
     }

services/backend/src/services/faculty/facultyThesisWriters.ts

@@ -1,17 +1,10 @@
-import { cloneDeep } from 'lodash'
+import { cloneDeep } from 'lodash-es'
 
-import { DegreeProgrammeType } from '@oodikone/shared/types'
+import { DegreeProgrammeType, type NameWithCode } from '@oodikone/shared/types'
 import { getGraduationStats, setGraduationStats } from '../analyticsService'
 import { getGraduationStatsForStudyTrack } from '../studyProgramme/studyProgrammeGraduations'
 import { getDegreeProgrammesOfFaculty, ProgrammesOfOrganization } from './faculty'
 
-type ProgrammeName = {
-  code: string
-  en?: string
-  fi?: string
-  sv?: string
-}
-
 const calculateCombinedStats = async (
   isAcademicYear: boolean,
   includeAllSpecials: boolean,
@@ -28,7 +21,7 @@ const calculateCombinedStats = async (
   const newStats: Array<Awaited<ReturnType<typeof getGraduationStatsForStudyTrack>>> = []
   const combinedTableStats: Array<Array<string | number>> = []
   let years: Array<string | number> = []
-  const programmeNames: Record<string, ProgrammeName> = {}
+  const programmeNames: Record<string, NameWithCode> = {}
 
   for (const programme of facultyProgrammes) {
     if (
@@ -90,6 +83,7 @@ const calculateCombinedStats = async (
       isBachelor ? wroteThesis : 0,
       isMaster ? wroteThesis : 0,
     ])
+
     programmeTableStats[progId] = tableStatsForProgramme
 
     if (!combinedTableStats.length) {
@@ -135,7 +129,5 @@ export const combineFacultyThesisWriters = async (
     programmeTableStats,
     titles: ['Code', 'All', 'Bachelors', 'Masters'],
     programmeNames,
-    status: 'Done',
-    lastUpdated: '',
   }
 }

services/backend/src/services/faculty/facultyUpdates.ts

@@ -18,22 +18,15 @@ import { combineFacultyThesisWriters } from './facultyThesisWriters'
 const yearOptions = ['CALENDAR_YEAR', 'ACADEMIC_YEAR'] as const
 const specialGroupOptions = ['SPECIAL_INCLUDED', 'SPECIAL_EXCLUDED'] as const
 const programmeFilterOptions = ['ALL_PROGRAMMES', 'NEW_DEGREE_PROGRAMMES'] as const
-const graduatedOptions = ['GRADUATED_INCLUDED', 'GRADUATED_EXCLUDED'] as const
 
 export const updateFacultyOverview = async (facultyCode: string, statsType: StatsType) => {
-  const all = await getDegreeProgrammesOfFaculty(facultyCode, false)
-  const onlyNew = await getDegreeProgrammesOfFaculty(facultyCode, true)
+  for (const programmeFilter of programmeFilterOptions) {
+    const programmes = await getDegreeProgrammesOfFaculty(facultyCode, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
 
-  for (const yearType of yearOptions) {
-    for (const specialGroups of specialGroupOptions) {
-      for (const programmeFilter of programmeFilterOptions) {
+    for (const yearType of yearOptions) {
+      for (const specialGroups of specialGroupOptions) {
         if (statsType === 'ALL' || statsType === 'STUDENT') {
-          const updatedStudentInfo = await combineFacultyBasics(
-            facultyCode,
-            programmeFilter === 'NEW_DEGREE_PROGRAMMES' ? onlyNew : all,
-            yearType,
-            specialGroups
-          )
+          const updatedStudentInfo = await combineFacultyBasics(facultyCode, programmes, yearType, specialGroups)
           await setBasicStats(updatedStudentInfo, yearType, programmeFilter, specialGroups)
         }
         if ((statsType === 'ALL' || statsType === 'CREDITS') && specialGroups !== 'SPECIAL_EXCLUDED') {
@@ -43,7 +36,7 @@ export const updateFacultyOverview = async (facultyCode: string, statsType: Stat
         if (statsType === 'ALL' || statsType === 'THESIS') {
           const updateThesisWriters = await combineFacultyThesisWriters(
             facultyCode,
-            programmeFilter === 'NEW_DEGREE_PROGRAMMES' ? onlyNew : all,
+            programmes,
             yearType,
             specialGroups
           )
@@ -51,30 +44,22 @@ export const updateFacultyOverview = async (facultyCode: string, statsType: Stat
         }
       }
     }
+
+    const updatedTimes = await countGraduationTimes(facultyCode, programmes)
+    await setGraduationStats(updatedTimes, programmeFilter)
   }
 
-  const updatedTimesAll = await countGraduationTimes(facultyCode, all)
-  await setGraduationStats(updatedTimesAll, 'ALL_PROGRAMMES')
-  const updatedTimesNew = await countGraduationTimes(facultyCode, onlyNew)
-  await setGraduationStats(updatedTimesNew, 'NEW_DEGREE_PROGRAMMES')
   return 'OK'
 }
 
 export const updateFacultyProgressOverview = async (facultyCode: string) => {
   const onlyNew = await getDegreeProgrammesOfFaculty(facultyCode, true)
 
-  for (const graduated of graduatedOptions) {
-    for (const specialGroups of specialGroupOptions) {
-      const updateFacultyStudentStats = await combineFacultyStudents(facultyCode, onlyNew, specialGroups, graduated)
-      await setFacultyStudentStats(updateFacultyStudentStats, specialGroups, graduated)
-      const updateFacultyProgressStats = await combineFacultyStudentProgress(
-        facultyCode,
-        onlyNew,
-        specialGroups,
-        graduated
-      )
-      await setFacultyProgressStats(updateFacultyProgressStats, specialGroups, graduated)
-    }
+  for (const specialGroups of specialGroupOptions) {
+    const updateFacultyStudentStats = await combineFacultyStudents(facultyCode, onlyNew, specialGroups)
+    await setFacultyStudentStats(updateFacultyStudentStats, specialGroups)
+    const updateFacultyProgressStats = await combineFacultyStudentProgress(facultyCode, onlyNew, specialGroups)
+    await setFacultyProgressStats(updateFacultyProgressStats, specialGroups)
   }
 
   return 'OK'