[Docker][Dev] fix permissions with node_modules...................

We use xlsx which is distributed by their own cdn rather than npm. Npm
v12 by defaults blocks this installation.

.github/actions/build/backend/action.yaml

@@ -0,0 +1,41 @@
+name: 'Build backend'
+description: 'Build backend in staging or production version'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+
+outputs:
+  image:
+    description: 'Image name provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.image }}
+  tags:
+    description: 'Tag names provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@v6
+
+    - name: Build backend image
+      id: build-image-action
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: oodikone-backend
+        tags: ${{ inputs.environment }} ${{ github.sha }}
+        context: services
+        containerfiles: |
+          services/backend/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          BASE_PATH=${{ case(inputs.environment == 'staging', '/oodikone', '') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}

.github/actions/build/frontend/action.yaml

@@ -0,0 +1,50 @@
+name: 'Build frontend'
+description: 'Build frontend in staging or production versions'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+  sentry_dns:
+    required: false
+    description: 'Sentry url to send error logs to (only production, enforced)'
+    default: ''
+
+outputs:
+  image:
+    description: 'Image name provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.image }}
+  tags:
+    description: 'Tag names provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - name: Get current time
+      id: get_current_time
+      shell: bash
+      run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
+
+    - name: Build frontend image
+      id: build-image-action
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: oodikone-frontend
+        tags: ${{ inputs.environment }} ${{ github.sha }}
+        context: services
+        containerfiles: |
+          services/frontend/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          BASE_PATH=${{ case(inputs.environment == 'staging', '/oodikone', '') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}
+          BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+          SENTRY_DSN=${{ case(inputs.environment == 'production', inputs.sentry_dns, '') }}

.github/actions/build/scheduler/action.yaml

@@ -0,0 +1,40 @@
+name: 'Build updater scheduler'
+description: 'Build updater scheduler in staging or production version'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+
+outputs:
+  image:
+    description: 'Image name provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.image }}
+  tags:
+    description: 'Tag names provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@v6
+
+    - name: Build scheduler image
+      id: build-image-action
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: updater-scheduler
+        tags: ${{ inputs.environment }} ${{ github.sha }}
+        context: updater/sis-updater-scheduler
+        containerfiles: |
+          updater/sis-updater-scheduler/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}

.github/actions/build/worker/action.yaml

@@ -0,0 +1,40 @@
+name: 'Build updater worker'
+description: 'Build updater worker in staging or production version'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+
+outputs:
+  image:
+    description: 'Image name provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.image }}
+  tags:
+    description: 'Tag names provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@v6
+
+    - name: Build worker image
+      id: build-image-action
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: updater-worker
+        tags: ${{ inputs.environment }} ${{ github.sha }}
+        context: updater/sis-updater-worker
+        containerfiles: |
+          updater/sis-updater-worker/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}

.github/actions/test/api/action.yaml

@@ -0,0 +1,37 @@
+name: 'Run api tests'
+description: 'Runs api tests in backend'
+
+inputs:
+  username:
+    required: true
+    description: 'Toska registry username'
+  password:
+    required: true
+    description: 'Toska registry password'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Login to toska docker registry
+      uses: docker/login-action@v4
+      with:
+        registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+        username: ${{ inputs.username }}
+        password: ${{ inputs.password }}
+
+    - name: Start oodikone (without frontend and backend)
+      shell: bash
+      run: docker compose --file docker-compose.ci.yml up --quiet-pull --detach --wait
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/backend
+
+    - name: Run api-tests
+      shell: bash
+      run: npm run test:api --prefix ./services/backend

.github/actions/test/component/action.yaml

@@ -0,0 +1,31 @@
+name: 'Run component tests'
+description: 'Runs frontend component tests'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/shared && npm ci --prefix ./services/frontend
+
+    - name: Install Playwright Browsers
+      shell: bash
+      run: npm exec -- playwright install --with-deps
+      working-directory: ./services/frontend
+
+    - name: Run Playwright tests
+      shell: bash
+      run: npm run test:component --prefix ./services/frontend
+
+    - uses: actions/upload-artifact@v4
+      if: |
+        !cancelled()
+      with:
+        name: playwright-report
+        path: playwright-report/
+        retention-days: 7

.github/actions/test/e2e/action.yaml

@@ -32,7 +32,7 @@ runs:
     - name: 'Run cypress'
       uses: cypress-io/github-action@v7
       with:
-        browser: chrome
+        browser: firefox
         spec: ${{ inputs.spec }}
       env:
         CYPRESS_baseUrl: http://localhost:1337
@@ -43,8 +43,9 @@ runs:
       with:
         name: 'Cypress videos - ${{ inputs.test_name }}'
         path: ./cypress/videos/*
+        retention-days: 7
 
     - name: 'Print logs on failure'
       shell: bash
       if: failure()
-      run: docker compose --file docker-compose.ci.yml --profile oodikone logs && sleep 15
+      run: npm run logs:ci && sleep 15

.github/actions/test/services/action.yaml

@@ -0,0 +1,37 @@
+name: 'Run service tests'
+description: 'Runs service tests in backend'
+
+inputs:
+  username:
+    required: true
+    description: 'Toska registry username'
+  password:
+    required: true
+    description: 'Toska registry password'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Login to toska docker registry
+      uses: docker/login-action@v4
+      with:
+        registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+        username: ${{ inputs.username }}
+        password: ${{ inputs.password }}
+
+    - name: Start oodikone (without frontend and backend)
+      shell: bash
+      run: docker compose --file docker-compose.ci.yml up --quiet-pull --detach --wait
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/backend
+
+    - name: Run service-tests
+      shell: bash
+      run: npm run test:services --prefix ./services/backend

.github/actions/test/unit/action.yaml

@@ -0,0 +1,22 @@
+name: 'Run unit tests'
+description: 'Runs unit tests in backend and frontend'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/shared && npm ci --prefix ./services/backend && npm ci --prefix ./services/frontend
+
+    - name: Run frontend unit tests
+      shell: bash
+      run: npm run test:unit --prefix ./services/frontend
+
+    - name: Run backend unit tests
+      shell: bash
+      run: npm run test:unit --prefix ./services/backend

.github/workflows/production.yaml

@@ -5,45 +5,6 @@ on:
     types: [published]
 
 jobs:
-  release_frontend:
-    name: 'Build and release frontend'
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: 'Get current time'
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-
-      - name: Extract release version
-        id: extract_version
-        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
-
-      - name: Build frontend image
-        id: build-image
-        uses: redhat-actions/buildah-build@v2
-        with:
-          image: oodikone-frontend
-          tags: production ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/frontend/Dockerfile
-          build-args: |
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
-            SENTRY_DSN=${{ secrets.OODIKONE_FRONTEND_SENTRY_DSN }}
-
-      - name: Push to quay.io
-        uses: redhat-actions/push-to-registry@v2
-        with:
-          image: ${{ steps.build-image.outputs.image }}
-          tags: ${{ steps.build-image.outputs.tags }}
-          registry: quay.io/toska
-          username: toska+github
-          password: ${{ secrets.QUAY_IO_TOKEN }}
-
   release_backend:
     name: 'Build and release backend'
     runs-on: ubuntu-latest
@@ -55,18 +16,40 @@ jobs:
         id: extract_version
         run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
 
-      - name: Build backend image
+      - name: 'Build backend'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/backend
         with:
-          image: oodikone-backend
-          tags: production ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/backend/Dockerfile
-          build-args: |
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
+          environment: 'production'
+          release_version: ${{ steps.extract_version.outputs.VERSION }}
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_frontend:
+    name: 'Build and release frontend'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Extract release version
+        id: extract_version
+        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+
+      - name: 'Build frontend'
+        id: build-image
+        uses: ./.github/actions/build/frontend
+        with:
+          environment: 'production'
+          release_version: ${{ steps.extract_version.outputs.VERSION }}
+          sentry_dns: ${{ secrets.OODIKONE_FRONTEND_SENTRY_DSN }} # Used only in production
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2

.github/workflows/pull_request.yaml

@@ -5,11 +5,16 @@ on:
     branches:
       - master
     paths:
+      - '.github/actions/**'
+      - '.github/workflows/pull_request.yaml'
+      - '.github/workflows/test.yaml'
+      - 'services/**'
+      - 'updater/**'
+      - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
+      - 'package*.json'
       - 'cypress/**'
       - 'cypress.config.json'
-      - 'services/**'
-      - 'docker-compose.ci.yml'
-      - 'package*.json'
 
 jobs:
   lint_packages:
@@ -22,36 +27,27 @@ jobs:
       - name: Lint backend, frontend and shared
         uses: ./.github/actions/lint
 
+  test_unit:
+    name: 'Run unit tests'
+    runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
+    needs:
+      - lint_packages
+
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/test/unit
+
   test_api:
     name: 'Test backend api'
     runs-on: ubuntu-latest
     if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
     needs:
       - lint_packages
-    strategy:
-      fail-fast: false
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-
-      - name: Install node-24 for cypress
-        uses: actions/setup-node@v6
-        with:
-          node-version: 24
-
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v4
-        with:
-          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-
-      - name: Start oodikone detached
-        run: npm run oodikone:ci:setup
-
-      - name: Run api-tests
-        run: npm run oodikone:ci:api
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/test/api
 
   test_end_to_end:
     name: ${{ matrix.name }}
@@ -91,11 +87,13 @@ jobs:
             spec: cypress/e2e/Study_guidance_groups.js
           - name: 'Test university'
             spec: cypress/e2e/University.js
+          - name: 'Test numbers match'
+            spec: cypress/e2e/Numbers_match.js
 
     steps:
       - uses: actions/checkout@v6
       - name: 'Run tests'
-        uses: ./.github/actions/run-tests
+        uses: ./.github/actions/test/e2e
         with:
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}

.github/workflows/staging.yaml

@@ -8,13 +8,13 @@ on:
       - '.github/actions/**'
       - '.github/workflows/staging.yaml'
       - '.github/workflows/test.yaml'
-      - 'cypress/**'
-      - 'cypress.config.json'
       - 'services/**'
-      - 'updater/sis-updater-worker/src/db/migrations/**'
+      - 'updater/**'
       - 'docker-compose.ci.yml'
       - 'docker-compose.test.yml'
       - 'package*.json'
+      - 'cypress/**'
+      - 'cypress.config.json'
 
 jobs:
   test:
@@ -23,31 +23,20 @@ jobs:
       username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
       password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
 
-  release_frontend:
-    name: 'Build and release frontend'
+  release_backend:
+    name: 'Build and release backend'
     runs-on: ubuntu-latest
     needs: test
 
     steps:
       - uses: actions/checkout@v6
 
-      - name: Get current time
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-
-      - name: Build frontend image
+      - name: 'Build backend'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/backend
         with:
-          image: oodikone-frontend
-          tags: staging ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/frontend/Dockerfile
-          build-args: |
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-            BASE_PATH=/oodikone
-            STAGING=true
+          environment: 'staging'
+          release_version: ${{ github.sha }}
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2
@@ -58,27 +47,68 @@ jobs:
           username: toska+github
           password: ${{ secrets.QUAY_IO_TOKEN }}
 
-  release_backend:
-    name: 'Build and release backend'
+  release_frontend:
+    name: 'Build and release frontend'
     runs-on: ubuntu-latest
     needs: test
 
     steps:
       - uses: actions/checkout@v6
 
-      - name: Build backend image
+      - name: 'Build frontend'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/frontend
         with:
-          image: oodikone-backend
-          tags: staging ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/backend/Dockerfile
-          build-args: |
-            BASE_PATH=/oodikone
-            GIT_SHA=${{ github.sha }}
-            STAGING=true
+          environment: 'staging'
+          release_version: ${{ github.sha }}
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_updater_worker:
+    name: 'Build and release updater worker'
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: 'Build worker'
+        id: build-image
+        uses: ./.github/actions/build/worker
+        with:
+          environment: 'staging'
+          release_version: ${{ github.sha }}
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_updater_scheduler:
+    name: 'Build and release updater scheduler'
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: 'Build scheduler'
+        id: build-image
+        uses: ./.github/actions/build/scheduler
+        with:
+          environment: 'staging'
+          release_version: ${{ github.sha }}
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2
@@ -96,6 +126,8 @@ jobs:
     needs:
       - release_frontend
       - release_backend
+      - release_updater_worker
+      - release_updater_scheduler
 
     steps:
       - uses: actions/checkout@v6
@@ -114,6 +146,8 @@ jobs:
     needs:
       - release_frontend
       - release_backend
+      - release_updater_worker
+      - release_updater_scheduler
 
     steps:
       - uses: actions/checkout@v6

.github/workflows/test.yaml

@@ -26,94 +26,22 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: docker/setup-buildx-action@v4
-      - name: 'Build image and export it'
-        uses: docker/build-push-action@v7
+      - name: Build backend
+        uses: ./.github/actions/build/backend
         with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          context: services
-          file: services/backend/Dockerfile
-          push: false
-          tags: oodikone-backend:staging
-          outputs: type=docker,dest=/tmp/backend.tar
-        env:
-          DOCKER_BUILD_SUMMARY: false
-
-      - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v7
-        with:
-          if-no-files-found: error
-          name: backend-image
-          path: /tmp/backend.tar
+          environment: 'staging'
+          release_version: ${{ github.sha }}
 
   build_frontend:
     name: 'Build frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: docker/setup-buildx-action@v4
-
-      - name: Get current time
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-
-      - name: Build frontend image
-        uses: docker/build-push-action@v7
+      - name: Build frontend
+        uses: ./.github/actions/build/frontend
         with:
-          build-args: |
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-            BASE_PATH=/oodikone
-            STAGING=true
-          context: services
-          file: services/frontend/Dockerfile
-          push: false
-          tags: oodikone-frontend:staging
-          outputs: type=docker,dest=/tmp/frontend.tar
-        env:
-          DOCKER_BUILD_SUMMARY: false
-
-      # - name: 'Upload image for other jobs'
-      #   uses: actions/upload-artifact@v7
-      #   with:
-      #     if-no-files-found: error
-      #     name: frontend-image
-      #     path: /tmp/frontend.tar
-
-  pull_docker_images:
-    name: 'Pull all needed docker images and cache them'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v6
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v4
-        with:
-          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-
-      - name: 'Pull required images'
-        run: docker compose --file docker-compose.ci.yml --profile oodikone --profile updater pull --ignore-buildable --quiet
-
-      - name: 'Save Docker images for downstream jobs'
-        run: |
-          docker save \
-            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db \
-            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db \
-            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db \
-            redis:7.2.6 \
-            postgres:15.10 \
-            "quay.io/toska/jami:production" \
-            nginx:latest \
-          | zstd -T0 -3 -o /tmp/docker-images.tar.zst
-
-      - name: 'Upload Docker images artifact'
-        uses: actions/upload-artifact@v7
-        with:
-          name: docker-images
-          path: /tmp/docker-images.tar.zst
-          retention-days: 1
+          environment: 'staging'
+          release_version: ${{ github.sha }}
 
   # === Update databases ===
   check_migration_files:
@@ -128,7 +56,7 @@ jobs:
       - uses: actions/checkout@v6
       - name: Check if new migrations have been added
         id: check_migrations
-        uses: dorny/paths-filter@v3
+        uses: dorny/paths-filter@v4
         with:
           filters: |
             kone_db_migrations:
@@ -144,10 +72,14 @@ jobs:
   build_and_push_anon_sis_db:
     name: 'Build anon sis-db and push to toska hub if new migrations have been added'
     runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.sis_db_migrations == 'true'
+    if: |
+      !cancelled() && !failure() &&
+      needs.check_migration_files.outputs.sis_db_migrations == 'true'
     needs:
-      - pull_docker_images
       - check_migration_files
+      - build_backend
+      - build_frontend
+      - lint_packages
 
     steps:
       - uses: actions/checkout@v6
@@ -158,15 +90,6 @@ jobs:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-
       - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
         run: |
           docker compose --file docker-compose.ci.yml up --detach --quiet-pull sis-updater-worker
@@ -176,31 +99,18 @@ jobs:
   build_and_push_anon_kone_db:
     name: 'Build anon kone-db and push to toska hub if new migrations have been added'
     runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.kone_db_migrations == 'true'
+    if: |
+      !cancelled() && !failure() &&
+      needs.check_migration_files.outputs.kone_db_migrations == 'true'
     needs:
-      - pull_docker_images
+      - check_migration_files
       - build_backend
       - build_frontend
-      - check_migration_files
+      - lint_packages
 
     steps:
       - uses: actions/checkout@v6
 
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-image
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
-
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v4
         with:
@@ -210,43 +120,24 @@ jobs:
 
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         run: |
-          docker compose --file docker-compose.ci.yml up --detach backend
+          docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
           sleep 30 # wait for backend to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
 
   build_and_push_anon_user_db:
     name: 'Build anon user-db and push to toska hub if new migrations have been added'
     runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.user_db_migrations == 'true'
+    if: |
+      !cancelled() && !failure() &&
+      needs.check_migration_files.outputs.user_db_migrations == 'true'
     needs:
-      - pull_docker_images
+      - check_migration_files
       - build_backend
       - build_frontend
-      - check_migration_files
+      - lint_packages
+
     steps:
       - uses: actions/checkout@v6
-      - uses: dorny/paths-filter@v3
-        id: check_migrations
-        with:
-          filters: |
-            migrations:
-              - 'services/backend/src/database/migrations_user/**'
-              - '.github/workflows/test.yaml'
-
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-image
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
 
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v4
@@ -257,41 +148,49 @@ jobs:
 
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         run: |
-          docker compose --file docker-compose.ci.yml up --detach backend
+          docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
           sleep 30 # wait for backend to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
 
-  # === Backend api-tests ===
-  test_api:
-    name: 'Run backend api tests'
+  # === Update sis-db with updater ===
+  check_updater_changed:
+    name: 'Check if updater has been updated'
     runs-on: ubuntu-latest
-    if: always() && !cancelled() && needs.lint_packages.result == 'success'
+    if: |
+      !cancelled() && !failure()
+    outputs:
+      has_updater_changed: ${{ steps.check_updater_changed.outputs.has_updater_changed }}
+
+    steps:
+      - uses: actions/checkout@v6
+      - uses: dorny/paths-filter@v4
+        id: check_updater_changed
+        with:
+          filters: |
+            has_updater_changed:
+              - 'updater/**'
+              - '.github/workflows/test.yaml'
+              - '.github/workflows/staging.yaml'
+              - 'docker-compose.ci.yml'
+
+  run_updater_updates:
+    name: 'Run updater tests and updates'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure() &&
+      (needs.check_updater_changed.outputs.has_updater_changed == 'true' ||
+      needs.check_migration_files.outputs.kone_db_migrations == 'true' ||
+      needs.check_migration_files.outputs.user_db_migrations == 'true' ||
+      needs.check_migration_files.outputs.sis_db_migrations == 'true')
     needs:
-      - pull_docker_images
+      - check_migration_files
+      - check_updater_changed
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
-      - lint_packages
-    strategy:
-      fail-fast: false
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-
-      - name: Install node-24
-        uses: actions/setup-node@v6
-        with:
-          node-version: 24
-
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
+      - uses: actions/checkout@v6
 
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v4
@@ -300,91 +199,103 @@ jobs:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: Start oodikone (without frontend and backend)
-        run: npm run oodikone:ci:setup -- --quiet-pull
+      - name: Start worker containers
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-worker --quiet-pull --detach --wait
 
-      - name: Run api-tests
-        run: npm run oodikone:ci:api
+      - name: Run hourly jobs
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
+        env:
+          SCHEDULE_IMMEDIATE: hourly
+          EXIT_AFTER_IMMEDIATES: yes
 
-  # === Frontend unit testing ===
-  test_unit_frontend:
-    name: 'Run frontend unit tests'
+      - name: Run weekly jobs
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
+        env:
+          SCHEDULE_IMMEDIATE: weekly
+          EXIT_AFTER_IMMEDIATES: yes
+
+      # TODO: Push the updated sis-db to toska-hub when updater is deterministic with test-data
+      # - name: Push newly build db to toska hub
+      #   if: success()
+      #   run: ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
+
+      - name: Dump logs on failure
+        if: failure()
+        run: npm run logs:ci
+
+  # === Backend api-tests ===
+  test_api:
+    name: 'Run backend api tests'
     runs-on: ubuntu-latest
-    if: always() && !cancelled() && needs.lint_packages.result == 'success'
+    if: |
+      !cancelled() && !failure()
     needs:
-      - pull_docker_images
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-      - lint_packages
+      - run_updater_updates
+    strategy:
+      fail-fast: false
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-
-      - name: Install node-24
-        uses: actions/setup-node@v6
+      - uses: actions/checkout@v6
+      - name: Run api tests
+        uses: ./.github/actions/test/api
         with:
-          node-version: 24
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+  #
+  # === Backend service-tests ===
+  test_services:
+    name: 'Run backend service-tests'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - run_updater_updates
+    strategy:
+      fail-fast: false
 
-      - name: Install dependencies
-        run: npm ci
-        working-directory: ./services/frontend
+    steps:
+      - uses: actions/checkout@v6
+      - name: Run service-tests
+        uses: ./.github/actions/test/services
+        with:
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
 
-      - name: Run unit tests
-        run: npm run test:unit
-        working-directory: ./services/frontend
+  # === Unit testing ===
+  test_unit:
+    name: 'Run unit tests'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - run_updater_updates
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Run unit tests (frontend and backend)
+        uses: ./.github/actions/test/unit
 
   # === Frontend component testing ===
   test_components:
     name: 'Test frontend components'
     runs-on: ubuntu-latest
-    if: always() && !cancelled() && needs.lint_packages.result == 'success'
+    if: |
+      !cancelled() && !failure()
     needs:
-      - pull_docker_images
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-      - lint_packages
+      - run_updater_updates
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-
-      - name: Install node-24
-        uses: actions/setup-node@v6
-        with:
-          node-version: 24
-
-      - name: Install dependencies
-        run: npm ci --prefix shared && npm ci --prefix frontend
-        working-directory: ./services
-
-      - name: Install Playwright Browsers
-        run: npm exec -- playwright install --with-deps
-        working-directory: ./services/frontend
-
-      - name: Run Playwright tests
-        run: npm run test:component
-        working-directory: ./services/frontend
-
-      - uses: actions/upload-artifact@v4
-        if: ${{ !cancelled() }}
-        with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30
+      - uses: actions/checkout@v6
+      - name: Run frontend component tests
+        uses: ./.github/actions/test/component
 
   # === End-to-end tests ===
   test_end_to_end:
     runs-on: ubuntu-latest
-    if: always() && !cancelled() && needs.lint_packages.result == 'success'
+    if: |
+      !cancelled() && !failure()
     needs:
-      - pull_docker_images
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-      - lint_packages
+      - run_updater_updates
     strategy:
       fail-fast: false
       matrix:
@@ -415,6 +326,10 @@ jobs:
             spec: cypress/e2e/Study_guidance_groups.js
           - name: 'Test university'
             spec: cypress/e2e/University.js
+          - name: 'Test numbers match'
+            spec: cypress/e2e/Numbers_match.js
+          - name: 'Test access rights'
+            spec: cypress/e2e/Routes.js
     name: ${{ matrix.name }}
     steps:
       - name: Checkout
@@ -425,29 +340,8 @@ jobs:
         with:
           node-version: 24
 
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-image
-      # - name: 'Download frontend image'
-      #   uses: actions/download-artifact@v8
-      #   with:
-      #     name: frontend-image
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
-      # - name: 'Load built frontend image'
-      #   run: docker load --input frontend.tar
-
       - name: 'Run tests'
-        uses: ./.github/actions/run-tests
+        uses: ./.github/actions/test/e2e
         with:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
@@ -458,13 +352,10 @@ jobs:
   test_service_exit_statuses:
     name: 'Check services run correctly'
     runs-on: ubuntu-latest
-    if: always() && !cancelled() && needs.lint_packages.result == 'success'
+    if: |
+      !cancelled() && !failure()
     needs:
-      - pull_docker_images
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-      - lint_packages
+      - run_updater_updates
 
     steps:
       - uses: actions/checkout@v6
@@ -475,29 +366,8 @@ jobs:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-image
-      # - name: 'Download frontend image'
-      #   uses: actions/download-artifact@v8
-      #   with:
-      #     name: frontend-image
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
-      # - name: 'Load built frontend image'
-      #   run: docker load --input frontend.tar
-
       - name: 'Run services, build rest of the needed images'
-        run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull
+        run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull --wait
 
       - name: 'Check that exit statuses are not errors'
         run: |
@@ -509,4 +379,4 @@ jobs:
 
       - name: 'Print logs on failure'
         if: failure()
-        run: docker compose --file docker-compose.ci.yml --profile oodikone logs && sleep 15
+        run: npm run logs:ci && sleep 15

.github/workflows/updater.yaml

@@ -5,8 +5,7 @@ on:
     types: [published]
     paths:
       - '.github/workflows/updater.yaml'
-      - 'updater/sis-updater-worker/**'
-      - 'updater/sis-updater-scheduler/**'
+      - 'updater/**'
 
 jobs:
   lint_scheduler_and_worker:
@@ -18,9 +17,6 @@ jobs:
         with:
           node-version: 24
 
-      - name: Install safe-chain
-        run: set -o pipefail && curl -fsSL https://github.com/AikidoSec/safe-chain/releases/latest/download/install-safe-chain.sh | sh -s -- --ci
-
       - name: 'Install dependencies'
         run: npm ci && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker
       - name: 'Run linter'
@@ -29,6 +25,7 @@ jobs:
   build_worker:
     name: 'Build Updater worker'
     runs-on: ubuntu-latest
+
     steps:
       - uses: actions/checkout@v6
       - uses: docker/setup-buildx-action@v4
@@ -41,15 +38,18 @@ jobs:
           tags: toska/sis-updater-worker:latest
         env:
           DOCKER_BUILD_SUMMARY: false
+
       - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v7
         with:
           name: updater-worker-image
           path: /tmp/worker.tar
+          retention-days: 7
 
   build_scheduler:
     name: 'Build Updater scheduler'
     runs-on: ubuntu-latest
+
     steps:
       - uses: actions/checkout@v6
       - uses: docker/setup-buildx-action@v4
@@ -62,11 +62,13 @@ jobs:
           tags: toska/sis-updater-scheduler:latest
         env:
           DOCKER_BUILD_SUMMARY: false
+
       - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v7
         with:
           name: updater-scheduler-image
           path: /tmp/scheduler.tar
+          retention-days: 7
 
   test_updater:
     name: 'Test Updater'
@@ -75,6 +77,7 @@ jobs:
       - lint_scheduler_and_worker
       - build_worker
       - build_scheduler
+
     steps:
       - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
@@ -83,34 +86,41 @@ jobs:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+
       - name: 'Download built images'
         uses: actions/download-artifact@v8
         with:
           pattern: 'updater-{worker,scheduler}-image'
           merge-multiple: true
+
       - name: 'Load worker image'
         run: docker load --input worker.tar
       - name: 'Load scheduler image'
         run: docker load --input scheduler.tar
+
       - name: 'Start worker'
-        run: docker compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
+        run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull --no-recreate --wait sis-updater-worker
+
       - name: 'Run hourly jobs'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: hourly
           EXIT_AFTER_IMMEDIATES: yes
+
       - name: 'Run weekly jobs'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: weekly
           EXIT_AFTER_IMMEDIATES: yes
+
       - name: 'Run prepurge'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: prepurge
           EXIT_AFTER_IMMEDIATES: yes
+
       - name: 'Run purge'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: purge
           EXIT_AFTER_IMMEDIATES: yes
@@ -122,18 +132,12 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - name: Build worker image
+      - name: 'Build updater worker'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/worker
         with:
-          image: updater-worker
-          tags: staging production ${{ github.sha }}
-          context: updater/sis-updater-worker
-          containerfiles: |
-            updater/sis-updater-worker/Dockerfile
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=production
+          environment: 'production'
+          release_version: ${{ github.sha }}
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2
@@ -151,18 +155,12 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - name: Build scheduler image
+      - name: 'Build udpater scheduler'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/scheduler
         with:
-          image: updater-scheduler
-          tags: staging production ${{ github.sha }}
-          context: updater/sis-updater-scheduler
-          containerfiles: |
-            updater/sis-updater-scheduler/Dockerfile
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=production
+          environment: 'production'
+          release_version: ${{ github.sha }}
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2

.gitignore

@@ -56,7 +56,7 @@ cypress/videos/
 coverage/
 instrumented/
 
-# Pycharm 
+# Pycharm
 .idea
 
 # VS Code
@@ -68,3 +68,5 @@ instrumented/
 # Target of shared symlink
 services/**/src/shared
 
+
+!services/backend/.env.test

README.md

@@ -123,6 +123,8 @@ Some useful commands are defined in `package.json` and can be run with `npm run
 
 - `npm run oodikone`: starts Oodikone with anonymized data
 - `npm run oodikone:real`: starts Oodikone with real data
+- `npm run oodikone:test`: starts Oodikone with non-persistent database (wipes after every reset)
+- `npm run oodikone:ci`: starts Oodikone in CI mode
 - `npm run updater`: starts updater with anonymized data
 - `npm run updater:real`: starts updater with real data
 - `npm run both`: starts Oodikone and updater with anonymized data
@@ -191,7 +193,7 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 
 ### Modules are missing after updating package.json
 
-You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container or `npm run install:docker` to install modules inside all packages (backend, frontend, sis-updater-scheduler and sis-updater-worker).
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Prefer `npm run ci:docker` or `npm run install:docker`, which also fix `node_modules` ownership for the non-root runtime user. If you run installs manually, use root inside the container and restore ownership afterwards (`chown -R 1001:0 node_modules`) to avoid permission issues.
 
 ### Study guidance groups don't work on my machine
 

anonyymioodi/docker-compose.yml

@@ -1,6 +1,11 @@
 x-defaults: &defaults
   restart: unless-stopped
 
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
 services:
   kone-db:
     <<: *defaults
@@ -10,6 +15,21 @@ services:
       POSTGRES_DB: kone-db
       POSTGRES_HOST_AUTH_METHOD: trust
     image: postgres:15.10
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
+
+  user-db:
+    <<: *defaults
+    container_name: oodikone-user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: postgres:15.10
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-db:
     <<: *defaults
@@ -20,6 +40,9 @@ services:
       POSTGRES_HOST_AUTH_METHOD: trust
     image: postgres:15.10
     shm_size: 2gb
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-importer-db:
     <<: *defaults
@@ -29,12 +52,6 @@ services:
       POSTGRES_DB: sis-importer-db
       POSTGRES_HOST_AUTH_METHOD: trust
     image: postgres:15.10
-
-  user-db:
-    <<: *defaults
-    container_name: oodikone-user-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: user-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults

cli.sh

@@ -49,7 +49,7 @@ set_custom_select_prompt() {
 
   options=(
     "Set up Oodikone from scratch"
-    "Reset all real data"
+    "Reset all data"
     "Reset single database"
     "Restore data from dumps"
     "Docker system prune"
@@ -68,8 +68,8 @@ while true; do
     case $opt in
       "Set up Oodikone from scratch")
         set_up_oodikone_from_scratch;;
-      "Reset all real data")
-        reset_all_real_data;;
+      "Reset all data")
+        reset_all_data;;
       "Reset single database")
         reset_single_database
         set_custom_select_prompt # Set prompt back to initial values

cypress/e2e/Course_statistics.js

@@ -468,10 +468,10 @@ describe('Basic user', () => {
       cy.cs('course-population-for-2019-2020').click()
 
       cy.contains('Tietorakenteet ja algoritmit')
-      cy.contains('Class of 2019-2020, 33 students')
+      cy.contains('Class of 2019-2020, 32 students')
       cy.contains('Include substitutions, Open and normal')
 
-      cy.contains('Students (33)').click()
+      cy.contains('Students (32)').click()
       cy.contains('td', '394776').siblings().eq(2).contains('3')
       cy.contains('td', '497388').siblings().eq(2).contains('2')
     })
@@ -486,10 +486,10 @@ describe('Basic user', () => {
       cy.cs('course-population-for-2019-2020').click()
 
       cy.contains('Tietorakenteet ja algoritmit')
-      cy.contains('Class of 2019-2020, 33 students')
+      cy.contains('Class of 2019-2020, 32 students')
       cy.contains('Include substitutions, Open and normal')
 
-      cy.contains('Students (33)').click()
+      cy.contains('Students (32)').click()
       cy.get('table tbody td').filter(':contains("Hidden")').should('have.length', 9)
     })
   })

cypress/e2e/Degree_programme_overview.js

@@ -46,7 +46,7 @@ describe('Degree programme overview', () => {
     beforeEach(() => {
       cy.init('/study-programme')
       cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click({ force: true })
-      cy.cs('year-toggle').click()
+      cy.cs('year-toggle').click() // NOTE: Tests are written for calendar years
     })
 
     // If the backend breaks for one of the sections, the section header is not rendered and this will fail
@@ -64,11 +64,11 @@ describe('Degree programme overview', () => {
         // [Year, Started studying, Accepted, Graduated, Transferred Away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
         [2023, 8, 8, 26, 0, 0],
-        [2022, 25, 26, 48, 1, 4],
-        [2021, 29, 32, 47, 0, 2],
+        [2022, 25, 26, 47, 1, 3],
+        [2021, 29, 32, 48, 0, 2],
         [2020, 26, 27, 12, 1, 3],
         [2019, 28, 34, 1, 0, 1],
-        [2018, 40, 45, 0, 0, 0],
+        [2018, 40, 45, 0, 0, 1],
         [2017, 41, 47, 0, 0, 0],
       ]
 
@@ -97,8 +97,8 @@ describe('Degree programme overview', () => {
         // [Year, Graduated, Wrote thesis]
         ...years.map(year => [year, 0, 0]),
         [2023, 26, 5],
-        [2022, 48, 13],
-        [2021, 47, 76],
+        [2022, 47, 13],
+        [2021, 48, 76],
         [2020, 12, 23],
         [2019, 1, 19],
         [2018, 0, 0],
@@ -115,8 +115,8 @@ describe('Degree programme overview', () => {
         // [Year, Started studying, Accepted, Graduated]
         ...years.map(year => [year, 0, 0, 0]),
         [2023, 8, 8, 24],
-        [2022, 25, 26, 44],
-        [2021, 29, 32, 46],
+        [2022, 25, 26, 43],
+        [2021, 29, 32, 47],
         [2020, 26, 27, 11],
         [2019, 28, 34, 1],
         [2018, 40, 45, 0],
@@ -129,8 +129,8 @@ describe('Degree programme overview', () => {
         // [Year, Graduated, Wrote thesis]
         ...years.map(year => [year, 0, 0]),
         [2023, 24, 5],
-        [2022, 44, 12],
-        [2021, 46, 69],
+        [2022, 43, 12],
+        [2021, 47, 69],
         [2020, 11, 22],
         [2019, 1, 18],
         [2018, 0, 0],
@@ -148,11 +148,11 @@ describe('Degree programme overview', () => {
         // [Year, Started studying, Accepted, Graduated, Transferred away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
         ['2023 - 2024', 8, 8, 4, 0, 0],
-        ['2022 - 2023', 25, 26, 36, 0, 0],
-        ['2021 - 2022', 29, 32, 57, 1, 6],
+        ['2022 - 2023', 25, 26, 35, 0, 0],
+        ['2021 - 2022', 29, 32, 58, 1, 5],
         ['2020 - 2021', 26, 27, 30, 0, 3],
         ['2019 - 2020', 28, 34, 7, 1, 1],
-        ['2018 - 2019', 40, 45, 0, 0, 0],
+        ['2018 - 2019', 40, 45, 0, 0, 1],
         ['2017 - 2018', 41, 47, 0, 0, 0],
       ]
       cy.checkTableStats(studentTableContents, 'students-of-the-study-programme')
@@ -192,19 +192,20 @@ describe('Degree programme overview', () => {
         .should('contain', 47)
         .should('contain', 76)
 
-      cy.cs('unset-breakdown-bar-chart')
-      cy.cs('graduation-time-toggle').click()
+      cy.cs('graduation-mode-selector').within(() => {
+        cy.cs('select-median').click()
+      })
       cy.cs('unset-median-bar-chart').within(() => {
-        cy.contains('48 graduated').should('be.visible')
-        cy.contains('48 graduated').hover()
+        cy.contains('47 graduated').should('be.visible')
+        cy.contains('47 graduated').hover()
       })
 
-      cy.get('.grad-vals').contains('48 students graduated in year 2022').should('be.visible')
+      cy.get('.grad-vals').contains('47 students graduated in year 2022').should('be.visible')
 
-      cy.contains('median study time: 6.5 semesters')
+      cy.contains('median study time: 6 semesters')
       cy.contains('24 graduated on time')
       cy.contains('12 graduated max year overtime')
-      cy.contains('12 graduated over year late')
+      cy.contains('11 graduated over year late')
 
       cy.cs('programmes-before-or-after-stacked-bar-chart-section')
         .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
@@ -217,15 +218,16 @@ describe('Degree programme overview', () => {
   })
 
   describe('Graduation times of master programmes', () => {
-    it('are split into two graphs', () => {
+    it('are split into three graphs', () => {
       cy.init('/study-programme')
       cy.contains('a', 'Matematiikan ja tilastotieteen maisteriohjelma').click({ force: true })
       cy.cs('year-toggle').click() // Tests are written for calendar year
 
-      cy.cs('unset-breakdown-bar-chart')
       cy.cs('unset-breakdown-bar-chart')
 
-      cy.cs('graduation-time-toggle').click()
+      cy.cs('graduation-mode-selector').within(() => {
+        cy.cs('select-average').click()
+      })
       cy.cs('unset-graduation-times-section')
         .eq(1)
         .within(() => {
@@ -235,7 +237,7 @@ describe('Degree programme overview', () => {
         })
 
       cy.get('.grad-vals').contains('2 students graduated in year 2021').should('be.visible')
-      cy.contains('median study time: 4.5 semesters')
+      cy.contains('average study time: 4.5 semesters')
       cy.contains('1 graduated on time')
       cy.contains('1 graduated max year overtime')
       cy.contains('0 graduated over year late')
@@ -249,7 +251,7 @@ describe('Degree programme overview', () => {
         })
 
       cy.get('.grad-vals').contains('11 students graduated in year 2023').should('be.visible')
-      cy.contains('median study time: 12 semesters')
+      cy.contains('average study time: 11.18 semesters')
       cy.contains('4 graduated on time')
       cy.contains('6 graduated max year overtime')
       cy.contains('1 graduated over year late')
@@ -293,15 +295,15 @@ describe('Degree programme overview', () => {
 
     it('Students of the degree programme are shown correctly', () => {
       const tableContents = [
-        // [Year, All, Started studying, Present, Absent, Passive, Graduated, Men, Women, Other/Unknown, Finland, Other]
-        ['2023 - 2024', 8, 8, 0, 0, 8, 0, 5, 3, 0, 8, 1],
-        ['2022 - 2023', 26, 25, 0, 0, 24, 2, 19, 7, 0, 25, 1],
-        ['2021 - 2022', 38, 29, 0, 0, 33, 5, 29, 9, 0, 36, 4],
-        ['2020 - 2021', 30, 26, 0, 0, 11, 19, 15, 15, 0, 29, 3],
-        ['2019 - 2020', 35, 28, 0, 0, 8, 27, 22, 13, 0, 35, 2],
-        ['2018 - 2019', 45, 40, 0, 0, 6, 39, 26, 19, 0, 44, 1],
-        ['2017 - 2018', 47, 41, 0, 0, 5, 42, 31, 16, 0, 47, 0],
-        ['Total', 229, 197, 0, 0, 95, 134, 147, 82, 0, 224, 12],
+        // [Year, All, Started studying, Present, Absent, Passive, Graduated, Has recent attainments, Men, Women, Other/Unknown, Finland, Other]
+        ['2023 - 2024', 8, 8, 0, 0, 8, 0, 0, 5, 3, 0, 8, 1],
+        ['2022 - 2023', 26, 25, 0, 0, 24, 2, 0, 19, 7, 0, 25, 1],
+        ['2021 - 2022', 37, 29, 0, 0, 33, 4, 0, 29, 8, 0, 35, 4],
+        ['2020 - 2021', 30, 26, 0, 0, 11, 19, 0, 15, 15, 0, 29, 3],
+        ['2019 - 2020', 35, 28, 0, 0, 8, 27, 0, 22, 13, 0, 35, 2],
+        ['2018 - 2019', 46, 40, 0, 0, 6, 40, 0, 26, 20, 0, 45, 1],
+        ['2017 - 2018', 47, 41, 0, 0, 5, 42, 0, 31, 16, 0, 47, 0],
+        ['Total', 229, 197, 0, 0, 95, 134, 0, 147, 82, 0, 224, 12],
       ]
 
       cy.checkTableStats(tableContents, 'study-tracks-and-class-statistics')
@@ -376,10 +378,10 @@ describe('Degree programme overview', () => {
         ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
         ['2023 - 2024', 8, 8, 0, 0, 0, 0, 0, 0, 0],
         ['2022 - 2023', 26, 9, 8, 4, 3, 0, 0, 0, 2],
-        ['2021 - 2022', 38, 8, 4, 9, 8, 4, 0, 0, 5],
+        ['2021 - 2022', 37, 8, 4, 9, 8, 4, 0, 0, 4],
         ['2020 - 2021', 30, 2, 1, 2, 5, 0, 0, 1, 19],
         ['2019 - 2020', 35, 1, 0, 2, 1, 0, 0, 4, 27],
-        ['2018 - 2019', 45, 0, 1, 1, 2, 0, 0, 2, 39],
+        ['2018 - 2019', 46, 0, 1, 1, 2, 0, 0, 2, 40],
         ['2017 - 2018', 47, 0, 1, 3, 0, 0, 1, 0, 42],
         ['Total', 229, 28, 15, 21, 19, 4, 1, 7, 134],
       ]
@@ -452,14 +454,14 @@ describe('Degree programme overview', () => {
       it('Students of the study track are shown correctly', () => {
         cy.cs('study-track-overview-section').contains('Students of the study track MAT-MAT by starting year')
         const tableContents = [
-          // [Year, All, Started studying, Present, Absent, Passive, Graduated, Men, Women, Other/Unknown, Finland, Other]
-          ['2022 - 2023', 3, 3, 0, 0, 1, 2, 2, 1, 0, 3, 0],
-          ['2021 - 2022', 5, 1, 0, 0, 1, 4, 3, 2, 0, 5, 0],
-          ['2020 - 2021', 13, 10, 0, 0, 1, 12, 7, 6, 0, 12, 3],
-          ['2019 - 2020', 17, 14, 0, 0, 0, 17, 10, 7, 0, 17, 1],
-          ['2018 - 2019', 23, 21, 0, 0, 2, 21, 11, 12, 0, 23, 0],
-          ['2017 - 2018', 28, 24, 0, 0, 1, 27, 15, 13, 0, 28, 0],
-          ['Total', 89, 73, 0, 0, 6, 83, 48, 41, 0, 88, 4],
+          // [Year, All, Started studying, Present, Absent, Passive, Graduated, Has recent attainment, Men, Women, Other/Unknown, Finland, Other]
+          ['2022 - 2023', 3, 3, 0, 0, 1, 2, 0, 2, 1, 0, 3, 0],
+          ['2021 - 2022', 4, 1, 0, 0, 1, 3, 0, 3, 1, 0, 4, 0],
+          ['2020 - 2021', 13, 10, 0, 0, 1, 12, 0, 7, 6, 0, 12, 3],
+          ['2019 - 2020', 17, 14, 0, 0, 0, 17, 0, 10, 7, 0, 17, 1],
+          ['2018 - 2019', 24, 21, 0, 0, 2, 22, 0, 11, 13, 0, 24, 0],
+          ['2017 - 2018', 28, 24, 0, 0, 1, 27, 0, 15, 13, 0, 28, 0],
+          ['Total', 89, 73, 0, 0, 6, 83, 0, 48, 41, 0, 88, 4],
         ]
         cy.checkTableStats(tableContents, 'study-tracks-and-class-statistics')
       })
@@ -705,7 +707,7 @@ describe('Degree programme overview', () => {
               cy.get('td').eq(0).contains('Course')
               cy.get('td').eq(1).contains('MAT11001')
               cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
-              cy.get('td').eq(3).contains('273')
+              cy.get('td').eq(3).contains('272')
             })
         })
       })

cypress/e2e/Filters.js

@@ -19,7 +19,7 @@ describe("Population statistics with a master's programme", { testIsolation: fal
 
   it('Study track filter works', () => {
     cy.cs('studyTrackFilter-header').click()
-    cy.selectFromDropdown('studyTrackFilter', 0, true)
+    cy.selectFromDropdown('studyTrackFilter', 'Matematiikka ja soveltava matematiikka (MAST-MSM)', true)
     checkFilteredStudentCount(15)
 
     cy.cs('studyTrackFilter-clear').click()

cypress/e2e/Language_center.js

@@ -44,7 +44,7 @@ describe('When language center is opened', () => {
         cy.get('table > tbody > tr:first').within(() => {
           cy.get('td').should('have.length', 15)
           cy.get('td').eq(0).contains('All courses total')
-          const numbers = [2076, 36, 9, 33, 757, 15, 40, 6, 48, 1, 24, 1059, 48, 0]
+          const numbers = [2076, 36, 9, 34, 759, 15, 40, 6, 51, 1, 24, 1059, 42, 0]
           numbers.forEach((number, index) => {
             cy.get('td')
               .eq(index + 1)
@@ -55,12 +55,12 @@ describe('When language center is opened', () => {
 
       it('Faculties tab "exceeding" button works', () => {
         cy.cs('difference-button').click()
-        checkNumbers([63, 2, 0, 3, 13, 0, 1, 1, 2, 0, 0, 30, 10, 1], 15, 'faculties')
+        checkNumbers([66, 3, 0, 4, 18, 0, 1, 1, 4, 0, 0, 31, 3, 1], 15, 'faculties')
       })
 
       it('Faculties tab semester selector changes numbers', () => {
         chooseSemester('Syksy 2020', 'from')
-        checkNumbers([1184, 28, 6, 27, 497, 4, 10, 2, 33, 0, 9, 535, 33, 0], 15, 'faculties')
+        checkNumbers([1184, 28, 6, 28, 499, 4, 10, 2, 36, 0, 9, 535, 27, 0], 15, 'faculties')
       })
     })
 

cypress/e2e/Numbers_match.js

@@ -0,0 +1,122 @@
+/// <reference types="cypress" />
+
+const selectCourseStatus = (courseCode, status) => {
+  cy.cs(`courseFilter-${courseCode}-selector`).click()
+  cy.get(`[role="listbox"][aria-labelledby=courseFilter-${courseCode}]`).within(() => {
+    cy.get('li').contains(status).click()
+  })
+}
+
+describe('Numbers should match between', () => {
+  describe('Programme courses, Course statistics and Course population (no substitutions)', () => {
+    const test = [
+      // Code, name, total students, passed, not completed, failed*, enrolled no grade* (not used by Programme courses)
+      ['Course', 'MAT11001', 'Johdatus yliopistomatematiikkaan', '272', '253', (1 + 18).toString(), '1', '18'],
+      ['Course', 'MAT11003', 'Raja-arvot', '270', '249', (1 + 20).toString(), '1', '20'],
+      ['Course', 'MAT11004', 'Differentiaalilaskenta', '262', '248', (1 + 13).toString(), '1', '13'],
+      ['Course', 'MAT21002', 'Sarjat', '256', '250', (0 + 6).toString(), '0', '6'],
+      ['Course', 'MAT11005', 'Integraalilaskenta', '251', '234', (0 + 17).toString(), '0', '17'],
+      ['Course', 'MAT11002', 'Lineaarialgebra ja matriisilaskenta I', '247', '242', (1 + 4).toString(), '1', '4'],
+      ['Course', 'MAT12003', 'Todennäköisyyslaskenta I', '234', '227', (3 + 4).toString(), '3', '4'],
+      ['Course', 'MAT21001', 'Lineaarialgebra ja matriisilaskenta II', '230', '216', (4 + 10).toString(), '4', '10'],
+      ['Course', 'MAT21003', 'Vektorianalyysi I', '228', '202', (3 + 23).toString(), '3', '23'],
+      ['Course', 'MAT20005', 'Akateemiset taidot', '201', '178', (0 + 23).toString(), '0', '23'],
+      ['Module', 'MAT110', 'Matematiikka, perusopinnot', '195', '195', (0 + 0).toString(), '0', '0'],
+      ['Course', 'MAT21014', 'Johdatus logiikkaan I', '188', '170', (0 + 18).toString(), '0', '18'],
+    ]
+
+    it('in Programme courses', () => {
+      // Move to Math bachelor programme's programme courses
+      cy.init('/study-programme')
+      cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click()
+      cy.cs('ProgrammeCoursesTab').click()
+
+      // Sort by Total students
+      cy.get('thead').within(() => {
+        cy.get('tr')
+          .eq(0)
+          .within(() => {
+            cy.get('th').eq(3).click()
+          })
+      })
+
+      cy.get('tbody').within(() => {
+        test.forEach((course, index) => {
+          cy.get('tr')
+            .eq(index)
+            .within(() => {
+              // Check a course's fields
+              course.forEach((field, fieldIndex) => {
+                if (fieldIndex <= 5) {
+                  // Failed, Enrolled no grade values not used
+                  cy.get('td').eq(fieldIndex).contains(field)
+                }
+              })
+
+              // Course link button, (name is second entry)
+              // cy.cs(`${course.at(1).toLocaleLowerCase()}-course-statistics-link-button`).click()
+            })
+        })
+      })
+    })
+
+    it('in Course statistics', () => {
+      test.forEach(course => {
+        // Move to Course statistics
+        cy.init(`/coursestatistics?courseCodes=%5B%22${course.at(1)}%22%5D&combineSubstitutions=false`, 'basic')
+        cy.url().should('include', `/coursestatistics?courseCodes=%5B%22${course.at(1)}%22%5D`)
+        cy.contains('Course statistics')
+        cy.contains(course.at(2)) // course name
+
+        cy.cs('FromYearSelector').click()
+        cy.cs('FromYearSelectorOption2017-2018').click()
+
+        cy.get('tbody').within(() => {
+          cy.get('tr')
+            .eq(0)
+            .within(() => {
+              cy.get('td').eq(1).contains(course.at(3))
+              cy.get('td').eq(2).contains(course.at(4))
+              cy.get('td').eq(3).contains(course.at(6)) // jump over combined Not completed
+              cy.get('td').eq(4).contains(course.at(7))
+            })
+        })
+      })
+    })
+
+    it('in Course population', () => {
+      test.forEach(course => {
+        // Academic years between 2017-2023
+        cy.init(
+          `/coursepopulation?from=68&to=74&coursecodes=%5B%22${course.at(1)}%22%5D&includeSubstitutions=false`,
+          'basic'
+        )
+        cy.contains(course.at(2)) // Course name
+
+        // Select course from Course-filter
+        cy.cs('courseFilter-filter-card').click()
+        cy.cs('courseFilter-selector').click()
+        cy.cs('courseFilter-popper').within(() => {
+          cy.contains('li', `${course.at(1)} - ${course.at(2)}`).click() // "code - name"
+        })
+
+        selectCourseStatus(course.at(1), 'All')
+        cy.contains(`Students (${course.at(3)})`)
+
+        selectCourseStatus(course.at(1), 'Passed')
+        cy.contains(`Students (${course.at(4)})`)
+
+        selectCourseStatus(course.at(1), 'Failed')
+        cy.contains(`Students (${course.at(6)})`)
+
+        selectCourseStatus(course.at(1), 'Enrolled, No Grade')
+        cy.contains(`Students (${course.at(7)})`)
+      })
+    })
+  })
+
+  describe('Course statistics and Course population', () => {
+    it.skip('when selecting one year')
+    it.skip('when selecting the all years')
+  })
+})

cypress/e2e/Routes.js

@@ -0,0 +1,85 @@
+import { userHeaders } from '../support/commands.js'
+
+const getBaseRoutes = [
+  ['/', {}],
+  ['/changelog', {}],
+  ['/completedcoursessearch', {}],
+  ['/university', {}],
+  ['/feedback', {}],
+  ['/populations', { roles: ['fullSisuAccess'], programmeRights: [] }],
+  ['/study-programme', { roles: ['fullSisuAccess'], programmeRights: [] }],
+  ['/study-programme/KH50_001', { roles: ['fullSisuAccess'], programmeRights: ['KH50_001'] }],
+  ['/coursepopulation', { roles: ['fullSisuAccess'], programmeRights: [] }],
+  ['/users', { roles: ['admin'] }],
+  ['/users/334', { roles: ['admin'] }],
+  ['/updater', { roles: ['admin'] }],
+  ['/coursestatistics', { roles: ['fullSisuAccess', 'courseStatistics'], programmeRights: [] }],
+  ['/faculties', { roles: ['fullSisuAccess', 'facultyStatistics'] }],
+  ['/faculties/hy-org-1000000911', { roles: ['fullSisuAccess', 'facultyStatistics'] }],
+  ['/students', { roles: ['fullSisuAccess', 'studyGuidanceGroups'], programmeRights: [] }],
+  ['/students/545400', { roles: ['fullSisuAccess', 'studyGuidanceGroups'], programmeRights: [] }],
+  ['/custompopulation', { roles: ['fullSisuAccess', 'studyGuidanceGroups'], programmeRights: [] }],
+  ['/teachers', { roles: ['teachers'] }],
+  ['/teachers/hy-hlo-1463447', { roles: ['teachers'] }],
+  ['/studyguidancegroups', { roles: ['studyGuidanceGroups'] }],
+  ['/studyguidancegroups/otm-b3b52ff5-88f6-4d52-bd18-0588023b6322', { userId: '', roles: ['studyGuidanceGroups'] }],
+  ['/close-to-graduation', { roles: ['fullSisuAccess', 'studyGuidanceGroups'] }],
+  ['/languagecenterview', { iamGroups: ['grp-kielikeskus-esihenkilot'] }],
+]
+
+for (const user of userHeaders) {
+  if (user.uid === 'admin') continue
+  describe(`${user.displayname} has access only to views that were intended`, () => {
+    let userInfo
+    before(() => {
+      cy.request({ method: 'GET', url: '/api/login', headers: user }).then(({ body }) => (userInfo = body?.user))
+    })
+
+    beforeEach(() => cy.wrap(userInfo).should('not.be.undefined'))
+    beforeEach(() => {
+      cy.intercept('**/api/**', req => {
+        req.on('response', res => {
+          expect(res.statusCode, `${req.method} ${req.url}`).to.be.lessThan(500)
+        })
+      })
+    })
+
+    for (const [route, requirements] of getBaseRoutes) {
+      it(`Checking access ${route}`, () => {
+        cy.init(route, user.uid)
+
+        const requiredUserId = !requirements.userId || requirements.userId === userInfo.userId
+
+        const requiredRoles = !requirements.roles || requirements.roles?.some(role => userInfo.roles?.includes(role))
+
+        const requiredRights =
+          !requirements.requireProgrammeRights ||
+          userInfo.roles?.includes('fullSisuAccess') ||
+          requirements.programmeRights.every(role => userInfo.programmeRights?.includes(role))
+
+        // This is defined in frontend ProtectedRoute component.
+        const specialNeedsRoutes = [
+          'populations',
+          'study-programme',
+          'coursepopulation',
+          'coursestatistics',
+          'students',
+          'custompopulation',
+        ]
+
+        const mankeli = specialNeedsRoutes.some(path => route.includes(path))
+          ? requiredRoles || requiredRights
+          : requiredRoles && requiredRights
+
+        const shouldHaveAccess = mankeli && requiredUserId
+
+        if (shouldHaveAccess) {
+          cy.contains('Something broke').should('not.exist')
+          cy.contains('Access denied').should('not.exist')
+        } else {
+          cy.contains('Access denied')
+        }
+      })
+    }
+  })
+}

cypress/e2e/Students.js

@@ -259,8 +259,10 @@ describe('Students tests', () => {
 
     it('When a study plan is selected, courses included in the study plan are highlighted with a blue background', () => {
       cy.visit('/students/550789')
-      cy.contains('table tbody tr', 'Kulttuurien tutkimuksen kandiohjelma (01.08.2020–30.06.2023)').within(() => {
-        cy.get('td').eq(0).click()
+      cy.contains('Study rights')
+      cy.get('[data-cy=KH40_005-0]').contains('Kulttuurien tutkimuksen kandiohjelma')
+      cy.get('[data-cy=KH40_005-0]').within(() => {
+        cy.get('td').eq(1).click()
       })
       cy.contains('table tbody tr', 'Kandidaatintutkielma (KUKA-LIS222)').should(
         'have.css',
@@ -269,27 +271,23 @@ describe('Students tests', () => {
       )
     })
 
-    it('When a study plan is selected, the time frame of the credit graph is updated', () => {
-      cy.clock(new Date('2024-08-30').getTime(), ['Date'])
-      cy.visit('/students/550789')
-      cy.contains('table tbody tr', 'Kulttuurien tutkimuksen kandiohjelma (01.08.2020–30.06.2023)').within(() => {
-        cy.get('td').eq(0).click()
-      })
-    })
-
     it("If there's a study plan corresponding to the degree programme, completed credits are displayed", () => {
       cy.visit('/students/550789')
-      cy.contains('table tbody tr', 'Kulttuurien tutkimuksen kandiohjelma (01.08.2020–30.06.2023)').within(() => {
+      cy.contains('Study rights')
+      cy.get('[data-cy=KH40_005-0]').contains('Kulttuurien tutkimuksen kandiohjelma')
+      cy.get('[data-cy=KH40_005-0]').within(() => {
         cy.get('td')
-          .eq(-1)
+          .eq(4)
           .contains(/^185 cr$/)
       })
     })
 
     it("If there's a study plan corresponding to the degree programme and the student hasn't graduated, percentage of completion is also displayed", () => {
       cy.visit('/students/458723')
-      cy.contains('table tbody tr', 'Oikeusnotaarin koulutusohjelma (01.08.2020–31.07.2027)').within(() => {
-        cy.get('td').eq(-1).contains('69% (125 cr)')
+      cy.contains('Study rights')
+      cy.get('[data-cy=KH20_001-0]').contains('Oikeusnotaarin koulutusohjelma')
+      cy.get('[data-cy=KH20_001-0]').within(() => {
+        cy.get('td').eq(4).contains('69% (125 cr)')
       })
     })
   })

cypress/e2e/Teachers.js

@@ -35,7 +35,7 @@ describe('Teachers page tests', () => {
     cy.cs('Search').click()
     cy.cs('teacher-search').type(teacher1.split(' ')[0])
     cy.get('table tbody tr').should('have.length', 4)
-    cy.get('table tbody tr').eq(1).contains('td', teacher1)
+    cy.get('table tbody tr').eq(2).contains('td', teacher1)
   })
 
   it('Can check teacher page', () => {

cypress/e2e/University.js

@@ -10,21 +10,18 @@ const checkProgressTables = () => {
   progressLevels.forEach(level => cy.cs(`${level}-faculty-progress-table`))
 }
 
-const graduationTimesLevels = ['bachelor', 'master', 'doctor'] // ? Bachelor + master seems to be missing in test data
+const graduationTimesLevels = ['bachelor', 'bcMsCombo', 'master', 'doctor']
 
-const checkAverageGraduationTimesBreakdownBarCharts = () => {
+const checkGraduationCharts = mode => {
   graduationTimesLevels.forEach(level => {
-    cy.cs(`${level}-breakdown-bar-chart`)
+    cy.cs(`${level}-${mode}-bar-chart`).within(() => {
+      // Check that bar has loaded
+      cy.contains('Loading content').should('not.exist')
+    })
   })
 }
 
-const checkAverageGraduationTimesMedianBarCharts = () => {
-  graduationTimesLevels.forEach(level => {
-    cy.cs(`${level}-median-bar-chart`)
-  })
-}
-
-describe('University view', () => {
+describe('University view', { defaultCommandTimeout: 60 * 1000 }, () => {
   beforeEach(() => {
     cy.init('/university')
     cy.contains('University')
@@ -73,17 +70,31 @@ describe('University view', () => {
       cy.cs('faculty-graduations-tab').click()
     })
 
-    it('has all the correct median time bar charts', () => {
-      checkAverageGraduationTimesBreakdownBarCharts()
-    })
+    describe('Different modes work', () => {
+      it('Breakdown', () => {
+        cy.cs('graduation-mode-selector').within(() => {
+          cy.cs('select-breakdown').click()
+        })
+        checkGraduationCharts('breakdown')
+      })
 
-    it("'Breakdown/Median times' toggle works", () => {
-      cy.cs('graduation-time-toggle').click()
-      checkAverageGraduationTimesMedianBarCharts()
+      it('Median', () => {
+        cy.cs('graduation-mode-selector').within(() => {
+          cy.cs('select-median').click()
+        })
+        checkGraduationCharts('median')
+      })
+
+      it('Average', () => {
+        cy.cs('graduation-mode-selector').within(() => {
+          cy.cs('select-average').click()
+        })
+        checkGraduationCharts('average')
+      })
     })
 
     it('info boxes contain correct information', () => {
-      cy.cs('average-graduation-times-info-box-button').click()
+      cy.cs('average-graduation-times-info-box-button').hover()
       cy.cs('average-graduation-times-info-box-content').contains('Opiskelijoiden keskimääräiset valmistumisajat')
     })
   })

cypress/support/commands.js

@@ -58,7 +58,7 @@ const onlyStudyGuidanceGroupsUser = {
   hypersonsisuid: 'hy-hlo-2222222',
 }
 
-const userHeaders = [
+export const userHeaders = [
   adminUserHeaders,
   basicUserHeaders,
   onlycoursestatisticsUserHeaders,

docker-compose.ci.yml

@@ -3,15 +3,28 @@ name: oodikone
 x-defaults: &defaults
   restart: unless-stopped
 
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
 services:
   backend:
     <<: *defaults
     container_name: oodikone-backend
+    image: oodikone-backend:staging
+    build:
+      context: services
+      dockerfile: backend/Dockerfile
     depends_on:
-      - kone-db
-      - redis
-      - sis-db
-      - user-db
+      kone-db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      user-db:
+        condition: service_healthy
     environment:
       CI: 'true'
       DB_URL_USER: postgres://postgres@user-db:5432/user-db
@@ -24,40 +37,68 @@ services:
       IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
       JAMI_URL: 'http://jami:3003/'
       CRYPT_KEY: 'e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'
-    build:
-      context: services
-      dockerfile: backend/Dockerfile
-    image: oodikone-backend:staging
+    ports:
+      - 8080:8080
     profiles:
       - oodikone
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8080/health'] # In CI/prod the routing happens at a different level, hence no /api/ needed
+      <<: *healthcheck-defaults
 
   frontend:
     <<: *defaults
     container_name: oodikone-frontend
+    image: oodikone-frontend:staging
     build:
       context: services
       dockerfile: frontend/Dockerfile
-    image: oodikone-frontend:staging
+    depends_on:
+      backend:
+        condition: service_healthy
+    ports:
+      - 5000:5000
     profiles:
       - oodikone
 
   kone-db:
     <<: *defaults
     container_name: oodikone-kone-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
     environment:
       PGDATA: /data
       POSTGRES_DB: kone-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
+    ports:
+      - 4321:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
+
+  user-db:
+    <<: *defaults
+    container_name: oodikone-user-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    ports:
+      - 4320:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   redis:
     <<: *defaults
     container_name: oodikone-redis
+    image: redis:7.2.6
     command: >
       sh -c 'redis-server --appendonly yes && redis-cli flushall'
-    image: redis:7.2.6
     ports:
       - 6379:6379
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping', '|', 'grep', 'PONG']
+      <<: *healthcheck-defaults
 
   sis-db:
     <<: *defaults
@@ -68,24 +109,35 @@ services:
       POSTGRES_DB: sis-db
       POSTGRES_HOST_AUTH_METHOD: trust
     shm_size: 2gb
+    ports:
+      - 12345:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
-  user-db:
+  sis-importer-db:
     <<: *defaults
-    container_name: oodikone-user-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: user-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    container_name: oodikone-sis-importer-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
+    profiles:
+      - updater
+    ports:
+      - 12346:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   jami:
     <<: *defaults
     container_name: oodikone-jami
     image: quay.io/toska/jami:production
+    depends_on:
+      jami-db:
+        condition: service_healthy
     environment:
       - npm_config_cache=/tmp
       - PORT=3003
-      - NODE_ENV=dev
+      - NODE_ENV=test # previously was development which borks CI
       - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
     ports:
       - 3003:3003
@@ -99,57 +151,38 @@ services:
       - POSTGRES_PASSWORD=postgres
     ports:
       - 5433:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
+  # This is still needed to make backend calls correctly route from localhost:5000/api to localhost:8080
   nginx:
     container_name: oodikone-nginx
     image: nginx:latest
     depends_on:
-      - frontend
-      - backend
+      frontend:
+        condition: service_started
+      backend:
+        condition: service_healthy
     ports:
       - 1337:80
-    volumes:
-      - ./nginx.conf:/etc/nginx/nginx.conf
     profiles:
       - oodikone
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
 
-  # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
-  sis-importer-db:
-    <<: *defaults
-    container_name: oodikone-sis-importer-db
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
-    profiles:
-      - updater
-
-  sis-updater-worker:
-    <<: *defaults
-    container_name: oodikone-updater-worker
-    build:
-      context: updater/sis-updater-worker
-    depends_on:
-      - redis
-      - sis-db
-      - sis-importer-db
-    environment:
-      CI: 'true'
-      DB_URL: postgres://postgres@sis-db:5432/sis-db
-      REDIS_HOST: redis
-      SIS_IMPORTER_DATABASE: sis-importer-db
-      SIS_IMPORTER_HOST: sis-importer-db
-      SIS_IMPORTER_PASSWORD: postgres
-      SIS_IMPORTER_USER: postgres
-      SIS_PASSWORD: postgres
-    profiles:
-      - updater
-
+  # Following services + sis-importer-db are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
   sis-updater-scheduler:
-    <<: *defaults
+    restart: no
     container_name: oodikone-updater-scheduler
     build:
       context: updater/sis-updater-scheduler
+      dockerfile: ./Dockerfile
     depends_on:
-      - redis
-      - sis-importer-db
+      redis:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
     environment:
       CI: 'true'
       REDIS_HOST: redis
@@ -164,3 +197,31 @@ services:
       - 8082:8082
     profiles:
       - updater
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8082/health']
+      <<: *healthcheck-defaults
+
+  sis-updater-worker:
+    <<: *defaults
+    container_name: oodikone-updater-worker
+    build:
+      context: updater/sis-updater-worker
+      dockerfile: ./Dockerfile
+    depends_on:
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
+    environment:
+      CI: 'true'
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_HOST: redis
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_PASSWORD: postgres
+    profiles:
+      - updater

docker-compose.test.yml

@@ -1,31 +1,27 @@
-services:
-  api-tests:
-    container_name: oodikone-api-tests
-    build:
-      context: services
-      dockerfile: backend/dev.Dockerfile
-    volumes:
-      - ./services/backend/:/opt/app-root/backend
-      - ./services/shared/:/opt/app-root/shared
-      - backend-tests-nodemod:/opt/app-root/backend/node_modules
-      - shared-tests-nodemod:/opt/app-root/shared/node_modules
-    environment:
-      CI: 'true'
-      NODE_ENV: 'test'
-      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
-      DB_URL_USER: postgres://postgres@user-db:5432/user-db
-      FRONT_URL: http://localhost:3000
-      REDIS: redis
-      SECRET_TOKEN: test
-      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
-      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      JAMI_URL: 'http://jami:3003/'
-      IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
-      CRYPT_KEY: 'e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'
-    command: npm run test
+name: oodikone
 
-volumes:
-  backend-tests-nodemod:
-    name: oodikone-backend-tests-nodemod
-  shared-tests-nodemod:
-    name: oodikone-shared-tests-nodemod
+x-defaults: &defaults
+  restart: unless-stopped
+
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
+services:
+  sis-db:
+    <<: *defaults
+    container_name: oodikone-sis-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
+    pull_policy: always
+    volumes: !reset [] # Remove previous volume definitions
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    shm_size: 2gb
+    ports:
+      - 12345:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults

docker-compose.yml

@@ -6,6 +6,11 @@ x-defaults: &defaults
 x-build_defaults: &build_defaults
   dockerfile: dev.Dockerfile
 
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
 services:
   adminer:
     <<: *defaults
@@ -25,10 +30,14 @@ services:
       dockerfile: backend/dev.Dockerfile
       context: services
     depends_on:
-      - kone-db
-      - redis
-      - sis-db
-      - user-db
+      kone-db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      user-db:
+        condition: service_healthy
     env_file:
       - .env
     tty: true
@@ -51,6 +60,9 @@ services:
       - ./services/shared/:/opt/app-root/shared
       - shared-nodemod:/opt/app-root/shared/node_modules
       - backend-nodemod:/opt/app-root/backend/node_modules
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8080/api/health']
+      <<: *healthcheck-defaults
 
   frontend:
     <<: *defaults
@@ -58,6 +70,9 @@ services:
     build:
       dockerfile: frontend/dev.Dockerfile
       context: services
+    depends_on:
+      backend:
+        condition: service_healthy
     ports:
       - 3000:3000
     profiles:
@@ -74,8 +89,27 @@ services:
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
     profiles:
       - oodikone
+    ports:
+      - 4321:5432
     volumes:
       - kone-db-data:/data
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
+
+  user-db:
+    <<: *defaults
+    container_name: oodikone-user-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    profiles:
+      - oodikone
+    ports:
+      - 4320:5432
+    volumes:
+      - user-db-data:/data
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   redis:
     <<: *defaults
@@ -83,9 +117,12 @@ services:
     command: 'redis-server --appendonly yes'
     image: redis:7.2.6
     ports:
-      - '6379:6379'
+      - 6379:6379
     volumes:
       - redis-data:/data
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping', '|', 'grep', 'PONG']
+      <<: *healthcheck-defaults
 
   sis-db:
     <<: *defaults
@@ -96,6 +133,9 @@ services:
     shm_size: 2gb
     ports:
       - 12345:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-importer-db:
     <<: *defaults
@@ -107,74 +147,23 @@ services:
       - sis-importer-db-data:/data
     ports:
       - 12346:5432
-
-  sis-updater-scheduler:
-    <<: *defaults
-    container_name: oodikone-updater-scheduler
-    build:
-      <<: *build_defaults
-      context: updater/sis-updater-scheduler
-    depends_on:
-      - redis
-      - sis-importer-db
-    environment:
-      REDIS_HOST: redis
-      SECRET_TOKEN: dev
-      SIS_IMPORTER_DATABASE: sis-importer-db
-      SIS_IMPORTER_HOST: sis-importer-db
-      SIS_IMPORTER_PASSWORD: postgres
-      SIS_IMPORTER_USER: postgres
-    ports:
-      - 8082:8082
-    profiles:
-      - updater
-    volumes:
-      - sis-updater-scheduler-nodemod:/opt/app-root/src/node_modules
-      - ./updater/sis-updater-scheduler/:/opt/app-root/src
-
-  sis-updater-worker:
-    <<: *defaults
-    container_name: oodikone-updater-worker
-    build:
-      <<: *build_defaults
-      context: updater/sis-updater-worker
-    depends_on:
-      - redis
-      - sis-db
-      - sis-importer-db
-    environment:
-      DB_URL: postgres://postgres@sis-db:5432/sis-db
-      REDIS_HOST: redis
-      SIS_IMPORTER_DATABASE: sis-importer-db
-      SIS_IMPORTER_HOST: sis-importer-db
-      SIS_IMPORTER_PASSWORD: postgres
-      SIS_IMPORTER_USER: postgres
-      SIS_PASSWORD: postgres
-    profiles:
-      - updater
-    volumes:
-      - sis-updater-worker-nodemod:/opt/app-root/src/node_modules
-      - ./updater/sis-updater-worker/:/opt/app-root/src
-
-  user-db:
-    <<: *defaults
-    container_name: oodikone-user-db
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
-    profiles:
-      - oodikone
-    volumes:
-      - user-db-data:/data
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   jami:
     <<: *defaults
     container_name: oodikone-jami
     image: quay.io/toska/jami:production
+    depends_on:
+      jami-db:
+        condition: service_healthy
     env_file:
       - .env
     environment:
       - npm_config_cache=/tmp
       - PORT=3003
-      - NODE_ENV=dev
+      - NODE_ENV=development
       - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/jami-db-real
     ports:
       - 3003:3003
@@ -190,6 +179,67 @@ services:
       - 5433:5432
     volumes:
       - jami-data:/data
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
+
+  sis-updater-worker:
+    <<: *defaults
+    container_name: oodikone-updater-worker
+    build:
+      <<: *build_defaults
+      context: updater/sis-updater-worker
+    depends_on:
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
+    environment:
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_HOST: redis
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_PASSWORD: postgres
+    profiles:
+      - updater
+    volumes:
+      - ./updater/sis-updater-worker/:/opt/app-root/src
+      - sis-updater-worker-nodemod:/opt/app-root/src/node_modules
+
+  sis-updater-scheduler:
+    <<: *defaults
+    container_name: oodikone-updater-scheduler
+    build:
+      <<: *build_defaults
+      context: updater/sis-updater-scheduler
+    depends_on:
+      redis:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
+    environment:
+      REDIS_HOST: redis
+      SECRET_TOKEN: dev
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
+    ports:
+      - 8082:8082
+    profiles:
+      - updater
+    volumes:
+      - ./updater/sis-updater-scheduler/:/opt/app-root/src
+      - sis-updater-scheduler-nodemod:/opt/app-root/src/node_modules
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8082/health']
+      <<: *healthcheck-defaults
 
 volumes:
   backend-nodemod:

eslint.config.mjs

@@ -264,6 +264,15 @@ export default tseslint.config(
     },
   },
 
+
+  // Backend tests
+  {
+    files: ["services/backend/tests/**/*.ts", "services/backend/vitest.config.ts"],
+    rules: {
+      'import-x/no-extraneous-dependencies': 'off',
+    }
+  },
+
   // Frontend tests
   {
     files: ['services/frontend/tests/**/*.{ts,tsx}', "services/frontend/playwright/**/*.{ts,tsx}", 'services/frontend/playwright-ct.config.ts'],

package.json

@@ -13,23 +13,25 @@
     "prettier": "prettier",
     "cli": "./cli.sh",
     "logs": "docker compose --file docker-compose.yml logs backend frontend sis-updater-worker sis-updater-scheduler jami -f",
+    "logs:all": "docker compose --file docker-compose.yml --profile oodikone --profile updater logs -f",
+    "logs:ci": "docker compose --file docker-compose.ci.yml logs backend frontend sis-updater-worker sis-updater-scheduler jami",
     "oodikone": "npm run docker:down && ./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
-    "oodikone:ci:setup": "docker compose --file docker-compose.ci.yml --profile tests up --detach",
-    "oodikone:ci:api": "docker compose --file docker-compose.test.yml up --exit-code-from api-tests",
+    "oodikone:test": "npm run docker:down && docker compose --profile oodikone up --detach && docker compose --file docker-compose.test.yml up --detach",
+    "oodikone:ci": "npm run docker:down && docker compose --file docker-compose.ci.yml --profile oodikone up",
     "updater": "./run.sh updater anon up --detach",
     "updater:real": "./run.sh updater real up --detach",
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
     "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
-    "testupdater": "docker compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start",
+    "testupdater": "docker compose --file docker-compose.ci.yml up --detach --no-recreate --wait sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker compose --file docker-compose.ci.yml up sis-updater-scheduler --exit-code-from sis-updater-scheduler",
     "flushredis": "docker exec -it oodikone-redis redis-cli flushall",
     "prepare": "husky",
     "install:local": "npm i && npm i --prefix services/shared && npm i --prefix services/frontend && npm i --prefix services/backend && npm i --prefix updater/sis-updater-scheduler && npm i --prefix updater/sis-updater-worker",
-    "install:docker": "docker exec oodikone-frontend npm i && docker exec oodikone-backend npm i && docker exec oodikone-updater-scheduler npm i && docker exec oodikone-updater-worker npm i",
+    "install:docker": "docker exec --user 0 oodikone-frontend sh -lc \"npm i --prefix /opt/app-root/shared && npm i --prefix /opt/app-root/frontend && chown -R 1001:0 /opt/app-root/shared/node_modules /opt/app-root/frontend/node_modules\" && docker exec --user 0 oodikone-backend sh -lc \"npm i --prefix /opt/app-root/backend && chown -R 1001:0 /opt/app-root/backend/node_modules\" && docker exec --user 0 oodikone-updater-scheduler sh -lc \"npm i --prefix /opt/app-root/src && chown -R 1001:0 /opt/app-root/src/node_modules\" && docker exec --user 0 oodikone-updater-worker sh -lc \"npm i --prefix /opt/app-root/src && chown -R 1001:0 /opt/app-root/src/node_modules\"",
     "install:both": "npm run install:local && npm run install:docker",
     "ci:local": "npm ci && npm ci --prefix services/shared && npm ci --prefix services/frontend && npm ci --prefix services/backend && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker",
-    "ci:docker": "docker exec oodikone-frontend npm ci && docker exec oodikone-backend npm ci && docker exec oodikone-updater-scheduler npm ci && docker exec oodikone-updater-worker npm ci",
+    "ci:docker": "docker exec --user 0 oodikone-frontend sh -lc \"npm ci --prefix /opt/app-root/shared && npm ci --prefix /opt/app-root/frontend && chown -R 1001:0 /opt/app-root/shared/node_modules /opt/app-root/frontend/node_modules\" && docker exec --user 0 oodikone-backend sh -lc \"npm ci --prefix /opt/app-root/backend && chown -R 1001:0 /opt/app-root/backend/node_modules\" && docker exec --user 0 oodikone-updater-scheduler sh -lc \"npm ci --prefix /opt/app-root/src && chown -R 1001:0 /opt/app-root/src/node_modules\" && docker exec --user 0 oodikone-updater-worker sh -lc \"npm ci --prefix /opt/app-root/src && chown -R 1001:0 /opt/app-root/src/node_modules\"",
     "ci:both": "npm run ci:local && npm run ci:docker"
   },
   "repository": {

scripts/functions_for_setup.sh

@@ -138,7 +138,7 @@ reset_databases() {
   successmsg "Database setup finished"
 }
 
-reset_all_real_data() {
+reset_all_data() {
   infomsg "Downloading real data dumps"
   for i in ${!DATABASES[*]}; do
     local database="${DATABASES[$i]}"

services/.dockerignore

@@ -1,7 +1,116 @@
-*/.dockerignore
-*/.eslintrc.json
-*/.gitignore
-*/*Dockerfile
-*/build
-*/dist
-*/node_modules
+# disallow directory named "private", it can be used to store things that should not end up in version control or images
+# NOTE: needs to be at the root of the project
+private/
+gitignored/
+
+# disallow version control, package managment, IDEs, tools etc
+**/node_modules
+**/.venv
+**/.git
+**/.github
+**/.vscode
+**/.idea
+**/.zed
+**/coverage/
+**/.eslintcache
+**/.DS_Store
+
+# do not include any data by default (except json)
+**/*.csv
+**/*.tsv
+
+**/*.db
+**/*.sqlite
+**/*.sql
+**/*.psql
+**/*.sqlz
+**/*.sqz
+**/*.sql.gz
+**/*.bson
+**/*.jsonl
+
+**/*.bin
+**/*.dat
+**/*.data
+
+**/*.swp
+**/*.tmp
+**/.~lock*
+**/*.pid
+**/*.seed
+**/*.lock
+
+**/*.gz
+**/*.tgz
+**/*.xz
+**/*.zip
+**/*.gzip
+**/*.tar
+**/*.xz
+**/*.7z
+**/*.rar
+
+**/*.docx
+**/*.doc
+**/*.xlsx
+**/*.xls
+**/*.pptx
+**/*.ppt
+**/*.odt
+**/*.odp
+**/*.ods
+**/*.pdf
+
+
+# disallow logs
+**/*.log
+**/*.log*
+**/logs/
+**/log/
+**/*.cpuprofile
+**/*.heapprofile
+**/*.heapsnapshot
+
+# disallow configuration
+**/*.yml
+**/*.yaml
+**/*.toml
+**/*.conf
+**/*.ini
+**/toska_login/
+**/toska_login/*
+**/cypress.env.json
+cypress/
+
+
+# disallow secrects
+**/.env
+**/.env*
+**/*.env.*
+**/.secrects
+**/*.s3cfg
+
+# disallow backups
+**/backups/*
+**/backups
+**/backup/*
+**/backup
+**/dump/
+**/dump
+**/dumps/
+**/dumps
+**/databasedumps/
+**/privateData
+**/temp
+**/minio-data/*
+
+
+# additional inclusions here
+
+# exclusions here
+**/.dockerignore
+**/.gitignore
+**/.eslintrc.json
+**/*Dockerfile
+**/build
+**/dist

services/backend/.env.test

@@ -0,0 +1,12 @@
+CI='true'
+NODE_ENV='test'
+DB_URL_KONE=postgres://postgres@localhost:4321/kone-db
+DB_URL_USER=postgres://postgres@localhost:4320/user-db
+FRONT_URL=http://localhost:3000
+REDIS=localhost
+SECRET_TOKEN=test
+SIS_DB_URL=postgres://postgres@localhost:12345/sis-db
+SIS_UPDATER_URL=http://localhost:8082
+JAMI_URL='http://localhost:3003/'
+IMPORTER_DB_API_URL='https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
+CRYPT_KEY='e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'

services/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi9/nodejs-24-minimal AS build
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal AS build
 
 ENV TZ="Europe/Helsinki"
 
@@ -29,7 +29,7 @@ RUN npm ci --include=dev --prefix shared && npm ci --include=dev --prefix backen
 RUN npm run build --prefix shared && npm run build --prefix backend
 
 
-FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 

services/backend/dev.Dockerfile

@@ -1,21 +1,21 @@
-FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 
 ENV NODE_ENV=development
 ENV NODE_OPTIONS=--max-old-space-size=4096
 
-USER 1001
+USER 0
 
 WORKDIR /opt/app-root/shared
-COPY --chown=1001:1001 ./shared .
-
-RUN npm ci
+COPY --chown=1001:0 ./shared .
+RUN npm ci && chown -R 1001:0 /opt/app-root/shared
 
 WORKDIR /opt/app-root/backend
-COPY --chown=1001:1001 ./backend .
-
-RUN npm ci
+COPY --chown=1001:0 ./backend .
+RUN npm ci && chown -R 1001:0 /opt/app-root/backend
 
 EXPOSE 8080
+
+USER 1001
 CMD ["node_modules/.bin/nodemon", "--exec", "node --import tsx index.ts", "--ext", "ts,js,json", "--watch", "src", "--watch", "index.ts", "--legacy-watch"]

services/backend/package.json

@@ -3,7 +3,11 @@
   "private": true,
   "scripts": {
     "build": "tsc",
-    "test": "node --import=tsx --test --test-force-exit --test-reporter spec tests/**/*.ts"
+    "test:all": "vitest",
+    "test:api": "vitest ./tests/api",
+    "test:services": "vitest ./tests/services",
+    "test:unit": "vitest ./tests/unit",
+    "test:silent": "SILENT_TESTING=true vitest"
   },
   "dependencies": {
     "@oodikone/shared": "file:../shared",
@@ -41,6 +45,7 @@
     "nodemon": "^3.1.11",
     "supertest": "^7.2.2",
     "tsx": "^4.19.4",
-    "typescript": "^5.9.2"
+    "typescript": "^5.9.2",
+    "vitest": "4.1.9"
   }
 }

services/backend/src/config/index.ts

@@ -6,6 +6,8 @@ export const isProduction = !isStaging && process.env.NODE_ENV === 'production'
 
 export const runningInCI = process.env.CI === 'true'
 
+export const silentTesting = process.env.SILENT_TESTING === 'true'
+
 // IAM group
 export const requiredGroup = ['grp-oodikone-users', 'grp-oodikone-basic-users']
 
@@ -36,9 +38,6 @@ export const { DB_URL_KONE, DB_URL_USER, SECRET_TOKEN, SIS_DB_URL, SIS_UPDATER_U
 
 export const rootOrgId = process.env.ROOT_ORG_ID ?? 'hy-university-root-id'
 
-export const languageCenterViewEnabled =
-  process.env.LANGUAGE_CENTER_VIEW_ENABLED == null ? true : process.env.LANGUAGE_CENTER_VIEW_ENABLED === 'true'
-
 export const concurrentWorkers = parseInt(process.env.CONCURRENT_WORKERS!, 10) || 2
 
 export const DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS!, 10) || 5 // sequelize's default

services/backend/src/database/connection.ts

@@ -24,6 +24,7 @@ import {
   TeacherModel,
 } from '../models'
 import {
+  BannerModel,
   CustomPopulationSearchModel,
   ExcludedCourseModel,
   OpenUniPopulationSearchModel,
@@ -107,6 +108,7 @@ const sequelizeKone = new Sequelize(conf.DB_URL_KONE!, {
   logging: false,
   password: conf.KONE_PASSWORD,
   models: [
+    BannerModel,
     CustomPopulationSearchModel,
     ExcludedCourseModel,
     OpenUniPopulationSearchModel,

services/backend/src/database/migrations_kone/20260709_01_create_table_banners.js

@@ -0,0 +1,46 @@
+const { INTEGER, STRING, DATE } = require('sequelize')
+
+module.exports = {
+  up: async ({ context: queryInterface }) => {
+    await queryInterface.createTable('banners', {
+      id: {
+        type: INTEGER,
+        primaryKey: true,
+        autoIncrement: true,
+      },
+      text: {
+        type: STRING,
+        allowNull: false,
+      },
+      color: {
+        type: STRING,
+        allowNull: false,
+      },
+      lightness: {
+        type: STRING,
+        allowNull: false,
+      },
+      start_date: {
+        type: DATE,
+        allowNull: false,
+      },
+      end_date: {
+        type: DATE,
+        allowNull: false,
+      },
+      last_modified_by: {
+        type: STRING,
+        allowNull: false,
+      },
+      created_at: {
+        type: DATE,
+      },
+      updated_at: {
+        type: DATE,
+      },
+    })
+  },
+  down: async ({ context: queryInterface }) => {
+    await queryInterface.dropTable('banners')
+  },
+}

services/backend/src/events.ts

@@ -1,6 +1,6 @@
 import { CronJob } from 'cron'
 
-import { isProduction, runningInCI, languageCenterViewEnabled, CRON_SCHEDULE } from './config'
+import { isProduction, runningInCI, CRON_SCHEDULE } from './config'
 import { getDegreeProgrammesOfFaculty } from './services/faculty/faculty'
 import { getFaculties } from './services/faculty/facultyHelpers'
 import { updateFacultyOverview, updateFacultyProgressOverview } from './services/faculty/facultyUpdates'
@@ -98,10 +98,7 @@ const dailyJobs = async () => {
     await refreshTeacherLeaderboard()
     await refreshProgrammesAndFaculties()
 
-    if (languageCenterViewEnabled) {
-      jobQueue.refreshLanguagecenter()
-    }
-
+    jobQueue.refreshLanguagecenter()
     jobQueue.refreshCloseToGraduation()
   } catch (error) {
     logger.error('Daily jobs failed', error)

services/backend/src/middleware/accessLogger.ts

@@ -38,6 +38,11 @@ const accessLogger = morgan((tokens, req: Request, res: Response): undefined =>
     meta['req-route'] = req.route.path
   }
 
+  // Ignore admin route that is frequently pinged for updates.
+  if (/.*\/updater\/jobs$/.test(meta.url ?? '')) {
+    return
+  }
+
   const message = [
     `${user.mockedBy ? '(mocking) ' : ''}${user.name}`,
     `${(tokens['response-time'](req, res) ?? '0').split('.')[0]} ms`.padEnd(8, ' '),

services/backend/src/middleware/currentUser.ts

@@ -2,7 +2,7 @@ import * as Sentry from '@sentry/node'
 import { NextFunction, Request, Response } from 'express'
 import { intersection } from 'lodash-es'
 
-import { requiredGroup } from '../config'
+import { requiredGroup, baseUrl } from '../config'
 import { getMockedUser, getOrganizationAccess, getUser } from '../services/userService'
 import { ApplicationError } from '../util/customErrors'
 import logger from '../util/logger'
@@ -36,6 +36,11 @@ export default async (req: Request, _: Response, next: NextFunction) => {
     uid: username,
   } = req.headers as Headers
 
+  if (req.path === `${baseUrl}/health`) {
+    next()
+    return
+  }
+
   if (!username || !sessionId)
     throw new ApplicationError(
       `${username ?? 'Anonymous user'} requested ${req.url} without authorization headers.`,

services/backend/src/models/kone/banner.ts

@@ -0,0 +1,33 @@
+import { Column, DataType, Model, PrimaryKey, Table, AutoIncrement } from 'sequelize-typescript'
+
+import type { Banner } from '@oodikone/shared/models/kone'
+
+@Table({
+  underscored: true,
+  modelName: 'banner',
+  tableName: 'banners',
+})
+export class BannerModel extends Model implements Banner {
+  @PrimaryKey
+  @AutoIncrement
+  @Column(DataType.INTEGER)
+  declare id: number
+
+  @Column(DataType.DATE)
+  declare startDate: Date
+
+  @Column(DataType.DATE)
+  declare endDate: Date
+
+  @Column(DataType.STRING)
+  declare text: string
+
+  @Column(DataType.STRING)
+  declare color: string
+
+  @Column(DataType.STRING)
+  declare lightness: 'light' | 'main' | 'dark'
+
+  @Column(DataType.STRING)
+  declare lastModifiedBy: string
+}

services/backend/src/models/kone/index.ts

@@ -1,3 +1,4 @@
+export { BannerModel } from './banner'
 export { CustomPopulationSearchModel } from './customPopulationSearch'
 export { ExcludedCourseModel } from './excludedCourse'
 export { OpenUniPopulationSearchModel } from './openUniPopulationSearch'

services/backend/src/routes.ts

@@ -3,12 +3,13 @@ import compression from 'compression'
 import cors from 'cors'
 import { Express, json as jsonExpress } from 'express'
 
-import { frontUrl, languageCenterViewEnabled } from './config'
+import { frontUrl } from './config'
 import accessLogger from './middleware/accessLogger'
 import * as auth from './middleware/auth'
 import currentUserMiddleware from './middleware/currentUser'
 import errorHandler from './middleware/errorHandler'
 import headersMiddleware from './middleware/headers'
+import banners from './routes/banners'
 import changelog from './routes/changelog'
 import closeToGraduation from './routes/closeToGraduation'
 import completedCoursesSearch from './routes/completedCoursesSearch'
@@ -18,6 +19,7 @@ import curriculumPeriods from './routes/curriculumPeriods'
 import customPopulationSearch from './routes/customPopulationSearch'
 import faculties from './routes/faculties'
 import feedback from './routes/feedback'
+import health from './routes/health'
 import languageCenterData from './routes/languageCenterData'
 import login from './routes/login'
 import population from './routes/population'
@@ -44,6 +46,7 @@ const routes = (app: Express, url: string) => {
   app.use(currentUserMiddleware)
   app.use(accessLogger)
   app.use(`${url}/login`, login)
+  app.use(`${url}/banners`, banners)
   app.use(url, courses)
   app.use(`${url}/students`, students)
   app.use(url, population)
@@ -56,9 +59,7 @@ const routes = (app: Express, url: string) => {
   app.use(`${url}/programme-criteria`, studyProgrammeCriteria)
   app.use(`${url}/changelog`, changelog)
   app.use(`${url}/completedcoursessearch`, completedCoursesSearch)
-  if (languageCenterViewEnabled) {
-    app.use(`${url}/languagecenterdata`, languageCenterData)
-  }
+  app.use(`${url}/languagecenterdata`, languageCenterData)
   app.use(`${url}/faculties`, faculties)
   app.use(`${url}/university`, university)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
@@ -70,6 +71,7 @@ const routes = (app: Express, url: string) => {
   app.use(`${url}/close-to-graduation`, auth.roles(['fullSisuAccess', 'studyGuidanceGroups']), closeToGraduation)
   app.use(`${url}/study-programme-pins`, studyProgrammePins)
   app.use(`${url}/curriculum-periods`, curriculumPeriods)
+  app.use(`${url}/health`, health)
   app.get('', (_, res) => {
     res.status(404).json({ error: 'unknown endpoint' })
   })

services/backend/src/routes/banners.ts

@@ -0,0 +1,53 @@
+import { Request, Response, Router } from 'express'
+
+import { Banner } from '@oodikone/shared/models/kone'
+import { saveBanner, getActiveBanners, getAllBanners } from '../services/banners'
+
+const router = Router()
+
+router.get<never, Banner[]>('/', async (_req: Request, res: Response) => {
+  const banners = await getActiveBanners()
+  res.json(banners).end()
+})
+
+router.get<never, Banner[]>('/all', async (req: Request, res: Response) => {
+  const { user } = req
+  if (!user.isAdmin || !user.iamGroups.includes('grp-toska')) {
+    return res.status(403).end()
+  }
+
+  const banners = await getAllBanners()
+  res.json(banners)
+})
+
+router.post('/new', async (req: Request, res: Response) => {
+  const { user } = req
+  if (!user.isAdmin || !user.iamGroups.includes('grp-toska')) {
+    return res.status(403).end()
+  }
+
+  const payload = {
+    ...req.body,
+    lastModifiedBy: user.username,
+  }
+
+  await saveBanner(payload)
+  res.status(201).end()
+})
+
+router.put('/update', async (req: Request, res: Response) => {
+  const { user } = req
+  if (!user.isAdmin || !user.iamGroups.includes('grp-toska')) {
+    return res.status(403).end()
+  }
+
+  const payload = {
+    ...req.body,
+    lastModifiedBy: user.username,
+  }
+
+  await saveBanner(payload)
+  res.status(201).end()
+})
+
+export default router

services/backend/src/routes/faculties.ts

@@ -36,6 +36,7 @@ import { combineFacultyStudentProgress } from '../services/faculty/facultyStuden
 import { combineFacultyStudents } from '../services/faculty/facultyStudents'
 import { combineFacultyThesisWriters } from '../services/faculty/facultyThesisWriters'
 import { updateFacultyOverview, updateFacultyProgressOverview } from '../services/faculty/facultyUpdates'
+import { objectKeyAnnihilator } from '../util'
 import { ApplicationError } from '../util/customErrors'
 
 // Faculty uses a lot of tools designed for Degree programme.
@@ -145,19 +146,6 @@ router.get<
   GetStatsReqBody,
   GetGraduationStatsQuery
 >('/:id/graduationtimes', auth.roles(['facultyStatistics']), async (req, res) => {
-  // Recursively clears graduation time arrays "times" from result.
-  // We don't need to send them to client,
-  // but university view (uses same data) needs it to compute medians.
-  const clearTimes = (data: any) => {
-    if (Array.isArray(data)) {
-      data.forEach(clearTimes)
-    } else if (!!data && typeof data === 'object') {
-      if (data.times) data.times = []
-      Object.values(data).forEach(clearTimes)
-    }
-    return data
-  }
-
   const { id: facultyId } = req.params
   const { programme_filter: programmeFilter } = req.query
 
@@ -165,7 +153,9 @@ router.get<
   if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
   const data = await getGraduationStats(faculty.code, programmeFilter)
-  if (data) return res.json(clearTimes(data))
+  // Times were needed to compute medians/averages serverside, but are not needed in frontend.
+  // They are still kept in redis as university view needs them for the same purpose.
+  if (data) return res.json(objectKeyAnnihilator(data, 'times', []))
 
   const programmes = await getDegreeProgrammesOfFaculty(faculty.code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
   if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
@@ -173,7 +163,7 @@ router.get<
   const updatedStats = await countGraduationTimes(faculty.code, programmes)
   if (updatedStats) await setGraduationStats(updatedStats, programmeFilter)
 
-  return res.json(clearTimes(updatedStats))
+  return res.json(objectKeyAnnihilator(updatedStats, 'times', []))
 })
 
 type GetProgressStatsResBody = {
@@ -224,7 +214,7 @@ export type DegreeProgramme = Pick<
   'code' | 'name' | 'degreeProgrammeType' | 'progId'
 >
 
-type GetStudetStatsResBody = {
+type GetStudentStatsResBody = {
   id: string
   years: string[]
   facultyTableStats: Record<string, (number | string)[]>
@@ -241,7 +231,7 @@ type GetStudetStatsResBody = {
 
 router.get<
   GetStatsParams,
-  CanError<GetStudetStatsResBody, GenericApplicationError>,
+  CanError<GetStudentStatsResBody, GenericApplicationError>,
   GetStatsReqBody,
   GetProgressStatsQuery
 >('/:id/studentstats', auth.roles(['facultyStatistics']), async (req, res) => {

services/backend/src/routes/health.ts

@@ -0,0 +1,10 @@
+import { Router } from 'express'
+
+const router = Router()
+
+/** Used for checking if backend is up eg. Docker healthchecks */
+router.get<never, { status: string }>('/', (_, res) => {
+  return res.status(200).json({ status: 'healthy' })
+})
+
+export default router

services/backend/src/routes/university.ts

@@ -17,6 +17,7 @@ import {
 import { combineFacultyStudentProgress, FacultyProgressData } from '../services/faculty/facultyStudentProgress'
 import { GraduationTarget } from '../services/graduationHelpers'
 import { getMedian } from '../services/studyProgramme/studyProgrammeHelpers'
+import { objectKeyAnnihilator } from '../util'
 import logger from '../util/logger'
 
 const router = Router()
@@ -200,9 +201,7 @@ router.get('/allgraduationstats', async (_req: Request, res: Response<University
       if (!facultyData[degreeType]) continue
 
       for (const facultyYearStats of facultyData[degreeType]) {
-        if (!mediansForUniversity[degreeType][facultyYearStats.name]) {
-          mediansForUniversity[degreeType][facultyYearStats.name] = { programmes: [], data: [] }
-        }
+        mediansForUniversity[degreeType][facultyYearStats.name] ??= { programmes: [], data: [] }
         const uniYearStats = mediansForUniversity[degreeType][facultyYearStats.name]
         if (!uniYearStats.programmes.find(prog => prog === facultyCode)) {
           uniYearStats.programmes.push(facultyCode)
@@ -233,6 +232,9 @@ router.get('/allgraduationstats', async (_req: Request, res: Response<University
     classSizes.programmes[facultyCode] = rest
   }
 
+  // Individual times are no longer needed as medians/averages were already computed.
+  objectKeyAnnihilator(byGradYear, 'times', [])
+
   res.status(200).json({
     goals,
     programmeNames,

services/backend/src/routes/updater.ts

@@ -1,6 +1,5 @@
 import { Request, Response, Router } from 'express'
 
-import { languageCenterViewEnabled } from '../config'
 import { refreshFaculties, refreshProgrammes } from '../events'
 import {
   abortUpdate,
@@ -109,8 +108,6 @@ router.post('/refresh_faculties_v2', async (req: Request, res: Response) => {
 
 router.post('/refresh_language_center_data', (req: Request, res: Response) => {
   logger.info(`${req.user.username} requested refresh of language center data`)
-  if (!languageCenterViewEnabled)
-    res.status(418).json({ error: 'The language center functionality is not activated in your environment.' })
   jobQueue.refreshLanguagecenter()
   res.status(200).json('Added job for refreshing language center data')
 })

services/backend/src/services/banners.ts

@@ -0,0 +1,28 @@
+import { Op } from 'sequelize'
+import { Banner } from '@oodikone/shared/models/kone'
+import { BannerModel } from '../models/kone/banner'
+import { SetOptional } from '../types'
+
+export const getActiveBanners = async () =>
+  await BannerModel.findAll({
+    attributes: ['id', 'text', 'lightness', 'color', 'startDate', 'endDate'],
+    where: {
+      endDate: {
+        [Op.gt]: new Date(),
+      },
+      startDate: {
+        [Op.lt]: new Date(),
+      },
+    },
+    raw: true,
+  })
+
+export const getAllBanners = async () => await BannerModel.findAll({ raw: true })
+
+export const saveBanner = async (payload: SetOptional<Banner, 'id'>) => {
+  if (!payload.id) {
+    await BannerModel.create(payload)
+  } else {
+    await BannerModel.upsert(payload)
+  }
+}

services/backend/src/services/courses/courseYearlyStatsCounter.ts

@@ -124,6 +124,7 @@ export class CourseYearlyStatsCounter {
     this.groups[groupCode].students.grades[studentNumber] = { grade, passed }
   }
 
+  /** NOTE: Duplicates are counted by year eg. you can be both in 2022 and 2023 enrollments */
   public markEnrollmentToGroup(
     studentNumber: string,
     enrollmentDateTime: Date,

services/backend/src/services/courses/creditsAndEnrollmentsOfCourse.ts

@@ -1,7 +1,7 @@
 import { Op } from 'sequelize'
 
 import { Credit, Enrollment } from '@oodikone/shared/models'
-import { EnrollmentState, Unification } from '@oodikone/shared/types'
+import { CreditTypeCode, EnrollmentState, Unification } from '@oodikone/shared/types'
 import { enrollmentTimeDateThreshold } from '@oodikone/shared/util'
 import {
   StudentModel,
@@ -40,6 +40,7 @@ export const getCreditsForCourses = async (codeGroups: string[][], unification:
             [Op.in]: allCourseCodes,
           },
           is_open: getIsOpen(unification),
+          credittypecode: { [Op.not]: CreditTypeCode.IMPROVED }, // We do not care about improved grades
         },
         order: [['attainment_date', 'ASC']],
         include: [

services/backend/src/services/courses/index.ts

@@ -126,15 +126,15 @@ const parseEnrollment = (
     studyright_id: studyRightId,
   } = enrollment
 
-  const programmeOfCredit: SISStudyRightElementModel | undefined =
+  const programmeOfEnrollment: SISStudyRightElementModel | undefined =
     studyRightElements.find(studyRightElement => studyRightElement.studyRightId === studyRightId) ??
     studyRightElements
       .filter(({ startDate, endDate }) => dateIsBetween(enrollmentDateTime, startDate, endDate))
       .sort((a, b) => b.startDate.getTime() - a.startDate.getTime())
       .at(0) // The newest studyRightElement
 
-  const programme = programmeOfCredit
-    ? formatStudyRightElement(programmeOfCredit)
+  const programme = programmeOfEnrollment
+    ? formatStudyRightElement(programmeOfEnrollment)
     : {
         code: 'OTHER',
         name: { en: 'Other', fi: 'Muu', sv: 'Andra' },
@@ -314,6 +314,11 @@ const getYearlyStatsOfNew = async (
   }
 }
 
+/**
+  Mostly-legacy function that prevents user from opening too big of a populations.
+  At the moment the threshold to prevent a population is so high it will never trigger.
+
+ */
 export const maxYearsToCreatePopulationFrom = async (courseCodes: string[], unification: Unification) => {
   const lastAttainmentDate = (await CourseModel.findOne({
     attributes: [[dbFn('MAX', dbCol('max_attainment_date')), 'date']],
@@ -337,7 +342,7 @@ export const maxYearsToCreatePopulationFrom = async (courseCodes: string[], unif
   })
 
   // MAGIC NUMBER
-  const maxAllowedAttainments = 1000000 // * Lower this value to get a smaller result if necessary
+  const maxAllowedAttainments = 1_000_000 // * Lower this value to get a smaller result if necessary
   return Math.max(1, maxAllowedAttainments / attainmentsWithinThreshold)
 }
 
@@ -426,8 +431,8 @@ export const getCourseYearlyStats = async (
   return statsRegular
 }
 
-export const getCourseProvidersForCourses = async (codes: string[]) => {
-  return (
+export const getCourseProvidersForCourses = async (codes: string[]) =>
+  (
     await OrganizationModel.findAll({
       attributes: ['code'],
       include: {
@@ -441,7 +446,6 @@ export const getCourseProvidersForCourses = async (codes: string[]) => {
       raw: true,
     })
   ).map(({ code }) => code)
-}
 
 export const getCourseDetails = async (codes: string[]) =>
   CourseModel.findAll({

services/backend/src/services/faculty/facultyGraduationTimes.ts

@@ -1,7 +1,7 @@
 import { Name, DegreeProgrammeType, FacultyGraduationStatistics, ClassSizes } from '@oodikone/shared/types'
 import type {
   GraduationStatistics,
-  MedianEntry,
+  GraduationStats,
   MediansByCategory,
   MediansByProgrammes,
   ProgrammeGraduationStats,
@@ -11,7 +11,7 @@ import { omitKeys } from '@oodikone/shared/util'
 import { getGraduationStats, getStudyTrackStats, setGraduationStats, setStudyTrackStats } from '../analyticsService'
 import { GraduationTarget } from '../graduationHelpers'
 import { getGraduationStatsForStudyTrack } from '../studyProgramme/studyProgrammeGraduations'
-import { getMedian } from '../studyProgramme/studyProgrammeHelpers'
+import { getAverage, getMedian } from '../studyProgramme/studyProgrammeHelpers'
 import { getStudyRightsInProgramme } from '../studyProgramme/studyRightFinders'
 import { getStudyTrackStatsForStudyProgramme } from '../studyProgramme/studyTrackStats'
 import type { ProgrammesOfOrganization } from './faculty'
@@ -22,20 +22,18 @@ export const programmeTypes = {
   [DegreeProgrammeType.DOCTOR]: 'doctor',
 } as const
 
-const setYearlyBreakdown = (code: string, progId: string, level: ProgrammeMedians, year: MedianEntry) => {
-  if (year.y === 0) return
+const setYearlyBreakdown = (code: string, name: string, level: ProgrammeMedians, year: GraduationStats) => {
+  if (!year.times.length) return
 
   level[year.name] ??= { data: [], programmes: [] }
   level[year.name].programmes.push(code)
-  level[year.name].data.push(
-    omitKeys({ ...structuredClone(year), name: progId, code, median: year.y }, ['y', 'classSize'])
-  )
+  level[year.name].data.push(omitKeys({ ...structuredClone(year), name, code }, ['classSize']))
 }
 
 const mergeStats = (
   medians: MediansByCategory,
   level: keyof MediansByCategory,
-  statsForYear: MedianEntry,
+  statsForYear: GraduationStats,
   classSizes?: ClassSizes
 ) => {
   if (classSizes) {
@@ -46,10 +44,11 @@ const mergeStats = (
   const gradStatsForYear = medians[level].find(entry => entry.name === statsForYear.name)
 
   if (!gradStatsForYear) {
-    medians[level].push({ ...omitKeys(structuredClone(statsForYear), ['y']), median: statsForYear.y })
+    medians[level].push(structuredClone(statsForYear))
   } else {
     gradStatsForYear.times.push(...statsForYear.times)
     gradStatsForYear.median = getMedian(gradStatsForYear.times)
+    gradStatsForYear.average = getAverage(gradStatsForYear.times)
     gradStatsForYear.amount += statsForYear.amount
     gradStatsForYear.statistics.onTime += statsForYear.statistics.onTime
     gradStatsForYear.statistics.yearOver += statsForYear.statistics.yearOver

services/backend/src/services/faculty/facultyStudents.ts

@@ -48,9 +48,7 @@ const calculateCombinedStats = (programmeCodes: string[], stats: StudyTrackStats
 
     if (!programmeOtherCountriesCount) continue
     for (const [year, countryStats] of Object.entries(programmeOtherCountriesCount)) {
-      if (!(year in facultyTableStatsExtra)) {
-        facultyTableStatsExtra[year] = {}
-      }
+      facultyTableStatsExtra[year] ??= {}
       facultyTableStatsExtra[year][programmeCode] = countryStats
     }
   }

services/backend/src/services/languageCenterData.ts

@@ -1,16 +1,18 @@
 import { orderBy } from 'lodash-es'
 import { Op } from 'sequelize'
 
+import { EnrollmentState } from '@oodikone/shared/types'
 import { CourseModel, CreditModel, EnrollmentModel, SISStudyRightModel } from '../models'
 import { redisClient } from './redis'
 
 export const LANGUAGE_CENTER_REDIS_KEY = 'LANGUAGE_CENTER_DATA'
 
-const isBetween = (start, date, end) => {
+const isBetween = (start: Date, date: Date, end: Date) => {
   return new Date(start).getTime() <= new Date(date).getTime() && new Date(date).getTime() <= new Date(end).getTime()
 }
 
-const findStudyRight = (studyRights, date) => {
+/** Assumes studyrights are in correct order */
+const findStudyRight = (studyRights: SISStudyRightModel[], date: Date) => {
   return studyRights?.find(studyRight => isBetween(studyRight.startDate, date, studyRight.endDate))
 }
 
@@ -23,7 +25,7 @@ const getDifference = stats => {
 }
 
 const getLanguageCenterCourses = async () => {
-  const courses = await CourseModel.findAll({
+  const courses: Pick<CourseModel, 'code' | 'name'>[] = await CourseModel.findAll({
     attributes: ['code', 'name'],
     where: {
       [Op.or]: [{ code: { [Op.like]: 'KK%' } }, { code: { [Op.like]: 'AYKK%' } }],
@@ -33,29 +35,37 @@ const getLanguageCenterCourses = async () => {
   return courses
 }
 
-export const createArrayOfCourses = async (attempts, courses) => {
+export const createArrayOfCourses = (attempts: Attempt[], courses: Pick<CourseModel, 'code' | 'name'>[]) => {
   const fields = { completions: 0, enrollments: 0, difference: 0, rejected: 0 }
-  const semesters = {}
-  const faculties = {}
-  const semesterStatsMap = attempts.reduce((obj, cur) => {
+  const semesters: Record<Attempt['semestercode'], boolean> = {}
+  // Remove undefined as a possibility for type
+  const faculties: Record<Exclude<Attempt['faculty'], undefined>, boolean> = {}
+
+  // TODO: Type this properly. Good luck :)
+  const semesterStatsMap = attempts.reduce((acc, cur) => {
     const semester = cur.semestercode
-    faculties[cur.faculty] = true
-    semesters[semester] = true
-    if (!obj[cur.courseCode]) {
-      obj[cur.courseCode] = {}
-    }
-    if (!obj[cur.courseCode][semester]) {
-      obj[cur.courseCode][semester] = { ...fields }
-    }
-    if (cur.faculty && !obj[cur.courseCode][semester][cur.faculty]) {
-      obj[cur.courseCode][semester][cur.faculty] = { ...fields }
-    }
-    const allFacultiesTotal = obj[cur.courseCode][semester]
-    const allStats = [allFacultiesTotal]
     if (cur.faculty) {
-      const semesterFacultyStats = obj[cur.courseCode][semester][cur.faculty]
+      faculties[cur.faculty] = true
+    }
+    semesters[semester] = true
+
+    acc[cur.courseCode] ??= {}
+    // NOTE: This has to be spread because otherwise a ref to fields is saved to
+    // acc[cur.courseCode][semester] which would in the end result in a massive circular object
+    acc[cur.courseCode][semester] ??= { ...fields }
+
+    if (cur.faculty) {
+      acc[cur.courseCode][semester][cur.faculty] ??= { ...fields }
+    }
+
+    const allFacultiesTotal = acc[cur.courseCode][semester]
+    const allStats = [allFacultiesTotal]
+
+    if (cur.faculty) {
+      const semesterFacultyStats = acc[cur.courseCode][semester][cur.faculty]
       allStats.push(semesterFacultyStats)
     }
+
     for (const stats of allStats) {
       if (cur.completed) {
         stats.completions += 1
@@ -65,7 +75,7 @@ export const createArrayOfCourses = async (attempts, courses) => {
         stats.rejected += 1
       }
     }
-    return obj
+    return acc
   }, {})
 
   const courseList = courses
@@ -88,55 +98,60 @@ export const createArrayOfCourses = async (attempts, courses) => {
   return courseList
 }
 
+type Attempt = {
+  studentNumber: string
+  courseCode: string
+  completed: boolean
+  date: Date
+  semestercode: number
+  faculty?: string
+  enrolled?: boolean
+}
+
 export const computeLanguageCenterData = async () => {
   const courses = await getLanguageCenterCourses()
   const autumnSemester2017 = 135
 
   const credits = await CreditModel.findAll({
+    raw: true,
     attributes: ['course_code', 'student_studentnumber', 'semestercode', 'attainment_date', 'studyright_id'],
     where: {
       [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
       semestercode: { [Op.gte]: autumnSemester2017 },
       credittypecode: 4,
     },
-    raw: true,
   })
 
   const enrollments = await EnrollmentModel.findAll({
+    raw: true,
     attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'studyright_id', 'state'],
     where: {
       [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
       state: { [Op.in]: ['ENROLLED', 'REJECTED'] },
-      // This doesn't need the `enrollmentTimeDateThreshold`, ColorizedDataTable doesn't use studyright_ids
+      // EnrollmentDateTimeThreshold need not be used here as we are not counting failed courses
     },
-    raw: true,
   })
 
-  const studyRights = await SISStudyRightModel.findAll({ raw: true })
+  const studyRights = await SISStudyRightModel.findAll({
+    raw: true,
+    order: [['end_date', 'DESC NULLS LAST']],
+  })
 
-  const attemptStudyRightToFacultyMap = studyRights.reduce((obj, cur) => {
-    if (!cur.id) return obj
-    obj[cur.id] = cur.facultyCode
-    return obj
+  const attemptStudyRightToFacultyMap = studyRights.reduce<
+    Record<SISStudyRightModel['id'], SISStudyRightModel['facultyCode']>
+  >((acc, cur) => {
+    if (!cur.id) return acc
+    acc[cur.id] = cur.facultyCode
+    return acc
   }, {})
 
-  credits.forEach(credit => {
-    credit.faculty = attemptStudyRightToFacultyMap[credit.studyright_id]
-  })
-
-  enrollments.forEach(enrollment => {
-    enrollment.faculty = attemptStudyRightToFacultyMap[enrollment.studyright_id]
-  })
-
-  const studentNumbers = new Set()
-  const attemptsByStudents = {}
+  const studentNumbers = new Set<string>()
+  const attemptsByStudents: Record<string, Attempt[]> = {}
 
   credits.forEach(credit => {
     const studentNumber = credit.student_studentnumber
     studentNumbers.add(studentNumber)
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
+    attemptsByStudents[studentNumber] ??= []
     attemptsByStudents[studentNumber].push({
       studentNumber,
       courseCode: credit.course_code,
@@ -149,10 +164,8 @@ export const computeLanguageCenterData = async () => {
 
   enrollments.forEach(enrollment => {
     const studentNumber = enrollment.studentnumber
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
     studentNumbers.add(studentNumber)
+    attemptsByStudents[studentNumber] ??= []
     if (
       attemptsByStudents[studentNumber].find(
         attempt =>
@@ -163,6 +176,7 @@ export const computeLanguageCenterData = async () => {
     ) {
       return
     }
+
     attemptsByStudents[studentNumber].push({
       studentNumber,
       courseCode: enrollment.course_code,
@@ -170,23 +184,25 @@ export const computeLanguageCenterData = async () => {
       date: enrollment.enrollment_date_time,
       faculty: attemptStudyRightToFacultyMap[enrollment.studyright_id],
       semestercode: enrollment.semestercode,
-      enrolled: enrollment.state === 'ENROLLED',
+      enrolled: enrollment.state === EnrollmentState.ENROLLED,
     })
   })
 
-  const studentNumberToStudyRightsMap = studyRights.reduce((obj, cur) => {
-    if (!obj[cur.studentNumber]) obj[cur.studentNumber] = []
-    obj[cur.studentNumber].push(cur)
-    return obj
+  const studentNumberToStudyRightsMap = studyRights.reduce<
+    Record<SISStudyRightModel['studentNumber'], SISStudyRightModel[]>
+  >((acc, cur) => {
+    acc[cur.studentNumber] ??= []
+    acc[cur.studentNumber].push(cur)
+    return acc
   }, {})
 
-  const attempts = []
+  const attempts: Attempt[] = []
   studentNumbers.forEach(studentNumber => attempts.push(...attemptsByStudents[studentNumber]))
   // 93033 Avoin yliopisto, yhteistyöoppilaitokset
   // 93034 Kesäyliopistot
-  const isOpenUni = facultyCode => ['9301', 'H930', '93033', '93034'].includes(facultyCode)
-  const isMisc = facultyCode => ['H906', 'H401'].includes(facultyCode) // Language center, Alexander institute (very few numbers)
-  const isHyFaculty = facultyCode => facultyCode.substring(0, 3).match('^H\\d')
+  const isOpenUni = (facultyCode: string) => ['9301', 'H930', '93033', '93034'].includes(facultyCode)
+  const isMisc = (facultyCode: string) => ['H906', 'H401'].includes(facultyCode) // Language center, Alexander institute (very few numbers)
+  const isHyFaculty = (facultyCode: string) => /^H\d/.exec(facultyCode.substring(0, 3))
 
   attempts.forEach(attempt => {
     if (attempt.faculty) {
@@ -210,7 +226,7 @@ export const computeLanguageCenterData = async () => {
     }
   })
 
-  const unorderedTableData = await createArrayOfCourses(attempts, courses)
+  const unorderedTableData = createArrayOfCourses(attempts, courses)
   const tableData = orderBy(unorderedTableData, 'code')
 
   const faculties = [...new Set(attempts.map(({ faculty }) => faculty))]

services/backend/src/services/students.ts

@@ -1,15 +1,15 @@
-import dayjs from 'dayjs'
-import { Op, QueryTypes } from 'sequelize'
+import { Op } from 'sequelize'
 
 import type { Credit, Student } from '@oodikone/shared/models'
-import { EnrollmentState, UnifyStatus } from '@oodikone/shared/types'
+import { CreditTypeCode, EnrollmentState, UnifyStatus } from '@oodikone/shared/types'
 import { FormattedStudentForSearch, StudentPageStudent } from '@oodikone/shared/types/studentData'
-import { enrollmentTimeDateThreshold, splitByEmptySpace } from '@oodikone/shared/util'
-import { dbConnections } from '../database/connection'
+import { enrollmentTimeDateThresholdAcademicYear, splitByEmptySpace } from '@oodikone/shared/util'
+import { dateIsBetween, dateMaxFromList } from '@oodikone/shared/util/datetime'
 import {
   StudentModel,
   CreditModel,
   CourseModel,
+  EnrollmentModel,
   ProgrammeModuleModel,
   SemesterModel,
   StudyplanModel,
@@ -18,8 +18,6 @@ import {
 } from '../models'
 import { TagModel, TagStudentModel } from '../models/kone'
 
-const { sequelize } = dbConnections
-
 const byStudentNumber = async (studentNumber: string) => {
   const [student, tags] = await Promise.all([
     StudentModel.findByPk(studentNumber, {
@@ -62,7 +60,14 @@ const byStudentNumber = async (studentNumber: string) => {
         },
         {
           model: StudyplanModel,
-          attributes: ['id', 'included_courses', 'programme_code', 'completed_credits', 'sis_study_right_id'],
+          attributes: [
+            'id',
+            'included_courses',
+            'includedModules',
+            'programme_code',
+            'completed_credits',
+            'sis_study_right_id',
+          ],
         },
       ],
       order: [
@@ -117,7 +122,13 @@ const getUnifyStatus = (unifyCourses: UnifyStatus): [boolean] | [true, false] =>
   }
 }
 
-/* from & to are semestercodes if separate = false, or yearcodes in case separate is true. */
+/**
+  Returns students whose latest "active" attainment/enrollment is between to and from.
+
+  NOTE: If you have failed a course in 2021 and passed the same course in 2023
+  you would not be included in from=2020 to=2022 query
+
+ */
 export const findByCourseAndSemesters = async (
   codes: string[],
   from: number,
@@ -125,6 +136,7 @@ export const findByCourseAndSemesters = async (
   separate: boolean,
   unifyCourses: UnifyStatus = 'unifyStats'
 ) => {
+  /* from & to are semestercodes if separate = false, or yearcodes in case separate is true. */
   const startSemester = await SemesterModel.findOne({
     where: {
       [separate ? 'semestercode' : 'yearcode']: from,
@@ -147,61 +159,108 @@ export const findByCourseAndSemesters = async (
     return []
   }
 
-  const { startdate, semestercode: fromSemester } = startSemester
-  const { enddate, semestercode: toSemester } = endSemester
+  const { startdate } = startSemester
+  const { enddate } = endSemester
 
   const unifyStatus = getUnifyStatus(unifyCourses)
 
-  type QueryResult = {
-    studentnumber: string
-  }[]
+  const credits = (await CreditModel.findAll({
+    raw: true,
+    attributes: [
+      ['student_studentnumber', 'studentnumber'],
+      'course_code',
+      ['attainment_date', 'date'],
+      'credittypecode',
+    ],
+    where: {
+      course_code: { [Op.in]: codes },
+      is_open: unifyStatus,
+      credittypecode: { [Op.not]: CreditTypeCode.IMPROVED }, // We do not care about improved grades
+    },
+    order: [
+      ['attainment_date', 'DESC'],
+      ['credittypecode', 'ASC'],
+    ],
+  })) as unknown as Array<{
+    studentnumber: CreditModel['student_studentnumber']
+    course_code: CreditModel['course_code']
+    date: CreditModel['attainment_date']
+    credittypecode: CreditModel['credittypecode']
+  }>
+  // HACK: Sequelize doesn't like renaming attributes so we have to force-cast
 
-  const queryResult: QueryResult = await sequelize.query(
-    `
-    SELECT
-      studentnumber
-    FROM
-      student s
-    WHERE
-      EXISTS (
-        SELECT
-          1
-        FROM
-          credit c
-        WHERE
-          c.student_studentnumber = s.studentnumber
-          AND c.course_code IN (:codes)
-          AND c.is_open IN (:isOpen)
-          AND c.attainment_date BETWEEN '${startdate.toISOString()}' AND '${enddate.toISOString()}'
+  const enrollments = (await EnrollmentModel.findAll({
+    raw: true,
+    attributes: ['studentnumber', 'course_code', ['enrollment_date_time', 'date']],
+    where: {
+      course_code: { [Op.in]: codes },
+      is_open: unifyStatus,
+      state: EnrollmentState.ENROLLED,
+    },
+    order: [['enrollment_date_time', 'DESC']],
+  })) as unknown as Array<{
+    studentnumber: EnrollmentModel['studentnumber']
+    course_code: EnrollmentModel['course_code']
+    date: EnrollmentModel['enrollment_date_time']
+  }>
+  // HACK: Sequelize doesn't like renaming attributes so we have to force-cast
+
+  const studentCreditsAndEnrollments: Record<
+    string,
+    Array<{
+      studentnumber: string
+      course_code: string
+      date: Date
+      type: 'credit' | 'enrollment'
+      credittypecode?: CreditTypeCode
+    }>
+  > = {}
+
+  for (const credit of credits) {
+    studentCreditsAndEnrollments[credit.studentnumber] ??= []
+    studentCreditsAndEnrollments[credit.studentnumber].push({ ...credit, type: 'credit' })
+  }
+  for (const enrollment of enrollments) {
+    studentCreditsAndEnrollments[enrollment.studentnumber] ??= []
+    studentCreditsAndEnrollments[enrollment.studentnumber].push({ ...enrollment, type: 'enrollment' })
+  }
+
+  const passedCreditTypes = [CreditTypeCode.PASSED, CreditTypeCode.APPROVED]
+
+  const filteredStudentNumbers = new Set<string>()
+  Object.entries(studentCreditsAndEnrollments).map(([studentNumber, creditsOrEnrollments]) => {
+    const latestCredit = creditsOrEnrollments.find(
+      entry => entry.type === 'credit' && codes.includes(entry.course_code)
+    )
+    const latestEnrollment = creditsOrEnrollments.find(
+      entry => entry.type === 'enrollment' && codes.includes(entry.course_code)
+    )
+
+    if (
+      latestCredit &&
+      passedCreditTypes.includes(latestCredit.credittypecode!) &&
+      dateIsBetween(latestCredit.date, startdate, enddate)
+    ) {
+      filteredStudentNumbers.add(studentNumber)
+    } else if (
+      latestCredit &&
+      latestCredit.credittypecode === CreditTypeCode.FAILED &&
+      dateIsBetween(latestCredit.date, startdate, enddate)
+    ) {
+      filteredStudentNumbers.add(studentNumber)
+    } else if (
+      latestEnrollment &&
+      dateIsBetween(
+        latestEnrollment.date,
+        dateMaxFromList(startdate, enrollmentTimeDateThresholdAcademicYear)!,
+        enddate
       )
-      OR EXISTS (
-        SELECT
-          1
-        FROM
-          enrollment e
-        WHERE
-          e.studentnumber = s.studentnumber
-          AND e.course_code IN (:codes)
-          AND e.semestercode BETWEEN ${fromSemester} AND ${toSemester}
-          AND e.enrollment_date_time >= '${dayjs(enrollmentTimeDateThreshold).format('YYYY-MM-DD')}'
-          AND e.state = :enrollmentState
-      );
-    `,
-    {
-      replacements: {
-        codes,
-        minYearCode: from,
-        maxYearCode: to,
-        isOpen: unifyStatus,
-        enrollmentState: EnrollmentState.ENROLLED,
-      },
-      type: QueryTypes.SELECT,
-      raw: true,
+    ) {
+      filteredStudentNumbers.add(studentNumber)
     }
-  )
+  })
 
-  const studentNumbers = queryResult.map(result => result.studentnumber)
-  return studentNumbers
+  return [...filteredStudentNumbers]
 }
 
 const formatSharedStudentData = ({

services/backend/src/services/studyProgramme/creditGetters.ts

@@ -39,7 +39,7 @@ export const getCreditsForProvider = async (provider: string, codes: string[], s
   }, {})
 
   const courseIdToShareMap = providers.reduce<Record<string, CourseProviderModel[]>>((obj, provider) => {
-    if (!obj[provider.coursecode]) obj[provider.coursecode] = []
+    obj[provider.coursecode] ??= []
     obj[provider.coursecode].push(provider)
     return obj
   }, {})

services/backend/src/services/studyProgramme/index.ts

@@ -1,14 +1,16 @@
 import { memoize } from 'lodash-es'
 import { Op, QueryTypes } from 'sequelize'
 
-import { Name, CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
-import { enrollmentTimeDateThreshold } from '@oodikone/shared/util'
+import { Name, EnrollmentState } from '@oodikone/shared/types'
+import { enrollmentTimeDateThresholdAcademicYear } from '@oodikone/shared/util'
+import { dateIsBetween } from '@oodikone/shared/util/datetime'
 import { dbConnections } from '../../database/connection'
 import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, ProgrammeModuleModel } from '../../models'
 import logger from '../../util/logger'
 
 export const getAllProgrammeCourses = async (providerCode: string) => {
   const courses: Pick<CourseModel, 'id' | 'code' | 'name' | 'substitution_groups'>[] = await CourseModel.findAll({
+    raw: true,
     attributes: ['id', 'code', 'name', 'substitution_groups'],
     include: [
       {
@@ -23,61 +25,76 @@ export const getAllProgrammeCourses = async (providerCode: string) => {
         },
       },
     ],
-    raw: true,
   })
   return courses
 }
 
-const removeOpenUniCodePrefix = (code: string) => {
-  if (code.startsWith('AY')) {
-    return code.replace('AY', '')
-  }
-  if (code.startsWith('A')) {
-    return code.replace('A', '')
-  }
-  return code
-}
-
 export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, programmeCourses: string[]) => {
   try {
-    const enrollmentsCourses = await EnrollmentModel.findAll({
+    const enrollments: Array<
+      Pick<EnrollmentModel, 'studentnumber' | 'course_code' | 'enrollment_date_time' | 'course'>
+    > = await EnrollmentModel.findAll({
       raw: true,
-      attributes: ['studentnumber', 'course_code'],
+      nest: true,
+      attributes: ['studentnumber', 'course_code', 'enrollment_date_time'],
+      include: {
+        model: CourseModel,
+        attributes: ['is_study_module'],
+      },
       where: {
         course_code: {
           [Op.in]: programmeCourses,
         },
         enrollment_date_time: {
-          [Op.between]: [from, to],
-          [Op.gte]: enrollmentTimeDateThreshold,
+          [Op.gte]: enrollmentTimeDateThresholdAcademicYear, // This has to be Academic year to match with Course statistics which doesn't display enrollments for ...-2021
         },
         state: EnrollmentState.ENROLLED,
       },
+      order: [['enrollment_date_time', 'DESC']],
     })
 
-    const allEnrollments = new Map<string, Set<string>>()
-    for (const { studentnumber, course_code: courseCode } of enrollmentsCourses) {
-      const code = removeOpenUniCodePrefix(courseCode)
-      if (!allEnrollments.has(code)) {
-        allEnrollments.set(code, new Set())
-      }
-      allEnrollments.get(code)!.add(studentnumber)
-    }
-
-    const credits = await CreditModel.findAll({
-      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'isStudyModule'],
+    // TODO: This is being called for each year, could be optimized to call only once
+    const credits: Array<
+      Pick<
+        CreditModel,
+        'course_code' | 'student_studentnumber' | 'credittypecode' | 'isStudyModule' | 'attainment_date'
+      >
+    > = await CreditModel.findAll({
+      raw: true,
+      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'isStudyModule', 'attainment_date'],
       where: {
         course_code: {
           [Op.in]: programmeCourses,
         },
-        attainment_date: {
-          [Op.between]: [from, to],
-        },
       },
+      order: [
+        ['attainment_date', 'DESC'],
+        ['credittypecode', 'ASC'],
+      ], // Passed credits for a student have to be handled first (only PASSED, and FAILED matter)
     })
 
+    const enrolledStudentsByCourseCode = new Map<string, Set<string>>()
+
+    // Pick only this (between to, from) year's credits
+    const filteredCredits = credits.filter(credit => dateIsBetween(credit.attainment_date, from, to))
+
+    const filteredEnrollments = enrollments
+      // Only include the first enrollment for a course for a student.
+      // This assumes that enrollments are sorted in DESC order by enrollment_date_time
+      .filter(({ course_code: courseCode, studentnumber: studentNumber }) => {
+        const courseEnrollments = enrolledStudentsByCourseCode.get(courseCode)
+        const hasEnrolled = courseEnrollments?.has(studentNumber) ?? false
+
+        if (!courseEnrollments) enrolledStudentsByCourseCode.set(courseCode, new Set<string>())
+        if (!hasEnrolled) enrolledStudentsByCourseCode.get(courseCode)!.add(studentNumber)
+
+        return !hasEnrolled
+      })
+      .filter(enrollment => dateIsBetween(enrollment.enrollment_date_time, from, to))
+
     const courseCodeToName = (
       await CourseModel.findAll({
+        raw: true,
         attributes: ['code', 'name'],
         where: {
           code: programmeCourses,
@@ -88,50 +105,102 @@ export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, p
       return acc
     }, new Map())
 
-    const creditCourses = credits.map(credit => ({
-      code: removeOpenUniCodePrefix(credit.course_code),
+    const creditCourses = filteredCredits.map(credit => ({
+      code: credit.course_code,
       studentNumber: credit.student_studentnumber,
       creditTypeCode: credit.credittypecode,
       courseName: courseCodeToName.get(credit.course_code)!,
       isStudyModule: credit.isStudyModule,
+      attainmentDate: credit.attainment_date,
     }))
 
     const passedByCourseCodes = new Map<string, Set<string>>()
+    const failedByCourseCodes = new Map<string, Set<string>>()
     const notCompletedByCourseCodes = new Map<string, Set<string>>()
     const courses = new Map<string, { name: Name; isStudyModule: boolean }>()
 
-    for (const { code, courseName, creditTypeCode, isStudyModule, studentNumber } of creditCourses) {
-      if (!courses.has(code)) {
-        courses.set(code, {
+    const studentHasPassedCourse = (studentNumber: string, courseCode: string): boolean =>
+      (passedByCourseCodes.get(courseCode)?.has(studentNumber) ?? false) ||
+      !!credits.find(
+        credit =>
+          credit.student_studentnumber === studentNumber &&
+          credit.course_code === courseCode &&
+          CreditModel.passed(credit)
+      )
+
+    /** Student has already been counted as failed or there is a newer failed credit */
+    const studentHasFailedCourse = (
+      studentNumber: string,
+      courseCode: string,
+      attainmentDate: Date = new Date(0)
+    ): boolean =>
+      (failedByCourseCodes.get(courseCode)?.has(studentNumber) ?? false) ||
+      !!credits.find(
+        credit =>
+          credit.student_studentnumber === studentNumber &&
+          credit.course_code === courseCode &&
+          CreditModel.failed(credit) &&
+          credit.attainment_date > attainmentDate
+      )
+
+    const studentHasCredit = (studentNumber: string, courseCode: string): boolean =>
+      studentHasPassedCourse(studentNumber, courseCode) || studentHasFailedCourse(studentNumber, courseCode)
+
+    for (const {
+      code: courseCode,
+      courseName,
+      creditTypeCode,
+      isStudyModule,
+      studentNumber,
+      attainmentDate,
+    } of creditCourses) {
+      if (!courses.has(courseCode)) {
+        courses.set(courseCode, {
           isStudyModule,
           name: courseName,
         })
-        passedByCourseCodes.set(code, new Set())
-        notCompletedByCourseCodes.set(code, new Set())
+        passedByCourseCodes.set(courseCode, new Set())
+        failedByCourseCodes.set(courseCode, new Set())
+        notCompletedByCourseCodes.set(courseCode, new Set())
       }
-      if ([CreditTypeCode.PASSED, CreditTypeCode.IMPROVED, CreditTypeCode.APPROVED].includes(creditTypeCode)) {
-        passedByCourseCodes.get(code)!.add(studentNumber)
+
+      if (CreditModel.passed({ credittypecode: creditTypeCode })) {
+        passedByCourseCodes.get(courseCode)!.add(studentNumber)
       }
-      if (creditTypeCode === CreditTypeCode.FAILED && !passedByCourseCodes.get(code)!.has(studentNumber)) {
-        notCompletedByCourseCodes.get(code)!.add(studentNumber)
+      if (
+        CreditModel.failed({ credittypecode: creditTypeCode }) &&
+        !studentHasPassedCourse(studentNumber, courseCode) &&
+        !studentHasFailedCourse(studentNumber, courseCode, attainmentDate)
+      ) {
+        failedByCourseCodes.get(courseCode)!.add(studentNumber)
       }
     }
 
-    // If student has enrollments, but no attainment for a particular course, they have no credit info.
-    programmeCourses.forEach(courseCode => {
-      allEnrollments.get(courseCode)?.forEach(studentnumber => {
-        if (!passedByCourseCodes.get(courseCode)?.has(studentnumber)) {
-          notCompletedByCourseCodes.get(courseCode)?.add(studentnumber)
-        }
-      })
-    })
+    // Add course details to courseMap if there are only enrollments
+    for (const { studentnumber: studentNumber, course_code: courseCode, course } of filteredEnrollments) {
+      if (!courses.has(courseCode)) {
+        courses.set(courseCode, {
+          isStudyModule: course.is_study_module,
+          name: courseCodeToName.get(courseCode)!,
+        })
+        passedByCourseCodes.set(courseCode, new Set())
+        failedByCourseCodes.set(courseCode, new Set())
+        notCompletedByCourseCodes.set(courseCode, new Set())
+      }
 
-    return [...courses.entries()].map(([code, { isStudyModule, name }]) => ({
-      code,
-      name,
-      isStudyModule,
-      allNotPassed: notCompletedByCourseCodes.get(code)?.size ?? 0,
-    }))
+      if (!studentHasCredit(studentNumber, courseCode)) {
+        notCompletedByCourseCodes.get(courseCode)!.add(studentNumber)
+      }
+    }
+
+    return [...courses.entries()].map(([code, { isStudyModule, name }]) => {
+      return {
+        code,
+        name,
+        isStudyModule,
+        allNotPassed: (notCompletedByCourseCodes.get(code)?.size ?? 0) + (failedByCourseCodes.get(code)?.size ?? 0),
+      }
+    })
   } catch (error) {
     logger.error(`getNotCompletedForProgrammeCourses failed ${error}`)
     return []

services/backend/src/services/studyProgramme/studyProgrammeCoursesBySemester.ts

@@ -47,9 +47,7 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
   credits.forEach(credit => {
     const studentNumber = credit.student_studentnumber
     studentList.add(studentNumber)
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
+    attemptsByStudents[studentNumber] ??= []
     attemptsByStudents[studentNumber].push({
       studentNumber,
       courseCode: credit.course_code,
@@ -61,9 +59,7 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
 
   enrollments.forEach(enrollment => {
     const studentNumber = enrollment.studentnumber
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
+    attemptsByStudents[studentNumber] ??= []
     studentList.add(studentNumber)
     if (
       attemptsByStudents[studentNumber].find(
@@ -88,7 +84,7 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
   const attemptsArray = [] as Attempt[]
   studentList.forEach(studentNumber => attemptsArray.push(...attemptsByStudents[studentNumber]))
 
-  const unorderedTableData = await createArrayOfCourses(attemptsArray, courses)
+  const unorderedTableData = createArrayOfCourses(attemptsArray, courses)
   const tableData = orderBy(unorderedTableData, 'code')
 
   return { tableData }

services/backend/src/services/studyProgramme/studyProgrammeGraduations.ts

@@ -34,6 +34,7 @@ import {
   getYearsObject,
   getStudyRightElementsWithPhase,
   hasTransferredFromOrToProgramme,
+  getAverage,
 } from './studyProgrammeHelpers'
 import { getStudyRightsInProgramme } from './studyRightFinders'
 
@@ -234,24 +235,22 @@ const getGraduationTimeAndThesisWriterStats = async ({
   const comboTimes: ChartGraduationTimes = { medians: [], goal: goal + GraduationTarget.THREE_YEARS }
 
   for (const year of years.toReversed()) {
-    const median = getMedian(graduationTimes[year])
-    const statistics = countTimeCategories(graduationTimes[year], goal)
     times.medians.push({
-      y: median,
+      median: getMedian(graduationTimes[year]),
+      average: getAverage(graduationTimes[year]),
       amount: graduationTimes[year].length,
       name: year.toString(),
-      statistics,
+      statistics: countTimeCategories(graduationTimes[year], goal),
       times: [...graduationTimes[year]],
     })
 
     if (doCombo) {
-      const median = getMedian(graduationTimesCombo[year])
-      const statistics = countTimeCategories(graduationTimesCombo[year], goal + GraduationTarget.THREE_YEARS)
       comboTimes.medians.push({
-        y: median,
+        median: getMedian(graduationTimesCombo[year]),
+        average: getAverage(graduationTimesCombo[year]),
         amount: graduationTimesCombo[year].length,
         name: year.toString(),
-        statistics,
+        statistics: countTimeCategories(graduationTimesCombo[year], goal + GraduationTarget.THREE_YEARS),
         times: [...graduationTimesCombo[year]],
       })
     }
@@ -302,13 +301,10 @@ const getProgrammesBeforeStarting = async ({
     if (studyRight.extentCode !== ExtentCode.BACHELOR_AND_MASTER) return acc
     const phase1Programmes = studyRight.studyRightElements.filter(elem => elem.phase === Phase.ANY)
     const [latestPhase1Programme] = orderBy(phase1Programmes, ['endDate'], ['desc'])
-    if (!acc[latestPhase1Programme.code]) {
-      const programmeWithYears: ProgrammeWithYears = {
-        code: latestPhase1Programme.code,
-        name: latestPhase1Programme.name,
-        ...getYearsObject({ years }),
-      }
-      acc[latestPhase1Programme.code] = programmeWithYears
+    acc[latestPhase1Programme.code] ??= {
+      code: latestPhase1Programme.code,
+      name: latestPhase1Programme.name,
+      ...getYearsObject({ years }),
     }
     const phase2Programmes = studyRight.studyRightElements.filter(elem => elem.phase === Phase.MASTER)
 
@@ -347,15 +343,11 @@ const getProgrammesAfterGraduation = async ({
     const [firstPhase2Programme] = orderBy(phase2Programmes, ['startDate'], ['asc'])
     if (!firstPhase2Programme) return acc
 
-    if (!acc[firstPhase2Programme.code]) {
-      const programmeWithYears: ProgrammeWithYears = {
-        code: firstPhase2Programme.code,
-        name: firstPhase2Programme.name,
-        ...getYearsObject({ years }),
-      }
-      acc[firstPhase2Programme.code] = programmeWithYears
+    acc[firstPhase2Programme.code] ??= {
+      code: firstPhase2Programme.code,
+      name: firstPhase2Programme.name,
+      ...getYearsObject({ years }),
     }
-
     ;(acc[firstPhase2Programme.code][defineYear(firstPhase2Programme.startDate, isAcademicYear)] as number) += 1
     return acc
   }, {})

services/backend/src/services/studyProgramme/studyProgrammeHelpers.ts

@@ -56,6 +56,11 @@ export const getMedian = (values: number[]) => {
   return (sorted[half - 1] + sorted[half]) / 2.0
 }
 
+export const getAverage = (values: number[]) => {
+  if (!values.length) return 0
+  return Number((values.reduce((acc, cur) => acc + cur, 0) / values.length).toFixed(2))
+}
+
 /** @returns academic starting year e.g. "2019-2020" */
 export function defineYear(date: Date, isAcademicYear: true): string
 export function defineYear(date: Date, isAcademicYear: false): number
@@ -259,7 +264,7 @@ export const tableTitles = {
     licentiate: ['Graduated bachelor', 'Graduated licentiate'],
     master: ['Graduated bachelor', 'Graduated master'],
   },
-  studytracksEnd: ['Men', 'Women', 'Other / Unknown', 'Finland', 'Other'],
+  studytracksEnd: ['Has recent attainment', 'Men', 'Women', 'Other / Unknown', 'Finland', 'Other'],
 } as const
 
 export const getId = (code: string) => {

services/backend/src/services/studyProgramme/studyRightFinders.ts

@@ -2,7 +2,7 @@ import { col, fn, Includeable, Op } from 'sequelize'
 
 import { SISStudyRight } from '@oodikone/shared/models'
 import { CreditTypeCode } from '@oodikone/shared/types'
-import { StudentModel, SISStudyRightModel, SISStudyRightElementModel, CreditModel } from '../../models'
+import { StudentModel, SISStudyRightModel, SISStudyRightElementModel, CreditModel, StudyplanModel } from '../../models'
 
 export const getStudyRightsInProgramme = async (
   programmeCode: string,
@@ -16,6 +16,7 @@ export const getStudyRightsInProgramme = async (
 
   const studyRightIds = (
     await SISStudyRightModel.findAll({
+      raw: true,
       attributes: ['id'],
       include: {
         model: SISStudyRightElementModel,
@@ -23,7 +24,6 @@ export const getStudyRightsInProgramme = async (
         attributes: [],
         where,
       },
-      raw: true,
     })
   ).map(sr => sr.id)
 
@@ -36,23 +36,34 @@ export const getStudyRightsInProgramme = async (
   ]
 
   if (includeStudentsAndCredits) {
-    include.push({
-      model: StudentModel,
-      attributes: ['gender_code', 'citizenships'],
-      include: [
-        {
-          model: CreditModel,
-          attributes: ['attainment_date', 'credits'],
-          where: {
-            isStudyModule: false,
-            credittypecode: {
-              [Op.in]: [CreditTypeCode.PASSED, CreditTypeCode.APPROVED],
+    include.push(
+      {
+        model: StudentModel,
+        attributes: ['gender_code', 'citizenships'],
+        include: [
+          {
+            model: CreditModel,
+            attributes: ['attainment_date', 'credits', 'course_code'],
+            required: false,
+            where: {
+              isStudyModule: false,
+              credittypecode: {
+                [Op.in]: [CreditTypeCode.PASSED, CreditTypeCode.APPROVED],
+              },
             },
           },
-          required: false,
+        ],
+      },
+      {
+        model: StudyplanModel,
+        as: 'studyPlans',
+        attributes: ['included_courses'],
+        required: false,
+        where: {
+          programme_code: programmeCode,
         },
-      ],
-    })
+      }
+    )
   }
 
   return (

services/backend/src/services/studyProgramme/studyTrackStats.ts

@@ -1,8 +1,7 @@
+import dayjs from 'dayjs'
 import { SISStudyRight, SISStudyRightElement } from '@oodikone/shared/models'
 import {
   GraduationTimes,
-  MedianEntry,
-  ProgrammeOrStudyTrackGraduationStats,
   StudyTrackStats,
   GenderCode,
   EnrollmentType,
@@ -17,6 +16,7 @@ import { calculateDurationOfStudies, shouldIncludeComboStats } from './studyProg
 import {
   computePercentiles,
   defineYear,
+  getAverage,
   getCreditCount,
   getGoal,
   getMedian,
@@ -40,7 +40,7 @@ const getGraduationTimeStats = async (
   combinedProgramme?: string
 ) => {
   const goal = await getGoal(combinedProgramme ?? studyProgramme)
-  const finalGraduationTimes = { goals: { basic: goal, combo: goal + GraduationTarget.THREE_YEARS } }
+  const finalGraduationTimes = { goals: { basic: goal, combo: goal + GraduationTarget.THREE_YEARS } } as GraduationTimes
 
   const calculateGraduationTimes = (
     graduationTimes: Record<string, Record<string, number[]>>,
@@ -48,7 +48,7 @@ const getGraduationTimeStats = async (
   ) => {
     for (const programmeOrTrack of Object.keys(graduationTimes)) {
       const stats = graduationTimes[programmeOrTrack]
-      finalGraduationTimes[programmeOrTrack] ??= { medians: { basic: [] as MedianEntry[], combo: [] as MedianEntry[] } }
+      finalGraduationTimes[programmeOrTrack] ??= { medians: { basic: [], combo: [] } }
       for (const [year, oneYearStats] of Object.entries(stats)) {
         const classSize = mainStatsByTrack[combinedProgramme ? studyProgramme : programmeOrTrack].find(
           stats => stats[0] === year
@@ -61,21 +61,20 @@ const getGraduationTimeStats = async (
           classSize,
           name: year,
           statistics: countTimeCategories(oneYearStats, finalGraduationTimes.goals[type]),
-          y: getMedian(oneYearStats),
+          median: getMedian(oneYearStats),
+          average: getAverage(oneYearStats),
           times: [...oneYearStats],
         }
-        ;(finalGraduationTimes[programmeOrTrack] as ProgrammeOrStudyTrackGraduationStats).medians[type].push(final)
+        finalGraduationTimes[programmeOrTrack].medians[type].push(final)
       }
-      ;(finalGraduationTimes[programmeOrTrack] as ProgrammeOrStudyTrackGraduationStats).medians[type].sort(
-        createLocaleComparator('name', true)
-      )
+      finalGraduationTimes[programmeOrTrack].medians[type].sort(createLocaleComparator('name', true))
     }
   }
 
   calculateGraduationTimes(graduationTimes, 'basic')
   calculateGraduationTimes(graduationTimesCombo, 'combo')
 
-  return finalGraduationTimes as GraduationTimes
+  return finalGraduationTimes
 }
 
 const getEmptyYear = () => ({
@@ -86,6 +85,7 @@ const getEmptyYear = () => ({
   passive: 0,
   graduated: 0,
   graduatedCombinedProgramme: 0,
+  attainmentWithinYear: 0,
   male: 0,
   female: 0,
   otherUnknown: 0,
@@ -133,6 +133,8 @@ const combineStats = (
         getPercentage(yearStats.passive, yearStats.all),
         yearStats.graduated,
         getPercentage(yearStats.graduated, yearStats.all),
+        yearStats.attainmentWithinYear,
+        getPercentage(yearStats.attainmentWithinYear, yearStats.all),
         yearStats.male,
         getPercentage(yearStats.male, yearStats.all),
         yearStats.female,
@@ -158,9 +160,7 @@ const combineStats = (
       ])
       mainStatsByTrack[programmeOrStudyTrack].push([year, ...yearArray])
       if (Object.keys(yearStats.otherCountriesCounts).length > 0) {
-        if (!otherCountriesCount[programmeOrStudyTrack]) {
-          otherCountriesCount[programmeOrStudyTrack] = {}
-        }
+        otherCountriesCount[programmeOrStudyTrack] ??= {}
         otherCountriesCount[programmeOrStudyTrack][year] = yearStats.otherCountriesCounts
       }
     }
@@ -182,6 +182,8 @@ const getMainStatsByTrackAndYear = async (
   const { semesters } = await getSemestersAndYears()
   const { semestercode: currentSemester } = Object.values(semesters).find(semester => semester.enddate >= new Date())!
 
+  const yearAgo = dayjs().subtract(1, 'year')
+
   const graduationTimes: Record<string, Record<string, number[]>> = {}
   const graduationTimesCombo: Record<string, Record<string, number[]>> = {}
   const graduationTimesCombinedProgrammeCombo: Record<string, Record<string, number[]>> = {}
@@ -213,6 +215,18 @@ const getMainStatsByTrackAndYear = async (
       }
     }
 
+    const latestAttainmentWithinYearAndInHops = dayjs(
+      studyRight.student.credits
+        .filter(credit => studyRight.studyPlans.some(sp => sp.included_courses.includes(credit.course_code)))
+        .reduce((latest, { attainment_date }) => {
+          return attainment_date > latest ? attainment_date : latest
+        }, new Date('1900-01-01'))
+    ).isSameOrAfter(yearAgo)
+
+    if (latestAttainmentWithinYearAndInHops) {
+      yearlyStats[year][programmeOrStudyTrack].attainmentWithinYear++
+    }
+
     /* Gender cols */
     if (studyRight.student.gender_code === GenderCode.MALE) {
       yearlyStats[year][programmeOrStudyTrack].male += 1
@@ -285,35 +299,27 @@ const getMainStatsByTrackAndYear = async (
     )
 
     if (countAsBachelorMaster) {
-      if (!graduationTimesCombo[programmeOrStudyTrack]) {
-        graduationTimesCombo[programmeOrStudyTrack] = getYearsObject({
-          years: years.filter(year => year !== 'Total'),
-          emptyArrays: true,
-        })
-      }
+      graduationTimesCombo[programmeOrStudyTrack] ??= getYearsObject({
+        years: years.filter(year => year !== 'Total'),
+        emptyArrays: true,
+      })
       const startYearInBachelor = defineYear(startDate, true)
-      if (graduationTimesCombo[programmeOrStudyTrack][startYearInBachelor]) {
-        graduationTimesCombo[programmeOrStudyTrack][startYearInBachelor].push(duration)
-      }
+      graduationTimesCombo[programmeOrStudyTrack][startYearInBachelor]?.push(duration)
     } else {
-      if (!graduationTimes[programmeOrStudyTrack]) {
-        graduationTimes[programmeOrStudyTrack] = getYearsObject({
-          years: years.filter(year => year !== 'Total'),
-          emptyArrays: true,
-        })
-      }
+      graduationTimes[programmeOrStudyTrack] ??= getYearsObject({
+        years: years.filter(year => year !== 'Total'),
+        emptyArrays: true,
+      })
       graduationTimes[programmeOrStudyTrack][year].push(duration)
     }
 
     if (!hasGraduatedFromCombinedProgramme) return
     const bachelorStartDate = getStudyRightElementsWithPhase(studyRight, 1)[0]?.startDate
     if (!bachelorStartDate) return
-    if (!graduationTimesCombinedProgrammeCombo[combinedProgramme!]) {
-      graduationTimesCombinedProgrammeCombo[combinedProgramme!] = getYearsObject({
-        years: years.filter(year => year !== 'Total'),
-        emptyArrays: true,
-      })
-    }
+    graduationTimesCombinedProgrammeCombo[combinedProgramme!] ??= getYearsObject({
+      years: years.filter(year => year !== 'Total'),
+      emptyArrays: true,
+    })
     const startYearInNewBachelorProgramme = defineYear(studyRightElement.startDate, true)
     const combinedDuration = calculateDurationOfStudies(
       bachelorStartDate,
@@ -322,9 +328,7 @@ const getMainStatsByTrackAndYear = async (
       semesters,
       studyRight.transferInfo
     )
-    if (graduationTimesCombinedProgrammeCombo[combinedProgramme!][startYearInNewBachelorProgramme]) {
-      graduationTimesCombinedProgrammeCombo[combinedProgramme!][startYearInNewBachelorProgramme].push(combinedDuration)
-    }
+    graduationTimesCombinedProgrammeCombo[combinedProgramme!][startYearInNewBachelorProgramme]?.push(combinedDuration)
   }
 
   const creditCounts: Record<string, number[]> = getYearsObject({

services/backend/src/services/teachers/index.ts

@@ -21,6 +21,7 @@ export const getTeachersBySearchTerm = async (searchTerm: string) => {
         [Op.and]: searchTerms.map(term => ({ [Op.iLike]: `%${term}%` })),
       },
     },
+    order: [['name', 'ASC']],
   })
 }
 

services/backend/src/services/teachers/top.ts

@@ -83,9 +83,7 @@ const updatedStats = (
   transferred: boolean
 ) => {
   const { id, name } = teacher
-  if (!teacherStats[id]) {
-    teacherStats[id] = { id, name, passed: 0, failed: 0, credits: 0, transferred: 0 }
-  }
+  teacherStats[id] ??= { id, name, passed: 0, failed: 0, credits: 0, transferred: 0 }
   const stats = teacherStats[id]
   if (passed) {
     stats.passed += 1

services/backend/src/types/index.ts

@@ -1 +1,4 @@
 export { ParsedCourse } from './course'
+
+/** Sets one or more field(s) of an object to optional */
+export type SetOptional<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>

services/backend/src/util/index.ts

@@ -108,3 +108,23 @@ export const safeJSONParse = async <T>(json: string): Promise<T | null> => {
 export const handleQueryArrays = <T>(requestField: T | T[]): T[] => {
   return requestField ? (Array.isArray(requestField) ? requestField : [requestField]) : []
 }
+
+/**
+ * Recursively deletes all values from object that match the given key.
+ * Alternatively instead of deleting, replaces the values with something else.
+ */
+export const objectKeyAnnihilator = (data: any, keyToNuke: string, replaceWith?: any) => {
+  if (Array.isArray(data)) {
+    data.forEach(item => objectKeyAnnihilator(item, keyToNuke, replaceWith))
+  } else if (!!data && typeof data === 'object') {
+    if (data.times) {
+      if (replaceWith) {
+        data.times = replaceWith
+      } else {
+        delete data.times
+      }
+    }
+    Object.values(data).forEach(item => objectKeyAnnihilator(item, keyToNuke, replaceWith))
+  }
+  return data
+}

services/backend/src/util/logger.ts

@@ -1,7 +1,7 @@
 import * as winston from 'winston'
 import LokiTransport from 'winston-loki'
 
-import { isProduction } from '../config'
+import { isProduction, silentTesting } from '../config'
 
 const transports: winston.transport[] = [new winston.transports.Console()]
 
@@ -30,6 +30,7 @@ const logger = winston.createLogger({
     })
   ),
   transports,
+  silent: silentTesting,
 })
 
 logger.on('error', error => console.error('Logging failed! Reason: ', error)) // eslint-disable-line no-console

services/backend/src/worker/processor.js

@@ -6,7 +6,6 @@ const {
   refreshCloseToGraduating,
 } = require('../events')
 const { updateSISStudyPlans } = require('../services/sisUpdaterService')
-const logger = require('../util/logger')
 
 // This bullmq worker processor has to be in a different file for it to be run in a sandboxed process.
 // Beware of creating a circular dependency, e.g. don't import worker file into the files that you import to this file.
@@ -23,14 +22,9 @@ const refreshers = {
 }
 
 module.exports = async job => {
-  try {
-    if (job.name === 'studyplansUpdate') {
-      await updateSISStudyPlans(job.data.days)
-      return
-    }
+  if (job.name === 'studyplansUpdate') {
+    await updateSISStudyPlans(job.data.days)
+  } else {
     await refreshers[job.id.split('-')[0]](job.data?.code)
-  } catch (error) {
-    logger.error(error.stack)
-    throw error
   }
 }

services/backend/src/worker/worker.ts

@@ -31,12 +31,14 @@ worker.on('completed', job => {
 })
 
 // If there is no error event listener, the worker stops taking jobs after any error.
+// This is only for some internal errors. If a job fails/throws, the below "failed" event is emitted instead.
 worker.on('error', error => {
   logger.error('Job returned error:')
   logger.error(error.toString())
   Sentry.captureException(error)
 })
 
+// Errors from the processor are propagated back here. Thus no logging needs to be done there.
 worker.on('failed', (job, error) => {
   logger.error(`Job ${job?.id ?? 'unknown id'} failed. ${error.stack ?? ''}`)
   Sentry.captureException(error)

services/backend/tests/api/changelog.test.ts

@@ -1,17 +1,16 @@
 import { Express } from 'express'
-import assert from 'node:assert/strict'
-import { describe, it, before } from 'node:test'
 import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
 
-import { initTests } from './utils'
+import { initTests } from '../utils'
 
-let app: Express
-before(async () => {
-  app = await initTests()
-})
+void describe('Changelog', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
 
-void describe('Changelog', async () => {
-  await it('should return dev data in dev env', async () => {
+  it('should return dev data in dev env', async () => {
     const res = await request(app)
       .get('/changelog')
       .set('shib-session-id', 'test')

services/backend/tests/api/courses/courseyearlystats.test.ts

@@ -0,0 +1,465 @@
+import { Express } from 'express'
+import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { Unarray } from '@oodikone/shared/types'
+import { CourseYearlyStatsResBody } from '../../../src/routes/courses'
+import { initTests, ResponseWithBody } from '../../utils'
+
+void describe('Course yearly statistics', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
+
+  it('should return nothing with missing parameters', async () => {
+    const res = await request(app)
+      .get('/courseyearlystats')
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')
+
+    assert.strictEqual(res.status, 422)
+    assert.strictEqual(res.body?.error, 'Missing required query parameters')
+  })
+
+  describe('should return correct amount of values for a course without substitutions (TKT10002)', () => {
+    let body: Unarray<CourseYearlyStatsResBody>
+    beforeAll(async () => {
+      const res = (await request(app)
+        .get('/courseyearlystats?codes=TKT10002&combineSubstitutions=false')
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
+
+      assert.strictEqual(res.status, 200)
+      assert.strictEqual(res.body.length, 1, 'Query to return anything')
+      body = res.body.at(0)!
+      assert(
+        'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
+        'All keys of courseyearlystats not defined'
+      )
+    })
+
+    it('(statistic years)', () => {
+      assert.strictEqual(body.unifyStats!.statistics.length, 8, 'unifyStats years')
+      assert.strictEqual(body.regularStats!.statistics.length, 8, 'regularStats years')
+      assert.strictEqual(body.openStats!.statistics.length, 4, 'openStats years')
+
+      // Faculties should have the same amount of years as normal statistics (above)
+      assert.strictEqual(Object.keys(body.unifyStats!.facultyStats).length, 8, 'unifyStats years')
+      assert.strictEqual(Object.keys(body.regularStats!.facultyStats).length, 8, 'regularStats years')
+      assert.strictEqual(Object.keys(body.openStats!.facultyStats).length, 4, 'openStats years')
+    })
+
+    describe('(statistic contents)', () => {
+      it('- 2016-2017', () => {
+        const stats2016 = body.unifyStats!.statistics.find(year => year.name === '2016-2017')!
+        assert.deepStrictEqual(
+          stats2016.students.studentNumbers,
+          ['484144'],
+          'Incorrect students marked to 2016 stats (students)'
+        )
+
+        assert.deepStrictEqual(stats2016.attempts.grades['4'], ['484144'], 'Incorrect grades marked to 2016 stats (4)')
+        assert.strictEqual(
+          stats2016.attempts.categories.passed.length,
+          1,
+          'Incorrect students marked as passed to 2016 stats (passed)'
+        )
+        assert.strictEqual(
+          stats2016.attempts.categories.failed.length,
+          0,
+          'Incorrect students marked as failed to 2016 stats (failed)'
+        )
+
+        assert('enrollments' in stats2016, 'Missing field enrollment in statsitics')
+        assert.strictEqual(stats2016.enrollments.length, 0, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2016.allEnrollments.length, 0, 'Enrollments found when none should exist')
+      })
+
+      it('- 2017-2018', () => {
+        const stats2017 = body.unifyStats!.statistics.find(year => year.name === '2017-2018')!
+
+        assert.strictEqual(stats2017.attempts.grades['5'].length, 11, 'Incorrect grades marked to 2017 stats (5)')
+        assert.strictEqual(stats2017.attempts.grades['4'].length, 5, 'Incorrect grades marked to 2017 stats, (4)')
+        assert.strictEqual(stats2017.attempts.grades['3'].length, 2, 'Incorrect grades marked to 2017 stats (3)')
+        assert.strictEqual(stats2017.attempts.grades['2'].length, 1, 'Incorrect grades marked to 2017 stats (2)')
+        assert(!('1' in stats2017.attempts.grades), 'Incorrect grades marked to 2017 stats (1)')
+        assert.strictEqual(stats2017.attempts.grades['Hyl.'].length, 4, 'Incorrect grades marked to 2017 stats (Hyl.)')
+
+        assert.strictEqual(stats2017.students.studentNumbers.length, 23, 'Incorrect amount of total students 2017')
+
+        assert.strictEqual(
+          stats2017.attempts.categories.passed.length,
+          11 + 5 + 2 + 1 + 0,
+          'Incorrect students marked as passed to 2017 stats'
+        )
+        assert.strictEqual(
+          stats2017.attempts.categories.failed.length,
+          4,
+          'Incorrect students marked as failed to 2017 stats'
+        )
+
+        // These should not exist before 21-22
+        assert('enrollments' in stats2017, 'Missing field enrollment in statsitics')
+        assert.strictEqual(stats2017.enrollments.length, 0, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2017.allEnrollments.length, 0, 'Enrollments found when none should exist')
+      })
+
+      it('- 2018-2019', () => {
+        const stats2018 = body.unifyStats!.statistics.find(year => year.name === '2018-2019')!
+
+        // There are more students that have passed this course later even though they have failed the course during 2018-2019
+        assert.deepStrictEqual(
+          stats2018.attempts.categories.failed,
+          ['454128', '546124'],
+          'Incorrect students marked as failed 2018'
+        )
+
+        assert.strictEqual(stats2018.attempts.grades['0'].length, 2)
+        assert.strictEqual((stats2018.attempts.grades['Hyl.'] ?? []).length, 0)
+        assert.strictEqual(stats2018.attempts.categories.failed.length, 2, 'Incorrect amount of failed students 2018')
+      })
+
+      it('- 2020-2021', () => {
+        const stats2020 = body.unifyStats!.statistics.find(year => year.name === '2020-2021')!
+
+        // These should not exist before 21-22
+        assert('enrollments' in stats2020, 'Missing field enrollment in statsitics')
+        assert.strictEqual(stats2020.enrollments.length, 0, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2020.allEnrollments.length, 0, 'Enrollments found when none should exist')
+      })
+
+      it('- 2022-2023', () => {
+        const stats2022 = body.unifyStats!.statistics.find(year => year.name === '2022-2023')!
+        assert('enrollments' in stats2022, 'Missing field enrollment in statsitics')
+
+        assert.strictEqual(stats2022.students.studentNumbers.length, 27, 'Incorrect amount of total students 2022')
+
+        assert.strictEqual(stats2022.enrollments.length, 34, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2022.allEnrollments.length, 43, 'Enrollments found when none should exist')
+      })
+
+      it('- 2023-2024', () => {
+        const stats2023 = body.unifyStats!.statistics.find(year => year.name === '2023-2024')!
+        assert.strictEqual(stats2023.students.studentNumbers.length, 1, 'Incorrect amount of total students 2023')
+
+        assert.deepStrictEqual(
+          stats2023.attempts.categories.passed,
+          ['521757'],
+          'Incorrect students marked as passed in 2023 stats'
+        )
+        assert.deepStrictEqual(
+          stats2023.attempts.grades['5'],
+          ['521757'],
+          "Correct student doesn't exist under correct grade"
+        )
+
+        assert('enrollments' in stats2023, 'Missing field enrollment in statsitics')
+        assert.strictEqual(stats2023.enrollments.length, 7, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2023.allEnrollments.length, 7, 'Enrollments found when none should exist')
+      })
+    })
+
+    describe('should work correctly in specific cases (TKT10002)', () => {
+      let body: Unarray<CourseYearlyStatsResBody>
+      beforeAll(async () => {
+        const res = (await request(app)
+          .get('/courseyearlystats?codes=TKT10002&combineSubstitutions=false')
+          .set('shib-session-id', 'test')
+          .set('uid', 'basic')
+          .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
+
+        assert.strictEqual(res.status, 200)
+        assert.strictEqual(res.body.length, 1, 'Query to return anything')
+        body = res.body.at(0)!
+        assert(
+          'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
+          'All keys of courseyearlystats not defined'
+        )
+      })
+
+      it('should include students who have failed a course and then completed it only once', () => {
+        const stats2017 = body.unifyStats!.statistics.find(year => year.name === '2017-2018')!
+        const { studentNumbers } = stats2017.students
+
+        assert(
+          !(
+            '457686' in studentNumbers || // SPRING 2019
+            '455478' in studentNumbers || // FALL 2020
+            '547994' in studentNumbers // FALL 2022
+          ),
+          "Students that have completed course later should not be included in the previous year's stats"
+        )
+
+        assert(
+          !body
+            .unifyStats!.statistics.find(year => year.name === '2017-2018')
+            ?.attempts.categories.passed.includes('457686') &&
+            !body
+              .unifyStats!.statistics.find(year => year.name === '2017-2018')
+              ?.attempts.categories.failed.includes('457686'),
+          "Student was incorrectly included to the failed course code's year stats (457686)"
+        )
+        assert(
+          body
+            .unifyStats!.statistics.find(year => year.name === '2018-2019')
+            ?.attempts.categories.passed.includes('457686') &&
+            body
+              .unifyStats!.statistics.find(year => year.name === '2018-2019')
+              ?.attempts.grades['5'].includes('457686'),
+          "Student was incorrectly excluded from the passed course's completion year stats (457686)"
+        )
+
+        assert(
+          !body
+            .unifyStats!.statistics.find(year => year.name === '2017-2018')
+            ?.attempts.categories.passed.includes('455478') &&
+            !body
+              .unifyStats!.statistics.find(year => year.name === '2017-2018')
+              ?.attempts.categories.failed.includes('455478'),
+          "Student was incorrectly included to the failed course code's year stats (455478)"
+        )
+        assert(
+          body
+            .unifyStats!.statistics.find(year => year.name === '2020-2021')
+            ?.attempts.categories.passed.includes('455478') &&
+            body
+              .unifyStats!.statistics.find(year => year.name === '2020-2021')
+              ?.attempts.grades['2'].includes('455478'),
+          "Student was incorrectly excluded from the passed course's completion year stats (455478)"
+        )
+
+        assert(
+          !body
+            .unifyStats!.statistics.find(year => year.name === '2017-2018')
+            ?.attempts.categories.passed.includes('547994') &&
+            !body
+              .unifyStats!.statistics.find(year => year.name === '2017-2018')
+              ?.attempts.categories.failed.includes('547994'),
+          "Student was incorrectly included to the failed course code's year stats (547994)"
+        )
+        assert(
+          body
+            .unifyStats!.statistics.find(year => year.name === '2022-2023')
+            ?.attempts.categories.passed.includes('547994') &&
+            body
+              .unifyStats!.statistics.find(year => year.name === '2022-2023')
+              ?.attempts.grades['5'].includes('547994'),
+          "Student was incorrectly excluded from the passed course's completion year stats (547994)"
+        )
+      })
+
+      it("should mark student's enrollment time correctly with mismatched semestercode and enrollment_date_time", () => {
+        const stats2021 = body.unifyStats!.statistics.find(year => year.name === '2021-2022')!
+        const stats2022 = body.unifyStats!.statistics.find(year => year.name === '2022-2023')!
+        assert('enrollments' in stats2021, 'Missing field enrollment in statsitics (2021)')
+        assert('enrollments' in stats2022, 'Missing field enrollment in statsitics (2022)')
+        // Student with enrollment_date_time outside of semestert start and end dates
+        assert(
+          !stats2021.enrollments.map(enrollment => enrollment.studentNumber).includes('455129') &&
+            stats2022.enrollments.map(enrollment => enrollment.studentNumber).includes('455129'),
+          'Student with incorrect semester (by updater) but a correct enrollment_date was not found in the correct year (by enrollment_date)'
+        )
+      })
+    })
+
+    describe('should work correctly in specific cases (MAT11001)', () => {
+      let body: Unarray<CourseYearlyStatsResBody>
+      beforeAll(async () => {
+        const res = (await request(app)
+          .get('/courseyearlystats?codes=MAT11001&combineSubstitutions=false')
+          .set('shib-session-id', 'test')
+          .set('uid', 'basic')
+          .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
+
+        assert.strictEqual(res.status, 200)
+        assert.strictEqual(res.body.length, 1)
+        body = res.body.at(0)!
+        assert(
+          'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
+          'All keys of courseyearlystats not defined'
+        )
+      })
+
+      it('where a student has failed a course', () => {
+        const stats = body.unifyStats?.statistics!.find(year => year.name === '2018-2019')
+        assert(stats && 'enrollments' in stats, 'Missing field enrollment in statsitics')
+
+        assert(
+          stats.attempts.categories.failed.includes('542874'),
+          "Stats didn't include student with only failed course attainment (failed)"
+        )
+        assert(
+          !stats.attempts.categories.passed.includes('542874'),
+          'Stats did incorrectly include student with only failed course attainment (passed)'
+        )
+        assert(
+          !stats.enrollments.map(({ studentNumber }) => studentNumber).includes('542874'),
+          "Stats didn't include student with only failed course attainment (enrollments)"
+        )
+      })
+    })
+
+    describe('should work correctly in specific cases (MAT11002)', () => {
+      let body: Unarray<CourseYearlyStatsResBody>
+      beforeAll(async () => {
+        const res = (await request(app)
+          .get('/courseyearlystats?codes=MAT11002&combineSubstitutions=false')
+          .set('shib-session-id', 'test')
+          .set('uid', 'basic')
+          .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
+
+        assert.strictEqual(res.status, 200)
+        assert.strictEqual(res.body.length, 1)
+        body = res.body.at(0)!
+        assert(
+          'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
+          'All keys of courseyearlystats not defined'
+        )
+      })
+
+      it('should not include students for AY code', () => {
+        const stats = body.unifyStats?.statistics!.find(year => year.name === '2017-2018')
+        assert(stats && 'enrollments' in stats, 'Missing field enrollment in statsitics')
+        assert(!('534980' in stats.attempts.categories.failed))
+      })
+
+      it('should not include student with failed grade after passed grade', () => {
+        const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
+        assert(year, 'Stats missing completely')
+        assert(
+          !year.attempts.categories.failed.includes('501716'),
+          'Failed students included incorrectly the student in question'
+        )
+        assert(
+          year.attempts.categories.passed.includes('501716'),
+          "Passed students didn't include student with a passed grade"
+        )
+        assert(year.attempts.grades['1'].includes('501716'), 'Grades should include the student in the correct grade')
+      })
+
+      it('should include student with failed grade and passed AY grade', () => {
+        const year = body.unifyStats?.statistics.find(year => year.name === '2020-2021')
+        assert(year, 'Stats missing completely')
+        assert(
+          year.attempts.categories.failed.includes('0011812135'),
+          "Failed students didn't incorrectly include student in question"
+        )
+        assert(
+          !year.attempts.categories.passed.includes('0011812135'),
+          'Passed students incorrectly included student without a passing grade'
+        )
+        assert(
+          year.attempts.grades['Hyl.'].includes('0011812135'),
+          'Grades should include the student in the correct grade'
+        )
+      })
+
+      it('should include student with only approved grade', () => {
+        const year = body.unifyStats?.statistics.find(year => year.name === '2022-2023')
+        assert(year, 'Stats missing completely')
+        assert(
+          !year.attempts.categories.failed.includes('543385'),
+          'Failed students included incorrectly the student in question'
+        )
+        assert(
+          year.attempts.categories.passed.includes('543385'),
+          "Passed students didn't include student with a passed grade"
+        )
+        assert(year.attempts.grades['5'].includes('543385'), 'Grades should include the student in the correct grade')
+      })
+
+      describe('should calculate enrollments correctly', () => {
+        it('- 2021-2022', () => {
+          const year = body.unifyStats?.statistics.find(year => year.name === '2021-2022')
+          assert(year && 'enrollments' in year, 'Missing field enrollment in statsitics')
+          assert.strictEqual(year.enrollments.length, 27, 'Incorrect amount of distinct enrolled students 2021')
+          assert.strictEqual(year.allEnrollments.length, 29, 'Incorrect amount of total enrolled students 2021')
+        })
+        it('- 2022-2023', () => {
+          const year = body.unifyStats?.statistics.find(year => year.name === '2022-2023')
+          assert(year && 'enrollments' in year, 'Missing field enrollment in statsitics')
+          assert.strictEqual(year.enrollments.length, 23, 'Incorrect amount of distinct enrolled students 2022')
+          assert.strictEqual(year.allEnrollments.length, 26, 'Incorrect amount of total enrolled students 2022')
+        })
+        it('- 2023-2024', () => {
+          const year = body.unifyStats?.statistics.find(year => year.name === '2023-2024')
+          assert(year && 'enrollments' in year, 'Missing field enrollment in statsitics')
+          assert.strictEqual(year.enrollments.length, 1, 'Incorrect amount of distinct enrolled students 2023')
+          assert.strictEqual(year.allEnrollments.length, 1, 'Incorrect amount of total enrolled students 2023')
+        })
+
+        it('- total', () => {
+          assert(
+            body.unifyStats?.statistics.every(year => 'enrollments' in year),
+            'Missing field enrollment in statsitics'
+          )
+          assert.strictEqual(
+            body.unifyStats?.statistics.reduce((acc, yearStats) => acc + yearStats.enrollments.length, 0),
+            27 + 23 + 1
+          )
+          assert.strictEqual(
+            body.unifyStats?.statistics.reduce((acc, yearStats) => acc + yearStats.allEnrollments.length, 0),
+            29 + 26 + 1
+          )
+          assert.strictEqual(
+            body.unifyStats?.statistics.reduce(
+              (acc, yearStats) => acc + yearStats.attempts.categories.failed.length,
+              0
+            ),
+            1
+          )
+          assert.strictEqual(
+            body.unifyStats?.statistics.reduce(
+              (acc, yearStats) => acc + yearStats.attempts.categories.passed.length,
+              0
+            ),
+            249
+          )
+        })
+      })
+    })
+
+    describe('should work correctly in specific cases (MAT21003)', () => {
+      let body: Unarray<CourseYearlyStatsResBody>
+      beforeAll(async () => {
+        const res = (await request(app)
+          .get('/courseyearlystats?codes=MAT21003&combineSubstitutions=false')
+          .set('shib-session-id', 'test')
+          .set('uid', 'basic')
+          .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
+
+        assert.strictEqual(res.status, 200)
+        assert.strictEqual(res.body.length, 1)
+        body = res.body.at(0)!
+        assert(
+          'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
+          'All keys of courseyearlystats not defined'
+        )
+      })
+
+      describe('should not count duplicate failed grades to different years', () => {
+        it('- 2017-2018 should not include failed grade', () => {
+          const year = body.unifyStats?.statistics.find(year => year.name === '2017-2018')
+          assert.strictEqual(year?.attempts.categories.failed.length, 0, 'Failed stats should not include any students')
+          assert.deepStrictEqual(year?.attempts.categories.failed, [], 'Failed stats should not include any students')
+        })
+        it('- 2018-2019 should include a failed grade', () => {
+          const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
+          assert.strictEqual(year?.attempts.categories.failed.length, 1, 'Failed stats should include only one student')
+          assert.deepStrictEqual(
+            year?.attempts.categories.failed,
+            ['539036'],
+            'Failed stats included the incorrect student'
+          )
+        })
+      })
+    })
+  })
+
+  it.todo('should return correct amount of students for a course with substitutions')
+  it.todo('should return correct amount of students for multiple coursecodes without substitutions')
+  it.todo('should return correct amount of students for multiple coursecodes with substitutions')
+})

services/backend/tests/api/population/populationStatistics.test.ts

@@ -1,22 +1,12 @@
 import { Express } from 'express'
-import assert from 'node:assert/strict'
-import { describe, it, before } from 'node:test'
 import { Sequelize } from 'sequelize'
-import request, { Response } from 'supertest'
+import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
 
+import { CanError } from '@oodikone/shared/routes'
 import { PopulationstatisticsResBody } from '@oodikone/shared/routes/populations'
-import { dbConnections } from '../../src/database/connection'
-import { initTests } from '../utils'
-
-let app: Express
-let connection: Sequelize
-before(async () => {
-  app = await initTests()
-  connection = dbConnections.sequelize
-})
-
-// Override Supertest's Response body with our own type
-type ResponseWithBody<T> = Omit<Response, 'body'> & { body: T }
+import { dbConnections } from '../../../src/database/connection'
+import { initTests, ResponseWithBody } from '../../utils'
 
 const populationUrl = (
   programme = '',
@@ -27,9 +17,17 @@ const populationUrl = (
   return `/populationstatistics?${programme && 'programme=' + programme}${years.map(year => '&years=' + year).join('')}${semesters.map(sem => '&semesters=' + sem).join('')}${students.map(s => '&studentStatuses=' + s).join('')}`
 }
 
-void describe('Population statistics', async () => {
-  await it('should fail when all fields not defined', async t => {
-    await t.test('(all)', async () => {
+void describe('Population statistics', () => {
+  let app: Express
+  let connection: Sequelize
+
+  beforeAll(async () => {
+    app = await initTests()
+    connection = dbConnections.sequelize
+  })
+
+  describe('should fail when all fields not defined', () => {
+    it('(all)', async () => {
       const resAllMissing = await request(app)
         .get(populationUrl())
         .set('shib-session-id', 'test')
@@ -38,7 +36,7 @@ void describe('Population statistics', async () => {
       assert.strictEqual(resAllMissing.status, 400)
     })
 
-    await t.test('(all but code)', async () => {
+    it('(all but code)', async () => {
       const resProgramme = await request(app)
         .get(populationUrl('KH50_001'))
         .set('shib-session-id', 'test')
@@ -48,7 +46,7 @@ void describe('Population statistics', async () => {
       assert.strictEqual(resProgramme.status, 400)
     })
 
-    await t.test('(all but code and years)', async () => {
+    it('(all but code and years)', async () => {
       const resProgrammeAndYears = await request(app)
         .get(populationUrl('KH50_001', ['2021', '2022']))
         .set('shib-session-id', 'test')
@@ -59,29 +57,29 @@ void describe('Population statistics', async () => {
     })
   })
 
-  await it('should not return any data to unauthorized user', async () => {
+  it('should not return any data to unauthorized user', async () => {
     const res = (await request(app)
       .get(populationUrl('KH50_001', ['2021', '2022'], ['SPRING', 'FALL']))
       .set('shib-session-id', 'test')
       .set('uid', 'norights')
-      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CanError<PopulationstatisticsResBody>>
 
     assert.deepStrictEqual(res.body, { error: 'Trying to request unauthorized students data' })
     assert.strictEqual(res.status, 403)
   })
 
-  await it('should fail when trying to access a programme that user has no permissions to', async () => {
+  it('should fail when trying to access a programme that user has no permissions to', async () => {
     const res = (await request(app)
       .get(populationUrl('MH50_001', ['2021', '2022'], ['SPRING', 'FALL']))
       .set('shib-session-id', 'test')
       .set('uid', 'onlyiamrights')
-      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CanError<PopulationstatisticsResBody>>
 
     assert.strictEqual(res.status, 403)
     assert.deepStrictEqual(res.body, { error: 'Trying to request unauthorized students data' })
   })
 
-  await it('should return students in the programme for authorized user', async () => {
+  it('should return students in the programme for authorized user', async () => {
     const res = (await request(app)
       .get(populationUrl('KH50_001', ['2021', '2022'], ['SPRING', 'FALL']))
       .set('shib-session-id', 'test')
@@ -92,8 +90,9 @@ void describe('Population statistics', async () => {
     assert.notStrictEqual(res.body.students.length, 0)
   })
 
-  await it('should work with the programme flag', async t => {
-    await t.test('(bachelor)', async () => {
+  describe('should work with the programme flag', () => {
+    // TODO: Fix this asap
+    it.skip('(bachelor)', async () => {
       const resBachelor = (await request(app)
         .get(populationUrl('KH50_001', ['2021'], ['SPRING', 'FALL']))
         .set('shib-session-id', 'test')
@@ -101,10 +100,10 @@ void describe('Population statistics', async () => {
         .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
 
       assert.strictEqual(resBachelor.status, 200)
-      assert.strictEqual(resBachelor.body.students.length, 38)
+      assert.strictEqual(resBachelor.body.students.length, 37)
     })
 
-    await t.test('(master)', async () => {
+    it('(master)', async () => {
       const resMaster = (await request(app)
         .get(populationUrl('MH50_001', ['2021'], ['SPRING', 'FALL']))
         .set('shib-session-id', 'test')
@@ -112,12 +111,12 @@ void describe('Population statistics', async () => {
         .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
 
       assert.strictEqual(resMaster.status, 200)
-      assert.strictEqual(resMaster.body.students.length, 36)
+      assert.strictEqual(resMaster.body.students.length, 37)
     })
   })
 
-  await it('should work with semester flag', async t => {
-    await t.test('(FALL)', async () => {
+  describe('should work with semester flag', () => {
+    it('(FALL)', async () => {
       const resFall = (await request(app)
         .get(populationUrl('KH50_001', ['2021'], ['FALL']))
         .set('shib-session-id', 'test')
@@ -128,7 +127,8 @@ void describe('Population statistics', async () => {
       assert.strictEqual(resFall.body.students.length, 34)
     })
 
-    await t.test('(SPRING)', async () => {
+    // TODO: Fix this asap
+    it.skip('(SPRING)', async () => {
       const resSpring = (await request(app)
         .get(populationUrl('KH50_001', ['2021'], ['SPRING']))
         .set('shib-session-id', 'test')
@@ -136,12 +136,17 @@ void describe('Population statistics', async () => {
         .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
 
       assert.strictEqual(resSpring.status, 200)
-      assert.strictEqual(resSpring.body.students.length, 4)
+      assert.strictEqual(resSpring.body.students.length, 3)
+      // 524071 had previously start date in math of 2022-05-27
+      assert(
+        !resSpring.body.students.map(({ studentNumber }) => studentNumber).includes('524071'),
+        'Most likely student with incorrectly calculated study right start date included in to students'
+      )
     })
   })
 
-  await it('should work with year flag correctly', async t => {
-    await t.test('(2021)', async () => {
+  describe('should work with year flag correctly', () => {
+    it('(2021)', async () => {
       const res2021 = (await request(app)
         .get(populationUrl('KH50_001', ['2021'], ['FALL', 'SPRING']))
         .set('shib-session-id', 'test')
@@ -156,7 +161,7 @@ void describe('Population statistics', async () => {
       assert.strictEqual(res2021.body.students.length, dbCount2021?.count)
     })
 
-    await t.test('(2022)', async () => {
+    it('(2022)', async () => {
       const res2022 = (await request(app)
         .get(populationUrl('KH50_001', ['2022'], ['FALL', 'SPRING']))
         .set('shib-session-id', 'test')
@@ -171,7 +176,7 @@ void describe('Population statistics', async () => {
       assert.strictEqual(res2022.body.students.length, dbCount2022?.count)
     })
 
-    await t.test('(2021-2022)', async () => {
+    it('(2021-2022)', async () => {
       const res2021_2022 = (await request(app)
         .get(populationUrl('KH50_001', ['2021', '2022'], ['FALL', 'SPRING']))
         .set('shib-session-id', 'test')
@@ -187,7 +192,7 @@ void describe('Population statistics', async () => {
     })
   })
 
-  await it.skip('should return only students who have transferred out of the programme when TRANSFERRED flag is set', async () => {
+  it.skip('should return only students who have transferred out of the programme when TRANSFERRED flag is set', async () => {
     // TODO: This doesn't work correctly...
     const resWithTransOut = (await request(app)
       .get(populationUrl('KH50_001', ['2019'], ['FALL', 'SPRING'], ['']))
@@ -216,6 +221,6 @@ void describe('Population statistics', async () => {
     assert.strictEqual(resWithTransOut.body.students.length, dbCountWithTransferredOut?.count)
     assert.strictEqual(resWithoutTransOut.body.students.length, dbCountWithoutTransferredOut?.count)
   })
-  await it.todo('should return only degree-students and non-degree students when NONDEGREE flag is set')
-  await it.todo('should return only degree-students and exchange students when EXCHANGE flag is set')
+  it.todo('should return only degree-students and non-degree students when NONDEGREE flag is set')
+  it.todo('should return only degree-students and exchange students when EXCHANGE flag is set')
 })

services/backend/tests/api/population/populationStatisticsByCourse.test.ts

@@ -1,17 +1,16 @@
 import { Express } from 'express'
-import assert from 'node:assert/strict'
-import { describe, it, before } from 'node:test'
 import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
 
-import { initTests } from '../utils'
+import { initTests } from '../../utils'
 
-let app: Express
-before(async () => {
-  app = await initTests()
-})
+void describe('Population statistics by course', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
 
-void describe('Population statistics by course', async () => {
-  await it('should not return anything with missing parameters', async () => {
+  it('should not return anything with missing parameters', async () => {
     const res = await request(app)
       .get('/populationstatisticsbycourse')
       .set('shib-session-id', 'test')
@@ -22,7 +21,7 @@ void describe('Population statistics by course', async () => {
     assert.deepEqual(res.body.error, 'The body should have a yearcode and coursecode defined')
   })
 
-  await it('should return correct amount of students for a course', async () => {
+  it('should return correct amount of students for a course', async () => {
     const res = await request(app)
       .get('/populationstatisticsbycourse?coursecodes=["MAT21005"]&from=70&to=70')
       .set('shib-session-id', 'test')
@@ -126,6 +125,16 @@ void describe('Population statistics by course', async () => {
 
     assert.strictEqual(res.status, 200)
     assert.strictEqual(res.body.students.length, 40)
-    assert.deepStrictEqual(res.body.students.sort((a, b) => b.studentNumber - a.studentNumber).at(-1), testStudent)
+    const responseStudent: typeof testStudent = res.body.students
+      .sort((a, b) => b.studentNumber - a.studentNumber)
+      .at(-1)
+
+    assert.strictEqual(responseStudent.name, testStudent.name)
+    assert.strictEqual(responseStudent.studentNumber, testStudent.studentNumber)
+    assert.strictEqual(responseStudent.credits, testStudent.credits)
+    assert.strictEqual(responseStudent.credits, testStudent.credits)
+    assert.strictEqual(responseStudent.started, testStudent.started)
+    assert.strictEqual(responseStudent.sis_person_id, testStudent.sis_person_id)
+    assert.deepStrictEqual(responseStudent.studyRights, testStudent.studyRights)
   })
 })

services/backend/tests/api/population/populationStatisticsByStudentNumbers.ts

@@ -1,17 +1,16 @@
 import { Express } from 'express'
-import assert from 'node:assert/strict'
-import { describe, it, before } from 'node:test'
 import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
 
-import { initTests } from '../utils'
+import { initTests } from '../../utils'
 
-let app: Express
-before(async () => {
-  app = await initTests()
-})
+void describe('Population statistics by student number', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
 
-void describe('Population statistics by student number', async () => {
-  await it('should not return anything with missing parameters', async () => {
+  it('should not return anything with missing parameters', async () => {
     const res = await request(app)
       .post('/populationstatisticsbystudentnumbers')
       .set('shib-session-id', 'test')

services/backend/tests/api/population/populationStudyProgrammes.test.ts

@@ -1,17 +1,16 @@
 import { Express } from 'express'
-import assert from 'node:assert/strict'
-import { describe, it, before } from 'node:test'
 import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
 
-import { initTests } from '../utils'
+import { initTests } from '../../utils'
 
-let app: Express
-before(async () => {
-  app = await initTests()
-})
+void describe('Population statistics (study programme)', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
 
-void describe('Population statistics (study programme)', async () => {
-  await it('should return correctly formatted responses', async () => {
+  it('should return correctly formatted responses', async () => {
     const res = await request(app)
       .get('/populationstatistics/studyprogrammes')
       .set('shib-session-id', 'test')
@@ -27,7 +26,7 @@ void describe('Population statistics (study programme)', async () => {
     assert.notStrictEqual(Object.keys(res.body.filteredProgrammes).length, 0)
   })
 
-  await it('should return correct programmes to allProgrammes', async () => {
+  it('should return correct programmes to allProgrammes', async () => {
     const res = await request(app)
       .get('/populationstatistics/studyprogrammes')
       .set('shib-session-id', 'test')
@@ -46,7 +45,7 @@ void describe('Population statistics (study programme)', async () => {
         'hy-lv-69',
         'hy-lv-68',
         'hy-lv-70',
-      ],
+      ].sort(),
       degreeProgrammeType: 'urn:code:degree-program-type:bachelors-degree',
       name: {
         en: "Bachelor's Programme in Mathematical Sciences",
@@ -56,11 +55,14 @@ void describe('Population statistics (study programme)', async () => {
       progId: 'MAT',
     }
 
+    // Sort curriculum periods so that order doesn't mess up tests
+    res.body.allProgrammes.KH50_001.curriculumPeriodIds.sort()
+
     assert.strictEqual(Object.keys(res.body.allProgrammes).length, 972)
     assert.deepStrictEqual(res.body.allProgrammes.KH50_001, testMathProgramme)
   })
 
-  await it('should return the programmes, that user has access to, to filteredProgrammes', async () => {
+  it('should return the programmes, that user has access to, to filteredProgrammes', async () => {
     const res = await request(app)
       .get('/populationstatistics/studyprogrammes')
       .set('shib-session-id', 'test')
@@ -79,7 +81,7 @@ void describe('Population statistics (study programme)', async () => {
         'hy-lv-69',
         'hy-lv-68',
         'hy-lv-70',
-      ],
+      ].sort(),
       degreeProgrammeType: 'urn:code:degree-program-type:bachelors-degree',
       name: {
         en: "Bachelor's Programme in Mathematical Sciences",
@@ -91,11 +93,15 @@ void describe('Population statistics (study programme)', async () => {
 
     assert.strictEqual(Object.keys(res.body.filteredProgrammes).length, 2) // Basic has access to math bachelor + master
 
+    // Sort curriculum periods so that order doesn't mess up tests
+    res.body.filteredProgrammes.KH50_001.curriculumPeriodIds.sort()
+    res.body.allProgrammes.KH50_001.curriculumPeriodIds.sort()
+
     assert.deepStrictEqual(res.body.filteredProgrammes.KH50_001, testMathProgramme)
     assert.deepStrictEqual(res.body.allProgrammes.KH50_001, testMathProgramme) // All programmes should still include filteredProgramme programmes
   })
 
-  await it.todo(
+  it.todo(
     'should return both degree levels of study programme when searching KH90_001 or MH90_001 (ELK, ELL) in filteredProgrammes'
   )
 })

services/backend/tests/api/studyProgramme/studyProgrammeCourseStats.test.ts

@@ -0,0 +1,208 @@
+import { Express } from 'express'
+import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { StudyProgrammeCourse } from '@oodikone/shared/types'
+import { initTests, ResponseWithBody } from '../../utils'
+
+const testData = [
+  {
+    // Johdatus yliopistomatematiikkaan, MAT11001
+    courseCode: 'MAT11001',
+    // year: [allPassed, allNotPassed]
+    2023: [0, 9],
+    2022: [17, 3],
+    2021: [29, 6],
+    2020: [39, 0],
+    2019: [48, 0],
+    2018: [69, 1],
+    2017: [51, 0],
+    // total: [allPassed, allNotPassed, allStudents]
+    total: [253, 18 + 1, 253 + 18 + 1],
+  },
+
+  {
+    // Lineaarialgebra ja matriisilaskenta I, MAT11002
+    courseCode: 'MAT11002',
+    // year: [allPassed, allNotPassed]
+    2023: [0, 1],
+    2022: [20, 1],
+    2021: [22, 2],
+    2020: [45, 1],
+    2019: [51, 0],
+    2018: [57, 0],
+    2017: [47, 0],
+    // total: [allPassed, allNotPassed, allStudents]
+    total: [242, 4 + 1, 242 + 4 + 1],
+  },
+  {
+    // Raja-arvot, MAT11003
+    courseCode: 'MAT11003',
+    // year: [allPassed, allNotPassed]
+    2023: [0, 10],
+    2022: [20, 5],
+    2021: [40, 5],
+    2020: [33, 0],
+    2019: [42, 1],
+    2018: [70, 0],
+    2017: [44, 0],
+    // total: [allPassed, allNotPassed, allStudents]
+    total: [249, 20 + 1, 249 + 20 + 1],
+  },
+
+  {
+    // Lineaarialgebra ja matriisilaskenta II, MAT21001
+    courseCode: 'MAT21001',
+    // year: [allPassed, allNotPassed]
+    2023: [1, 0],
+    2022: [22, 6],
+    2021: [20, 4],
+    2020: [36, 4],
+    2019: [45, 0],
+    2018: [56, 0],
+    2017: [36, 0],
+    // total: [allPassed, allNotPassed, allStudents]
+    total: [216, 4 + 10, 216 + 4 + 10],
+  },
+
+  {
+    // Vektorianalyysi I, MAT21001
+    courseCode: 'MAT21003',
+    // year: [allPassed, allNotPassed]
+    2023: [1, 20],
+    2022: [21, 1],
+    2021: [35, 2],
+    2020: [62, 1],
+    2019: [56, 1],
+    2018: [27, 1],
+    2017: [0, 0],
+    // total: [allPassed, allNotPassed, allStudents]
+    total: [202, 3 + 23, 202 + 3 + 23],
+  },
+]
+
+/** This mostly checks that values are the same as in Course stats */
+void describe("Study programme's course stats", () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
+
+  it('should return data for correct courses', async () => {
+    const res = (await request(app)
+      .get('/studyprogrammes/KH50_001/coursestats?yearType=ACADEMIC_YEAR')
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<StudyProgrammeCourse[]>
+
+    assert.strictEqual(res.status, 200)
+    assert.strictEqual(res.body.length, 102, 'Response included incorrect amount of courses') // Not checked number
+
+    const courseCodes = res.body.map(({ code }) => code)
+    assert(
+      ['MAT11003', 'MAT11001', 'MAT11004', 'AYMAT11003', 'AYMAT11001', 'AYMAT11004'].every(code =>
+        courseCodes.includes(code)
+      ),
+      'Required courses or their open uni variants missing'
+    )
+
+    assert(courseCodes.includes('MAT11015'), 'Course with no passing grades missing from statistics')
+  })
+
+  describe.each(testData)(
+    'should return correct data for a course ($courseCode)',
+    ({ courseCode, total, ...years }) => {
+      let course: StudyProgrammeCourse
+      beforeAll(async () => {
+        const res = (await request(app)
+          .get('/studyprogrammes/KH50_001/coursestats?yearType=ACADEMIC_YEAR')
+          .set('shib-session-id', 'test')
+          .set('uid', 'basic')
+          .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<StudyProgrammeCourse[]>
+
+        assert.strictEqual(res.status, 200)
+        assert(!!res.body.find(course => course.code === courseCode), "Didn't find required course")
+        course = res.body.find(course => course.code === courseCode)!
+
+        assert.deepStrictEqual(
+          Object.keys(course.years),
+          ['2017', '2018', '2019', '2020', '2021', '2022', '2023'], // This doesn't necessarily hold for all courses
+          'Years should also include years where no credits were completed but enrollments were made'
+        )
+        assert.strictEqual(Object.keys(course.years).length, 7, 'Incorrect amount of years')
+      })
+
+      it.each(Object.entries(years))('- %s', (year, stats) => {
+        const yearStats = course.years[parseInt(year)]
+        assert.strictEqual(yearStats.allPassed, stats.at(0), `Incorrect amount of Passed students (${year}})`)
+        assert.strictEqual(yearStats.allNotPassed, stats.at(1), `Incorrect amount of Not passed students (${year})`)
+      })
+
+      it('- total', () => {
+        const years = Object.values(course.years)
+        assert.strictEqual(
+          years.reduce((acc, year) => acc + year.allPassed, 0),
+          total.at(0),
+          'Total number of passed students incorrect (total)'
+        )
+        assert.strictEqual(
+          years.reduce((acc, year) => acc + year.allNotPassed, 0),
+          total.at(1),
+          'Total number of failed or not enrolled students incorrect (total)'
+        )
+        assert.strictEqual(
+          years.reduce((acc, year) => acc + year.allStudents, 0),
+          total.at(2),
+          'Total number of students incorrect (total)'
+        )
+      })
+    }
+  )
+
+  describe('should work correctly in specific cases', () => {
+    // Lineaarialgebra ja matriisilaskenta I (MAT11002)
+    it('Student with AY code attainment should not be calculated into original course code stats (MAT11002, 534980)', async () => {
+      const res = (await request(app)
+        .get('/studyprogrammes/KH50_001/coursestats?yearType=ACADEMIC_YEAR')
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<StudyProgrammeCourse[]>
+
+      assert.strictEqual(res.status, 200)
+      assert(!!res.body.find(course => course.code === 'MAT11002'), "Didn't find required course")
+      const course = res.body.find(course => course.code === 'MAT11002')!
+
+      assert.strictEqual(
+        course.years['2017'].allNotPassed,
+        0,
+        'Most likely student with AY code is calculated into stats (not passed)'
+      )
+      assert.strictEqual(
+        course.years['2017'].allPassed,
+        47,
+        'Most likely student with AY code is calculated into stats (passed)'
+      )
+      assert.strictEqual(
+        course.years['2017'].allStudents,
+        47,
+        'Most likely student with AY code is calculated into stats (total)'
+      )
+    })
+
+    // Todennäköisyyslaskenta I (MAT11001)
+    it('Student with multiple enrollments in different years should only be counted once (MAT11001)', async () => {
+      const res = (await request(app)
+        .get('/studyprogrammes/KH50_001/coursestats?yearType=ACADEMIC_YEAR')
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<StudyProgrammeCourse[]>
+
+      assert.strictEqual(res.status, 200)
+      assert(!!res.body.find(course => course.code === 'MAT11001'), "Didn't find required course")
+      const year = res.body.find(course => course.code === 'MAT11001')?.years['2022']
+      assert.strictEqual(year?.allPassed, 17, 'Incorrect amount of Passed')
+      assert.strictEqual(year?.allNotPassed, 3, 'Incorrect amount of Not completed')
+      assert.strictEqual(year?.allStudents, 20, 'Incorrect amount of Total students')
+    })
+  })
+})

services/backend/tests/courses/courseyearlystats.test.ts

@@ -1,180 +0,0 @@
-import { Express } from 'express'
-import assert from 'node:assert/strict'
-import { describe, it, before, after } from 'node:test'
-import { Sequelize } from 'sequelize'
-import request, { Response } from 'supertest'
-
-import { dbConnections } from '../../src/database/connection'
-import { CourseYearlyStatsResBody } from '../../src/routes/courses'
-import { initTests } from '../utils'
-
-let app: Express
-let connection: Sequelize
-before(async () => {
-  app = await initTests()
-  connection = dbConnections.sequelize
-})
-
-after(async () => {
-  await connection.close()
-})
-
-// Override Supertest's Response body with our own type
-type ResponseWithBody<T> = Omit<Response, 'body'> & { body: T }
-
-void describe('Course yearly statistics', async () => {
-  await it('should return nothing with missing parameters', async () => {
-    const res = await request(app)
-      .get('/courseyearlystats')
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')
-
-    assert.strictEqual(res.status, 422)
-    assert.strictEqual(res.body?.error, 'Missing required query parameters')
-  })
-
-  await it('should return correct amount of values for a course without substitutions', async t => {
-    const res = (await request(app)
-      .get('/courseyearlystats?codes=TKT10002&combineSubstitutions=false')
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
-
-    assert.strictEqual(res.status, 200)
-    assert.strictEqual(res.body.length, 1, 'Query to return anything')
-    const body = res.body.at(0)!
-    assert(
-      'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
-      'All keys of courseyearlystats not defined'
-    )
-
-    await t.test('(statistic years)', () => {
-      assert.strictEqual(body.unifyStats!.statistics.length, 8, 'unifyStats years')
-      assert.strictEqual(body.regularStats!.statistics.length, 8, 'regularStats years')
-      assert.strictEqual(body.openStats!.statistics.length, 4, 'openStats years')
-
-      // Faculties should have the same amount of years as normal statistics (above)
-      assert.strictEqual(Object.keys(body.unifyStats!.facultyStats).length, 8, 'unifyStats years')
-      assert.strictEqual(Object.keys(body.regularStats!.facultyStats).length, 8, 'regularStats years')
-      assert.strictEqual(Object.keys(body.openStats!.facultyStats).length, 4, 'openStats years')
-    })
-
-    await t.test('(statistic contents)', async tt => {
-      await tt.test('- 2016-2017', () => {
-        const stats2016 = body.unifyStats!.statistics.find(year => year.name === '2016-2017')!
-        assert.deepStrictEqual(stats2016.students.studentNumbers, ['484144'], 'Incorrect students marked to 2016 stats')
-
-        assert.deepStrictEqual(stats2016.attempts.grades['4'], ['484144'], 'Incorrect grades marked to 2016 stats')
-        assert.strictEqual(
-          stats2016.attempts.categories.passed.length,
-          1,
-          'Incorrect students marked as passed to 2016 stats'
-        )
-        assert.strictEqual(
-          stats2016.attempts.categories.failed.length,
-          0,
-          'Incorrect students marked as failed to 2016 stats'
-        )
-
-        assert('enrollments' in stats2016, 'Missing field enrollment in statsitics')
-        assert.strictEqual(stats2016.enrollments.length, 0, 'Enrollments found when none should exist')
-        assert.strictEqual(stats2016.allEnrollments.length, 0, 'Enrollments found when none should exist')
-      })
-
-      await tt.test('- 2017-2018', () => {
-        const stats2017 = body.unifyStats!.statistics.find(year => year.name === '2017-2018')!
-        const { studentNumbers } = stats2017.students
-        assert(
-          !('509165' in studentNumbers || '455478' in studentNumbers || '457686' in studentNumbers),
-          "Students that have completed course later should not be included in the previous year's stats"
-        )
-        assert.strictEqual(stats2017.students.studentNumbers.length, 23, 'Incorrect amount of total students 2017')
-
-        assert.strictEqual(stats2017.attempts.grades['5'].length, 11, 'Incorrect grades marked to 2017 stats (5)')
-        assert.strictEqual(stats2017.attempts.grades['4'].length, 5, 'Incorrect grades marked to 2017 stats, (4)')
-        assert.strictEqual(stats2017.attempts.grades['3'].length, 2, 'Incorrect grades marked to 2017 stats (3)')
-        assert.strictEqual(stats2017.attempts.grades['2'].length, 1, 'Incorrect grades marked to 2017 stats (2)')
-        assert(!('1' in stats2017.attempts.grades), 'Incorrect grades marked to 2017 stats (1)')
-        assert.strictEqual(stats2017.attempts.grades['Hyl.'].length, 4, 'Incorrect grades marked to 2017 stats (Hyl.)')
-
-        assert(
-          !stats2017.attempts.grades['Hyl.'].includes('455478'),
-          "Specific student's failed grade is not marked to the correct year (should be only in passed 2020-2021)"
-        )
-
-        assert.strictEqual(
-          stats2017.attempts.categories.passed.length,
-          11 + 5 + 2 + 1 + 0,
-          'Incorrect students marked as passed to 2017 stats'
-        )
-        assert.strictEqual(
-          stats2017.attempts.categories.failed.length,
-          4,
-          'Incorrect students marked as failed to 2017 stats'
-        )
-
-        // These should not exist before 21-22
-        assert('enrollments' in stats2017, 'Missing field enrollment in statsitics')
-        assert.strictEqual(stats2017.enrollments.length, 0, 'Enrollments found when none should exist')
-        assert.strictEqual(stats2017.allEnrollments.length, 0, 'Enrollments found when none should exist')
-      })
-
-      await tt.test('- 2020-2021', () => {
-        const stats2020 = body.unifyStats!.statistics.find(year => year.name === '2020-2021')!
-        assert(
-          stats2020.attempts.categories.passed.includes('455478'),
-          "Specific student's passed course is not marked to the correct year (passed)"
-        )
-        assert(
-          stats2020.attempts.grades['2'].includes('455478'),
-          "Specific student's passed course is not marked to the correct year (grade)"
-        )
-
-        // These should not exist before 21-22
-        assert('enrollments' in stats2020, 'Missing field enrollment in statsitics')
-        assert.strictEqual(stats2020.enrollments.length, 0, 'Enrollments found when none should exist')
-        assert.strictEqual(stats2020.allEnrollments.length, 0, 'Enrollments found when none should exist')
-      })
-
-      // TODO: This should be fixed by course_unit_realisation
-      await tt.test('- 2022-2023', () => {
-        const stats2022 = body.unifyStats!.statistics.find(year => year.name === '2022-2023')!
-        assert.strictEqual(stats2022.students.studentNumbers.length, 27, 'Incorrect amount of total students 2022')
-
-        assert('enrollments' in stats2022, 'Missing field enrollment in statsitics')
-        // Student with enrollment_date_time outside of semestert start and end dates
-        assert(
-          !!stats2022.enrollments.find(enrollment => enrollment.studentNumber === '455129'),
-          'Student with incorrect semester (by updater) but a correct enrollment_date was not found in the correct year (by enrollment_date)'
-        )
-        assert.strictEqual(stats2022.enrollments.length, 34, 'Enrollments found when none should exist')
-        assert.strictEqual(stats2022.allEnrollments.length, 43, 'Enrollments found when none should exist')
-      })
-
-      await tt.test('- 2023-2024', () => {
-        const stats2023 = body.unifyStats!.statistics.find(year => year.name === '2023-2024')!
-        assert.strictEqual(stats2023.students.studentNumbers.length, 1, 'Incorrect amount of total students 2023')
-
-        assert.deepStrictEqual(
-          stats2023.attempts.categories.passed,
-          ['521757'],
-          'Incorrect students marked as passed in 2023 stats'
-        )
-        assert.deepStrictEqual(
-          stats2023.attempts.grades['5'],
-          ['521757'],
-          "Correct student doesn't exist under correct grade"
-        )
-
-        assert('enrollments' in stats2023, 'Missing field enrollment in statsitics')
-        assert.strictEqual(stats2023.enrollments.length, 7, 'Enrollments found when none should exist')
-        assert.strictEqual(stats2023.allEnrollments.length, 7, 'Enrollments found when none should exist')
-      })
-    })
-
-    await it.todo('should return correct amount of students for a course with substitutions')
-    await it.todo('should return correct amount of students for multiple coursecodes without substitutions')
-    await it.todo('should return correct amount of students for multiple coursecodes with substitutions')
-  })
-})

services/backend/tests/services/courses/index.test.ts

@@ -0,0 +1,153 @@
+import { describe, it, assert, beforeAll, vi } from 'vitest'
+import { Unification } from '@oodikone/shared/types'
+import { initializeDatabaseConnection } from '../../../src/database/connection'
+import {
+  maxYearsToCreatePopulationFrom,
+  searchAndCombineSubstitutionGroupsToCodes,
+} from '../../../src/services/courses'
+import { findByCourseAndSemesters } from '../../../src/services/students'
+
+void describe.concurrent('Search and complete substitution groups to codes', () => {
+  beforeAll(async () => {
+    await initializeDatabaseConnection()
+  })
+
+  it('should return correct course codes for MAT11001 (only single-length groups)', async () => {
+    const res = await searchAndCombineSubstitutionGroupsToCodes(['MAT11001'])
+    assert.deepStrictEqual(res, ['MAT11001', '57033', 'AYMAT11001', 'A57033'])
+  })
+
+  it('should return correct course codes for MAT21016 (also multicourse groups)', async () => {
+    const res = await searchAndCombineSubstitutionGroupsToCodes(['MAT21016'])
+    assert.deepStrictEqual(res, ['MAT21016', '57282', 'MFK-M310', 'MAT21023'])
+  })
+  it('should return correct course codes for MAT11001 + MAT21016', async () => {
+    const res = await searchAndCombineSubstitutionGroupsToCodes(['MAT11001', 'MAT21016'])
+    assert.deepStrictEqual(res, [
+      'MAT11001',
+      'MAT21016',
+      '57033',
+      'AYMAT11001',
+      'A57033',
+      '57282',
+      'MFK-M310',
+      'MAT21023',
+    ])
+  })
+  it.todo('should return correct course codes for X + Y (duplicate courses in substitution groups)')
+})
+
+void describe.skip('Max years to generate population from', () => {
+  beforeAll(async () => {
+    await initializeDatabaseConnection()
+  })
+
+  it('should return no years without a course code', async () => {
+    const res = await maxYearsToCreatePopulationFrom([], Unification.UNIFY)
+    assert.strictEqual(res, 0, 'Got something else than 0 years')
+  })
+
+  it('should return correct years for MAT11001', async () => {
+    const res = await maxYearsToCreatePopulationFrom(['MAT11001'], Unification.UNIFY)
+    assert.strictEqual(res, 6)
+  })
+
+  it.todo('should should return the same max years as before', () => {
+    const date = new Date(2020)
+    vi.useFakeTimers()
+    vi.setSystemTime(date)
+
+    vi.useRealTimers()
+  })
+})
+
+type TestList = [string, number, number][]
+
+const semesterToSemesterCode = (semesterYear: number, springOrFall: 'spring' | 'fall') => {
+  return (semesterYear - 1950) * 2 + (springOrFall === 'spring' ? 0 : 1)
+}
+const yearToYearCode = (year: number) => year - 1949
+
+void describe.concurrent('Find by course and semester', () => {
+  it.each([
+    ['semester', semesterToSemesterCode(2019, 'fall'), semesterToSemesterCode(2023, 'spring')],
+    ['year', yearToYearCode(2017), yearToYearCode(2023)],
+  ] as TestList)(
+    'should return correct student numbers for a short course (CSM14204, 2019-2023) by $0',
+    async (separate, to, from) => {
+      const res = (await findByCourseAndSemesters(['CSM14204'], to, from, separate === 'semester', 'unifyStats')).sort()
+      assert.deepStrictEqual(res, ['458079', '478837', '483126'])
+    }
+  )
+
+  it.each([
+    ['semester', semesterToSemesterCode(2020, 'fall'), semesterToSemesterCode(2023, 'spring')],
+    ['year', yearToYearCode(2020), yearToYearCode(2023)],
+  ] as TestList)('should filter out students outside of 2020-2023 by $0', async (separate, to, from) => {
+    const res = (await findByCourseAndSemesters(['CSM14204'], to, from, separate === 'semester', 'unifyStats')).sort()
+    assert.deepStrictEqual(res, ['478837', '483126'])
+  })
+
+  it.each([
+    ['semester', semesterToSemesterCode(2017, 'fall'), semesterToSemesterCode(2021, 'spring')],
+    ['year', yearToYearCode(2017), yearToYearCode(2020)],
+  ] as TestList)(
+    'should return correct student numbers for a longer course (MAT21018, 2017-2021) by $0',
+    async (separate, to, from) => {
+      const res = (await findByCourseAndSemesters(['MAT21018'], to, from, separate === 'semester', 'unifyStats')).sort()
+      const passed = ['457686', '495976', '484997', '491970', '461485', '501442', '508370']
+      const failed = ['484541', '487566', '493344', '520906', '541350', '544750', '550840']
+      assert.deepStrictEqual(res, passed.concat(failed).sort())
+    }
+  )
+
+  it.each([
+    ['semester', semesterToSemesterCode(2021, 'fall'), semesterToSemesterCode(2023, 'spring')],
+    ['year', yearToYearCode(2021), yearToYearCode(2022)],
+  ] as TestList)(
+    'should return correct student numbers for a longer course (MAT21018, 2021-2023) by $0',
+    async (separate, to, from) => {
+      const res = (await findByCourseAndSemesters(['MAT21018'], to, from, separate === 'semester', 'unifyStats')).sort()
+      const passed = [
+        '509745',
+        '504315',
+        '520805',
+        '509881',
+        '511089',
+        '519527',
+        '522321',
+        '527445',
+        '538399',
+        '534980',
+        '474789',
+        '478837',
+        '482406',
+        '486809',
+        '547552',
+        '493345',
+        '495398',
+        '498558',
+      ]
+      const enrolledNoGrade = ['529866', '488481', '479440', '518062']
+
+      assert.deepStrictEqual(res, passed.concat(enrolledNoGrade).sort())
+      assert.strictEqual(res.length, passed.length + enrolledNoGrade.length)
+    }
+  )
+
+  it.todo('should return correct student numbers for multiple courses (CSM14204+MAT21018)')
+
+  describe('should work with specific cases', () => {
+    it('Student with enrollment in -21 and passed grade in -23 should not be included in -21 stats (522321)', async () => {
+      const res21 = (
+        await findByCourseAndSemesters(['MAT21018'], yearToYearCode(2021), yearToYearCode(2021), false, 'unifyStats')
+      ).sort()
+      const res23 = (
+        await findByCourseAndSemesters(['MAT21018'], yearToYearCode(2022), yearToYearCode(2022), false, 'unifyStats')
+      ).sort()
+
+      assert.notInclude(res21, '522321', '522321 incorrectly included in -21 stats')
+      assert.include(res23, '522321')
+    })
+  })
+})

services/backend/tests/setup.ts

@@ -0,0 +1,5 @@
+import { loadEnvFile } from 'node:process'
+
+// We need to load the local .env file to access some variables needed when
+// testing.
+loadEnvFile('.env.test')

services/backend/tests/unit/placeholder.test.ts

@@ -0,0 +1,5 @@
+import { describe, it } from 'vitest'
+
+void describe('Placeholder test', () => {
+  it.todo('should make unit tests not fail')
+})

services/backend/tests/utils.ts

@@ -1,10 +1,18 @@
 import express from 'express'
+import { Response } from 'supertest'
 
-import { baseUrl } from '../src/config'
+import { baseUrl, silentTesting } from '../src/config'
 import { initializeDatabaseConnection, dbConnections } from '../src/database/connection'
 import routes from '../src/routes'
+import logger from '../src/util/logger'
+
+/** Override Supertest's Response body with our own type */
+export type ResponseWithBody<T> = Omit<Response, 'body'> & { body: T }
+
+export const initTests = async (silent = silentTesting) => {
+  /** Enable or disable logging from app */
+  logger.silent = silent
 
-export const initTests = async () => {
   // Copied from "app.ts"
   await initializeDatabaseConnection()
 

services/backend/tsconfig.json

@@ -19,6 +19,6 @@
     "inlineSources": true,
     "resolveJsonModule": true
   },
-  "include": ["index.ts", "src", "tests"],
+  "include": ["index.ts", "src", "tests", "vitest.config.ts"],
   "exclude": ["../frontend"]
 }

services/backend/vitest.config.ts

@@ -0,0 +1,14 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    reporters: ['tree'],
+    exclude: ['./dist', 'node_modules'],
+    setupFiles: ['./tests/setup.ts'],
+    pool: 'threads',
+    testTimeout: 10_000, // 10s timeout
+    sequence: {
+      concurrent: true, // Enable parallel tests by default
+    },
+  },
+})

services/frontend/.npmrc

@@ -1,3 +1,4 @@
 ignore-scripts=true  
 min-release-age=7  
 save-prefix=  
+allow-remote=root

services/frontend/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 

services/frontend/dev.Dockerfile

@@ -1,18 +1,18 @@
-FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 
-USER 1001
+USER 0
 
 WORKDIR /opt/app-root/shared
-COPY --chown=1001:1001 ./shared .
-
-RUN npm ci
+COPY --chown=1001:0 ./shared .
+RUN npm ci && chown -R 1001:0 /opt/app-root/shared
 
 WORKDIR /opt/app-root/frontend
-COPY --chown=1001:1001 ./frontend .
-
-RUN npm ci
+COPY --chown=1001:0 ./frontend .
+RUN npm ci && chown -R 1001:0 /opt/app-root/frontend
 
 EXPOSE 3000
+
+USER 1001
 CMD ["node_modules/.bin/vite"]

services/frontend/package.json

@@ -44,9 +44,9 @@
     "@types/node": "^24.10.1",
     "@types/react": "^18.3.23",
     "@types/react-dom": "^18.3.7",
-    "@vitejs/plugin-react-swc": "^4.2.3",
+    "@vitejs/plugin-react": "^6.0.3",
     "tsx": "4.22.4",
     "typescript": "^5.9.2",
-    "vite": "^7.3.1"
+    "vite": "^8.1.4"
   }
 }

services/frontend/src/apiConnection/index.ts

@@ -65,6 +65,7 @@ export const callApi = async (
 export const RTKApi = createApi({
   reducerPath: 'api',
   tagTypes: [
+    'Banners',
     'CompletedCoursesSearchList',
     'CustomPopulationSearches',
     'ProgressCriteria',

services/frontend/src/common/InfoToolTips/faculty.ts

@@ -1,4 +1,4 @@
-import { studyProgrammeToolTips } from './studyProgramme'
+import { studyProgrammeToolTips } from '@/common/InfoToolTips/studyProgramme'
 
 export const facultyToolTips = {
   common: {

services/frontend/src/common/InfoToolTips/index.ts

@@ -1,14 +1,14 @@
-import { closeToGraduationToolTips } from './closeToGraduation'
-import { coursePopulationToolTips } from './coursePopulation'
-import { courseStatisticsToolTips } from './courseStatistics'
-import { facultyToolTips } from './faculty'
-import { filterToolTips } from './filters'
-import { populationStatisticsToolTips } from './populationStatistics'
-import { populationStudentsToolTips } from './populationStudents'
-import { studentToolTips } from './students'
-import { studyProgrammeToolTips } from './studyProgramme'
-import { updaterToolTips } from './updater'
-import { userToolTips } from './users'
+import { closeToGraduationToolTips } from '@/common/InfoToolTips/closeToGraduation'
+import { coursePopulationToolTips } from '@/common/InfoToolTips/coursePopulation'
+import { courseStatisticsToolTips } from '@/common/InfoToolTips/courseStatistics'
+import { facultyToolTips } from '@/common/InfoToolTips/faculty'
+import { filterToolTips } from '@/common/InfoToolTips/filters'
+import { populationStatisticsToolTips } from '@/common/InfoToolTips/populationStatistics'
+import { populationStudentsToolTips } from '@/common/InfoToolTips/populationStudents'
+import { studentToolTips } from '@/common/InfoToolTips/students'
+import { studyProgrammeToolTips } from '@/common/InfoToolTips/studyProgramme'
+import { updaterToolTips } from '@/common/InfoToolTips/updater'
+import { userToolTips } from '@/common/InfoToolTips/users'
 
 export {
   closeToGraduationToolTips,

services/frontend/src/common/InfoToolTips/populationStatistics.ts

@@ -1,4 +1,4 @@
-import { populationStudentsToolTips } from './populationStudents'
+import { populationStudentsToolTips } from '@/common/InfoToolTips/populationStudents'
 
 export const populationStatisticsToolTips = {
   curriculumPicker: {