[CI][Updater] Replace eslint with lint

Clarify the description for clearing node_modules.

.github/actions/build/backend/action.yaml

@@ -0,0 +1,61 @@
+name: 'Build backend'
+description: 'Build backend in staging or production version'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+  push:
+    required: false
+    description: Also push built image to toska registry
+    default: 'false'
+  registry_password:
+    required: false
+    description: Password to quay.io registry
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v4
+
+    - name: Docker meta
+      id: meta
+      uses: docker/metadata-action@v6
+      with:
+        images: |
+          quay.io/toska/oodikone-backend
+        tags: |
+          ${{ inputs.environment }}
+          ${{ github.sha }}
+
+    - name: Login to quay.io
+      if: ${{ inputs.push == 'true' }}
+      uses: docker/login-action@v4
+      with:
+        registry: quay.io/toska
+        username: toska+github
+        password: ${{ inputs.registry_password }}
+
+    - name: 'Build image and export it'
+      id: build-image-action
+      uses: docker/build-push-action@v7
+      with:
+        context: services
+        file: services/backend/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          BASE_PATH=${{ case(inputs.environment == 'staging', '/oodikone', '') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}
+        push: ${{ inputs.push }}
+        tags: ${{ steps.meta.outputs.tags }}
+      env:
+        DOCKER_BUILD_SUMMARY: false

.github/actions/build/frontend/action.yaml

@@ -0,0 +1,72 @@
+name: 'Build frontend'
+description: 'Build frontend in staging or production versions'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+  sentry_dns:
+    required: false
+    description: 'Sentry url to send error logs to (only production, enforced)'
+    default: ''
+  push:
+    required: false
+    description: Also push built image to toska registry
+    default: 'false'
+  registry_password:
+    required: false
+    description: Password to quay.io registry
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v4
+
+    - name: Login to quay.io
+      if: ${{ inputs.push == 'true' }}
+      uses: docker/login-action@v4
+      with:
+        registry: quay.io/toska
+        username: toska+github
+        password: ${{ inputs.registry_password }}
+
+    - name: Docker meta
+      id: meta
+      uses: docker/metadata-action@v6
+      with:
+        images: |
+          quay.io/toska/oodikone-frontend
+        tags: |
+          ${{ inputs.environment }}
+          ${{ github.sha }}
+
+    - name: Get current time
+      id: get_current_time
+      shell: bash
+      run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
+
+    - name: 'Build image and export it'
+      id: build-image-action
+      uses: docker/build-push-action@v7
+      with:
+        context: services
+        file: services/frontend/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          BASE_PATH=${{ case(inputs.environment == 'staging', '/oodikone', '') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}
+          BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+          SENTRY_DSN=${{ case(inputs.environment == 'production', inputs.sentry_dns, '') }}
+        push: ${{ inputs.push }}
+        tags: ${{ steps.meta.outputs.tags }}
+      env:
+        DOCKER_BUILD_SUMMARY: false

.github/actions/build/scheduler/action.yaml

@@ -0,0 +1,60 @@
+name: 'Build updater scheduler'
+description: 'Build updater scheduler in staging or production version'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+  push:
+    required: false
+    description: Also push built image to toska registry
+    default: 'false'
+  registry_password:
+    required: false
+    description: Password to quay.io registry
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v4
+
+    - name: Login to quay.io
+      if: ${{ inputs.push == 'true' }}
+      uses: docker/login-action@v4
+      with:
+        registry: quay.io/toska
+        username: toska+github
+        password: ${{ inputs.registry_password }}
+
+    - name: Docker meta
+      id: meta
+      uses: docker/metadata-action@v6
+      with:
+        images: |
+          quay.io/toska/updater-scheduler
+        tags: |
+          ${{ inputs.environment }}
+          ${{ github.sha }}
+
+    - name: 'Build image and export it'
+      id: build-image-action
+      uses: docker/build-push-action@v7
+      with:
+        context: updater/sis-updater-scheduler
+        file: updater/sis-updater-scheduler/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}
+        push: ${{ inputs.push }}
+        tags: ${{ steps.meta.outputs.tags }}
+      env:
+        DOCKER_BUILD_SUMMARY: false

.github/actions/build/worker/action.yaml

@@ -0,0 +1,60 @@
+name: 'Build updater worker'
+description: 'Build updater worker in staging or production version'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+  push:
+    required: false
+    description: Also push built image to toska registry
+    default: 'false'
+  registry_password:
+    required: false
+    description: Password to quay.io registry
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v4
+
+    - name: Login to quay.io
+      if: ${{ inputs.push == 'true' }}
+      uses: docker/login-action@v4
+      with:
+        registry: quay.io/toska
+        username: toska+github
+        password: ${{ inputs.registry_password }}
+
+    - name: Docker meta
+      id: meta
+      uses: docker/metadata-action@v6
+      with:
+        images: |
+          quay.io/toska/updater-worker
+        tags: |
+          ${{ inputs.environment }}
+          ${{ github.sha }}
+
+    - name: 'Build image and export it'
+      id: build-image-action
+      uses: docker/build-push-action@v7
+      with:
+        context: updater/sis-updater-worker
+        file: updater/sis-updater-worker/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}
+        push: ${{ inputs.push }}
+        tags: ${{ steps.meta.outputs.tags }}
+      env:
+        DOCKER_BUILD_SUMMARY: false

.github/actions/lint/action.yaml

@@ -1,10 +1,10 @@
 name: 'Lint code'
-description: 'Runs ESLint and Stylelint for frontend and backend'
+description: 'Runs Oxlint and Stylelint for frontend and backend'
 
 runs:
   using: composite
   steps:
-    - uses: actions/setup-node@v6
+    - uses: actions/setup-node@v7
       with:
         node-version: 24
     - name: 'Install dependencies'
@@ -13,6 +13,6 @@ runs:
     - name: 'Run Stylelint for CSS files'
       shell: bash
       run: npm run stylelint "services/frontend/src/**/*.css"
-    - name: 'Run ESLint for JS and TS files'
+    - name: 'Run Oxlint for the whole repository'
       shell: bash
-      run: npm run eslint -- services
+      run: npm run lint

.github/actions/teardown/action.yml

@@ -0,0 +1,9 @@
+name: Teardown docker compose (docker-compose.ci.yml)
+description: 'Teardown all containers started by docker (docker-compose.ci.yml)'
+
+runs:
+  using: composite
+  steps:
+    - name: 'Teardown docker compose'
+      shell: bash
+      run: docker compose --file docker-compose.ci.yml --profile="*" down --remove-orphans

.github/actions/test/api/action.yaml

@@ -0,0 +1,56 @@
+name: 'Run api tests'
+description: 'Runs api tests in backend'
+
+inputs:
+  username:
+    required: true
+    description: 'Toska registry username'
+  password:
+    required: true
+    description: 'Toska registry password'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Login to toska docker registry
+      uses: docker/login-action@v4
+      with:
+        registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+        username: ${{ inputs.username }}
+        password: ${{ inputs.password }}
+
+    - name: Restore cached docker images
+      id: cache-images-restore
+      uses: actions/cache/restore@v6
+      with:
+        path: |
+          /tmp/docker-images.tar.zst
+        key: docker-images-${{ github.sha }}
+
+    - name: Load Docker images (or skip if missing)
+      shell: bash
+      run: |
+        if [ -f /tmp/docker-images.tar.zst ]; then
+          zstd -d /tmp/docker-images.tar.zst --stdout | docker load
+        fi
+
+    - name: Start oodikone (without frontend and backend)
+      shell: bash
+      run: docker compose --file docker-compose.ci.yml up --quiet-pull --detach --wait
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/backend
+
+    - name: Run api-tests
+      shell: bash
+      run: npm run test:api --prefix ./services/backend
+
+    - name: Teardown containers started by compose
+      if: always()
+      uses: ./.github/actions/teardown

.github/actions/test/component/action.yaml

@@ -0,0 +1,31 @@
+name: 'Run component tests'
+description: 'Runs frontend component tests'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/shared && npm ci --prefix ./services/frontend
+
+    - name: Install Playwright Browsers
+      shell: bash
+      run: npm exec -- playwright install --with-deps
+      working-directory: ./services/frontend
+
+    - name: Run Playwright tests
+      shell: bash
+      run: npm run test:component --prefix ./services/frontend
+
+    - uses: actions/upload-artifact@v4
+      if: |
+        !cancelled()
+      with:
+        name: playwright-report
+        path: playwright-report/
+        retention-days: 7

.github/actions/test/e2e/action.yaml

@@ -29,22 +29,25 @@ runs:
       shell: bash
       run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull
 
-    - name: 'Run cypress'
-      uses: cypress-io/github-action@v7
-      with:
-        browser: chrome
-        spec: ${{ inputs.spec }}
-      env:
-        CYPRESS_baseUrl: http://localhost:1337
+    - name: Run Cypress inside docker container
+      shell: bash
+      # Permission errors? See: https://github.com/cypress-io/cypress-docker-images/blob/master/examples/included-as-non-root/README.md#run-as-non-root
+      run: |
+        docker run --user node --rm -v $PWD:/e2e -w /e2e --add-host host.docker.internal:host-gateway -e CYPRESS_VIDEO -e CYPRESS_baseUrl=http://host.docker.internal:1337 --entrypoint cypress cypress/included:15.18.1 run --browser firefox --spec ${{ inputs.spec }}
 
     - name: 'Upload video if tests fail'
       uses: actions/upload-artifact@v7
       if: failure()
       with:
         name: 'Cypress videos - ${{ inputs.test_name }}'
-        path: ./cypress/videos/*
+        path: /tmp/cypress/videos/*
+        retention-days: 7
 
     - name: 'Print logs on failure'
       shell: bash
       if: failure()
-      run: docker compose --file docker-compose.ci.yml --profile oodikone logs && sleep 15
+      run: npm run logs:ci && sleep 15
+
+    - name: Teardown containers started by compose
+      if: always()
+      uses: ./.github/actions/teardown

.github/actions/test/services/action.yaml

@@ -0,0 +1,56 @@
+name: 'Run service tests'
+description: 'Runs service tests in backend'
+
+inputs:
+  username:
+    required: true
+    description: 'Toska registry username'
+  password:
+    required: true
+    description: 'Toska registry password'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Login to toska docker registry
+      uses: docker/login-action@v4
+      with:
+        registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+        username: ${{ inputs.username }}
+        password: ${{ inputs.password }}
+
+    - name: Restore cached docker images
+      id: cache-images-restore
+      uses: actions/cache/restore@v6
+      with:
+        path: |
+          /tmp/docker-images.tar.zst
+        key: docker-images-${{ github.sha }}
+
+    - name: Load Docker images (or skip if missing)
+      shell: bash
+      run: |
+        if [ -f /tmp/docker-images.tar.zst ]; then
+          zstd -d /tmp/docker-images.tar.zst --stdout | docker load
+        fi
+
+    - name: Start oodikone (without frontend and backend)
+      shell: bash
+      run: docker compose --file docker-compose.ci.yml up --quiet-pull --detach --wait
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/backend
+
+    - name: Run service-tests
+      shell: bash
+      run: npm run test:services --prefix ./services/backend
+
+    - name: Teardown containers started by compose
+      if: always()
+      uses: ./.github/actions/teardown

.github/actions/test/unit/action.yaml

@@ -0,0 +1,26 @@
+name: 'Run unit tests'
+description: 'Runs unit tests in backend and frontend'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/shared && npm ci --prefix ./services/backend && npm ci --prefix ./services/frontend
+
+    - name: Run shared unit tests
+      shell: bash
+      run: npm run test:unit --prefix ./services/shared
+
+    - name: Run frontend unit tests
+      shell: bash
+      run: npm run test:unit --prefix ./services/frontend
+
+    - name: Run backend unit tests
+      shell: bash
+      run: npm run test:unit --prefix ./services/backend

.github/workflows/production.yaml

@@ -5,77 +5,46 @@ on:
     types: [published]
 
 jobs:
-  release_frontend:
-    name: 'Build and release frontend'
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: 'Get current time'
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-
-      - name: Extract release version
-        id: extract_version
-        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
-
-      - name: Build frontend image
-        id: build-image
-        uses: redhat-actions/buildah-build@v2
-        with:
-          image: oodikone-frontend
-          tags: production ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/frontend/Dockerfile
-          build-args: |
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
-            SENTRY_DSN=${{ secrets.OODIKONE_FRONTEND_SENTRY_DSN }}
-
-      - name: Push to quay.io
-        uses: redhat-actions/push-to-registry@v2
-        with:
-          image: ${{ steps.build-image.outputs.image }}
-          tags: ${{ steps.build-image.outputs.tags }}
-          registry: quay.io/toska
-          username: toska+github
-          password: ${{ secrets.QUAY_IO_TOKEN }}
-
   release_backend:
     name: 'Build and release backend'
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Extract release version
         id: extract_version
         run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
 
-      - name: Build backend image
+      - name: 'Build backend'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/backend
         with:
-          image: oodikone-backend
-          tags: production ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/backend/Dockerfile
-          build-args: |
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
+          environment: 'production'
+          release_version: ${{ steps.extract_version.outputs.VERSION }}
+          push: true
+          registry_password: ${{ secrets.QUAY_IO_TOKEN }}
 
-      - name: Push to quay.io
-        uses: redhat-actions/push-to-registry@v2
+  release_frontend:
+    name: 'Build and release frontend'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Extract release version
+        id: extract_version
+        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+
+      - name: 'Build frontend'
+        id: build-image
+        uses: ./.github/actions/build/frontend
         with:
-          image: ${{ steps.build-image.outputs.image }}
-          tags: ${{ steps.build-image.outputs.tags }}
-          registry: quay.io/toska
-          username: toska+github
-          password: ${{ secrets.QUAY_IO_TOKEN }}
+          environment: 'production'
+          release_version: ${{ steps.extract_version.outputs.VERSION }}
+          sentry_dns: ${{ secrets.OODIKONE_FRONTEND_SENTRY_DSN }} # Used only in production
+          push: true
+          registry_password: ${{ secrets.QUAY_IO_TOKEN }}
 
   sentry_release_frontend:
     name: 'Create Sentry release for frontend'
@@ -85,7 +54,7 @@ jobs:
       - release_backend
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Extract release version
         id: extract_version
@@ -119,7 +88,7 @@ jobs:
       - release_backend
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Extract release version
         id: extract_version
@@ -157,7 +126,7 @@ jobs:
     if: success()
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
@@ -178,7 +147,7 @@ jobs:
     if: failure()
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:

.github/workflows/pull_request.yaml

@@ -5,56 +5,60 @@ on:
     branches:
       - master
     paths:
+      - '.github/actions/**'
+      - '.github/workflows/pull_request.yaml'
+      - '.github/workflows/test.yaml'
+      - 'services/**'
+      - 'updater/**'
+      - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
+      - 'package*.json'
       - 'cypress/**'
       - 'cypress.config.json'
-      - 'services/**'
-      - 'docker-compose.ci.yml'
-      - 'package*.json'
 
 jobs:
   lint_packages:
     name: 'Lint packages'
     runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
+
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Lint backend, frontend and shared
         uses: ./.github/actions/lint
 
-  test_api:
+  test_unit:
+    name: 'Run unit tests'
+    runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
     needs:
       - lint_packages
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-24.04
+
     steps:
-      - name: Checkout
-        uses: actions/checkout@v6
+      - uses: actions/checkout@v7
+      - uses: ./.github/actions/test/unit
 
-      - name: Install node-24 for cypress
-        uses: actions/setup-node@v6
-        with:
-          node-version: 24
+  test_api:
+    name: 'Test backend api'
+    runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
+    needs:
+      - lint_packages
 
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v4
-        with:
-          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-
-      - name: Start oodikone detached
-        run: npm run oodikone:ci:setup
-
-      - name: Run api-tests
-        run: npm run oodikone:ci:api
+    steps:
+      - uses: actions/checkout@v7
+      - uses: ./.github/actions/test/api
 
   test_end_to_end:
+    name: ${{ matrix.name }}
     runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
     needs:
       - lint_packages
       - test_api
     strategy:
       fail-fast: false
+
       matrix:
         include:
           - name: 'Test course statistics'
@@ -83,11 +87,13 @@ jobs:
             spec: cypress/e2e/Study_guidance_groups.js
           - name: 'Test university'
             spec: cypress/e2e/University.js
-    name: ${{ matrix.name }}
+          - name: 'Test numbers match'
+            spec: cypress/e2e/Numbers_match.js
+
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: 'Run tests'
-        uses: ./.github/actions/run-tests
+        uses: ./.github/actions/test/e2e
         with:
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}

.github/workflows/staging.yaml

@@ -8,12 +8,13 @@ on:
       - '.github/actions/**'
       - '.github/workflows/staging.yaml'
       - '.github/workflows/test.yaml'
+      - 'services/**'
+      - 'updater/**'
+      - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
+      - 'package*.json'
       - 'cypress/**'
       - 'cypress.config.json'
-      - 'services/**'
-      - 'updater/sis-updater-worker/src/db/migrations/**'
-      - 'docker-compose.ci.yml'
-      - 'package*.json'
 
 jobs:
   test:
@@ -22,83 +23,86 @@ jobs:
       username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
       password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
 
-  release_frontend:
-    name: 'Build and release frontend'
-    runs-on: ubuntu-latest
-    needs: test
-
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: Get current time
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-
-      - name: Build frontend image
-        id: build-image
-        uses: redhat-actions/buildah-build@v2
-        with:
-          image: oodikone-frontend
-          tags: staging ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/frontend/Dockerfile
-          build-args: |
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-            BASE_PATH=/oodikone
-            STAGING=true
-
-      - name: Push to quay.io
-        uses: redhat-actions/push-to-registry@v2
-        with:
-          image: ${{ steps.build-image.outputs.image }}
-          tags: ${{ steps.build-image.outputs.tags }}
-          registry: quay.io/toska
-          username: toska+github
-          password: ${{ secrets.QUAY_IO_TOKEN }}
-
   release_backend:
     name: 'Build and release backend'
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     needs: test
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
-      - name: Build backend image
+      - name: 'Build backend'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/backend
         with:
-          image: oodikone-backend
-          tags: staging ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/backend/Dockerfile
-          build-args: |
-            BASE_PATH=/oodikone
-            GIT_SHA=${{ github.sha }}
-            STAGING=true
+          environment: 'staging'
+          release_version: ${{ github.sha }}
+          push: true
+          registry_password: ${{ secrets.QUAY_IO_TOKEN }}
 
-      - name: Push to quay.io
-        uses: redhat-actions/push-to-registry@v2
+  release_frontend:
+    name: 'Build and release frontend'
+    runs-on: self-hosted
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: 'Build frontend'
+        id: build-image
+        uses: ./.github/actions/build/frontend
         with:
-          image: ${{ steps.build-image.outputs.image }}
-          tags: ${{ steps.build-image.outputs.tags }}
-          registry: quay.io/toska
-          username: toska+github
-          password: ${{ secrets.QUAY_IO_TOKEN }}
+          environment: 'staging'
+          release_version: ${{ github.sha }}
+          push: true
+          registry_password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_updater_worker:
+    name: 'Build and release updater worker'
+    runs-on: self-hosted
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: 'Build worker'
+        id: build-image
+        uses: ./.github/actions/build/worker
+        with:
+          environment: 'staging'
+          release_version: ${{ github.sha }}
+          push: true
+          registry_password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_updater_scheduler:
+    name: 'Build and release updater scheduler'
+    runs-on: self-hosted
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: 'Build scheduler'
+        id: build-image
+        uses: ./.github/actions/build/scheduler
+        with:
+          environment: 'staging'
+          release_version: ${{ github.sha }}
+          push: true
+          registry_password: ${{ secrets.QUAY_IO_TOKEN }}
 
   report_success:
     name: 'Report deployment success to the "oodikone_notifications" channel'
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
+    if: success()
     needs:
       - release_frontend
       - release_backend
-
-    if: success()
+      - release_updater_worker
+      - release_updater_scheduler
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
@@ -109,15 +113,16 @@ jobs:
 
   report_failure:
     name: 'Report deployment failure to the "oodikone_notifications" channel'
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
+    if: failure()
     needs:
       - release_frontend
       - release_backend
-
-    if: failure()
+      - release_updater_worker
+      - release_updater_scheduler
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:

.github/workflows/test.yaml

@@ -14,120 +14,102 @@ jobs:
   # === Static tests / linting ===
   lint_packages:
     name: 'Lint packages'
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Lint backend, frontend and shared
         uses: ./.github/actions/lint
 
   # === Build database images ===
   build_backend:
     name: 'Build backend'
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
-      - uses: actions/checkout@v6
-      - uses: docker/setup-buildx-action@v4
-      - name: 'Build image and export it'
-        uses: docker/build-push-action@v7
+      - uses: actions/checkout@v7
+      - name: Build backend
+        uses: ./.github/actions/build/backend
         with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          context: services
-          file: services/backend/Dockerfile
+          environment: 'staging'
+          release_version: ${{ github.sha }}
           push: false
-          tags: oodikone-backend:staging
-          outputs: type=docker,dest=/tmp/backend.tar
-        env:
-          DOCKER_BUILD_SUMMARY: false
-
-      - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v7
-        with:
-          if-no-files-found: error
-          name: backend-image
-          path: /tmp/backend.tar
 
   build_frontend:
     name: 'Build frontend'
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
-      - uses: actions/checkout@v6
-      - uses: docker/setup-buildx-action@v4
-
-      - name: Get current time
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-
-      - name: Build frontend image
-        uses: docker/build-push-action@v7
+      - uses: actions/checkout@v7
+      - name: Build frontend
+        uses: ./.github/actions/build/frontend
         with:
-          build-args: |
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-            BASE_PATH=/oodikone
-            STAGING=true
-          context: services
-          file: services/frontend/Dockerfile
+          environment: 'staging'
+          release_version: ${{ github.sha }}
           push: false
-          tags: oodikone-frontend:staging
-          outputs: type=docker,dest=/tmp/frontend.tar
-        env:
-          DOCKER_BUILD_SUMMARY: false
-
-      # - name: 'Upload image for other jobs'
-      #   uses: actions/upload-artifact@v7
-      #   with:
-      #     if-no-files-found: error
-      #     name: frontend-image
-      #     path: /tmp/frontend.tar
 
+  # === Cache docker images ===
+  # TODO: Disabled when running in self-hosted runner
   pull_docker_images:
-    name: 'Pull all needed docker images and cache them'
-    runs-on: ubuntu-latest
+    name: Pull all needed docker images and cache them (if dbs have not changed)
+    runs-on: self-hosted
+    if: |
+      !cancelled() && !failure() && false &&
+      !(
+        needs.check_updater_changed.outputs.has_updater_changed == 'true' ||
+        needs.check_migration_files.outputs.kone_db_migrations == 'true' ||
+        needs.check_migration_files.outputs.user_db_migrations == 'true' ||
+        needs.check_migration_files.outputs.sis_db_migrations == 'true'
+      )
+    needs:
+      - check_updater_changed
+      - check_migration_files
+
     steps:
-      - uses: actions/checkout@v6
-      - name: 'Login to toska docker registry'
+      - uses: actions/checkout@v7
+      - name: Login to toska docker registry
         uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: 'Pull required images'
+      - name: Set up cache for pulled docker images (saved in Post-step)
+        id: cache-images
+        uses: actions/cache@v6
+        with:
+          path: /tmp/docker-images.tar.zst
+          key: docker-images-${{ github.sha }}
+
+      - name: Pull required images
+        if: steps.cache-images.outputs.cache-hit != 'true'
         run: docker compose --file docker-compose.ci.yml --profile oodikone --profile updater pull --ignore-buildable --quiet
 
-      - name: 'Save Docker images for downstream jobs'
+      - name: Save Docker images for downstream jobs
+        if: steps.cache-images.outputs.cache-hit != 'true'
         run: |
           docker save \
             registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db \
-            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db \
             registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db \
+            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db \
+            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db \
+            "quay.io/toska/jami:production" \
             redis:7.2.6 \
             postgres:15.10 \
-            "quay.io/toska/jami:production" \
             nginx:latest \
           | zstd -T0 -3 -o /tmp/docker-images.tar.zst
 
-      - name: 'Upload Docker images artifact'
-        uses: actions/upload-artifact@v7
-        with:
-          name: docker-images
-          path: /tmp/docker-images.tar.zst
-          retention-days: 1
-
   # === Update databases ===
   check_migration_files:
     name: Check if new migrations have been added
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     outputs:
       kone_db_migrations: ${{ steps.check_migrations.outputs.kone_db_migrations }}
       user_db_migrations: ${{ steps.check_migrations.outputs.user_db_migrations }}
       sis_db_migrations: ${{ steps.check_migrations.outputs.sis_db_migrations }}
+
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Check if new migrations have been added
         id: check_migrations
-        uses: dorny/paths-filter@v3
+        uses: dorny/paths-filter@v4
         with:
           filters: |
             kone_db_migrations:
@@ -142,13 +124,19 @@ jobs:
 
   build_and_push_anon_sis_db:
     name: 'Build anon sis-db and push to toska hub if new migrations have been added'
+    runs-on: self-hosted
+    if: |
+      !cancelled() && !failure() &&
+      needs.check_migration_files.outputs.sis_db_migrations == 'true'
     needs:
-      - pull_docker_images
       - check_migration_files
-    runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.sis_db_migrations == 'true'
+      - build_backend
+      - build_frontend
+      - lint_packages
+      - test_unit
+
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v4
         with:
@@ -156,47 +144,31 @@ jobs:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-
       - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
         run: |
           docker compose --file docker-compose.ci.yml up --detach --quiet-pull sis-updater-worker
           sleep 30 # wait for sis-updater-worker to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
 
+      - name: Teardown containers started by compose
+        if: always()
+        uses: ./.github/actions/teardown
+
   build_and_push_anon_kone_db:
     name: 'Build anon kone-db and push to toska hub if new migrations have been added'
+    runs-on: self-hosted
+    if: |
+      !cancelled() && !failure() &&
+      needs.check_migration_files.outputs.kone_db_migrations == 'true'
     needs:
-      - pull_docker_images
+      - check_migration_files
       - build_backend
       - build_frontend
-      - check_migration_files
-    runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.kone_db_migrations == 'true'
+      - lint_packages
+      - test_unit
+
     steps:
-      - uses: actions/checkout@v6
-
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-image
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
+      - uses: actions/checkout@v7
 
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v4
@@ -207,43 +179,29 @@ jobs:
 
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         run: |
-          docker compose --file docker-compose.ci.yml up --detach backend
+          docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
           sleep 30 # wait for backend to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
 
+      - name: Teardown containers started by compose
+        if: always()
+        uses: ./.github/actions/teardown
+
   build_and_push_anon_user_db:
     name: 'Build anon user-db and push to toska hub if new migrations have been added'
+    runs-on: self-hosted
+    if: |
+      !cancelled() && !failure() &&
+      needs.check_migration_files.outputs.user_db_migrations == 'true'
     needs:
-      - pull_docker_images
+      - check_migration_files
       - build_backend
       - build_frontend
-      - check_migration_files
-    runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.user_db_migrations == 'true'
+      - lint_packages
+      - test_unit
+
     steps:
-      - uses: actions/checkout@v6
-      - uses: dorny/paths-filter@v3
-        id: check_migrations
-        with:
-          filters: |
-            migrations:
-              - 'services/backend/src/database/migrations_user/**'
-              - '.github/workflows/test.yaml'
-
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-image
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
+      - uses: actions/checkout@v7
 
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v4
@@ -254,39 +212,49 @@ jobs:
 
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         run: |
-          docker compose --file docker-compose.ci.yml up --detach backend
+          docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
           sleep 30 # wait for backend to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
 
-  # === Backend api-tests ===
-  test_api:
-    needs:
-      - pull_docker_images
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-      - lint_packages
-    if: always() && !cancelled() && needs.lint_packages.result == 'success'
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-24.04
+      - name: Teardown containers started by compose
+        if: always()
+        uses: ./.github/actions/teardown
+
+  # === Update sis-db with updater ===
+  check_updater_changed:
+    name: 'Check if updater has been updated'
+    runs-on: self-hosted
+    if: |
+      !cancelled() && !failure()
+    outputs:
+      has_updater_changed: ${{ steps.check_updater_changed.outputs.has_updater_changed }}
+
     steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-
-      - name: Install node-24
-        uses: actions/setup-node@v6
+      - uses: actions/checkout@v7
+      - uses: dorny/paths-filter@v4
+        id: check_updater_changed
         with:
-          node-version: 24
+          filters: |
+            has_updater_changed:
+              - 'updater/**'
+              - '.github/workflows/test.yaml'
+              - '.github/workflows/staging.yaml'
+              - 'docker-compose.ci.yml'
 
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
+  run_updater_updates:
+    name: 'Run updater tests and updates'
+    runs-on: self-hosted
+    if: |
+      !cancelled() && !failure() &&
+      (needs.check_updater_changed.outputs.has_updater_changed == 'true' ||
+      needs.check_migration_files.outputs.sis_db_migrations == 'true')
+    needs:
+      - check_migration_files
+      - check_updater_changed
+      - build_and_push_anon_sis_db
 
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
+    steps:
+      - uses: actions/checkout@v7
 
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v4
@@ -295,21 +263,117 @@ jobs:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: Start oodikone (without frontend and backend)
-        run: npm run oodikone:ci:setup -- --quiet-pull
+      - name: Start worker containers
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-worker --scale sis-updater-worker=3 --quiet-pull --detach --wait
 
-      - name: Run api-tests
-        run: npm run oodikone:ci:api
+      - name: Run hourly jobs
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
+        env:
+          SCHEDULE_IMMEDIATE: hourly
+          EXIT_AFTER_IMMEDIATES: yes
+
+      - name: Run weekly jobs
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
+        env:
+          SCHEDULE_IMMEDIATE: weekly
+          EXIT_AFTER_IMMEDIATES: yes
+
+      # TODO: Push the updated sis-db to toska-hub when updater is deterministic with test-data
+      # - name: Push newly build db to toska hub
+      #   if: success()
+      #   run: ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
+
+      - name: Dump logs on failure
+        if: failure()
+        run: docker compose --file docker-compose.ci.yml --profile updater logs
+
+      - name: Teardown containers started by compose
+        if: always()
+        uses: ./.github/actions/teardown
+
+  # === Backend api-tests ===
+  test_api:
+    name: 'Run backend api tests'
+    runs-on: self-hosted
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - pull_docker_images
+      - run_updater_updates
+      - build_and_push_anon_user_db
+      - build_and_push_anon_kone_db
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Run api tests
+        uses: ./.github/actions/test/api
+        with:
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+  #
+  # === Backend service-tests ===
+  test_services:
+    name: 'Run backend service-tests'
+    runs-on: self-hosted
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - pull_docker_images
+      - run_updater_updates
+      - build_and_push_anon_user_db
+      - build_and_push_anon_kone_db
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Run service-tests
+        uses: ./.github/actions/test/services
+        with:
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+
+  # === Unit testing ===
+  test_unit:
+    name: 'Run unit tests'
+    runs-on: self-hosted
+    if: |
+      !cancelled() && !failure()
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Run unit tests (frontend and backend)
+        uses: ./.github/actions/test/unit
+
+  # === Frontend component testing ===
+  test_components:
+    name: 'Test frontend components'
+    runs-on: self-hosted
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - pull_docker_images
+      - run_updater_updates
+      - build_and_push_anon_user_db
+      - build_and_push_anon_kone_db
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Run frontend component tests
+        uses: ./.github/actions/test/component
 
   # === End-to-end tests ===
   test_end_to_end:
+    runs-on: self-hosted
+    if: |
+      !cancelled() && !failure()
     needs:
       - pull_docker_images
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
+      - run_updater_updates
       - build_and_push_anon_user_db
-      - lint_packages
-    if: always() && !cancelled() && needs.lint_packages.result == 'success'
+      - build_and_push_anon_kone_db
     strategy:
       fail-fast: false
       matrix:
@@ -328,10 +392,10 @@ jobs:
             spec: cypress/e2e/Completed_courses.js
           - name: 'Test degree programme overview'
             spec: cypress/e2e/Degree_programme_overview.js
-          - name: 'Test teachers and users'
-            spec: |
-              cypress/e2e/Teachers.js
-              cypress/e2e/Users.js
+          - name: 'Test teachers'
+            spec: cypress/e2e/Teachers.js
+          - name: 'Test users'
+            spec: cypress/e2e/Users.js
           - name: 'Test faculty statistics'
             spec: cypress/e2e/Faculty_statistics.js
           - name: 'Test language center view'
@@ -340,41 +404,36 @@ jobs:
             spec: cypress/e2e/Study_guidance_groups.js
           - name: 'Test university'
             spec: cypress/e2e/University.js
-
+          - name: 'Test numbers match'
+            spec: cypress/e2e/Numbers_match.js
+          - name: 'Test access rights'
+            spec: cypress/e2e/Routes.js
     name: ${{ matrix.name }}
-    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Install node-24 for cypress
-        uses: actions/setup-node@v6
+        uses: actions/setup-node@v7
         with:
           node-version: 24
 
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
+      - name: Restore cached docker images
+        id: cache-images-restore
+        uses: actions/cache/restore@v6
         with:
-          name: docker-images
-          path: /tmp
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-image
-      # - name: 'Download frontend image'
-      #   uses: actions/download-artifact@v8
-      #   with:
-      #     name: frontend-image
+          path: |
+            /tmp/docker-images.tar.zst
+          key: docker-images-${{ github.sha }}
 
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
-      # - name: 'Load built frontend image'
-      #   run: docker load --input frontend.tar
+      - name: Load Docker images (or skip if missing)
+        run: |
+          if [ -f /tmp/docker-images.tar.zst ]; then
+            zstd -d /tmp/docker-images.tar.zst --stdout | docker load
+          fi
 
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
+      - name: Run tests
+        uses: ./.github/actions/test/e2e
         with:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
@@ -384,16 +443,17 @@ jobs:
   # === Other tests ===
   test_service_exit_statuses:
     name: 'Check services run correctly'
+    runs-on: self-hosted
+    if: |
+      !cancelled() && !failure()
     needs:
       - pull_docker_images
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
+      - run_updater_updates
       - build_and_push_anon_user_db
-      - lint_packages
-    if: always() && !cancelled() && needs.lint_packages.result == 'success'
-    runs-on: ubuntu-latest
+      - build_and_push_anon_kone_db
+
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v4
         with:
@@ -401,29 +461,22 @@ jobs:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
+      - name: Restore cached docker images
+        id: cache-images-restore
+        uses: actions/cache/restore@v6
         with:
-          name: docker-images
-          path: /tmp
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-image
-      # - name: 'Download frontend image'
-      #   uses: actions/download-artifact@v8
-      #   with:
-      #     name: frontend-image
+          path: |
+            /tmp/docker-images.tar.zst
+          key: docker-images-${{ github.sha }}
 
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
-      # - name: 'Load built frontend image'
-      #   run: docker load --input frontend.tar
+      - name: Load Docker images (or skip if missing)
+        run: |
+          if [ -f /tmp/docker-images.tar.zst ]; then
+            zstd -d /tmp/docker-images.tar.zst --stdout | docker load
+          fi
 
       - name: 'Run services, build rest of the needed images'
-        run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull
+        run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull --wait
 
       - name: 'Check that exit statuses are not errors'
         run: |
@@ -435,4 +488,4 @@ jobs:
 
       - name: 'Print logs on failure'
         if: failure()
-        run: docker compose --file docker-compose.ci.yml --profile oodikone logs && sleep 15
+        run: npm run logs:ci && sleep 15

.github/workflows/updater.yaml

@@ -5,32 +5,29 @@ on:
     types: [published]
     paths:
       - '.github/workflows/updater.yaml'
-      - 'updater/sis-updater-worker/**'
-      - 'updater/sis-updater-scheduler/**'
+      - 'updater/**'
 
 jobs:
   lint_scheduler_and_worker:
     name: 'Lint scheduler and worker'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
-      - uses: actions/setup-node@v6
+      - uses: actions/checkout@v7
+      - uses: actions/setup-node@v7
         with:
           node-version: 24
 
-      - name: Install safe-chain
-        run: set -o pipefail && curl -fsSL https://github.com/AikidoSec/safe-chain/releases/latest/download/install-safe-chain.sh | sh -s -- --ci
-
       - name: 'Install dependencies'
         run: npm ci && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker
       - name: 'Run linter'
-        run: npm run eslint -- updater
+        run: npm run lint -- updater
 
   build_worker:
     name: 'Build Updater worker'
     runs-on: ubuntu-latest
+
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: docker/setup-buildx-action@v4
       - name: 'Build the image'
         uses: docker/build-push-action@v7
@@ -41,17 +38,20 @@ jobs:
           tags: toska/sis-updater-worker:latest
         env:
           DOCKER_BUILD_SUMMARY: false
+
       - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v7
         with:
           name: updater-worker-image
           path: /tmp/worker.tar
+          retention-days: 7
 
   build_scheduler:
     name: 'Build Updater scheduler'
     runs-on: ubuntu-latest
+
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: docker/setup-buildx-action@v4
       - name: 'Build the image'
         uses: docker/build-push-action@v7
@@ -62,11 +62,13 @@ jobs:
           tags: toska/sis-updater-scheduler:latest
         env:
           DOCKER_BUILD_SUMMARY: false
+
       - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v7
         with:
           name: updater-scheduler-image
           path: /tmp/scheduler.tar
+          retention-days: 7
 
   test_updater:
     name: 'Test Updater'
@@ -75,42 +77,50 @@ jobs:
       - lint_scheduler_and_worker
       - build_worker
       - build_scheduler
+
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+
       - name: 'Download built images'
         uses: actions/download-artifact@v8
         with:
           pattern: 'updater-{worker,scheduler}-image'
           merge-multiple: true
+
       - name: 'Load worker image'
         run: docker load --input worker.tar
       - name: 'Load scheduler image'
         run: docker load --input scheduler.tar
+
       - name: 'Start worker'
-        run: docker compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
+        run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull --no-recreate --wait sis-updater-worker
+
       - name: 'Run hourly jobs'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: hourly
           EXIT_AFTER_IMMEDIATES: yes
+
       - name: 'Run weekly jobs'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: weekly
           EXIT_AFTER_IMMEDIATES: yes
+
       - name: 'Run prepurge'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: prepurge
           EXIT_AFTER_IMMEDIATES: yes
+
       - name: 'Run purge'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: purge
           EXIT_AFTER_IMMEDIATES: yes
@@ -120,55 +130,29 @@ jobs:
     runs-on: ubuntu-latest
     needs: test_updater
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
-      - name: Build worker image
+      - name: 'Build updater worker'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/worker
         with:
-          image: updater-worker
-          tags: staging production ${{ github.sha }}
-          context: updater/sis-updater-worker
-          containerfiles: |
-            updater/sis-updater-worker/Dockerfile
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=production
-
-      - name: Push to quay.io
-        uses: redhat-actions/push-to-registry@v2
-        with:
-          image: ${{ steps.build-image.outputs.image }}
-          tags: ${{ steps.build-image.outputs.tags }}
-          registry: quay.io/toska
-          username: toska+github
-          password: ${{ secrets.QUAY_IO_TOKEN }}
+          environment: 'production'
+          release_version: ${{ github.sha }}
+          push: true
+          registry_password: ${{ secrets.QUAY_IO_TOKEN }}
 
   release_scheduler:
     name: 'Build and release Updater scheduler'
     runs-on: ubuntu-latest
     needs: test_updater
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
-      - name: Build scheduler image
+      - name: 'Build udpater scheduler'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/scheduler
         with:
-          image: updater-scheduler
-          tags: staging production ${{ github.sha }}
-          context: updater/sis-updater-scheduler
-          containerfiles: |
-            updater/sis-updater-scheduler/Dockerfile
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=production
-
-      - name: Push to quay.io
-        uses: redhat-actions/push-to-registry@v2
-        with:
-          image: ${{ steps.build-image.outputs.image }}
-          tags: ${{ steps.build-image.outputs.tags }}
-          registry: quay.io/toska
-          username: toska+github
-          password: ${{ secrets.QUAY_IO_TOKEN }}
+          environment: 'production'
+          release_version: ${{ github.sha }}
+          push: true
+          registry_password: ${{ secrets.QUAY_IO_TOKEN }}

.gitignore

@@ -56,7 +56,7 @@ cypress/videos/
 coverage/
 instrumented/
 
-# Pycharm 
+# Pycharm
 .idea
 
 # VS Code
@@ -68,3 +68,5 @@ instrumented/
 # Target of shared symlink
 services/**/src/shared
 
+
+!services/backend/.env.test

.lintstagedrc.mjs

@@ -5,25 +5,29 @@ const dockerCmdBase = `docker run --rm --volume ${cwd}:/oodikone --workdir /oodi
 const relativeFilePaths = files => [...files.map(file => relative(cwd, file))].join(' ')
 
 export default {
-  '{services,updater}/**/*.{js,jsx,ts,tsx}': () => 'eslint --fix --report-unused-disable-directives',
-  'services/backend/**/*.{js.jsx,ts,tsx}': () => 'npx tsc --noEmit --project services/backend/tsconfig.json',
-  'services/frontend/**/*.{js,jsx,ts,tsx}': () => 'npx tsc --noEmit --project services/frontend/tsconfig.json',
-  '*.{js,jsx,ts,tsx,json,md,yml,yaml,html,css}': () => 'prettier . --write',
-  '*.css': files => `stylelint --fix ${files.join(' ')}`,
-  Dockerfile: files =>
-    `${dockerCmdBase} hadolint/hadolint:v2.12.0 hadolint --ignore DL3006 ${relativeFilePaths(files)}`,
-  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck:v0.7.2 ${relativeFilePaths(files)} -x`,
-  'docker-compose*': files => {
-    const composeFiles = file => {
-      if (['docker-compose.ci.yml', 'docker-compose.test.yml'].some(f => file.includes(f))) {
-        return `--file ${file}`
-      }
-      if (file.includes('docker-compose.real.yml')) {
-        return `--file ${cwd}/docker-compose.yml --file ${file}`
-      }
-      return ''
-    }
+  // INFO: oxlint --type-check is equivalent to running tsc --noEmit
+  '{services,updater}/**/*.{js,jsx,ts,tsx}': files =>
+    `oxlint --fix --type-aware --type-check --no-error-on-unmatched-pattern --quiet ${files.join(' ')}`,
 
-    return files.map(file => `docker-compose ${composeFiles(file)} config --quiet`)
-  },
+  '*.{js,jsx,ts,tsx,json,md,yml,yaml,html,css}': files =>
+    `oxfmt --no-error-on-unmatched-pattern ${files.join(' ')} `,
+
+  '*.css': files => `stylelint --fix ${files.join(' ')}`,
+
+  Dockerfile: files =>
+    `${dockerCmdBase} hadolint/hadolint:v2.14.0 hadolint --ignore DL3006 ${relativeFilePaths(files)}`,
+    '*.sh': files => `${dockerCmdBase} koalaman/shellcheck:v0.11.0 ${relativeFilePaths(files)} -x`,
+    'docker-compose*': files => {
+      const composeFiles = file => {
+        if (['docker-compose.ci.yml', 'docker-compose.test.yml'].some(f => file.includes(f))) {
+          return `--file ${file}`
+        }
+        if (file.includes('docker-compose.real.yml')) {
+          return `--file ${cwd}/docker-compose.yml --file ${file}`
+        }
+        return ''
+      }
+
+      return files.map(file => `docker compose ${composeFiles(file)} config --quiet`)
+    },
 }

.oxfmtrc.json

@@ -0,0 +1,14 @@
+{
+  "$schema": "./node_modules/oxfmt/configuration_schema.json",
+  "semi": false,
+  "arrowParens": "avoid",
+  "singleQuote": true,
+  "printWidth": 120,
+  "tabWidth": 2,
+  "useTabs": false,
+  "jsxSingleQuote": false,
+  "trailingComma": "es5",
+  "endOfLine": "lf",
+  "sortPackageJson": false,
+  "ignorePatterns": []
+}

.oxlintrc.json

@@ -0,0 +1,405 @@
+{
+  "$schema": "./node_modules/oxlint/configuration_schema.json",
+  "plugins": ["import", "typescript"],
+  "categories": {
+    "correctness": "off"
+  },
+  "options": {
+    "typeAware": true,
+    "typeCheck": true
+  },
+  "env": {
+    "builtin": true,
+    "es2021": true,
+    "browser": true,
+    "node": true
+  },
+  "settings": {
+    "import-x/extensions": [".ts", ".tsx", ".cts", ".mts", ".js", ".jsx", ".cjs", ".mjs"],
+    "import-x/external-module-folders": ["node_modules", "node_modules/@types"],
+    "import-x/parsers": {
+      "@typescript-eslint/parser": [".ts", ".tsx", ".cts", ".mts"]
+    },
+    "import-x/resolver": {
+      "typescript": true
+    },
+    "import-x/resolver-next": [
+      {
+        "interfaceVersion": 3,
+        "name": "eslint-import-resolver-typescript"
+      },
+      {
+        "interfaceVersion": 3,
+        "name": "eslint-plugin-import-x:node"
+      }
+    ]
+  },
+  "ignorePatterns": [
+    "eslint.config.mjs",
+    "**/build/**",
+    "**/dist/**",
+    "**/node_modules/**",
+    "**/instrumented/**",
+    "**/coverage/**",
+    "nyc-config.cjs",
+    ".lintstagedrc.mjs",
+    "cypress.config.js",
+    "cypress/**/*.js"
+  ],
+  "rules": {
+    "constructor-super": "error",
+    "for-direction": "error",
+    "getter-return": "error",
+    "no-async-promise-executor": "error",
+    "no-case-declarations": "error",
+    "no-class-assign": "error",
+    "no-compare-neg-zero": "error",
+    "no-cond-assign": "error",
+    "no-const-assign": "error",
+    "no-constant-binary-expression": "error",
+    "no-constant-condition": "error",
+    "no-control-regex": "error",
+    "no-debugger": "error",
+    "no-delete-var": "error",
+    "no-dupe-class-members": "error",
+    "no-dupe-else-if": "error",
+    "no-dupe-keys": "error",
+    "no-duplicate-case": "error",
+    "no-empty": "error",
+    "no-empty-character-class": "error",
+    "no-empty-pattern": "error",
+    "no-empty-static-block": "error",
+    "no-ex-assign": "error",
+    "no-extra-boolean-cast": "error",
+    "no-fallthrough": "error",
+    "no-func-assign": "error",
+    "no-global-assign": "error",
+    "no-import-assign": "error",
+    "no-invalid-regexp": "error",
+    "no-irregular-whitespace": "error",
+    "no-loss-of-precision": "error",
+    "no-misleading-character-class": "error",
+    "no-new-native-nonconstructor": "error",
+    "no-nonoctal-decimal-escape": "error",
+    "no-obj-calls": "error",
+    "no-prototype-builtins": "error",
+    "no-redeclare": "error",
+    "no-regex-spaces": "error",
+    "no-self-assign": "error",
+    "no-setter-return": "error",
+    "no-shadow-restricted-names": "error",
+    "no-sparse-arrays": "error",
+    "no-this-before-super": "error",
+    "no-undef": "error",
+    "no-unexpected-multiline": "error",
+    "no-unreachable": "error",
+    "no-unsafe-finally": "error",
+    "no-unsafe-negation": "error",
+    "no-unsafe-optional-chaining": "error",
+    "no-unused-labels": "error",
+    "no-unused-private-class-members": "error",
+    "no-unused-vars": [
+      "error",
+      {
+        "argsIgnorePattern": "^_",
+        "caughtErrors": "none",
+        "ignoreRestSiblings": true,
+        "varsIgnorePattern": "^_"
+      }
+    ],
+    "no-useless-backreference": "error",
+    "no-useless-catch": "error",
+    "no-useless-escape": "error",
+    "no-with": "error",
+    "require-yield": "error",
+    "use-isnan": "error",
+    "valid-typeof": "error",
+    "no-array-constructor": "error",
+    "no-unused-expressions": "error",
+    "no-empty-function": "error",
+    "class-methods-use-this": "error",
+    "no-alert": "error",
+    "no-await-in-loop": "error",
+    "no-console": "error",
+    "no-param-reassign": "error",
+    "no-promise-executor-return": "error",
+    "no-return-assign": "error",
+    "object-shorthand": ["error", "always"],
+    "prefer-const": "error",
+    "no-restricted-imports": [
+      "error",
+      {
+        "patterns": [
+          {
+            "regex": "^@mui/[^/]+$",
+            "message": "Import only the needed module. This has no effect in prod due to tree-shake, but speeds up development builds."
+          },
+          {
+            "regex": "^src/.*",
+            "message": "Root imports will not resolve in production build (instead silently crash)."
+          }
+        ]
+      }
+    ],
+    "import/namespace": "error",
+    "import/default": "error",
+    "import/export": "error",
+    "import/no-named-as-default": "warn",
+    "import/no-named-as-default-member": "warn",
+    "import/no-duplicates": "warn",
+    "import/no-commonjs": "error",
+    "import/no-default-export": "error",
+    "typescript/await-thenable": "error",
+    "typescript/ban-ts-comment": "error",
+    "typescript/no-array-delete": "error",
+    "typescript/no-base-to-string": "error",
+    "typescript/no-duplicate-enum-values": "error",
+    "typescript/no-duplicate-type-constituents": "error",
+    "typescript/no-empty-object-type": [
+      "error",
+      {
+        "allowInterfaces": "with-single-extends"
+      }
+    ],
+    "typescript/no-extra-non-null-assertion": "error",
+    "typescript/no-floating-promises": "error",
+    "typescript/no-for-in-array": "error",
+    "typescript/no-implied-eval": "error",
+    "typescript/no-misused-new": "error",
+    "typescript/no-misused-promises": "error",
+    "typescript/no-namespace": "error",
+    "typescript/no-non-null-asserted-optional-chain": "error",
+    "typescript/no-redundant-type-constituents": "error",
+    "typescript/no-require-imports": "error",
+    "typescript/no-this-alias": "error",
+    "typescript/no-unnecessary-type-assertion": "error",
+    "typescript/no-unnecessary-type-constraint": "error",
+    "typescript/no-unsafe-declaration-merging": "error",
+    "typescript/no-unsafe-enum-comparison": "error",
+    "typescript/no-unsafe-function-type": "error",
+    "typescript/no-unsafe-unary-minus": "error",
+    "typescript/no-wrapper-object-types": "error",
+    "typescript/only-throw-error": "error",
+    "typescript/prefer-as-const": "error",
+    "typescript/prefer-namespace-keyword": "error",
+    "typescript/prefer-promise-reject-errors": "error",
+    "typescript/require-await": "error",
+    "typescript/restrict-plus-operands": "error",
+    "typescript/triple-slash-reference": "error",
+    "typescript/unbound-method": "error",
+    "typescript/adjacent-overload-signatures": "error",
+    "typescript/ban-tslint-comment": "error",
+    "typescript/class-literal-property-style": "error",
+    "typescript/consistent-generic-constructors": "error",
+    "typescript/consistent-indexed-object-style": "error",
+    "typescript/consistent-type-assertions": "error",
+    "typescript/dot-notation": "error",
+    "typescript/no-confusing-non-null-assertion": "error",
+    "typescript/no-inferrable-types": "error",
+    "typescript/non-nullable-type-assertion-style": "error",
+    "typescript/prefer-find": "error",
+    "typescript/prefer-for-of": "error",
+    "typescript/prefer-function-type": "error",
+    "typescript/prefer-includes": "error",
+    "typescript/prefer-nullish-coalescing": "warn",
+    "typescript/prefer-optional-chain": "error",
+    "typescript/prefer-regexp-exec": "error",
+    "typescript/prefer-string-starts-ends-with": "error",
+    "typescript/consistent-return": "error"
+  },
+  "overrides": [
+    {
+      "files": ["**/*.ts", "**/*.tsx", "**/*.mts", "**/*.cts"],
+      "rules": {
+        "constructor-super": "off",
+        "getter-return": "off",
+        "no-class-assign": "off",
+        "no-const-assign": "off",
+        "no-dupe-class-members": "off",
+        "no-dupe-keys": "off",
+        "no-func-assign": "off",
+        "no-import-assign": "off",
+        "no-new-native-nonconstructor": "off",
+        "no-obj-calls": "off",
+        "no-redeclare": "off",
+        "no-setter-return": "off",
+        "no-this-before-super": "off",
+        "no-undef": "off",
+        "no-unreachable": "off",
+        "no-unsafe-negation": "off",
+        "no-var": "error",
+        "no-with": "off",
+        "prefer-rest-params": "error",
+        "prefer-spread": "error"
+      }
+    },
+    {
+      "files": ["services/backend/src/routes/*.{ts,js}"],
+      "rules": {
+        "typescript/no-misused-promises": "off",
+        "typescript/consistent-return": "off"
+      }
+    },
+    {
+      "files": ["{updater,services}/**/*.{js,cjs}"],
+      "rules": {
+        "typescript/await-thenable": "off",
+        "typescript/consistent-return": "off",
+        "typescript/consistent-type-exports": "off",
+        "typescript/dot-notation": "off",
+        "typescript/no-array-delete": "off",
+        "typescript/no-base-to-string": "off",
+        "typescript/no-confusing-void-expression": "off",
+        "typescript/no-deprecated": "off",
+        "typescript/no-duplicate-type-constituents": "off",
+        "typescript/no-floating-promises": "off",
+        "typescript/no-for-in-array": "off",
+        "typescript/no-implied-eval": "off",
+        "typescript/no-meaningless-void-operator": "off",
+        "typescript/no-misused-promises": "off",
+        "typescript/no-misused-spread": "off",
+        "typescript/no-mixed-enums": "off",
+        "typescript/no-redundant-type-constituents": "off",
+        "typescript/no-unnecessary-boolean-literal-compare": "off",
+        "typescript/no-unnecessary-condition": "off",
+        "typescript/no-unnecessary-qualifier": "off",
+        "typescript/no-unnecessary-template-expression": "off",
+        "typescript/no-unnecessary-type-arguments": "off",
+        "typescript/no-unnecessary-type-assertion": "off",
+        "typescript/no-unnecessary-type-conversion": "off",
+        "typescript/no-unnecessary-type-parameters": "off",
+        "typescript/no-unsafe-enum-comparison": "off",
+        "typescript/no-unsafe-type-assertion": "off",
+        "typescript/no-unsafe-unary-minus": "off",
+        "typescript/non-nullable-type-assertion-style": "off",
+        "typescript/only-throw-error": "off",
+        "typescript/prefer-find": "off",
+        "typescript/prefer-includes": "off",
+        "typescript/prefer-nullish-coalescing": "off",
+        "typescript/prefer-optional-chain": "off",
+        "typescript/prefer-promise-reject-errors": "off",
+        "typescript/prefer-readonly": "off",
+        "typescript/prefer-readonly-parameter-types": "off",
+        "typescript/prefer-reduce-type-parameter": "off",
+        "typescript/prefer-regexp-exec": "off",
+        "typescript/prefer-return-this-type": "off",
+        "typescript/prefer-string-starts-ends-with": "off",
+        "typescript/promise-function-async": "off",
+        "typescript/related-getter-setter-pairs": "off",
+        "typescript/require-array-sort-compare": "off",
+        "typescript/require-await": "off",
+        "typescript/restrict-plus-operands": "off",
+        "typescript/return-await": "off",
+        "typescript/strict-boolean-expressions": "off",
+        "typescript/switch-exhaustiveness-check": "off",
+        "typescript/unbound-method": "off",
+        "typescript/use-unknown-in-catch-callback-variable": "off"
+      }
+    },
+    {
+      "files": ["services/frontend/**/*.{js,jsx,mjs,cjs,ts,tsx}"],
+      "rules": {
+        "react/react-compiler": "warn",
+
+        "react/display-name": "error",
+        "react/jsx-key": "error",
+        "react/jsx-no-comment-textnodes": "error",
+        "react/jsx-no-duplicate-props": "error",
+        "react/jsx-no-target-blank": "error",
+        "react/jsx-no-undef": "error",
+        "react/no-children-prop": "error",
+        "react/no-danger-with-children": "error",
+        "react/no-direct-mutation-state": "error",
+        "react/no-find-dom-node": "error",
+        "react/no-is-mounted": "error",
+        "react/no-render-return-value": "error",
+        "react/no-string-refs": "error",
+        "react/no-unescaped-entities": "off",
+        "react/no-unknown-property": "error",
+        "react/no-unsafe": "off",
+        "react/react-in-jsx-scope": "off",
+        "react/require-render-return": "error",
+        "react/rules-of-hooks": "error",
+        "react/exhaustive-deps": "warn",
+        "react/no-array-index-key": "error",
+        "react/no-unstable-nested-components": "warn",
+
+        "react/no-this-in-sfc": "error",
+        "react/jsx-boolean-value": ["error", "never"],
+        "react/jsx-filename-extension": [
+          "error",
+          {
+            "allow": "as-needed",
+            "extensions": [".jsx", ".tsx"]
+          }
+        ],
+        "react/jsx-no-useless-fragment": "error"
+      },
+      "plugins": ["react"]
+    },
+    {
+      "files": ["cypress/**/*.js"],
+      "rules": {
+        "cypress/no-assigning-return-values": "error",
+        "cypress/no-unnecessary-waiting": "error",
+        "cypress/no-async-tests": "error",
+        "cypress/unsafe-to-chain-command": "error"
+      },
+      "jsPlugins": ["eslint-plugin-cypress"],
+      "globals": {
+        "cy": "readonly",
+        "Cypress": "readonly",
+        "expect": "readonly",
+        "assert": "readonly",
+        "chai": "readonly"
+      },
+      "env": {
+        "mocha": true
+      }
+    },
+    {
+      "files": ["services/backend/tests/**/*.ts"],
+      "rules": {
+        "no-console": "off"
+      }
+    },
+    {
+      "files": [
+        "services/backend/**/*.js",
+        "updater/sis-updater-worker/src/db/migrations/*.cjs",
+        "updater/sis-updater-scheduler/**/*.js"
+      ],
+      "rules": {
+        "import/no-commonjs": "off",
+        "typescript/no-require-imports": "off"
+      }
+    },
+    {
+      "files": ["updater/sis-updater-worker/src/db/migrations/*.cjs", "services/backend/src/database/**/*.js"],
+      "rules": {
+        "no-empty-function": "off",
+        "typescript/require-await": "off"
+      }
+    },
+    {
+      "files": ["services/backend/**/*.{js,ts}", "updater/**/*.js"],
+      "rules": {
+        "no-await-in-loop": "off",
+        "no-promise-executor-return": "off",
+        "import/no-default-export": "off"
+      }
+    },
+    {
+      "files": [
+        "services/frontend/tests/**/*.{ts,tsx}",
+        "services/frontend/playwright/**/*.{ts,tsx}",
+        "services/frontend/playwright-ct.config.ts"
+      ],
+      "rules": {
+        "react/jsx-filename-extension": "off",
+        "import/no-default-export": "off"
+      },
+      "plugins": ["react"]
+    }
+  ]
+}

README.md

@@ -56,6 +56,10 @@ What different CLI options do:
    - Runs `docker system prune -a && docker volume prune -a`
    - This will remove all your Docker data, also related to any other applications than Oodikone!
    - Sometimes necessary after strange errors due to caching
+6. Clear node_nodules
+   - Deletes "nodemod" docker volumes and all local node_modules from each service
+   - Useful when resolving permission/installation/library issues without the need to reset all data
+   - Runs `npm ci` for every service locally
 
 ## ⌨️ Development
 
@@ -123,6 +127,8 @@ Some useful commands are defined in `package.json` and can be run with `npm run
 
 - `npm run oodikone`: starts Oodikone with anonymized data
 - `npm run oodikone:real`: starts Oodikone with real data
+- `npm run oodikone:test`: starts Oodikone with non-persistent database (wipes after every reset)
+- `npm run oodikone:ci`: starts Oodikone in CI mode
 - `npm run updater`: starts updater with anonymized data
 - `npm run updater:real`: starts updater with real data
 - `npm run both`: starts Oodikone and updater with anonymized data
@@ -131,8 +137,9 @@ Some useful commands are defined in `package.json` and can be run with `npm run
 - `npm run testupdater`: runs tests for updater
 - `npm run flushredis`: clears Redis, which forces big calculations to be redone in e.g. studyprogramme overview and faculty views
 - `npm run install:local`: installs `node_modules` for each package (backend, frontend, sis-updater-scheduler and sis-updater-worker) locally
-- `npm run install:docker`: installs `node_modules` for each package inside containers
-- `npm run install:both`: runs both `npm run install:local` and `npm run install:docker`
+- `npm run ci:local`: same as above, but sticks to the package versions pinned in package-lock
+- `npm run ci:docker`: installs `node_modules` for each package inside containers
+- `npm run ci:both`: runs both `npm run ci:local` and `npm run ci:docker`
 
 Once you have ran setup for Oodikone , you can just execute the first one (`npm run oodikone`). After starting and waiting for a while for containers to compile, Oodikone can be accessed at [http://localhost:3000](http://localhost:3000/) and Adminer (database investigation tool) at [http://localhost:5050](http://localhost:5050/). Adminer requires you to login with username `postgres` and with any password you choose (for example `p`).
 
@@ -149,8 +156,8 @@ It is recommended to spend some time to become familiar with `docker` and `docke
 - `docker compose ps`: view the containers in the running environment
 - `docker compose logs frontend`: print logs for just frontend
 - `docker compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
-- `docker exec -it backend sh`: open bash terminal inside backend container
-- `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database
+- `docker exec -it oodikone-backend sh`: open bash terminal inside backend container
+- `docker exec -it oodikone-sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database
 
 ### User types for development
 
@@ -189,9 +196,13 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 
 ## ❓FAQ
 
-### Modules are missing after updating package.json
+### How to install a new package
 
-You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container or `npm run install:docker` to install modules inside all packages (backend, frontend, sis-updater-scheduler and sis-updater-worker).
+Install the package locally to the relevant service with `npm install` followed by the script `npm run ci:docker`. The application running inside container uses your host-mounted package-lock; therefore when making any changes to packages do them first locally. Avoid running `npm install` inside docker containers. Our docker images run as non-root, and cannot write on the host mounted package-lock files (without sudo'ing and changing back the permissions later). The CLI has an option to clear all node_modules should the need arise.
+
+### Modules are missing after an update to package.json
+
+Module might be missing for example when someone else installs a new library and you pull the changes to the package files, without installing the acual packages. To sync the node_modules with the package files run `npm run ci:both`.
 
 ### Study guidance groups don't work on my machine
 
@@ -238,6 +249,20 @@ npm run cypress:run
 
 Remember to revert the `baseUrl` in [cypress.config.js](./cypress.config.js) to its original value when you're done.
 
+## How to read clients minds
+
+Search for client queries in grafana using a variation of this query:
+
+```
+{app="oodikone"} | regexp `\s(?P<payload>\{.*\}$)` | line_format `{{.payload}}` | json userId, mockedBy | userId = `client` | mockedBy != `admin`
+```
+
+For quick access to _any and all params_ under certain path:
+
+```
+{app="oodikone"} |= `path` | regexp `\s(?P<payload>\{.*\}$)` | line_format `{{.payload}}` | json userId, mockedBy | userId = `client` | mockedBy != `admin`
+```
+
 ## ✌🏼 Maintainers and contribution
 
 [Toska](https://toska.dev) - University of Helsinki

anonyymioodi/commit_and_push_to_toska_hub.sh

@@ -43,6 +43,6 @@ fi
 
 infomsg "Committing and pushing to registry"
 database=${args[0]}
-docker commit "$database" "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
+docker commit "oodikone-$database" "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
 docker push "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
 successmsg "Succesfully pushed"

anonyymioodi/create_container_from_dump.sh

@@ -52,13 +52,13 @@ fi
 
 
 infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
-database=${args[0]}
-docker-compose down --remove-orphans
-docker-compose up -d "$database"
+database="${args[0]}"
+docker compose down --remove-orphans
+docker compose up -d "$database"
 
 database_dump="$database.sqz"
 infomsg "Attempting to restore database $database from dump $database_dump"
-retry docker-compose exec "$database" pg_isready --dbname="$database"
-docker exec -i "$database" /bin/bash -c "pg_restore --username=postgres \
+retry docker compose exec "$database" pg_isready --dbname="$database"
+docker exec -i "oodikone-$database" /bin/bash -c "pg_restore --username=postgres \
 --format=custom --dbname=$database --no-owner" < "$database_dump"
 successmsg "Database $database succesfully created"

anonyymioodi/docker-compose.yml

@@ -1,40 +1,57 @@
 x-defaults: &defaults
   restart: unless-stopped
 
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
 services:
   kone-db:
     <<: *defaults
-    container_name: kone-db
+    container_name: oodikone-kone-db
     environment:
       PGDATA: /data
       POSTGRES_DB: kone-db
       POSTGRES_HOST_AUTH_METHOD: trust
     image: postgres:15.10
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
+
+  user-db:
+    <<: *defaults
+    container_name: oodikone-user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: postgres:15.10
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-db:
     <<: *defaults
-    container_name: sis-db
+    container_name: oodikone-sis-db
     environment:
       PGDATA: /data
       POSTGRES_DB: sis-db
       POSTGRES_HOST_AUTH_METHOD: trust
     image: postgres:15.10
     shm_size: 2gb
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-importer-db:
     <<: *defaults
-    container_name: sis-importer-db
+    container_name: oodikone-sis-importer-db
     environment:
       PGDATA: /data
       POSTGRES_DB: sis-importer-db
       POSTGRES_HOST_AUTH_METHOD: trust
     image: postgres:15.10
-
-  user-db:
-    <<: *defaults
-    container_name: user-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: user-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults

anonyymioodi/update_staging_db.sh

@@ -56,7 +56,7 @@ function create_dumps() {
 function create_dump_from_db() {
   echo "Dumping db $1..."
 
-  docker exec "$1" pg_dump -Fc -U postgres "$1" > "$dump_directory/$1.sqz" || die "Failed to dump db $1"
+  docker exec "oodikone-$1" pg_dump -Fc -U postgres "$1" > "$dump_directory/$1.sqz" || die "Failed to dump db $1"
 }
 
 

cli.sh

@@ -18,11 +18,11 @@ source "$PROJECT_ROOT"/scripts/functions_for_setup.sh
 # Try to run cleanup function if things fail.
 trap cleanup SIGINT SIGTERM ERR EXIT
 
-# Run docker-compose down on cleanup
+# Run docker compose down on cleanup
 cleanup() {
   trap - SIGINT SIGTERM ERR EXIT
-  warningmsg "Trying to run docker-compose down and remove orphans"
-  docker-compose --profile oodikone --profile updater down --remove-orphans
+  warningmsg "Trying to run docker compose down and remove orphans"
+  docker compose --profile oodikone --profile updater down --remove-orphans
 }
 
 # === CLI ===
@@ -49,10 +49,11 @@ set_custom_select_prompt() {
 
   options=(
     "Set up Oodikone from scratch"
-    "Reset all real data"
+    "Reset all data"
     "Reset single database"
     "Restore data from dumps"
     "Docker system prune"
+    "Nuke local node_modules"
     "Quit"
   )
 }
@@ -68,8 +69,8 @@ while true; do
     case $opt in
       "Set up Oodikone from scratch")
         set_up_oodikone_from_scratch;;
-      "Reset all real data")
-        reset_all_real_data;;
+      "Reset all data")
+        reset_all_data;;
       "Reset single database")
         reset_single_database
         set_custom_select_prompt # Set prompt back to initial values
@@ -78,6 +79,8 @@ while true; do
       restore_data_from_dumps;;
       "Docker system prune")
         docker_prune;;
+      "Nuke local node_modules")
+        nuke_node_modules;;
       "Quit")
         break 2;;
       *) msg "${RED}Invalid option:${NOFORMAT} $REPLY

cypress.config.js

@@ -4,6 +4,13 @@ const { defineConfig } = require('cypress')
 
 module.exports = defineConfig({
   projectId: 'c3jsph',
+
+  // Fixes Github <-> Cypress permission errors in CI
+  downloadsFolder: '/tmp/cypress/downloads',
+  screenshotsFolder: '/tmp/cypress/screenshots',
+  videosFolder: '/tmp/cypress/videos',
+  video: true,
+
   defaultCommandTimeout: 30000,
   requestTimeout: 30000,
   videoCompression: false,
@@ -19,5 +26,4 @@ module.exports = defineConfig({
     baseUrl: 'http://localhost:3000',
     specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}',
   },
-  video: true,
 })

cypress/e2e/Completed_courses.js

@@ -153,7 +153,8 @@ describe('Courselist saving-related functions work', () => {
     cy.cs('course-list-input').contains(courses.join(', '))
   })
 
-  it('Course list can be deleted', () => {
+  // TODO: This will be re-enabled when migrating to playwright
+  it.skip('Course list can be deleted', () => {
     const courseList = generateCourseListName()
     createCourseList(['CSM14204', 'TKT10004'], courseList)
     openSearch()

cypress/e2e/Course_statistics.js

@@ -537,7 +537,8 @@ describe('Basic user', () => {
         cy.contains('Show population').should('be.enabled')
       })
 
-      describe('Students tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Students tab', () => {
         describe('Info boxes', () => {
           it('Student statistics table', () => {
             cy.cs('StudentStatistics-info-box-button').click()
@@ -639,7 +640,8 @@ describe('Basic user', () => {
         })
       })
 
-      describe('Attempts tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Attempts tab', () => {
         beforeEach(() => {
           openAttemptsTab()
         })
@@ -774,7 +776,8 @@ describe('Basic user', () => {
         cy.contains('Show population').should('be.enabled')
       })
 
-      describe('Students tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Students tab', () => {
         it('Show grades off, Separate by semesters off', () => {
           const tableContents = [
             // [Time, Total students, Passed, Failed, Enrolled no grade, Pass rate, Fail rate]
@@ -960,7 +963,8 @@ describe('Basic user', () => {
         })
       })
 
-      describe('Attempts tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Attempts tab', () => {
         beforeEach(() => {
           openAttemptsTab()
         })
@@ -1206,7 +1210,8 @@ describe('Basic user', () => {
   })
 })
 
-describe('Only course statistics', () => {
+// TODO: Writing these as backend api tests
+describe.skip('Only course statistics', () => {
   beforeEach(() => {
     cy.init('/coursestatistics', 'onlycoursestatistics')
     cy.url().should('include', '/coursestatistics')

cypress/e2e/Degree_programme_overview.js

@@ -32,36 +32,21 @@ const deleteTag = tagName => {
 
 describe('Degree programme overview', () => {
   describe('Degree programme selector', () => {
-    beforeEach(() => {
-      cy.init('/study-programme', 'admin')
-    })
-
-    it('Correct categories', () => {
-      cy.contains('Bachelor programmes')
-      cy.contains('Master programmes')
-      cy.contains('Combined programmes')
-      cy.contains('Doctoral programmes')
-    })
-
     it('Degree programme search filter', () => {
+      cy.init('/study-programme', 'admin')
       cy.contains('Tietojenkäsittelytieteen kandiohjelma').should('exist')
       cy.contains('Matemaattisten tieteiden kandiohjelma').should('exist')
       cy.cs('study-programme-filter').type('Tietojenkäsittelytieteen')
       cy.contains('Tietojenkäsittelytieteen kandiohjelma').should('exist')
       cy.contains('Matemaattisten tieteiden kandiohjelma').should('not.exist')
     })
-
-    it('Old and specialized programmes filter', () => {
-      cy.contains('Biotekniikka (ylempi)').should('not.exist')
-      cy.cs('filter-old-programmes-toggle').click()
-      cy.contains('Biotekniikka (ylempi)').should('exist')
-    })
   })
 
   describe('Basic information tab works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
       cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click({ force: true })
+      cy.cs('year-toggle').click() // NOTE: Tests are written for calendar years
     })
 
     // If the backend breaks for one of the sections, the section header is not rendered and this will fail
@@ -76,15 +61,15 @@ describe('Degree programme overview', () => {
     it('Basic information contains correct students', () => {
       const years = getEmptyYears()
       const tableContents = [
-        // [Year, Started studying, Accepted, Graduated, Transferred Away, Transferred to]
+        // [Year, Started studying, Accepted, Graduated, Cancelled, Transferred Away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
-        [2023, 8, 8, 26, 0, 0],
-        [2022, 25, 26, 48, 1, 4],
-        [2021, 29, 32, 47, 0, 2],
-        [2020, 26, 27, 12, 1, 3],
-        [2019, 28, 34, 1, 0, 1],
-        [2018, 40, 45, 0, 0, 0],
-        [2017, 41, 47, 0, 0, 0],
+        [2023, 8, 8, 26, 2, 0, 0],
+        [2022, 25, 26, 47, 0, 1, 3],
+        [2021, 29, 32, 48, 1, 0, 2],
+        [2020, 26, 27, 12, 0, 1, 3],
+        [2019, 28, 34, 1, 0, 0, 1],
+        [2018, 40, 45, 0, 0, 0, 1],
+        [2017, 41, 47, 0, 0, 0, 0],
       ]
 
       cy.checkTableStats(tableContents, 'students-of-the-study-programme')
@@ -112,8 +97,8 @@ describe('Degree programme overview', () => {
         // [Year, Graduated, Wrote thesis]
         ...years.map(year => [year, 0, 0]),
         [2023, 26, 5],
-        [2022, 48, 13],
-        [2021, 47, 76],
+        [2022, 47, 13],
+        [2021, 48, 76],
         [2020, 12, 23],
         [2019, 1, 19],
         [2018, 0, 0],
@@ -130,8 +115,8 @@ describe('Degree programme overview', () => {
         // [Year, Started studying, Accepted, Graduated]
         ...years.map(year => [year, 0, 0, 0]),
         [2023, 8, 8, 24],
-        [2022, 25, 26, 44],
-        [2021, 29, 32, 46],
+        [2022, 25, 26, 43],
+        [2021, 29, 32, 47],
         [2020, 26, 27, 11],
         [2019, 28, 34, 1],
         [2018, 40, 45, 0],
@@ -144,8 +129,8 @@ describe('Degree programme overview', () => {
         // [Year, Graduated, Wrote thesis]
         ...years.map(year => [year, 0, 0]),
         [2023, 24, 5],
-        [2022, 44, 12],
-        [2021, 46, 69],
+        [2022, 43, 12],
+        [2021, 47, 69],
         [2020, 11, 22],
         [2019, 1, 18],
         [2018, 0, 0],
@@ -160,15 +145,15 @@ describe('Degree programme overview', () => {
       const isAcademicYear = true
       const years = getEmptyYears(isAcademicYear)
       const studentTableContents = [
-        // [Year, Started studying, Accepted, Graduated, Transferred away, Transferred to]
+        // [Year, Started studying, Accepted, Graduated, Cancelled, Transferred away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
-        ['2023 - 2024', 8, 8, 4, 0, 0],
-        ['2022 - 2023', 25, 26, 36, 0, 0],
-        ['2021 - 2022', 29, 32, 57, 1, 6],
-        ['2020 - 2021', 26, 27, 30, 0, 3],
-        ['2019 - 2020', 28, 34, 7, 1, 1],
-        ['2018 - 2019', 40, 45, 0, 0, 0],
-        ['2017 - 2018', 41, 47, 0, 0, 0],
+        ['2023 - 2024', 8, 8, 4, 1, 0, 0],
+        ['2022 - 2023', 25, 26, 35, 1, 0, 0],
+        ['2021 - 2022', 29, 32, 58, 1, 1, 5],
+        ['2020 - 2021', 26, 27, 30, 0, 0, 3],
+        ['2019 - 2020', 28, 34, 7, 0, 1, 1],
+        ['2018 - 2019', 40, 45, 0, 0, 0, 1],
+        ['2017 - 2018', 41, 47, 0, 0, 0, 0],
       ]
       cy.checkTableStats(studentTableContents, 'students-of-the-study-programme')
 
@@ -207,19 +192,20 @@ describe('Degree programme overview', () => {
         .should('contain', 47)
         .should('contain', 76)
 
-      cy.cs('unset-breakdown-bar-chart')
-      cy.cs('graduation-time-toggle').click()
+      cy.cs('graduation-mode-selector').within(() => {
+        cy.cs('select-median').click()
+      })
       cy.cs('unset-median-bar-chart').within(() => {
-        cy.contains('48 graduated').should('be.visible')
-        cy.contains('48 graduated').hover()
+        cy.contains('47 graduated').should('be.visible')
+        cy.contains('47 graduated').hover()
       })
 
-      cy.get('.grad-vals').contains('48 students graduated in year 2022').should('be.visible')
+      cy.get('.grad-vals').contains('47 students graduated in year 2022').should('be.visible')
 
-      cy.contains('median study time: 6.5 semesters')
+      cy.contains('median study time: 6 semesters')
       cy.contains('24 graduated on time')
       cy.contains('12 graduated max year overtime')
-      cy.contains('12 graduated over year late')
+      cy.contains('11 graduated over year late')
 
       cy.cs('programmes-before-or-after-stacked-bar-chart-section')
         .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
@@ -232,14 +218,16 @@ describe('Degree programme overview', () => {
   })
 
   describe('Graduation times of master programmes', () => {
-    it('are split into two graphs', () => {
+    it('are split into three graphs', () => {
       cy.init('/study-programme')
       cy.contains('a', 'Matematiikan ja tilastotieteen maisteriohjelma').click({ force: true })
+      cy.cs('year-toggle').click() // Tests are written for calendar year
 
-      cy.cs('unset-breakdown-bar-chart')
       cy.cs('unset-breakdown-bar-chart')
 
-      cy.cs('graduation-time-toggle').click()
+      cy.cs('graduation-mode-selector').within(() => {
+        cy.cs('select-average').click()
+      })
       cy.cs('unset-graduation-times-section')
         .eq(1)
         .within(() => {
@@ -249,7 +237,7 @@ describe('Degree programme overview', () => {
         })
 
       cy.get('.grad-vals').contains('2 students graduated in year 2021').should('be.visible')
-      cy.contains('median study time: 4.5 semesters')
+      cy.contains('average study time: 4.5 semesters')
       cy.contains('1 graduated on time')
       cy.contains('1 graduated max year overtime')
       cy.contains('0 graduated over year late')
@@ -263,7 +251,7 @@ describe('Degree programme overview', () => {
         })
 
       cy.get('.grad-vals').contains('11 students graduated in year 2023').should('be.visible')
-      cy.contains('median study time: 12 semesters')
+      cy.contains('average study time: 11.18 semesters')
       cy.contains('4 graduated on time')
       cy.contains('6 graduated max year overtime')
       cy.contains('1 graduated over year late')
@@ -292,7 +280,9 @@ describe('Degree programme overview', () => {
         cy.cs('progress-of-students-info-box-content').contains('Kuvaa koulutusohjelmassa')
       })
 
+      // HACK: Do this properly
       it('Average graduation times section', () => {
+        cy.wait(500)
         cy.cs('average-graduation-times-info-box-button').click()
         cy.cs('average-graduation-times-info-box-content').contains('Yksittäinen palkki')
       })
@@ -307,15 +297,15 @@ describe('Degree programme overview', () => {
 
     it('Students of the degree programme are shown correctly', () => {
       const tableContents = [
-        // [Year, All, Started studying, Present, Absent, Passive, Graduated, Men, Women, Other/Unknown, Finland, Other]
-        ['2023 - 2024', 8, 8, 0, 0, 8, 0, 5, 3, 0, 8, 1],
-        ['2022 - 2023', 26, 25, 0, 0, 24, 2, 19, 7, 0, 25, 1],
-        ['2021 - 2022', 38, 29, 0, 0, 33, 5, 29, 9, 0, 36, 4],
-        ['2020 - 2021', 30, 26, 0, 0, 11, 19, 15, 15, 0, 29, 3],
-        ['2019 - 2020', 35, 28, 0, 0, 8, 27, 22, 13, 0, 35, 2],
-        ['2018 - 2019', 45, 40, 0, 0, 6, 39, 26, 19, 0, 44, 1],
-        ['2017 - 2018', 47, 41, 0, 0, 5, 42, 31, 16, 0, 47, 0],
-        ['Total', 229, 197, 0, 0, 95, 134, 147, 82, 0, 224, 12],
+        // [Year, All, Started studying, Present, Absent, Passive, Graduated, Has recent attainments, Men, Women, Other/Unknown, Finland, Other]
+        ['2023 - 2024', 8, 8, 0, 0, 8, 0, 0, 5, 3, 0, 8, 1],
+        ['2022 - 2023', 26, 25, 0, 0, 24, 2, 0, 19, 7, 0, 25, 1],
+        ['2021 - 2022', 37, 29, 0, 0, 33, 4, 0, 29, 8, 0, 35, 4],
+        ['2020 - 2021', 30, 26, 0, 0, 11, 19, 0, 15, 15, 0, 29, 3],
+        ['2019 - 2020', 35, 28, 0, 0, 8, 27, 0, 22, 13, 0, 35, 2],
+        ['2018 - 2019', 46, 40, 0, 0, 6, 40, 0, 26, 20, 0, 45, 1],
+        ['2017 - 2018', 47, 41, 0, 0, 5, 42, 0, 31, 16, 0, 47, 0],
+        ['Total', 229, 197, 0, 0, 95, 134, 0, 147, 82, 0, 224, 12],
       ]
 
       cy.checkTableStats(tableContents, 'study-tracks-and-class-statistics')
@@ -390,10 +380,10 @@ describe('Degree programme overview', () => {
         ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
         ['2023 - 2024', 8, 8, 0, 0, 0, 0, 0, 0, 0],
         ['2022 - 2023', 26, 9, 8, 4, 3, 0, 0, 0, 2],
-        ['2021 - 2022', 38, 8, 4, 9, 8, 4, 0, 0, 5],
+        ['2021 - 2022', 37, 8, 4, 9, 8, 4, 0, 0, 4],
         ['2020 - 2021', 30, 2, 1, 2, 5, 0, 0, 1, 19],
         ['2019 - 2020', 35, 1, 0, 2, 1, 0, 0, 4, 27],
-        ['2018 - 2019', 45, 0, 1, 1, 2, 0, 0, 2, 39],
+        ['2018 - 2019', 46, 0, 1, 1, 2, 0, 0, 2, 40],
         ['2017 - 2018', 47, 0, 1, 3, 0, 0, 1, 0, 42],
         ['Total', 229, 28, 15, 21, 19, 4, 1, 7, 134],
       ]
@@ -466,14 +456,14 @@ describe('Degree programme overview', () => {
       it('Students of the study track are shown correctly', () => {
         cy.cs('study-track-overview-section').contains('Students of the study track MAT-MAT by starting year')
         const tableContents = [
-          // [Year, All, Started studying, Present, Absent, Passive, Graduated, Men, Women, Other/Unknown, Finland, Other]
-          ['2022 - 2023', 3, 3, 0, 0, 1, 2, 2, 1, 0, 3, 0],
-          ['2021 - 2022', 5, 1, 0, 0, 1, 4, 3, 2, 0, 5, 0],
-          ['2020 - 2021', 13, 10, 0, 0, 1, 12, 7, 6, 0, 12, 3],
-          ['2019 - 2020', 17, 14, 0, 0, 0, 17, 10, 7, 0, 17, 1],
-          ['2018 - 2019', 23, 21, 0, 0, 2, 21, 11, 12, 0, 23, 0],
-          ['2017 - 2018', 28, 24, 0, 0, 1, 27, 15, 13, 0, 28, 0],
-          ['Total', 89, 73, 0, 0, 6, 83, 48, 41, 0, 88, 4],
+          // [Year, All, Started studying, Present, Absent, Passive, Graduated, Has recent attainment, Men, Women, Other/Unknown, Finland, Other]
+          ['2022 - 2023', 3, 3, 0, 0, 1, 2, 0, 2, 1, 0, 3, 0],
+          ['2021 - 2022', 4, 1, 0, 0, 1, 3, 0, 3, 1, 0, 4, 0],
+          ['2020 - 2021', 13, 10, 0, 0, 1, 12, 0, 7, 6, 0, 12, 3],
+          ['2019 - 2020', 17, 14, 0, 0, 0, 17, 0, 10, 7, 0, 17, 1],
+          ['2018 - 2019', 24, 21, 0, 0, 2, 22, 0, 11, 13, 0, 24, 0],
+          ['2017 - 2018', 28, 24, 0, 0, 1, 27, 0, 15, 13, 0, 28, 0],
+          ['Total', 89, 73, 0, 0, 6, 83, 0, 48, 41, 0, 88, 4],
         ]
         cy.checkTableStats(tableContents, 'study-tracks-and-class-statistics')
       })
@@ -530,6 +520,8 @@ describe('Degree programme overview', () => {
 
     describe('By credit type tab', () => {
       beforeEach(() => {
+        cy.cs('show-credits-students-toggle').click()
+        cy.cs('year-toggle').click()
         cy.get('thead').within(() => {
           cy.get('tr')
             .eq(0)
@@ -544,7 +536,7 @@ describe('Degree programme overview', () => {
               cy.get('td').eq(0).contains('Course')
               cy.get('td').eq(1).contains('MAT11004')
               cy.get('td').eq(2).contains('Differentiaalilaskenta')
-              cy.get('td').eq(3).contains('1170')
+              cy.get('td').eq(3).contains('1240')
             })
         })
       })
@@ -568,7 +560,7 @@ describe('Degree programme overview', () => {
       })
 
       it('year toggle works', () => {
-        cy.cs('year-toggle').click()
+        cy.cs('year-toggle').click() // Default view now Academic years
 
         cy.cs('from-year-select').click()
         cy.cs('from-year-select-option-2017').click()
@@ -582,7 +574,7 @@ describe('Degree programme overview', () => {
               cy.get('td').eq(0).contains('Course')
               cy.get('td').eq(1).contains('MAT11001')
               cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
-              cy.get('td').eq(3).contains('1220')
+              cy.get('td').eq(3).contains('1265')
             })
         })
       })
@@ -641,6 +633,8 @@ describe('Degree programme overview', () => {
       })
 
       it("'Show credits/Show students' toggle works", () => {
+        cy.cs('year-toggle').click()
+        cy.wait(500) // So that the toggle has time to load
         cy.get('thead tr')
           .eq(0)
           .within(() => {
@@ -715,7 +709,7 @@ describe('Degree programme overview', () => {
               cy.get('td').eq(0).contains('Course')
               cy.get('td').eq(1).contains('MAT11001')
               cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
-              cy.get('td').eq(3).contains('375')
+              cy.get('td').eq(3).contains('272')
             })
         })
       })

cypress/e2e/Faculty_statistics.js

@@ -110,15 +110,15 @@ describe('Faculty statistics', () => {
       })
 
       it('Toggle years works', () => {
-        cy.cs('credits-produced-by-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('thesis-writers-of-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('students-of-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('graduated-of-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('year-toggle').click()
         cy.cs('credits-produced-by-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
         cy.cs('thesis-writers-of-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
         cy.cs('students-of-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
         cy.cs('graduated-of-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
+        cy.cs('year-toggle').click()
+        cy.cs('credits-produced-by-the-faculty-interactive-data-table').should('contain', '2022')
+        cy.cs('thesis-writers-of-the-faculty-interactive-data-table').should('contain', '2022')
+        cy.cs('students-of-the-faculty-interactive-data-table').should('contain', '2022')
+        cy.cs('graduated-of-the-faculty-interactive-data-table').should('contain', '2022')
       })
 
       it('Toggle programmes works', () => {

cypress/e2e/Filters.js

@@ -19,7 +19,7 @@ describe("Population statistics with a master's programme", { testIsolation: fal
 
   it('Study track filter works', () => {
     cy.cs('studyTrackFilter-header').click()
-    cy.selectFromDropdown('studyTrackFilter', 0, true)
+    cy.selectFromDropdown('studyTrackFilter', 'Matematiikka ja soveltava matematiikka (MAST-MSM)', true)
     checkFilteredStudentCount(15)
 
     cy.cs('studyTrackFilter-clear').click()

cypress/e2e/Language_center.js

@@ -44,7 +44,7 @@ describe('When language center is opened', () => {
         cy.get('table > tbody > tr:first').within(() => {
           cy.get('td').should('have.length', 15)
           cy.get('td').eq(0).contains('All courses total')
-          const numbers = [2076, 36, 9, 33, 757, 15, 40, 6, 48, 1, 24, 1059, 48, 0]
+          const numbers = [2076, 36, 9, 34, 759, 15, 40, 6, 51, 1, 24, 1059, 42, 0]
           numbers.forEach((number, index) => {
             cy.get('td')
               .eq(index + 1)
@@ -55,12 +55,12 @@ describe('When language center is opened', () => {
 
       it('Faculties tab "exceeding" button works', () => {
         cy.cs('difference-button').click()
-        checkNumbers([63, 2, 0, 3, 13, 0, 1, 1, 2, 0, 0, 30, 10, 1], 15, 'faculties')
+        checkNumbers([66, 3, 0, 4, 18, 0, 1, 1, 4, 0, 0, 31, 3, 1], 15, 'faculties')
       })
 
       it('Faculties tab semester selector changes numbers', () => {
         chooseSemester('Syksy 2020', 'from')
-        checkNumbers([1184, 28, 6, 27, 497, 4, 10, 2, 33, 0, 9, 535, 33, 0], 15, 'faculties')
+        checkNumbers([1184, 28, 6, 28, 499, 4, 10, 2, 36, 0, 9, 535, 27, 0], 15, 'faculties')
       })
     })
 

cypress/e2e/Numbers_match.js

@@ -0,0 +1,226 @@
+/// <reference types="cypress" />
+
+const selectCourseStatus = (courseCode, status) => {
+  cy.cs(`courseFilter-${courseCode}-selector`).click()
+  cy.get(`[role="listbox"][aria-labelledby=courseFilter-${courseCode}]`).within(() => {
+    cy.get('li').contains(status).click()
+  })
+}
+
+describe('Numbers should match between', () => {
+  describe('Programme courses, Course statistics and Course population (no substitutions, all years)', () => {
+    const test = [
+      // Code, name, total students, passed, not completed, failed*, enrolled no grade* (not used by Programme courses)
+      ['Course', 'MAT11001', 'Johdatus yliopistomatematiikkaan', 272, 253, 1 + 18, 1, 18],
+      ['Course', 'MAT11003', 'Raja-arvot', 270, 249, 1 + 20, 1, 20],
+      ['Course', 'MAT11004', 'Differentiaalilaskenta', 262, 248, 1 + 13, 1, 13],
+      ['Course', 'MAT21002', 'Sarjat', 256, 250, 0 + 6, 0, 6],
+      ['Course', 'MAT11005', 'Integraalilaskenta', 251, 234, 0 + 17, 0, 17],
+      ['Course', 'MAT11002', 'Lineaarialgebra ja matriisilaskenta I', 247, 242, 1 + 4, 1, 4],
+      ['Course', 'MAT12003', 'Todennäköisyyslaskenta I', 234, 227, 3 + 4, 3, 4],
+      ['Course', 'MAT21001', 'Lineaarialgebra ja matriisilaskenta II', 230, 216, 4 + 10, 4, 10],
+      ['Course', 'MAT21003', 'Vektorianalyysi I', 228, 202, 3 + 23, 3, 23],
+      ['Course', 'MAT20005', 'Akateemiset taidot', 201, 178, 0 + 23, 0, 23],
+      ['Module', 'MAT110', 'Matematiikka, perusopinnot', 195, 195, 0 + 0, 0, 0],
+      ['Course', 'MAT21014', 'Johdatus logiikkaan I', 188, 170, 0 + 18, 0, 18],
+    ]
+
+    // Check test data is correct
+    test.forEach(entry => {
+      assert.strictEqual(entry.length, 8)
+      assert.strictEqual(entry.at(6) + entry.at(7), entry.at(5))
+    })
+
+    it('in Programme courses', () => {
+      // Move to Math bachelor programme's programme courses
+      cy.init('/study-programme')
+      cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click()
+      cy.cs('ProgrammeCoursesTab').click()
+
+      // Wait for tab to load
+      cy.contains('Programme courses by credit type')
+
+      // Sort by Total students
+      cy.get('thead').within(() => {
+        cy.get('tr')
+          .eq(0)
+          .within(() => {
+            cy.get('th').eq(3).click()
+          })
+      })
+
+      cy.get('tbody').within(() => {
+        test.forEach((course, index) => {
+          cy.get('tr')
+            .eq(index)
+            .within(() => {
+              // Check a course's fields
+              course.forEach((field, fieldIndex) => {
+                // Failed, Enrolled no grade values not used
+                if (fieldIndex <= 5) {
+                  cy.get('td').eq(fieldIndex).contains(field)
+                }
+              })
+            })
+        })
+      })
+    })
+
+    it('in Course statistics', () => {
+      test.forEach(([_courseOrModule, code, name, total, passed, _notCompleted, failed, enrolledNoGrade]) => {
+        // Move to Course statistics
+        cy.init(`/coursestatistics?courseCodes=%5B%22${code}%22%5D&combineSubstitutions=false`, 'basic')
+        cy.url().should('include', `/coursestatistics?courseCodes=%5B%22${code}%22%5D`)
+        cy.contains('Course statistics')
+        cy.contains(name) // course name
+
+        cy.cs('FromYearSelector').click()
+        cy.cs('FromYearSelectorOption2017-2018').click()
+
+        cy.get('tbody').within(() => {
+          cy.get('tr')
+            .eq(0)
+            .within(() => {
+              cy.get('td').eq(1).contains(total)
+              cy.get('td').eq(2).contains(passed)
+              cy.get('td').eq(3).contains(failed) // jump over combined Not completed
+              cy.get('td').eq(4).contains(enrolledNoGrade)
+            })
+        })
+      })
+    })
+
+    it('in Course population', () => {
+      test.forEach(([_courseOrModule, code, name, total, passed, _notCompleted, failed, enrolledNoGrade]) => {
+        // Academic years between 2017-2023
+        cy.init(
+          // year - 1949 yields the correct year code
+          `/coursepopulation?from=${2017 - 1949}&to=${2023 - 1949}&coursecodes=%5B%22${code}%22%5D&includeSubstitutions=false`,
+          'basic'
+        )
+        cy.contains(name) // Course name
+
+        // Select course from Course-filter
+        cy.cs('courseFilter-filter-card').click()
+        cy.cs('courseFilter-selector').click()
+        cy.cs('courseFilter-popper').within(() => {
+          cy.contains('li', `${code} - ${name}`).click() // "code - name"
+        })
+
+        selectCourseStatus(code, 'All')
+        cy.contains(`Students (${total})`)
+
+        selectCourseStatus(code, 'Passed')
+        cy.contains(`Students (${passed})`)
+
+        selectCourseStatus(code, 'Failed')
+        cy.contains(`Students (${failed})`)
+
+        selectCourseStatus(code, 'Enrolled, No Grade')
+        cy.contains(`Students (${enrolledNoGrade})`)
+      })
+    })
+  })
+
+  describe('Programme courses, Course statistics and Course population (no substitutions, single years)', () => {
+    // MAT21003, Vektorianalyysi I
+    const test = [
+      // Year, Total students, Passed, Failed, Enrolled no grade, Not completed (failed + enrolled n.g.)
+      [2017, 1, 0, 1, 0, 1 + 0],
+      [2018, 31, 27, 4, 0, 4 + 0],
+      [2019, 58, 56, 2, 0, 2 + 0],
+      [2020, 63, 62, 1, 0, 1 + 0],
+      [2021, 41, 35, 0, 6, 0 + 6],
+      [2022, 28, 21, 0, 7, 0 + 7],
+      [2023, 21, 1, 0, 20, 0 + 20],
+    ]
+
+    // Check test data is correct
+    test.forEach(entry => {
+      assert.strictEqual(entry.length, 6)
+      assert.strictEqual(entry.at(3) + entry.at(4), entry.at(5))
+    })
+
+    // TODO: Re-enable when migrated to playwright
+    it.skip('in Programme courses', () => {
+      cy.init('/study-programme')
+      cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click()
+      cy.cs('ProgrammeCoursesTab').click()
+
+      test.forEach(([year, total, passed, _failed, _enrolledNoGrade, notCompleted]) => {
+        cy.cs('to-year-select').click()
+        cy.cs(`to-year-select-option-${year}`).click()
+
+        cy.cs('from-year-select').click()
+        cy.cs(`from-year-select-option-${year}`).click()
+
+        cy.get('tbody')
+          .contains('tr', 'MAT21003')
+          .scrollIntoView()
+          .within(() => {
+            cy.get('td').eq(3).contains(total)
+            cy.get('td').eq(4).contains(passed)
+            cy.get('td').eq(5).contains(notCompleted)
+          })
+      })
+    })
+
+    it('in Course statistics', () => {
+      test.forEach(([year, total, passed, failed, enrolledNoGrade, _notCompleted]) => {
+        cy.init(`/coursestatistics?courseCodes=%5B%22MAT21003%22%5D&combineSubstitutions=false`, 'basic')
+
+        const yearString = `${year}-${year + 1}`
+
+        cy.cs('ToYearSelector').click()
+        cy.cs(`ToYearSelectorOption${yearString}`).click()
+
+        cy.cs('FromYearSelector').click()
+        cy.cs(`FromYearSelectorOption${yearString}`).click()
+
+        cy.contains('Student statistics')
+
+        // Check that only total and the selected year are visible
+        cy.get('tbody tr').should('have.length', 2)
+
+        // Take the total entry in the table to have enrolled, no grade always be a number
+        cy.get('tbody')
+          .contains('tr', 'Total')
+          .within(() => {
+            cy.get('td').eq(1).contains(total)
+            cy.get('td').eq(2).contains(passed)
+            cy.get('td').eq(3).contains(failed)
+            cy.get('td').eq(4).contains(enrolledNoGrade)
+          })
+      })
+    })
+    it('in Course population', () => {
+      test.forEach(([year, total, passed, failed, enrolledNoGrade, _notCompleted]) => {
+        cy.init(
+          // year - 1949 yields the correct year code
+          `/coursepopulation?from=${year - 1949}&to=${year - 1949}&coursecodes=%5B%22MAT21003%22%5D&includeSubstitutions=false`,
+          'basic'
+        )
+        cy.contains('Vektorianalyysi I') // Course name
+
+        // Select course from Course-filter
+        cy.cs('courseFilter-filter-card').click()
+        cy.cs('courseFilter-selector').click()
+        cy.cs('courseFilter-popper').within(() => {
+          cy.contains('li', 'MAT21003 - Vektorianalyysi I').click() // "code - name"
+        })
+
+        selectCourseStatus('MAT21003', 'All')
+        cy.contains(`Students (${total})`)
+
+        selectCourseStatus('MAT21003', 'Passed')
+        cy.contains(`Students (${passed})`)
+
+        selectCourseStatus('MAT21003', 'Failed')
+        cy.contains(`Students (${failed})`)
+
+        selectCourseStatus('MAT21003', 'Enrolled, No Grade')
+        cy.contains(`Students (${enrolledNoGrade})`)
+      })
+    })
+  })
+})

cypress/e2e/Routes.js

@@ -0,0 +1,85 @@
+import { userHeaders } from '../support/commands.js'
+
+const getBaseRoutes = [
+  ['/', {}],
+  ['/changelog', {}],
+  ['/completedcoursessearch', {}],
+  ['/university', {}],
+  ['/feedback', {}],
+  ['/populations', { roles: ['fullSisuAccess'], programmeRights: [] }],
+  ['/study-programme', { roles: ['fullSisuAccess'], programmeRights: [] }],
+  ['/study-programme/KH50_001', { roles: ['fullSisuAccess'], programmeRights: ['KH50_001'] }],
+  ['/coursepopulation', { roles: ['fullSisuAccess'], programmeRights: [] }],
+  ['/users', { roles: ['admin'] }],
+  ['/users/334', { roles: ['admin'] }],
+  ['/updater', { roles: ['admin'] }],
+  ['/coursestatistics', { roles: ['fullSisuAccess', 'courseStatistics'], programmeRights: [] }],
+  ['/faculties', { roles: ['fullSisuAccess', 'facultyStatistics'] }],
+  ['/faculties/hy-org-1000000911', { roles: ['fullSisuAccess', 'facultyStatistics'] }],
+  ['/students', { roles: ['fullSisuAccess', 'studyGuidanceGroups'], programmeRights: [] }],
+  ['/students/545400', { roles: ['fullSisuAccess', 'studyGuidanceGroups'], programmeRights: [] }],
+  ['/custompopulation', { roles: ['fullSisuAccess', 'studyGuidanceGroups'], programmeRights: [] }],
+  ['/teachers', { roles: ['teachers'] }],
+  ['/teachers/hy-hlo-1463447', { roles: ['teachers'] }],
+  ['/studyguidancegroups', { roles: ['studyGuidanceGroups'] }],
+  ['/studyguidancegroups/otm-b3b52ff5-88f6-4d52-bd18-0588023b6322', { userId: '', roles: ['studyGuidanceGroups'] }],
+  ['/close-to-graduation', { roles: ['fullSisuAccess', 'studyGuidanceGroups'] }],
+  ['/languagecenterview', { iamGroups: ['grp-kielikeskus-esihenkilot'] }],
+]
+
+for (const user of userHeaders) {
+  if (user.uid === 'admin') continue
+  describe(`${user.displayname} has access only to views that were intended`, () => {
+    let userInfo
+    before(() => {
+      cy.request({ method: 'GET', url: '/api/login', headers: user }).then(({ body }) => (userInfo = body?.user))
+    })
+
+    beforeEach(() => cy.wrap(userInfo).should('not.be.undefined'))
+    beforeEach(() => {
+      cy.intercept('**/api/**', req => {
+        req.on('response', res => {
+          expect(res.statusCode, `${req.method} ${req.url}`).to.be.lessThan(500)
+        })
+      })
+    })
+
+    for (const [route, requirements] of getBaseRoutes) {
+      it(`Checking access ${route}`, () => {
+        cy.init(route, user.uid)
+
+        const requiredUserId = !requirements.userId || requirements.userId === userInfo.userId
+
+        const requiredRoles = !requirements.roles || requirements.roles?.some(role => userInfo.roles?.includes(role))
+
+        const requiredRights =
+          !requirements.requireProgrammeRights ||
+          userInfo.roles?.includes('fullSisuAccess') ||
+          requirements.programmeRights.every(role => userInfo.programmeRights?.includes(role))
+
+        // This is defined in frontend ProtectedRoute component.
+        const specialNeedsRoutes = [
+          'populations',
+          'study-programme',
+          'coursepopulation',
+          'coursestatistics',
+          'students',
+          'custompopulation',
+        ]
+
+        const mankeli = specialNeedsRoutes.some(path => route.includes(path))
+          ? requiredRoles || requiredRights
+          : requiredRoles && requiredRights
+
+        const shouldHaveAccess = mankeli && requiredUserId
+
+        if (shouldHaveAccess) {
+          cy.contains('Something broke').should('not.exist')
+          cy.contains('Access denied').should('not.exist')
+        } else {
+          cy.contains('Access denied')
+        }
+      })
+    }
+  })
+}

cypress/e2e/Students.js

@@ -259,8 +259,10 @@ describe('Students tests', () => {
 
     it('When a study plan is selected, courses included in the study plan are highlighted with a blue background', () => {
       cy.visit('/students/550789')
-      cy.contains('table tbody tr', 'Kulttuurien tutkimuksen kandiohjelma (01.08.2020–30.06.2023)').within(() => {
-        cy.get('td').eq(0).click()
+      cy.contains('Study rights')
+      cy.get('[data-cy=KH40_005-0]').contains('Kulttuurien tutkimuksen kandiohjelma')
+      cy.get('[data-cy=KH40_005-0]').within(() => {
+        cy.get('td').eq(1).click()
       })
       cy.contains('table tbody tr', 'Kandidaatintutkielma (KUKA-LIS222)').should(
         'have.css',
@@ -269,27 +271,23 @@ describe('Students tests', () => {
       )
     })
 
-    it('When a study plan is selected, the time frame of the credit graph is updated', () => {
-      cy.clock(new Date('2024-08-30').getTime(), ['Date'])
-      cy.visit('/students/550789')
-      cy.contains('table tbody tr', 'Kulttuurien tutkimuksen kandiohjelma (01.08.2020–30.06.2023)').within(() => {
-        cy.get('td').eq(0).click()
-      })
-    })
-
     it("If there's a study plan corresponding to the degree programme, completed credits are displayed", () => {
       cy.visit('/students/550789')
-      cy.contains('table tbody tr', 'Kulttuurien tutkimuksen kandiohjelma (01.08.2020–30.06.2023)').within(() => {
+      cy.contains('Study rights')
+      cy.get('[data-cy=KH40_005-0]').contains('Kulttuurien tutkimuksen kandiohjelma')
+      cy.get('[data-cy=KH40_005-0]').within(() => {
         cy.get('td')
-          .eq(-1)
+          .eq(4)
           .contains(/^185 cr$/)
       })
     })
 
     it("If there's a study plan corresponding to the degree programme and the student hasn't graduated, percentage of completion is also displayed", () => {
       cy.visit('/students/458723')
-      cy.contains('table tbody tr', 'Oikeusnotaarin koulutusohjelma (01.08.2020–31.07.2027)').within(() => {
-        cy.get('td').eq(-1).contains('69% (125 cr)')
+      cy.contains('Study rights')
+      cy.get('[data-cy=KH20_001-0]').contains('Oikeusnotaarin koulutusohjelma')
+      cy.get('[data-cy=KH20_001-0]').within(() => {
+        cy.get('td').eq(4).contains('69% (125 cr)')
       })
     })
   })

cypress/e2e/Teachers.js

@@ -35,7 +35,7 @@ describe('Teachers page tests', () => {
     cy.cs('Search').click()
     cy.cs('teacher-search').type(teacher1.split(' ')[0])
     cy.get('table tbody tr').should('have.length', 4)
-    cy.get('table tbody tr').eq(1).contains('td', teacher1)
+    cy.get('table tbody tr').eq(2).contains('td', teacher1)
   })
 
   it('Can check teacher page', () => {

cypress/e2e/University.js

@@ -10,21 +10,18 @@ const checkProgressTables = () => {
   progressLevels.forEach(level => cy.cs(`${level}-faculty-progress-table`))
 }
 
-const graduationTimesLevels = ['bachelor', 'master', 'doctor'] // ? Bachelor + master seems to be missing in test data
+const graduationTimesLevels = ['bachelor', 'bcMsCombo', 'master', 'doctor']
 
-const checkAverageGraduationTimesBreakdownBarCharts = () => {
+const checkGraduationCharts = mode => {
   graduationTimesLevels.forEach(level => {
-    cy.cs(`${level}-breakdown-bar-chart`)
+    cy.cs(`${level}-${mode}-bar-chart`).within(() => {
+      // Check that bar has loaded
+      cy.contains('Loading content').should('not.exist')
+    })
   })
 }
 
-const checkAverageGraduationTimesMedianBarCharts = () => {
-  graduationTimesLevels.forEach(level => {
-    cy.cs(`${level}-median-bar-chart`)
-  })
-}
-
-describe('University view', () => {
+describe('University view', { defaultCommandTimeout: 60 * 1000 }, () => {
   beforeEach(() => {
     cy.init('/university')
     cy.contains('University')
@@ -73,17 +70,31 @@ describe('University view', () => {
       cy.cs('faculty-graduations-tab').click()
     })
 
-    it('has all the correct median time bar charts', () => {
-      checkAverageGraduationTimesBreakdownBarCharts()
-    })
+    describe('Different modes work', () => {
+      it('Breakdown', () => {
+        cy.cs('graduation-mode-selector').within(() => {
+          cy.cs('select-breakdown').click()
+        })
+        checkGraduationCharts('breakdown')
+      })
 
-    it("'Breakdown/Median times' toggle works", () => {
-      cy.cs('graduation-time-toggle').click()
-      checkAverageGraduationTimesMedianBarCharts()
+      it('Median', () => {
+        cy.cs('graduation-mode-selector').within(() => {
+          cy.cs('select-median').click()
+        })
+        checkGraduationCharts('median')
+      })
+
+      it('Average', () => {
+        cy.cs('graduation-mode-selector').within(() => {
+          cy.cs('select-average').click()
+        })
+        checkGraduationCharts('average')
+      })
     })
 
     it('info boxes contain correct information', () => {
-      cy.cs('average-graduation-times-info-box-button').click()
+      cy.cs('average-graduation-times-info-box-button').hover()
       cy.cs('average-graduation-times-info-box-content').contains('Opiskelijoiden keskimääräiset valmistumisajat')
     })
   })

cypress/e2e/Users.js

@@ -61,7 +61,8 @@ describe('Users tests', () => {
       userButtonWorks('admin')
     })
 
-    describe('can mock as other users', { retries: 3 }, () => {
+    // TODO: This will be re-enabled when migrating to playwright
+    describe.skip('can mock as other users', { retries: 3 }, () => {
       beforeEach(() => {
         cy.cs('user-page-button-basic').click()
         cy.cs('mock-button').click()
@@ -71,6 +72,7 @@ describe('Users tests', () => {
         userButtonWorks('basic', true)
       })
 
+
       it("only the mocked user's programmes are visible", () => {
         cy.intercept('/api/populationstatistics/studyprogrammes').as('studyprogrammes')
         cy.visit('/populations')

cypress/support/commands.js

@@ -58,7 +58,7 @@ const onlyStudyGuidanceGroupsUser = {
   hypersonsisuid: 'hy-hlo-2222222',
 }
 
-const userHeaders = [
+export const userHeaders = [
   adminUserHeaders,
   basicUserHeaders,
   onlycoursestatisticsUserHeaders,
@@ -74,7 +74,7 @@ Cypress.Commands.add('init', { prevSubject: false }, (path, userId = 'basic') =>
   const headersToUse = userHeaders.find(({ uid }) => uid === userId)
   if (!headersToUse) throw Error(`${userId} is not valid user id!`)
 
-  cy.intercept({ hostname: 'localhost', url: '**', method: '*' }, req => {
+  cy.intercept('*', req => {
     req.headers = headersToUse
   })
 

docker-compose.ci.yml

@@ -1,15 +1,30 @@
+name: oodikone
+
 x-defaults: &defaults
   restart: unless-stopped
 
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
 services:
   backend:
     <<: *defaults
-    container_name: backend
+    container_name: oodikone-backend
+    image: oodikone-backend:staging
+    build:
+      context: services
+      dockerfile: backend/Dockerfile
     depends_on:
-      - kone-db
-      - redis
-      - sis-db
-      - user-db
+      kone-db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      user-db:
+        condition: service_healthy
     environment:
       CI: 'true'
       DB_URL_USER: postgres://postgres@user-db:5432/user-db
@@ -22,132 +37,152 @@ services:
       IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
       JAMI_URL: 'http://jami:3003/'
       CRYPT_KEY: 'e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'
-    build:
-      context: services
-      dockerfile: backend/Dockerfile
-    image: oodikone-backend:staging
+    ports:
+      - 8080:8080
     profiles:
       - oodikone
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8080/health'] # In CI/prod the routing happens at a different level, hence no /api/ needed
+      <<: *healthcheck-defaults
 
   frontend:
     <<: *defaults
-    container_name: frontend
+    container_name: oodikone-frontend
+    image: oodikone-frontend:staging
     build:
       context: services
       dockerfile: frontend/Dockerfile
-    image: oodikone-frontend:staging
+    depends_on:
+      backend:
+        condition: service_healthy
+    ports:
+      - 5000:5000
     profiles:
       - oodikone
 
   kone-db:
     <<: *defaults
-    container_name: kone-db
+    container_name: oodikone-kone-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
     environment:
       PGDATA: /data
       POSTGRES_DB: kone-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
+    ports:
+      - 4321:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
+
+  user-db:
+    <<: *defaults
+    container_name: oodikone-user-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    ports:
+      - 4320:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   redis:
     <<: *defaults
+    container_name: oodikone-redis
+    image: redis:7.2.6
     command: >
       sh -c 'redis-server --appendonly yes && redis-cli flushall'
-    image: redis:7.2.6
-    container_name: redis
     ports:
       - 6379:6379
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping', '|', 'grep', 'PONG']
+      <<: *healthcheck-defaults
 
   sis-db:
     <<: *defaults
-    container_name: sis-db
+    container_name: oodikone-sis-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
     environment:
       PGDATA: /data
       POSTGRES_DB: sis-db
       POSTGRES_HOST_AUTH_METHOD: trust
     shm_size: 2gb
+    ports:
+      - 12345:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
-  user-db:
+  sis-importer-db:
     <<: *defaults
-    container_name: user-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: user-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    container_name: oodikone-sis-importer-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
+    profiles:
+      - updater
+    ports:
+      - 12346:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   jami:
     <<: *defaults
+    container_name: oodikone-jami
     image: quay.io/toska/jami:production
+    depends_on:
+      jami-db:
+        condition: service_healthy
     environment:
       - npm_config_cache=/tmp
       - PORT=3003
-      - NODE_ENV=dev
+      - NODE_ENV=test # previously was development which borks CI
       - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
     ports:
       - 3003:3003
-    container_name: jami
 
   jami-db:
     <<: *defaults
+    container_name: oodikone-jami-db
     image: postgres:15.10
     environment:
       - PGDATA=/data
       - POSTGRES_PASSWORD=postgres
-    container_name: jami-db
     ports:
       - 5433:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
+  # This is still needed to make backend calls correctly route from localhost:5000/api to localhost:8080
   nginx:
+    container_name: oodikone-nginx
     image: nginx:latest
-    container_name: nginx
     depends_on:
-      - frontend
-      - backend
+      frontend:
+        condition: service_started
+      backend:
+        condition: service_healthy
     ports:
       - 1337:80
-    volumes:
-      - ./nginx.conf:/etc/nginx/nginx.conf
     profiles:
       - oodikone
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
 
-  # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
-  sis-importer-db:
-    <<: *defaults
-    container_name: sis-importer-db
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
-    profiles:
-      - updater
-
-  sis-updater-worker:
-    <<: *defaults
-    build:
-      context: updater/sis-updater-worker
-    container_name: sis-updater-worker
-    depends_on:
-      - redis
-      - sis-db
-      - sis-importer-db
-    environment:
-      CI: 'true'
-      DB_URL: postgres://postgres@sis-db:5432/sis-db
-      REDIS_HOST: redis
-      SIS_IMPORTER_DATABASE: sis-importer-db
-      SIS_IMPORTER_HOST: sis-importer-db
-      SIS_IMPORTER_PASSWORD: postgres
-      SIS_IMPORTER_USER: postgres
-      SIS_PASSWORD: postgres
-    profiles:
-      - updater
-
+  # Following services + sis-importer-db are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
   sis-updater-scheduler:
-    <<: *defaults
+    restart: no
+    container_name: oodikone-updater-scheduler
     build:
       context: updater/sis-updater-scheduler
-    container_name: sis-updater-scheduler
+      dockerfile: ./Dockerfile
     depends_on:
-      - redis
-      - sis-importer-db
+      redis:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
     environment:
       CI: 'true'
       REDIS_HOST: redis
@@ -162,3 +197,32 @@ services:
       - 8082:8082
     profiles:
       - updater
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8082/health']
+      <<: *healthcheck-defaults
+
+  sis-updater-worker:
+    <<: *defaults
+    # Disabled so that container can be scaled
+    # container_name: oodikone-updater-worker
+    build:
+      context: updater/sis-updater-worker
+      dockerfile: ./Dockerfile
+    depends_on:
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
+    environment:
+      CI: 'true'
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_HOST: redis
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_PASSWORD: postgres
+    profiles:
+      - updater

docker-compose.real.yml

@@ -1,3 +1,5 @@
+name: oodikone
+
 services:
   backend:
     environment:
@@ -20,3 +22,4 @@ services:
 
 volumes:
   redis-data-real:
+    name: oodikone-redis-data-real

docker-compose.test.yml

@@ -1,26 +1,27 @@
-services:
-  api-tests:
-    volumes:
-      - backend-tests-nodemod:/opt/app-root/backend/node_modules
-      - ./services/backend/:/opt/app-root/backend
-      - ./services/shared/:/opt/app-root/shared
-    environment:
-      CI: 'true'
-      NODE_ENV: 'test'
-      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
-      DB_URL_USER: postgres://postgres@user-db:5432/user-db
-      FRONT_URL: http://localhost:3000
-      REDIS: redis
-      SECRET_TOKEN: test
-      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
-      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      JAMI_URL: 'http://jami:3003/'
-      IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
-      CRYPT_KEY: 'e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'
-    build:
-      context: services/backend
-      dockerfile: dev.Dockerfile
-    command: npm run test
+name: oodikone
 
-volumes:
-  backend-tests-nodemod:
+x-defaults: &defaults
+  restart: unless-stopped
+
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
+services:
+  sis-db:
+    <<: *defaults
+    container_name: oodikone-sis-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
+    pull_policy: always
+    volumes: !reset [] # Remove previous volume definitions
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    shm_size: 2gb
+    ports:
+      - 12345:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults

docker-compose.yml

@@ -1,13 +1,20 @@
+name: oodikone
+
 x-defaults: &defaults
   restart: unless-stopped
 
 x-build_defaults: &build_defaults
   dockerfile: dev.Dockerfile
 
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
 services:
   adminer:
     <<: *defaults
-    container_name: adminer
+    container_name: oodikone-adminer
     environment:
       ADMINER_DESIGN: pepa-linha
     image: adminer:4.8.1
@@ -18,15 +25,19 @@ services:
 
   backend:
     <<: *defaults
+    container_name: oodikone-backend
     build:
-      <<: *build_defaults
-      context: services/backend
-    container_name: backend
+      dockerfile: backend/dev.Dockerfile
+      context: services
     depends_on:
-      - kone-db
-      - redis
-      - sis-db
-      - user-db
+      kone-db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      user-db:
+        condition: service_healthy
     env_file:
       - .env
     tty: true
@@ -47,14 +58,21 @@ services:
     volumes:
       - ./services/backend/:/opt/app-root/backend
       - ./services/shared/:/opt/app-root/shared
+      - shared-nodemod:/opt/app-root/shared/node_modules
       - backend-nodemod:/opt/app-root/backend/node_modules
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8080/api/health']
+      <<: *healthcheck-defaults
 
   frontend:
     <<: *defaults
+    container_name: oodikone-frontend
     build:
-      <<: *build_defaults
-      context: services/frontend
-    container_name: frontend
+      dockerfile: frontend/dev.Dockerfile
+      context: services
+    depends_on:
+      backend:
+        condition: service_healthy
     ports:
       - 3000:3000
     profiles:
@@ -62,40 +80,66 @@ services:
     volumes:
       - ./services/frontend/:/opt/app-root/frontend
       - ./services/shared/:/opt/app-root/shared
+      - shared-nodemod:/opt/app-root/shared/node_modules
       - frontend-nodemod:/opt/app-root/frontend/node_modules
 
   kone-db:
     <<: *defaults
-    container_name: kone-db
+    container_name: oodikone-kone-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
     profiles:
       - oodikone
+    ports:
+      - 4321:5432
     volumes:
       - kone-db-data:/data
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
+
+  user-db:
+    <<: *defaults
+    container_name: oodikone-user-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    profiles:
+      - oodikone
+    ports:
+      - 4320:5432
+    volumes:
+      - user-db-data:/data
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   redis:
     <<: *defaults
+    container_name: oodikone-redis
     command: 'redis-server --appendonly yes'
-    container_name: redis
     image: redis:7.2.6
     ports:
-      - '6379:6379'
+      - 6379:6379
     volumes:
       - redis-data:/data
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping', '|', 'grep', 'PONG']
+      <<: *healthcheck-defaults
 
   sis-db:
     <<: *defaults
-    container_name: sis-db
+    container_name: oodikone-sis-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
     volumes:
       - sis-db-data:/data
     shm_size: 2gb
     ports:
       - 12345:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-importer-db:
     <<: *defaults
-    container_name: sis-importer-db
+    container_name: oodikone-sis-importer-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
     profiles:
       - updater
@@ -103,41 +147,55 @@ services:
       - sis-importer-db-data:/data
     ports:
       - 12346:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
-  sis-updater-scheduler:
+  jami:
     <<: *defaults
-    build:
-      <<: *build_defaults
-      context: updater/sis-updater-scheduler
-    container_name: sis-updater-scheduler
+    container_name: oodikone-jami
+    image: quay.io/toska/jami:production
     depends_on:
-      - redis
-      - sis-importer-db
+      jami-db:
+        condition: service_healthy
+    env_file:
+      - .env
     environment:
-      REDIS_HOST: redis
-      SECRET_TOKEN: dev
-      SIS_IMPORTER_DATABASE: sis-importer-db
-      SIS_IMPORTER_HOST: sis-importer-db
-      SIS_IMPORTER_PASSWORD: postgres
-      SIS_IMPORTER_USER: postgres
+      - npm_config_cache=/tmp
+      - PORT=3003
+      - NODE_ENV=development
+      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/jami-db-real
     ports:
-      - 8082:8082
-    profiles:
-      - updater
+      - 3003:3003
+
+  jami-db:
+    <<: *defaults
+    container_name: oodikone-jami-db
+    image: postgres:15.10
+    environment:
+      - PGDATA=/data
+      - POSTGRES_PASSWORD=postgres
+    ports:
+      - 5433:5432
     volumes:
-      - sis-updater-scheduler-nodemod:/opt/app-root/src/node_modules
-      - ./updater/sis-updater-scheduler/:/opt/app-root/src
+      - jami-data:/data
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-updater-worker:
     <<: *defaults
+    container_name: oodikone-updater-worker
     build:
       <<: *build_defaults
       context: updater/sis-updater-worker
-    container_name: sis-updater-worker
     depends_on:
-      - redis
-      - sis-db
-      - sis-importer-db
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
     environment:
       DB_URL: postgres://postgres@sis-db:5432/sis-db
       REDIS_HOST: redis
@@ -149,52 +207,60 @@ services:
     profiles:
       - updater
     volumes:
-      - sis-updater-worker-nodemod:/opt/app-root/src/node_modules
       - ./updater/sis-updater-worker/:/opt/app-root/src
+      - sis-updater-worker-nodemod:/opt/app-root/src/node_modules
 
-  user-db:
+  sis-updater-scheduler:
     <<: *defaults
-    container_name: user-db
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    container_name: oodikone-updater-scheduler
+    build:
+      <<: *build_defaults
+      context: updater/sis-updater-scheduler
+    depends_on:
+      redis:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
+    environment:
+      REDIS_HOST: redis
+      SECRET_TOKEN: dev
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
+    ports:
+      - 8082:8082
     profiles:
-      - oodikone
+      - updater
     volumes:
-      - user-db-data:/data
-
-  jami:
-    <<: *defaults
-    image: quay.io/toska/jami:production
-    env_file:
-      - .env
-    environment:
-      - npm_config_cache=/tmp
-      - PORT=3003
-      - NODE_ENV=dev
-      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/jami-db-real
-    ports:
-      - 3003:3003
-    container_name: jami
-
-  jami-db:
-    <<: *defaults
-    image: postgres:15.10
-    environment:
-      - PGDATA=/data
-      - POSTGRES_PASSWORD=postgres
-    container_name: jami-db
-    ports:
-      - 5433:5432
-    volumes:
-      - jami-data:/data
+      - ./updater/sis-updater-scheduler/:/opt/app-root/src
+      - sis-updater-scheduler-nodemod:/opt/app-root/src/node_modules
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8082/health']
+      <<: *healthcheck-defaults
 
 volumes:
   backend-nodemod:
+    name: oodikone-backend-nodemod
   frontend-nodemod:
+    name: oodikone-frontend-nodemod
+  shared-nodemod:
+    name: oodikone-shared-nodemod
   kone-db-data:
+    name: oodikone-kone-db-data
   redis-data:
+    name: oodikone-redis-data
   sis-db-data:
+    name: oodikone-sis-db-data
   sis-importer-db-data:
+    name: oodikone-sis-importer-db-data
   sis-updater-scheduler-nodemod:
+    name: oodikone-updater-scheduler-nodemod
   sis-updater-worker-nodemod:
+    name: oodikone-updater-worker-nodemod
   user-db-data:
+    name: oodikone-user-db-data
   jami-data:
+    name: oodikone-jami-data

documentation/migrations.md

@@ -8,7 +8,7 @@ These instructions explain how to update the schemas for the following databases
 
 The steps to **add, remove, or modify columns** are mostly the same for each database, but the file paths and container names differ. Always **double-check** that you have the correct function name, column name, data type, and other relevant details in your migration file.
 
-> [!NOTE]  
+> [!NOTE]
 > If you need to change the `sis-importer-db` schema, see the instructions in the [sis-importer repo](https://github.com/UniversityOfHelsinkiCS/sis-importer/blob/master/README.md#tricks--tips).
 
 ## General guidelines
@@ -25,7 +25,7 @@ The steps to **add, remove, or modify columns** are mostly the same for each dat
 - Use **snake_case** in the actual database columns.
 - If a model has `underscored: true`, Sequelize automatically converts camelCase to snake_case in all queries. Otherwise, specify the column name explicitly.
 
-> [!IMPORTANT]  
+> [!IMPORTANT]
 > The column name **in the database** (snake_case) must be used in the migration file, not the camelCase name from your model.
 
 ### Verifying the migration locally
@@ -73,7 +73,7 @@ If you need to **revert a migration locally**, follow these steps:
 
 ## `sis-db`
 
-> [!NOTE]  
+> [!NOTE]
 > These instructions assume that needed data is already in the `sis-importer-db`. If not, you must add the data there first. See the [sis-importer repo](https://github.com/UniversityOfHelsinkiCS/sis-importer/blob/master/README.md#tricks--tips) for more details.
 
 1. **Update the updater model:**

eslint.config.mjs

@@ -1,267 +0,0 @@
-// @ts-check
-
-import eslint from '@eslint/js'
-import tseslint from 'typescript-eslint'
-
-import globals from 'globals'
-import eslintPluginCypress from 'eslint-plugin-cypress'
-import eslintPluginPrettierRecommended from 'eslint-plugin-prettier/recommended'
-import eslintPluginReact from 'eslint-plugin-react'
-import eslintPluginReactHooks from 'eslint-plugin-react-hooks'
-import eslintPluginImportX from 'eslint-plugin-import-x'
-import { createNodeResolver } from 'eslint-plugin-import-x'
-import { createTypeScriptImportResolver } from 'eslint-import-resolver-typescript'
-
-export default tseslint.config(
-  {
-    // Global ignores
-    ignores: [
-      'eslint.config.mjs',
-      '**/build/**',
-      '**/dist/**',
-      '**/node_modules/**',
-      '**/instrumented/**',
-      '**/coverage/**',
-      'nyc-config.cjs',
-      '.lintstagedrc.mjs',
-      'cypress.config.js',
-    ],
-  },
-
-  // Global options (excl. e2e)
-  {
-    ignores: ['cypress/**/*.js'],
-    extends: [
-      eslint.configs.recommended,
-      eslintPluginImportX.flatConfigs.recommended,
-      eslintPluginImportX.flatConfigs.typescript,
-      tseslint.configs.recommendedTypeChecked,
-      tseslint.configs.stylisticTypeChecked,
-    ],
-    languageOptions: {
-      ecmaVersion: 2021,
-      sourceType: 'module',
-      parser: tseslint.parser,
-      parserOptions: {
-        projectService: {
-          allowDefaultProject: ['*.{js,cjs}'],
-        },
-      },
-      globals: {
-        ...globals.browser,
-        ...globals.node,
-      },
-    },
-    plugins: {
-      'import-x': eslintPluginImportX,
-    },
-    linterOptions: {
-      reportUnusedDisableDirectives: 'error',
-    },
-    rules: {
-      camelcase: 'off', // TODO: Enable. old db/backend logic is still using snake_case every now and then and needs a big refactor
-      'class-methods-use-this': 'off', // Disabled in favor of @typescript-eslint
-      'consistent-return': 'off', // Disabled in favor of @typescript-eslint
-      'id-denylist': ['error', 'err'],
-      'no-alert': 'error',
-      'no-await-in-loop': 'error',
-      'no-console': 'error',
-      'no-implied-eval': 'off', // Disabled in favor of @typescript-eslint
-      'no-param-reassign': 'error',
-      'no-promise-executor-return': 'error',
-      'no-return-assign': 'error',
-      'no-unused-vars': 'off', // Disabled in favor of @typescript-eslint
-      'object-shorthand': ['error', 'always'],
-      'prefer-const': 'error',
-      'prefer-destructuring': 'off', // Disabled in favor of @typescript-eslint
-      'dot-notation': 'off', // Disabled in favor of @typescript-eslint
-      'no-restricted-imports': [
-        'error',
-        {
-          patterns: [
-            {
-              regex: '^@mui/[^/]+$',
-              message:
-                'Import only the needed module. This has no effect in prod due to tree-shake, but speeds up development builds.',
-            },
-            {
-              regex: '^src/.*',
-              message: 'Root imports will not resolve in production build (instead silently crash).',
-            },
-          ],
-        },
-      ],
-      quotes: ['error', 'single', { avoidEscape: true, allowTemplateLiterals: false }],
-
-      'import-x/no-commonjs': 'error',
-      'import-x/no-default-export': 'error',
-      'import-x/no-extraneous-dependencies': ['error', { devDependencies: false }],
-      'import-x/no-unused-modules': [
-        'error',
-        {
-          missingExports: true,
-          unusedExports: true,
-          src: ['{services,updater}/**/*.{js,jsx,ts,tsx}'],
-        },
-      ],
-      'import-x/order': [
-        'error',
-        {
-          alphabetize: { order: 'asc', caseInsensitive: true },
-          groups: [['builtin', 'external'], ['internal'], ['parent'], ['sibling', 'index']],
-        },
-      ],
-
-      // TypeScript
-      '@typescript-eslint/no-unused-vars': [
-        'error',
-        { argsIgnorePattern: '^_', caughtErrors: 'none', ignoreRestSiblings: true, varsIgnorePattern: '^_' },
-      ],
-      '@typescript-eslint/array-type': 'off',
-      '@typescript-eslint/class-methods-use-this': 'error',
-      // '@typescript-eslint/consistent-indexed-object-style': 'off', // TODO: keep as error/record (default)
-      '@typescript-eslint/consistent-return': 'error', // NOTE: This is weaker than tsconfig.json noImplicitReturns, and should be replaced once ts migration complete
-      '@typescript-eslint/consistent-type-definitions': 'off',
-      '@typescript-eslint/dot-notation': 'error',
-      '@typescript-eslint/prefer-destructuring': ['error', { VariableDeclarator: { object: true } }],
-      '@typescript-eslint/no-empty-object-type': ['error', { allowInterfaces: 'with-single-extends' }],
-      '@typescript-eslint/no-explicit-any': 'off', // TODO: enable when feeling brave
-      '@typescript-eslint/no-implied-eval': 'error',
-      '@typescript-eslint/no-shadow': 'off', // TODO: enable?
-      '@typescript-eslint/no-unsafe-argument': 'off', // TODO: enable
-      '@typescript-eslint/no-unsafe-assignment': 'off', // TODO: enable
-      '@typescript-eslint/no-unsafe-call': 'off', // TODO: enable
-      '@typescript-eslint/no-unsafe-member-access': 'off', // TODO: enable
-      '@typescript-eslint/no-unsafe-return': 'off', // TODO: enable
-      '@typescript-eslint/restrict-template-expressions': 'off',
-    },
-    settings: {
-      'import-x/internal-regex': '^@oodikone/shared/',
-      'import-x/resolver-next': [
-        createTypeScriptImportResolver({
-          project: [
-            'services/frontend/tsconfig.json',
-            'services/frontend/tsconfig.node.json',
-            'services/backend/tsconfig.json',
-            'services/shared/tsconfig.json',
-          ],
-        }),
-        createNodeResolver({
-          extensions: ['.js', '.jsx', 'cjs', 'mjs', '.ts', '.tsx'],
-        }),
-      ],
-    },
-  },
-
-  {
-    files: ['services/backend/src/routes/*.{ts,js}'],
-    rules: {
-      '@typescript-eslint/no-misused-promises': 'off', // Errors come mainly from Express route handlers that are handled correctly by express-async-errors
-      '@typescript-eslint/consistent-return': 'off',
-    },
-  },
-
-  // Disable type-aware linting for javascript files
-  {
-    files: ['{updater,services}/**/*.{js,cjs}'],
-    extends: [tseslint.configs.disableTypeChecked],
-  },
-
-  // Frontend camelCase and react
-  {
-    files: ['services/frontend/**/*.{js,jsx,mjs,cjs,ts,tsx}'],
-    plugins: {
-      react: eslintPluginReact,
-    },
-    extends: [
-      eslintPluginReact.configs.flat.recommended,
-      eslintPluginReact.configs.flat['jsx-runtime'],
-      eslintPluginReactHooks.configs['recommended-latest'],
-    ],
-    languageOptions: {
-      parserOptions: {
-        ecmaFeatures: {
-          jsx: true,
-        },
-      },
-    },
-    rules: {
-      camelcase: 'warn',
-      'react/prop-types': 'off', // Typescript but worse
-      'react/no-unescaped-entities': 'off',
-      'react/function-component-definition': [
-        'error',
-        { namedComponents: 'arrow-function', unnamedComponents: 'arrow-function' },
-      ],
-      'react/no-array-index-key': 'error',
-      'react/no-unstable-nested-components': 'warn', // TODO: turn into error, reduces re-renders
-      'react/no-this-in-sfc': 'error',
-      'react/prefer-stateless-function': 'error',
-      'react/jsx-boolean-value': ['error', 'never'],
-      'react/jsx-filename-extension': ['error', { allow: 'as-needed', extensions: ['.jsx', '.tsx'] }],
-      'react/jsx-no-leaked-render': 'warn', // TODO: enable, tho not that useful after full ts migration
-      'react/jsx-no-useless-fragment': 'error',
-      'react/jsx-props-no-spreading': 'warn', // TODO: enable, improves readability
-      'react/jsx-sort-props': ['error', { reservedFirst: false }],
-      'react/jsx-tag-spacing': 'warn',
-    },
-    settings: {
-      react: {
-        version: 'detect',
-      },
-    },
-  },
-
-  // Cypress should only use rules provided by the plugin below
-  {
-    files: ['cypress/**/*.js'],
-    extends: [eslintPluginCypress.configs.recommended],
-  },
-
-  // Override for backend api tests
-  {
-    files: ['services/backend/tests/**/*.ts'],
-    rules: {
-      'import-x/no-extraneous-dependencies': 'off',
-      'no-console': 'off',
-    },
-  },
-
-  // Overrides for CommonJS
-  {
-    files: [
-      'services/backend/**/*.js',
-      'updater/sis-updater-worker/src/db/migrations/*.cjs',
-      'updater/sis-updater-scheduler/**/*.js',
-    ],
-    languageOptions: {
-      sourceType: 'commonjs',
-    },
-    rules: {
-      'import-x/no-commonjs': 'off',
-      '@typescript-eslint/no-require-imports': 'off',
-    },
-  },
-
-  // Migration files
-  {
-    files: ['updater/sis-updater-worker/src/db/migrations/*.cjs', 'services/backend/src/database/**/*.js'],
-    rules: {
-      'import-x/no-unused-modules': 'off',
-      '@typescript-eslint/no-empty-function': 'off',
-      '@typescript-eslint/require-await': 'off',
-    },
-  },
-
-  // TODO: See if these can be disabled and or fix
-  {
-    files: ['services/backend/**/*.{js,ts}', 'updater/**/*.js'],
-    rules: {
-      'import-x/no-default-export': 'off',
-      'import-x/no-unused-modules': 'off',
-      'no-await-in-loop': 'off',
-      'no-promise-executor-return': 'off',
-    },
-  },
-  eslintPluginPrettierRecommended // This must be the last plugin so it can override other configs
-)

package.json

@@ -7,28 +7,32 @@
   "scripts": {
     "cypress": "cypress",
     "cypress:run": "cypress run --browser firefox",
-    "eslint": "eslint",
-    "lint:all": "eslint services/ updater/",
+    "lint": "oxlint",
+    "lint:fix": "oxlint --fix",
+    "lint:quiet": "oxlint --quiet",
+    "fmt": "oxfmt",
+    "fmt:check": "oxfmt --check",
     "stylelint": "stylelint",
     "prettier": "prettier",
     "cli": "./cli.sh",
-    "oodikone": "./run.sh oodikone anon up --detach",
+    "logs": "docker compose --file docker-compose.yml logs backend frontend sis-updater-worker sis-updater-scheduler jami -f",
+    "logs:all": "docker compose --file docker-compose.yml --profile oodikone --profile updater logs -f",
+    "logs:ci": "docker compose --file docker-compose.ci.yml logs backend frontend sis-updater-worker sis-updater-scheduler jami",
+    "oodikone": "npm run docker:down && ./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
-    "oodikone:ci:setup": "docker compose --file docker-compose.ci.yml --profile tests up --detach",
-    "oodikone:ci:api": "docker compose --file docker-compose.test.yml up --exit-code-from api-tests",
+    "oodikone:test": "npm run docker:down && docker compose --profile oodikone up --detach && docker compose --file docker-compose.test.yml up --detach",
+    "oodikone:ci": "npm run docker:down && docker compose --file docker-compose.ci.yml --profile oodikone up",
     "updater": "./run.sh updater anon up --detach",
     "updater:real": "./run.sh updater real up --detach",
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
     "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
-    "testupdater": "docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start",
-    "flushredis": "docker exec -it redis redis-cli flushall",
+    "testupdater": "docker compose --file docker-compose.ci.yml up --detach --no-recreate --wait sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker compose --file docker-compose.ci.yml up sis-updater-scheduler --exit-code-from sis-updater-scheduler",
+    "flushredis": "docker exec -it oodikone-redis redis-cli flushall",
     "prepare": "husky",
     "install:local": "npm i && npm i --prefix services/shared && npm i --prefix services/frontend && npm i --prefix services/backend && npm i --prefix updater/sis-updater-scheduler && npm i --prefix updater/sis-updater-worker",
-    "install:docker": "docker exec frontend npm i && docker exec backend npm i && docker exec sis-updater-scheduler npm i && docker exec sis-updater-worker npm i",
-    "install:both": "npm run install:local && npm run install:docker",
     "ci:local": "npm ci && npm ci --prefix services/shared && npm ci --prefix services/frontend && npm ci --prefix services/backend && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker",
-    "ci:docker": "docker exec frontend npm ci && docker exec backend npm ci && docker exec sis-updater-scheduler npm ci && docker exec sis-updater-worker npm ci",
+    "ci:docker": "docker exec oodikone-frontend sh -lc \"npm ci --prefix /opt/app-root/shared && npm ci\" && docker exec oodikone-backend npm ci && docker exec oodikone-updater-scheduler npm ci && docker exec oodikone-updater-worker npm ci",
     "ci:both": "npm run ci:local && npm run ci:docker"
   },
   "repository": {
@@ -39,24 +43,24 @@
   "bugs": {
     "url": "https://github.com/UniversityOfHelsinkiCS/oodikone/issues"
   },
+  "allowScripts": {
+    "cypress": true
+  },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@eslint/js": "^9.28.0",
-    "cypress": "^15.13.0",
-    "eslint": "^9.28.0",
-    "eslint-config-prettier": "^10.1.5",
-    "eslint-import-resolver-typescript": "^4.4.3",
+    "cypress": "15.18.1",
     "eslint-plugin-cypress": "^6.2.3",
     "eslint-plugin-import-x": "^4.16.1",
-    "eslint-plugin-prettier": "^5.4.1",
     "eslint-plugin-react": "^7.37.5",
     "eslint-plugin-react-hooks": "^5.2.0",
     "globals": "^17.4.0",
     "husky": "^9.1.7",
     "lint-staged": "^16.1.0",
+    "oxfmt": "0.58.0",
+    "oxlint": "1.73.0",
+    "oxlint-tsgolint": "0.25.0",
     "prettier": "3.5.3",
     "stylelint": "^17.6.0",
-    "stylelint-config-standard": "^40.0.0",
-    "typescript-eslint": "^8.34.0"
+    "stylelint-config-standard": "^40.0.0"
   }
 }

run.sh

@@ -23,7 +23,7 @@ Usage: $(basename "${BASH_SOURCE[0]}") option [version] command --flag
 Parameters:
 * Option: oodikone/updater/both
 * Version: anon/real. Not necessary when running down/logs/pull.
-* Command: will be passed to docker-compose.
+* Command: will be passed to docker compose.
 EOF
 }
 

scripts/functions_for_setup.sh

@@ -12,7 +12,7 @@ DUMP_DIR="$PROJECT_ROOT/.databasedumps"
 S3_CONFIG_FILE=~/.s3cfg
 S3_BUCKET="s3://psyduck"
 
-## Following the naming convention in docker-compose, these are names for database
+## Following the naming convention in docker compose, these are names for database
 ## services. The real data databases inside service will have names suffixed by "-real".
 KONE_DB_NAME="kone-db"
 SIS_DB_NAME="sis-db"
@@ -112,11 +112,11 @@ reset_databases() {
   infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
 
   npm run docker:down
-  docker-compose up -d "${databases[@]}"
+  docker compose up -d "${databases[@]}"
 
   for database in "${databases[@]}"; do
     local database_dump="$database_dump_dir/$database.sql.gz"
-    local database_container="$database"
+    local database_container="oodikone-$database"
     local database_name="$database-real"
 
     infomsg "Attempting to create database $database_name from dump $database_dump inside container $database_container"
@@ -138,7 +138,7 @@ reset_databases() {
   successmsg "Database setup finished"
 }
 
-reset_all_real_data() {
+reset_all_data() {
   infomsg "Downloading real data dumps"
   for i in ${!DATABASES[*]}; do
     local database="${DATABASES[$i]}"
@@ -209,12 +209,54 @@ reset_single_database() {
   fi
 }
 
+nuke_node_modules() {
+  folders_to_set_up=(
+    "$PROJECT_ROOT"
+    "$PROJECT_ROOT/services/frontend"
+    "$PROJECT_ROOT/services/backend"
+    "$PROJECT_ROOT/services/shared"
+    "$PROJECT_ROOT/updater/sis-updater-scheduler"
+    "$PROJECT_ROOT/updater/sis-updater-worker"
+  )
+
+  named_volumes=(
+    "oodikone-shared-nodemod"
+    "oodikone-frontend-nodemod"
+    "oodikone-backend-nodemod"
+    "oodikone-updater-worker-nodemod"
+    "oodikone-updater-scheduler-nodemod"
+  )
+
+  infomsg "Running docker compose down"
+  "$PROJECT_ROOT"/run.sh both down --remove-orphans || infomsg "brr"
+  for folder in "${folders_to_set_up[@]}"; do
+    cd "$folder" || die "Couldn't change directory to folder $folder"
+    rm -rf node_modules || die "Couldn't remove node_modules in folder $folder"
+  done
+
+  for volume in "${named_volumes[@]}"; do
+    docker volume rm "$volume" || infomsg "Couldn't delete volume $volume (already gone?)"
+  done
+
+  infomsg "Installing npm packages locally to root and each subfolder to enable linting and formatting (and LSP)"
+
+  for folder in "${folders_to_set_up[@]}"; do
+    cd "$folder" || die "Couldn't change directory to folder $folder"
+    ([[ -d node_modules ]] && die "$folder already has node_modules, exiting!") || npm ci
+  done
+
+  cd "$PROJECT_ROOT" || die "Couldn't change directory to project root"
+
+  successmsg "Node_module nuke performed"
+}
+
 set_up_oodikone_from_scratch() {
   draw_mopo
   folders_to_set_up=(
     "$PROJECT_ROOT"
     "$PROJECT_ROOT/services/frontend"
     "$PROJECT_ROOT/services/backend"
+    "$PROJECT_ROOT/services/shared"
     "$PROJECT_ROOT/updater/sis-updater-scheduler"
     "$PROJECT_ROOT/updater/sis-updater-worker"
   )
@@ -223,7 +265,7 @@ set_up_oodikone_from_scratch() {
   "$PROJECT_ROOT"/run.sh both down --remove-orphans --volumes --rmi all || infomsg "Cleaning errored, but will continue"
   for folder in "${folders_to_set_up[@]}"; do
     cd "$folder" || die "Couldn't change directory to folder $folder"
-    rm -rf node_modules|| die "Couldn't remove node_modules in folder $folder"
+    rm -rf node_modules || die "Couldn't remove node_modules in folder $folder"
   done
 
   infomsg "Installing npm packages locally to root and each subfolder to enable linting and formatting"

services/.dockerignore

@@ -1,7 +1,116 @@
-*/.dockerignore
-*/.eslintrc.json
-*/.gitignore
-*/*Dockerfile
-*/build
-*/dist
-*/node_modules
+# disallow directory named "private", it can be used to store things that should not end up in version control or images
+# NOTE: needs to be at the root of the project
+private/
+gitignored/
+
+# disallow version control, package managment, IDEs, tools etc
+**/node_modules
+**/.venv
+**/.git
+**/.github
+**/.vscode
+**/.idea
+**/.zed
+**/coverage/
+**/.eslintcache
+**/.DS_Store
+
+# do not include any data by default (except json)
+**/*.csv
+**/*.tsv
+
+**/*.db
+**/*.sqlite
+**/*.sql
+**/*.psql
+**/*.sqlz
+**/*.sqz
+**/*.sql.gz
+**/*.bson
+**/*.jsonl
+
+**/*.bin
+**/*.dat
+**/*.data
+
+**/*.swp
+**/*.tmp
+**/.~lock*
+**/*.pid
+**/*.seed
+**/*.lock
+
+**/*.gz
+**/*.tgz
+**/*.xz
+**/*.zip
+**/*.gzip
+**/*.tar
+**/*.xz
+**/*.7z
+**/*.rar
+
+**/*.docx
+**/*.doc
+**/*.xlsx
+**/*.xls
+**/*.pptx
+**/*.ppt
+**/*.odt
+**/*.odp
+**/*.ods
+**/*.pdf
+
+
+# disallow logs
+**/*.log
+**/*.log*
+**/logs/
+**/log/
+**/*.cpuprofile
+**/*.heapprofile
+**/*.heapsnapshot
+
+# disallow configuration
+**/*.yml
+**/*.yaml
+**/*.toml
+**/*.conf
+**/*.ini
+**/toska_login/
+**/toska_login/*
+**/cypress.env.json
+cypress/
+
+
+# disallow secrects
+**/.env
+**/.env*
+**/*.env.*
+**/.secrects
+**/*.s3cfg
+
+# disallow backups
+**/backups/*
+**/backups
+**/backup/*
+**/backup
+**/dump/
+**/dump
+**/dumps/
+**/dumps
+**/databasedumps/
+**/privateData
+**/temp
+**/minio-data/*
+
+
+# additional inclusions here
+
+# exclusions here
+**/.dockerignore
+**/.gitignore
+**/.eslintrc.json
+**/*Dockerfile
+**/build
+**/dist

services/backend/.env.test

@@ -0,0 +1,12 @@
+CI='true'
+NODE_ENV='test'
+DB_URL_KONE=postgres://postgres@localhost:4321/kone-db
+DB_URL_USER=postgres://postgres@localhost:4320/user-db
+FRONT_URL=http://localhost:3000
+REDIS=localhost
+SECRET_TOKEN=test
+SIS_DB_URL=postgres://postgres@localhost:12345/sis-db
+SIS_UPDATER_URL=http://localhost:8082
+JAMI_URL='http://localhost:3003/'
+IMPORTER_DB_API_URL='https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
+CRYPT_KEY='e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'

services/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi9/nodejs-24-minimal AS build
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal AS build
 
 ENV TZ="Europe/Helsinki"
 
@@ -29,7 +29,7 @@ RUN npm ci --include=dev --prefix shared && npm ci --include=dev --prefix backen
 RUN npm run build --prefix shared && npm run build --prefix backend
 
 
-FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 
@@ -50,10 +50,13 @@ ENV SENTRY_RELEASE=$SENTRY_RELEASE
 
 WORKDIR /opt/app-root/shared
 COPY --from=build /opt/app-root/shared/dist .
+COPY --from=build /opt/app-root/shared/.npmrc .
+COPY --from=build /opt/app-root/shared/package*.json ./
+RUN npm ci --omit=dev
 
 WORKDIR /opt/app-root/backend
 COPY --from=build /opt/app-root/backend/.npmrc .
-COPY --from=build /opt/app-root/backend/package*.json .
+COPY --from=build /opt/app-root/backend/package*.json ./
 COPY --from=build /opt/app-root/backend/dist ./dist
 
 RUN npm ci --omit=dev

services/backend/dev.Dockerfile

@@ -1,15 +1,20 @@
-FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 
 ENV NODE_ENV=development
 ENV NODE_OPTIONS=--max-old-space-size=4096
 
-WORKDIR /opt/app-root/backend
-COPY --chown=1001:1001 . .
+WORKDIR /opt/app-root/shared
 
-USER 1001
+COPY ./shared .
+RUN npm ci
+
+WORKDIR /opt/app-root/backend
+
+COPY ./backend .
 RUN npm ci
 
 EXPOSE 8080
+
 CMD ["node_modules/.bin/nodemon", "--exec", "node --import tsx index.ts", "--ext", "ts,js,json", "--watch", "src", "--watch", "index.ts", "--legacy-watch"]

services/backend/package.json

@@ -3,13 +3,17 @@
   "private": true,
   "scripts": {
     "build": "tsc",
-    "test": "node --import tsx --test --test-reporter spec --test-force-exit"
+    "test:all": "vitest",
+    "test:api": "vitest ./tests/api",
+    "test:services": "vitest ./tests/services",
+    "test:unit": "vitest ./tests/unit",
+    "test:silent": "SILENT_TESTING=true vitest"
   },
   "dependencies": {
     "@oodikone/shared": "file:../shared",
     "@sentry/node": "^10.43.0",
     "@sentry/profiling-node": "^10.43.0",
-    "axios": "^1.9.0",
+    "axios": "^1.18.1",
     "bullmq": "^5.53.2",
     "compression": "^1.8.0",
     "cors": "^2.8.5",
@@ -18,17 +22,16 @@
     "express": "^5.2.1",
     "lodash-es": "^4.18.1",
     "lru-cache": "^11.1.0",
-    "morgan": "^1.10.0",
+    "morgan": "^1.11.0",
     "papaparse": "^5.5.3",
     "pg": "^8.20.0",
     "redis": "^4.7.1",
     "sequelize": "^6.37.8",
     "sequelize-typescript": "^2.1.6",
-    "umzug": "^3.8.2",
+    "umzug": "^3.8.3",
     "unfuck-utf8-headers-middleware": "^1.0.1",
-    "winston": "^3.17.0",
-    "winston-gelf-transporter": "^1.0.2",
-    "winston-loki": "^6.1.3"
+    "winston": "^3.19.0",
+    "winston-loki": "^6.1.5"
   },
   "devDependencies": {
     "@types/compression": "^1.8.1",
@@ -42,11 +45,7 @@
     "nodemon": "^3.1.11",
     "supertest": "^7.2.2",
     "tsx": "^4.19.4",
-    "typescript": "^5.9.2"
-  },
-  "overrides": {
-    "winston-gelf-transporter": {
-      "gelf-pro": "^1.4.0"
-    }
+    "typescript": "^7.0.2",
+    "vitest": "4.1.9"
   }
 }

services/backend/src/config/index.ts

@@ -6,6 +6,8 @@ export const isProduction = !isStaging && process.env.NODE_ENV === 'production'
 
 export const runningInCI = process.env.CI === 'true'
 
+export const silentTesting = process.env.SILENT_TESTING === 'true'
+
 // IAM group
 export const requiredGroup = ['grp-oodikone-users', 'grp-oodikone-basic-users']
 
@@ -36,9 +38,6 @@ export const { DB_URL_KONE, DB_URL_USER, SECRET_TOKEN, SIS_DB_URL, SIS_UPDATER_U
 
 export const rootOrgId = process.env.ROOT_ORG_ID ?? 'hy-university-root-id'
 
-export const languageCenterViewEnabled =
-  process.env.LANGUAGE_CENTER_VIEW_ENABLED == null ? true : process.env.LANGUAGE_CENTER_VIEW_ENABLED === 'true'
-
 export const concurrentWorkers = parseInt(process.env.CONCURRENT_WORKERS!, 10) || 2
 
 export const DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS!, 10) || 5 // sequelize's default

services/backend/src/database/connection.ts

@@ -24,6 +24,7 @@ import {
   TeacherModel,
 } from '../models'
 import {
+  BannerModel,
   CustomPopulationSearchModel,
   ExcludedCourseModel,
   OpenUniPopulationSearchModel,
@@ -107,6 +108,7 @@ const sequelizeKone = new Sequelize(conf.DB_URL_KONE!, {
   logging: false,
   password: conf.KONE_PASSWORD,
   models: [
+    BannerModel,
     CustomPopulationSearchModel,
     ExcludedCourseModel,
     OpenUniPopulationSearchModel,

services/backend/src/database/migrations_kone/20260709_01_create_table_banners.js

@@ -0,0 +1,46 @@
+const { INTEGER, STRING, DATE } = require('sequelize')
+
+module.exports = {
+  up: async ({ context: queryInterface }) => {
+    await queryInterface.createTable('banners', {
+      id: {
+        type: INTEGER,
+        primaryKey: true,
+        autoIncrement: true,
+      },
+      text: {
+        type: STRING,
+        allowNull: false,
+      },
+      color: {
+        type: STRING,
+        allowNull: false,
+      },
+      lightness: {
+        type: STRING,
+        allowNull: false,
+      },
+      start_date: {
+        type: DATE,
+        allowNull: false,
+      },
+      end_date: {
+        type: DATE,
+        allowNull: false,
+      },
+      last_modified_by: {
+        type: STRING,
+        allowNull: false,
+      },
+      created_at: {
+        type: DATE,
+      },
+      updated_at: {
+        type: DATE,
+      },
+    })
+  },
+  down: async ({ context: queryInterface }) => {
+    await queryInterface.dropTable('banners')
+  },
+}

services/backend/src/events.ts

@@ -1,6 +1,6 @@
 import { CronJob } from 'cron'
 
-import { isProduction, runningInCI, languageCenterViewEnabled, CRON_SCHEDULE } from './config'
+import { isProduction, runningInCI, CRON_SCHEDULE } from './config'
 import { getDegreeProgrammesOfFaculty } from './services/faculty/faculty'
 import { getFaculties } from './services/faculty/facultyHelpers'
 import { updateFacultyOverview, updateFacultyProgressOverview } from './services/faculty/facultyUpdates'
@@ -13,7 +13,7 @@ import { updateBasicView, updateStudyTrackView } from './services/studyProgramme
 import { findAndSaveTeachers } from './services/teachers/top'
 import { deleteOutdatedUsers } from './services/userService'
 import logger from './util/logger'
-import { jobMaker, addToFlow } from './worker/queue'
+import { jobQueue } from './worker/queue'
 
 const schedule = (cronTime: string, onTick: () => void) => {
   const onComplete = null
@@ -41,7 +41,7 @@ export const refreshFaculties = async () => {
   logger.info('Adding jobs to refresh all faculties')
   const faculties = await getFaculties()
   for (const faculty of faculties) {
-    jobMaker.faculty(faculty.code)
+    jobQueue.refreshFaculty(faculty.code)
   }
 }
 
@@ -53,12 +53,13 @@ export const refreshLanguageCenterData = async () => {
 }
 
 const refreshProgrammesAndFaculties = async () => {
+  logger.info('Refreshing all degree programmes and faculties')
   const facultyCodes = (await getFaculties()).map(faculty => faculty.code)
   for (const faculty of facultyCodes) {
     const programmeCodes = (await getDegreeProgrammesOfFaculty(faculty, true))
       .map(programme => programme.code)
       .filter(code => isRelevantProgramme(code))
-    await addToFlow(faculty, programmeCodes)
+    await jobQueue.addToFlow(faculty, programmeCodes)
   }
 }
 
@@ -82,7 +83,7 @@ export const refreshProgrammes = async () => {
   }
   for (const code of programmeCodes) {
     // If combined programme is given, this updates only the bachelor programme
-    jobMaker.programme(code)
+    jobQueue.refreshProgramme(code)
   }
 }
 
@@ -96,8 +97,9 @@ const dailyJobs = async () => {
   try {
     await refreshTeacherLeaderboard()
     await refreshProgrammesAndFaculties()
-    if (languageCenterViewEnabled) jobMaker.languagecenter()
-    jobMaker.closeToGraduation()
+
+    jobQueue.refreshLanguagecenter()
+    jobQueue.refreshCloseToGraduation()
   } catch (error) {
     logger.error('Daily jobs failed', error)
   }
@@ -118,11 +120,11 @@ export const startCron = () => {
     })
     schedule('0 19 * * 1', () => {
       logger.info('Updating students whose studyplans have not been updated recently')
-      jobMaker.studyplansUpdate(4)
+      jobQueue.refreshStudyplans(4)
     })
     schedule('0 10 * * 2', () => {
       logger.info('Updating students whose studyplans have not been updated recently')
-      jobMaker.studyplansUpdate(5)
+      jobQueue.refreshStudyplans(5)
     })
   }
 }

services/backend/src/middleware/accessLogger.ts

@@ -38,6 +38,11 @@ const accessLogger = morgan((tokens, req: Request, res: Response): undefined =>
     meta['req-route'] = req.route.path
   }
 
+  // Ignore admin route that is frequently pinged for updates.
+  if (/.*\/updater\/jobs$/.test(meta.url ?? '')) {
+    return
+  }
+
   const message = [
     `${user.mockedBy ? '(mocking) ' : ''}${user.name}`,
     `${(tokens['response-time'](req, res) ?? '0').split('.')[0]} ms`.padEnd(8, ' '),

services/backend/src/middleware/currentUser.ts

@@ -2,7 +2,7 @@ import * as Sentry from '@sentry/node'
 import { NextFunction, Request, Response } from 'express'
 import { intersection } from 'lodash-es'
 
-import { requiredGroup } from '../config'
+import { requiredGroup, baseUrl } from '../config'
 import { getMockedUser, getOrganizationAccess, getUser } from '../services/userService'
 import { ApplicationError } from '../util/customErrors'
 import logger from '../util/logger'
@@ -36,6 +36,11 @@ export default async (req: Request, _: Response, next: NextFunction) => {
     uid: username,
   } = req.headers as Headers
 
+  if (req.path === `${baseUrl}/health`) {
+    next()
+    return
+  }
+
   if (!username || !sessionId)
     throw new ApplicationError(
       `${username ?? 'Anonymous user'} requested ${req.url} without authorization headers.`,

services/backend/src/models/kone/banner.ts

@@ -0,0 +1,33 @@
+import { Column, DataType, Model, PrimaryKey, Table, AutoIncrement } from 'sequelize-typescript'
+
+import type { Banner } from '@oodikone/shared/models/kone'
+
+@Table({
+  underscored: true,
+  modelName: 'banner',
+  tableName: 'banners',
+})
+export class BannerModel extends Model implements Banner {
+  @PrimaryKey
+  @AutoIncrement
+  @Column(DataType.INTEGER)
+  declare id: number
+
+  @Column(DataType.DATE)
+  declare startDate: Date
+
+  @Column(DataType.DATE)
+  declare endDate: Date
+
+  @Column(DataType.STRING)
+  declare text: string
+
+  @Column(DataType.STRING)
+  declare color: string
+
+  @Column(DataType.STRING)
+  declare lightness: 'light' | 'main' | 'dark'
+
+  @Column(DataType.STRING)
+  declare lastModifiedBy: string
+}

services/backend/src/models/kone/index.ts

@@ -1,3 +1,4 @@
+export { BannerModel } from './banner'
 export { CustomPopulationSearchModel } from './customPopulationSearch'
 export { ExcludedCourseModel } from './excludedCourse'
 export { OpenUniPopulationSearchModel } from './openUniPopulationSearch'

services/backend/src/routes.ts

@@ -3,12 +3,13 @@ import compression from 'compression'
 import cors from 'cors'
 import { Express, json as jsonExpress } from 'express'
 
-import { frontUrl, languageCenterViewEnabled } from './config'
+import { frontUrl } from './config'
 import accessLogger from './middleware/accessLogger'
 import * as auth from './middleware/auth'
 import currentUserMiddleware from './middleware/currentUser'
 import errorHandler from './middleware/errorHandler'
 import headersMiddleware from './middleware/headers'
+import banners from './routes/banners'
 import changelog from './routes/changelog'
 import closeToGraduation from './routes/closeToGraduation'
 import completedCoursesSearch from './routes/completedCoursesSearch'
@@ -18,6 +19,7 @@ import curriculumPeriods from './routes/curriculumPeriods'
 import customPopulationSearch from './routes/customPopulationSearch'
 import faculties from './routes/faculties'
 import feedback from './routes/feedback'
+import health from './routes/health'
 import languageCenterData from './routes/languageCenterData'
 import login from './routes/login'
 import population from './routes/population'
@@ -44,6 +46,7 @@ const routes = (app: Express, url: string) => {
   app.use(currentUserMiddleware)
   app.use(accessLogger)
   app.use(`${url}/login`, login)
+  app.use(`${url}/banners`, banners)
   app.use(url, courses)
   app.use(`${url}/students`, students)
   app.use(url, population)
@@ -56,9 +59,7 @@ const routes = (app: Express, url: string) => {
   app.use(`${url}/programme-criteria`, studyProgrammeCriteria)
   app.use(`${url}/changelog`, changelog)
   app.use(`${url}/completedcoursessearch`, completedCoursesSearch)
-  if (languageCenterViewEnabled) {
-    app.use(`${url}/languagecenterdata`, languageCenterData)
-  }
+  app.use(`${url}/languagecenterdata`, languageCenterData)
   app.use(`${url}/faculties`, faculties)
   app.use(`${url}/university`, university)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
@@ -70,6 +71,7 @@ const routes = (app: Express, url: string) => {
   app.use(`${url}/close-to-graduation`, auth.roles(['fullSisuAccess', 'studyGuidanceGroups']), closeToGraduation)
   app.use(`${url}/study-programme-pins`, studyProgrammePins)
   app.use(`${url}/curriculum-periods`, curriculumPeriods)
+  app.use(`${url}/health`, health)
   app.get('', (_, res) => {
     res.status(404).json({ error: 'unknown endpoint' })
   })

services/backend/src/routes/banners.ts

@@ -0,0 +1,53 @@
+import { Request, Response, Router } from 'express'
+
+import { Banner } from '@oodikone/shared/models/kone'
+import { saveBanner, getActiveBanners, getAllBanners } from '../services/banners'
+
+const router = Router()
+
+router.get<never, Banner[]>('/', async (_req: Request, res: Response) => {
+  const banners = await getActiveBanners()
+  res.json(banners).end()
+})
+
+router.get<never, Banner[]>('/all', async (req: Request, res: Response) => {
+  const { user } = req
+  if (!user.isAdmin || !user.iamGroups.includes('grp-toska')) {
+    return res.status(403).end()
+  }
+
+  const banners = await getAllBanners()
+  res.json(banners)
+})
+
+router.post('/new', async (req: Request, res: Response) => {
+  const { user } = req
+  if (!user.isAdmin || !user.iamGroups.includes('grp-toska')) {
+    return res.status(403).end()
+  }
+
+  const payload = {
+    ...req.body,
+    lastModifiedBy: user.username,
+  }
+
+  await saveBanner(payload)
+  res.status(201).end()
+})
+
+router.put('/update', async (req: Request, res: Response) => {
+  const { user } = req
+  if (!user.isAdmin || !user.iamGroups.includes('grp-toska')) {
+    return res.status(403).end()
+  }
+
+  const payload = {
+    ...req.body,
+    lastModifiedBy: user.username,
+  }
+
+  await saveBanner(payload)
+  res.status(201).end()
+})
+
+export default router

services/backend/src/routes/completedCoursesSearch.ts

@@ -2,8 +2,9 @@ import { Router } from 'express'
 import { difference, intersection } from 'lodash-es'
 
 import { CanError } from '@oodikone/shared/routes'
+import type { CompletedCoursesStudent, CompletedCoursesCourse } from '@oodikone/shared/types'
 import { tryCatch } from '@oodikone/shared/util'
-import { Courses, FormattedStudent, getCompletedCourses } from '../services/completedCoursesSearch'
+import { getCompletedCourses } from '../services/completedCoursesSearch'
 import {
   getUserSearches,
   createNewUserSearch,
@@ -19,11 +20,14 @@ const router = Router()
 
 const importerClient = getImporterClient()
 
-export type SearchResBody = CanError<
-  { discardedStudentNumbers: string[] } & { students: Omit<FormattedStudent, 'allEnrollments'>[]; courses: Courses }
+type SearchResBody = CanError<
+  { discardedStudentNumbers: string[] } & {
+    students: Omit<CompletedCoursesStudent, 'allEnrollments'>[]
+    courses: CompletedCoursesCourse[]
+  }
 >
-export type SearchReqBody = never
-export type SearchQuery = {
+type SearchReqBody = never
+type SearchQuery = {
   studentlist: string
   courselist: string
 }

services/backend/src/routes/courses.ts

@@ -9,7 +9,7 @@ import {
   CourseYearlyStatsReqBody,
   CourseYearlyStatsQuery,
 } from '@oodikone/shared/routes/courses'
-import type { CourseWithSubsId } from '@oodikone/shared/types/course'
+import type { CourseWithSubsDetails } from '@oodikone/shared/types/course'
 import { getCourseDetails, getCourseYearlyStats } from '../services/courses'
 import { getCoursesByNameAndOrCode } from '../services/courses/courseFinders'
 import {
@@ -31,7 +31,7 @@ router.get<never, CanError<CoursesMultiResBody>, CoursesMultiReqBody, CoursesMul
 
     const courses = await getCoursesByNameAndOrCode(name, code, includeSpecial === 'true')
 
-    const mergedCourses = new Map<string, CourseWithSubsId>()
+    const mergedCourses = new Map<string, CourseWithSubsDetails>()
 
     for (const course of courses) {
       if (!(course.max_attainment_date && course.min_attainment_date)) continue
@@ -51,12 +51,12 @@ router.get<never, CanError<CoursesMultiResBody>, CoursesMultiReqBody, CoursesMul
   }
 )
 
-type CourseYearlyStatsResBody = Awaited<ReturnType<typeof getCourseYearlyStats>>
+export type CourseYearlyStatsResBody = Awaited<ReturnType<typeof getCourseYearlyStats>>
 
 router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody, CourseYearlyStatsQuery>(
   '/courseyearlystats',
   async (req, res) => {
-    const { codes, combineSubstitutions, separate } = req.query
+    const { codes, combineSubstitutions, separate, fromYearCode, toYearCode } = req.query
 
     const courseCodes = handleQueryArrays(codes)
 
@@ -82,8 +82,15 @@ router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody,
     const useCombineSubstitutions = combineSubstitutions !== 'false'
     const useSeparate = separate === 'true'
 
-    const results = await getCourseYearlyStats(courseCodes, useSeparate, anonymizationSalt, useCombineSubstitutions)
-    res.json(results)
+    const results = await getCourseYearlyStats(
+      courseCodes,
+      useSeparate,
+      anonymizationSalt,
+      useCombineSubstitutions,
+      fromYearCode,
+      toYearCode
+    )
+    return res.json(results)
   }
 )
 

services/backend/src/routes/faculties.ts

@@ -2,6 +2,7 @@ import { Router } from 'express'
 
 import type { CanError, GenericApplicationError } from '@oodikone/shared/routes'
 import type {
+  FacultyGraduationStatistics,
   Graduated,
   Name,
   NameWithCode,
@@ -22,7 +23,6 @@ import {
   setBasicStats,
   getThesisWritersStats,
   setThesisWritersStats,
-  getGraduationStats,
   setGraduationStats,
   getFacultyStudentStats,
   setFacultyStudentStats,
@@ -30,12 +30,13 @@ import {
   setFacultyProgressStats,
   type BasicData,
   type ThesisWriterData,
-  GraduationData,
+  getGraduationStats,
 } from '../services/faculty/facultyService'
 import { combineFacultyStudentProgress } from '../services/faculty/facultyStudentProgress'
 import { combineFacultyStudents } from '../services/faculty/facultyStudents'
 import { combineFacultyThesisWriters } from '../services/faculty/facultyThesisWriters'
 import { updateFacultyOverview, updateFacultyProgressOverview } from '../services/faculty/facultyUpdates'
+import { objectKeyAnnihilator } from '../util'
 import { ApplicationError } from '../util/customErrors'
 
 // Faculty uses a lot of tools designed for Degree programme.
@@ -135,14 +136,13 @@ router.get<
   return res.json(await getFacultyCredits(faculty.code, yearType === 'ACADEMIC_YEAR'))
 })
 
-type GetGraduationStatsResBody = GraduationData
 type GetGraduationStatsQuery = {
   programme_filter: ProgrammeFilter
 }
 
 router.get<
   GetStatsParams,
-  CanError<GetGraduationStatsResBody, GenericApplicationError>,
+  CanError<FacultyGraduationStatistics, GenericApplicationError>,
   GetStatsReqBody,
   GetGraduationStatsQuery
 >('/:id/graduationtimes', auth.roles(['facultyStatistics']), async (req, res) => {
@@ -153,7 +153,9 @@ router.get<
   if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
   const data = await getGraduationStats(faculty.code, programmeFilter)
-  if (data) return res.json(data)
+  // Times were needed to compute medians/averages serverside, but are not needed in frontend.
+  // They are still kept in redis as university view needs them for the same purpose.
+  if (data) return res.json(objectKeyAnnihilator(data, 'times', []))
 
   const programmes = await getDegreeProgrammesOfFaculty(faculty.code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
   if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
@@ -161,7 +163,7 @@ router.get<
   const updatedStats = await countGraduationTimes(faculty.code, programmes)
   if (updatedStats) await setGraduationStats(updatedStats, programmeFilter)
 
-  return res.json(updatedStats)
+  return res.json(objectKeyAnnihilator(updatedStats, 'times', []))
 })
 
 type GetProgressStatsResBody = {
@@ -212,7 +214,7 @@ export type DegreeProgramme = Pick<
   'code' | 'name' | 'degreeProgrammeType' | 'progId'
 >
 
-type GetStudetStatsResBody = {
+type GetStudentStatsResBody = {
   id: string
   years: string[]
   facultyTableStats: Record<string, (number | string)[]>
@@ -229,7 +231,7 @@ type GetStudetStatsResBody = {
 
 router.get<
   GetStatsParams,
-  CanError<GetStudetStatsResBody, GenericApplicationError>,
+  CanError<GetStudentStatsResBody, GenericApplicationError>,
   GetStatsReqBody,
   GetProgressStatsQuery
 >('/:id/studentstats', auth.roles(['facultyStatistics']), async (req, res) => {

services/backend/src/routes/health.ts

@@ -0,0 +1,10 @@
+import { Router } from 'express'
+
+const router = Router()
+
+/** Used for checking if backend is up eg. Docker healthchecks */
+router.get<never, { status: string }>('/', (_, res) => {
+  return res.status(200).json({ status: 'healthy' })
+})
+
+export default router

services/backend/src/routes/population.ts

@@ -29,7 +29,7 @@ import { getStudentTagMap } from '../services/populations/getStudentData'
 import { parseDateRangeFromParams } from '../services/populations/shared'
 import { statisticsOf } from '../services/populations/statisticsOf'
 import { getStudentNumbersWithStudyRights } from '../services/populations/studentNumbersWithStudyRights'
-import { findByCourseAndSemesters } from '../services/students'
+import { findByCourseAndSemesters, getStartAndEndDates } from '../services/students'
 import { getFullStudyProgrammeRights, handleQueryArrays, hasFullAccessToStudentData } from '../util'
 
 const router = Router()
@@ -112,7 +112,7 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
     const studentStatuses = handleQueryArrays(req.query.studentStatuses) // NOTE: TRANSFERRED means transferred out of the program
 
     const requiredFields = [years, semesters, programme]
-    if (requiredFields.some(field => !field)) {
+    if (requiredFields.some(field => !field.length)) {
       return res.status(400).json({ error: 'The query should have years, semesters and a programme defined' })
     }
 
@@ -150,8 +150,9 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
 
     const studyRights = [programme]
     const tagMap = await getStudentTagMap(studyRights, studentNumbers, userId)
+    const filterCreditsAndEnrollmentsByDate = false
 
-    const result = await statisticsOf(studentNumbers, studyRights, tagMap, startDate)
+    const result = await statisticsOf(studentNumbers, studyRights, tagMap, filterCreditsAndEnrollmentsByDate, startDate)
     const processed = obfuscateStuff({
       result,
 
@@ -208,9 +209,19 @@ router.get<
     ? new Set(studentNumbers)
     : new Set(allStudentsUserCanAccess)
 
+  const { startDate: startDate, endDate: endDate } = await getStartAndEndDates(Number(from), Number(to), isSeparate)
+
   const studyRights = []
   const tagMap = await getStudentTagMap(studyRights, studentNumbers, userId)
-  const result = await statisticsOf(studentNumbers, studyRights, tagMap)
+  const filterCreditsAndEnrollmentsByDate = true
+  const result = await statisticsOf(
+    studentNumbers,
+    studyRights,
+    tagMap,
+    filterCreditsAndEnrollmentsByDate,
+    startDate?.toISOString(),
+    endDate?.toISOString()
+  )
   const processed = obfuscateStuff({
     result,
 
@@ -223,14 +234,18 @@ router.get<
 })
 
 // Used in custom population and single study guidance groups
-router.post<never, GetCustomPopulationResBody, CustomPopulationQuery>(
+router.post<never, CanError<GetCustomPopulationResBody>, CustomPopulationQuery>(
   '/populationstatisticsbystudentnumbers',
   async (req, res) => {
-    const { studentNumbers, tags } = req.body
+    const { studentNumbers, tags } = req.body ?? { studentNumbers: [] }
     const { id: userId, roles, studentsUserCanAccess } = req.user
 
     const students = handleQueryArrays(studentNumbers)
 
+    if (!students.length) {
+      return res.status(422).json({ error: 'Body should include student numbers' })
+    }
+
     const filteredStudentNumbers = hasFullAccessToStudentData(roles)
       ? students
       : intersection(students, studentsUserCanAccess)
@@ -239,12 +254,19 @@ router.post<never, GetCustomPopulationResBody, CustomPopulationQuery>(
 
     const studyRights = [studyProgrammeCode].filter(value => value !== undefined)
     const tagMap = await getStudentTagMap(studyRights, filteredStudentNumbers, userId)
+    const filterCreditsAndEnrollmentsByDate = false
 
     const { startDate } = tags?.year
       ? parseDateRangeFromParams({ semesters: ['FALL', 'SPRING'], years: [tags?.year] })
       : { startDate: undefined }
 
-    const result = await statisticsOf(filteredStudentNumbers, studyRights, tagMap, startDate)
+    const result = await statisticsOf(
+      filteredStudentNumbers,
+      studyRights,
+      tagMap,
+      filterCreditsAndEnrollmentsByDate,
+      startDate
+    )
 
     const resultWithStudyProgramme = { ...result, studyProgramme: tags?.studyProgramme }
     const discardedStudentNumbers = difference(students, filteredStudentNumbers)
@@ -282,13 +304,13 @@ router.get<
   PopulationstatisticsMaxYearsToCreatePopulationFormQuery
 >('/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
   const courseCodes = JSON.parse(req.query.courseCodes) as string[]
-  const maxYearsToCreatePopulationFromOpen = await maxYearsToCreatePopulationFrom(courseCodes, Unification.OPEN)
-  const maxYearsToCreatePopulationFromUni = await maxYearsToCreatePopulationFrom(courseCodes, Unification.REGULAR)
   const maxYearsToCreatePopulationFromBoth = await maxYearsToCreatePopulationFrom(courseCodes, Unification.UNIFY)
+  const maxYearsToCreatePopulationFromUni = await maxYearsToCreatePopulationFrom(courseCodes, Unification.REGULAR)
+  const maxYearsToCreatePopulationFromOpen = await maxYearsToCreatePopulationFrom(courseCodes, Unification.OPEN)
   return res.json({
-    openCourses: maxYearsToCreatePopulationFromOpen,
-    uniCourses: maxYearsToCreatePopulationFromUni,
     unifyCourses: maxYearsToCreatePopulationFromBoth,
+    uniCourses: maxYearsToCreatePopulationFromUni,
+    openCourses: maxYearsToCreatePopulationFromOpen,
   })
 })
 

services/backend/src/routes/studyProgramme.ts

@@ -20,6 +20,7 @@ import { getStudyRightsInProgramme, getStudyTracksForProgramme } from '../servic
 import { getStudyTrackStatsForStudyProgramme } from '../services/studyProgramme/studyTrackStats'
 import logger from '../util/logger'
 import { logInfoForGrafana } from '../util/logInfoForGrafana'
+import { yearToYearCode } from '@oodikone/shared/util'
 
 const router = Router()
 
@@ -103,16 +104,25 @@ router.get<StudyProgrammeParams, unknown, never, StatsRequestQuery>('/:id/gradua
 type CourseStatsQuery = {
   combinedProgramme: string
   yearType: YearType
+  fromYearCode?: string
+  toYearCode?: string
 }
 
 router.get<StudyProgrammeParams, unknown, never, CourseStatsQuery>('/:id/coursestats', async (req, res) => {
   const code = req.params.id
-  const { combinedProgramme, yearType } = req.query
+  const { combinedProgramme, yearType, fromYearCode, toYearCode } = req.query
   const date = new Date()
   date.setHours(23, 59, 59, 999)
   void logInfoForGrafana(code, combinedProgramme)
   try {
-    const data = await getStudyProgrammeCoursesForStudyTrack(date.getTime(), code, yearType, combinedProgramme)
+    const data = await getStudyProgrammeCoursesForStudyTrack(
+      date.getTime(),
+      code,
+      yearType,
+      combinedProgramme,
+      fromYearCode ?? '1', // 1950
+      toYearCode ?? yearToYearCode(new Date().getFullYear()).toString() // Current year
+    )
     return res.json(data)
   } catch (error) {
     logger.error({ message: `Failed to get code ${code} programme courses stats`, meta: `${error}` })

services/backend/src/routes/university.ts

@@ -1,11 +1,12 @@
 import { Request, Response, Router } from 'express'
 import { cloneDeep } from 'lodash-es'
 
-import { Graduated, NameWithCode } from '@oodikone/shared/types'
+import { FacultyGraduationStatistics, Graduated, NameWithCode } from '@oodikone/shared/types'
+import { MediansByCategory, UniversityGraduationStatistics } from '@oodikone/shared/types/graduations'
 import { magicFacultyCode } from '../config/organizationConstants'
 import { OrganizationModel } from '../models'
 import { getDegreeProgrammesOfFaculty } from '../services/faculty/faculty'
-import { countGraduationTimes, LevelGraduationStats } from '../services/faculty/facultyGraduationTimes'
+import { countGraduationTimes } from '../services/faculty/facultyGraduationTimes'
 import { getSortedFaculties } from '../services/faculty/facultyHelpers'
 import {
   getFacultyProgressStats,
@@ -16,19 +17,19 @@ import {
 import { combineFacultyStudentProgress, FacultyProgressData } from '../services/faculty/facultyStudentProgress'
 import { GraduationTarget } from '../services/graduationHelpers'
 import { getMedian } from '../services/studyProgramme/studyProgrammeHelpers'
+import { objectKeyAnnihilator } from '../util'
 import logger from '../util/logger'
 
 const router = Router()
 
 const degreeNames = ['bachelor', 'bachelorMaster', 'master', 'doctor'] as const
 
-const getProgrammeNames = (faculties: OrganizationModel[]) => {
-  return faculties.reduce<Record<string, NameWithCode>>((obj, faculty) => {
+const getProgrammeNames = (faculties: OrganizationModel[]) =>
+  faculties.reduce<Record<string, NameWithCode>>((obj, faculty) => {
     const { name, code } = faculty.toJSON()
     obj[faculty.code] = { code, ...name }
     return obj
   }, {})
-}
 
 interface GetProgressStatsRequest extends Request {
   query: {
@@ -120,16 +121,20 @@ router.get('/allprogressstats', async (req: GetProgressStatsRequest, res: Respon
   res.status(200).json(universityData)
 })
 
-router.get('/allgraduationstats', async (_req: Request, res: Response) => {
-  const degreeNames = ['bachelor', 'bcMsCombo', 'master', 'doctor'] as const
+router.get('/allgraduationstats', async (_req: Request, res: Response<UniversityGraduationStatistics>) => {
+  const degreeTypes = ['bachelor', 'bcMsCombo', 'master', 'doctor'] as const
+  /** University view does not support old degree programmes */
+  const programmeFilter = 'NEW_DEGREE_PROGRAMMES' as const
+
   const allFaculties = await getSortedFaculties()
   const facultyCodes = allFaculties.map(faculty => faculty.code)
-  const facultyData: Record<string, Awaited<ReturnType<typeof countGraduationTimes>>> = {}
-  const programmeFilter = 'NEW_DEGREE_PROGRAMMES'
+  const facultyData: Record<string, FacultyGraduationStatistics> = {}
+
   for (const facultyCode of facultyCodes) {
-    let data: any = await getGraduationStats(facultyCode, programmeFilter, true)
+    let data = await getGraduationStats(facultyCode, programmeFilter, true)
+
     if (!data) {
-      logger.info(`Data missing from server: Refreshing graduation faculty data for faculty ${facultyCode}`)
+      logger.info(`Data missing from redis: Refreshing graduation faculty data for faculty ${facultyCode}`)
       const programmes = await getDegreeProgrammesOfFaculty(facultyCode, true)
       data = await countGraduationTimes(facultyCode, programmes)
       await setGraduationStats(data, programmeFilter)
@@ -137,48 +142,50 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
     facultyData[facultyCode] = data
   }
 
-  const universityData = {
-    goals: {
-      bachelor: GraduationTarget.THREE_YEARS,
-      bcMsCombo: GraduationTarget.FIVE_YEARS,
-      master: GraduationTarget.TWO_YEARS,
-      doctor: GraduationTarget.FOUR_YEARS,
+  const goals: UniversityGraduationStatistics['goals'] = {
+    bachelor: GraduationTarget.THREE_YEARS,
+    bcMsCombo: GraduationTarget.FIVE_YEARS,
+    master: GraduationTarget.TWO_YEARS,
+    doctor: GraduationTarget.FOUR_YEARS,
+  }
+  const programmeNames = getProgrammeNames(allFaculties)
+
+  const byGradYear: UniversityGraduationStatistics['byGradYear'] = {
+    medians: {
+      bachelor: [],
+      bcMsCombo: [],
+      master: [],
+      doctor: [],
     },
-    programmeNames: getProgrammeNames(allFaculties),
-    byGradYear: {
-      medians: {} as Record<string, LevelGraduationStats[]>,
-      programmes: {
-        medians: {} as Record<
-          string,
-          Record<string, { programmes: string[]; data: Array<LevelGraduationStats & { name: string; code: string }> }>
-        >,
+    programmes: {
+      medians: {
+        bachelor: {},
+        bcMsCombo: {},
+        master: {},
+        doctor: {},
       },
     },
-    classSizes: {
-      bachelor: null as Record<string, number> | null,
-      bcMsCombo: null as Record<string, number> | null,
-      master: null as Record<string, number> | null,
-      doctor: null as Record<string, number> | null,
-      programmes: {} as Record<string, Record<string, Record<string, number>>>,
-    },
+  }
+  const classSizes: UniversityGraduationStatistics['classSizes'] = {
+    programmes: {},
   }
 
-  const unifyTotals = (facultyData: Record<string, LevelGraduationStats[]>, isLast: boolean) => {
-    const mediansForUniversity = universityData.byGradYear.medians
-    for (const degree of degreeNames) {
-      if (!mediansForUniversity[degree]) mediansForUniversity[degree] = []
-      if (!facultyData[degree]) continue
-      for (const yearStats of facultyData[degree]) {
-        const universityStats = mediansForUniversity[degree]
-        const universityYearStats = mediansForUniversity[degree].find(stats => stats.name === yearStats.name)
+  /** Computes university level stats */
+  const unifyTotals = (facultyData: MediansByCategory, isLast: boolean) => {
+    const mediansForUniversity = byGradYear.medians
+    for (const degreeType of degreeTypes) {
+      if (!facultyData[degreeType]) continue
+
+      for (const facultyYearStats of facultyData[degreeType]) {
+        const universityYearStats = mediansForUniversity[degreeType].find(stats => stats.name === facultyYearStats.name)
         if (!universityYearStats) {
-          universityStats.push(yearStats)
+          mediansForUniversity[degreeType].push(facultyYearStats)
         } else {
-          universityYearStats.times.push(...yearStats.times)
-          universityYearStats.amount += yearStats.amount
-          universityYearStats.statistics.onTime += yearStats.statistics.onTime
-          universityYearStats.statistics.yearOver += yearStats.statistics.yearOver
-          universityYearStats.statistics.wayOver += yearStats.statistics.wayOver
+          universityYearStats.times.push(...facultyYearStats.times)
+          universityYearStats.amount += facultyYearStats.amount
+          universityYearStats.statistics.onTime += facultyYearStats.statistics.onTime
+          universityYearStats.statistics.yearOver += facultyYearStats.statistics.yearOver
+          universityYearStats.statistics.wayOver += facultyYearStats.statistics.wayOver
           if (isLast) {
             universityYearStats.median = getMedian(universityYearStats.times)
           }
@@ -187,29 +194,28 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
     }
   }
 
-  const unifyProgrammeStats = (facultyData: Record<string, LevelGraduationStats[]>, facultyCode: string) => {
-    const mediansForUniversity = universityData.byGradYear.programmes.medians
-    for (const degree of degreeNames) {
-      if (!facultyData[degree]) continue
-      if (!mediansForUniversity[degree]) mediansForUniversity[degree] = {}
-      for (const yearData of facultyData[degree]) {
-        if (!mediansForUniversity[degree][yearData.name]) {
-          mediansForUniversity[degree][yearData.name] = { programmes: [], data: [] }
-        }
-        const uniYearStats = mediansForUniversity[degree][yearData.name]
+  /** Computes faculty breakdown */
+  const unifyFacultyStats = (facultyData: MediansByCategory, facultyCode: string) => {
+    const mediansForUniversity = byGradYear.programmes.medians
+    for (const degreeType of degreeTypes) {
+      if (!facultyData[degreeType]) continue
+
+      for (const facultyYearStats of facultyData[degreeType]) {
+        mediansForUniversity[degreeType][facultyYearStats.name] ??= { programmes: [], data: [] }
+        const uniYearStats = mediansForUniversity[degreeType][facultyYearStats.name]
         if (!uniYearStats.programmes.find(prog => prog === facultyCode)) {
           uniYearStats.programmes.push(facultyCode)
         }
         const uniYearFacultyStats = uniYearStats.data.find(item => item.code === facultyCode)
         if (!uniYearFacultyStats) {
-          uniYearStats.data.push({ ...cloneDeep(yearData), name: facultyCode, code: facultyCode })
+          uniYearStats.data.push({ ...cloneDeep(facultyYearStats), name: facultyCode, code: facultyCode })
         } else {
           uniYearFacultyStats.times.push(...uniYearFacultyStats.times)
           uniYearFacultyStats.median = getMedian(uniYearFacultyStats.times)
-          uniYearFacultyStats.amount += yearData.amount
-          uniYearFacultyStats.statistics.onTime += yearData.statistics.onTime
-          uniYearFacultyStats.statistics.yearOver += yearData.statistics.yearOver
-          uniYearFacultyStats.statistics.wayOver += yearData.statistics.wayOver
+          uniYearFacultyStats.amount += facultyYearStats.amount
+          uniYearFacultyStats.statistics.onTime += facultyYearStats.statistics.onTime
+          uniYearFacultyStats.statistics.yearOver += facultyYearStats.statistics.yearOver
+          uniYearFacultyStats.statistics.wayOver += facultyYearStats.statistics.wayOver
         }
       }
     }
@@ -218,27 +224,23 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
   for (let i = 0; i < facultyCodes.length; i++) {
     const facultyCode = facultyCodes[i]
     const data = facultyData[facultyCode]
-    unifyTotals(data.byGradYear.medians, i === facultyCodes.length - 1)
-    unifyProgrammeStats(data.byGradYear.medians, facultyCode)
-    for (const degree of degreeNames) {
-      if (!data.classSizes[degree]) {
-        continue
-      }
-      const facultyClassSizes = data.classSizes[degree] as Record<string, number>
-      if (!universityData.classSizes[degree]) {
-        universityData.classSizes[degree] = facultyClassSizes
-      } else {
-        Object.entries(facultyClassSizes).forEach(([key, value]) => {
-          universityData.classSizes[degree]![key] += value
-        })
-      }
-    }
 
-    const { programmes, ...rest } = data.classSizes
-    universityData.classSizes.programmes[facultyCode] = rest as Record<string, Record<string, number>>
+    unifyTotals(data.byGradYear.medians, i === facultyCodes.length - 1)
+    unifyFacultyStats(data.byGradYear.medians, facultyCode)
+
+    const { programmes: _, ...rest } = data.classSizes
+    classSizes.programmes[facultyCode] = rest
   }
 
-  res.status(200).json(universityData)
+  // Individual times are no longer needed as medians/averages were already computed.
+  objectKeyAnnihilator(byGradYear, 'times', [])
+
+  res.status(200).json({
+    goals,
+    programmeNames,
+    byGradYear,
+    classSizes,
+  })
 })
 
 export default router

services/backend/src/routes/updater.ts

@@ -1,9 +1,9 @@
 import { Request, Response, Router } from 'express'
 
-import { languageCenterViewEnabled } from '../config'
 import { refreshFaculties, refreshProgrammes } from '../events'
 import {
   abortUpdate,
+  nukeRedis,
   updateSISCoursesByCourseCode,
   updateSISMetadata,
   updateSISProgrammes,
@@ -12,17 +12,17 @@ import {
   updateSISStudentsByStudentNumber,
 } from '../services/sisUpdaterService'
 import logger from '../util/logger'
-import { getJobs, jobMaker, removeWaitingJobs } from '../worker/queue'
+import { jobQueue } from '../worker/queue'
 
 const router = Router()
 
 const refreshFacultiesByList = (facultyCodes: string[]) => {
-  facultyCodes.forEach(code => jobMaker.faculty(code))
+  facultyCodes.forEach(code => jobQueue.refreshFaculty(code))
   return 'Added jobs for refreshing faculties'
 }
 
 const refreshProgrammesByList = (programmeCodes: string[]) => {
-  programmeCodes.forEach(code => jobMaker.programme(code))
+  programmeCodes.forEach(code => jobQueue.refreshProgramme(code))
   return 'Added jobs for refreshing programme'
 }
 
@@ -80,11 +80,17 @@ router.get('/refresh_redis_cache', async (req: Request, res: Response) => {
   }
 })
 
+router.get('/nuke_redis', async (req: Request, res: Response) => {
+  logger.info(`${req.user.username} requested complete wipe of redis`)
+  const response = await nukeRedis()
+  if (response) res.status(200).json('Complete flush of redis scheduled')
+})
+
 router.post('/refresh-teacher-leaderboard', (req: Request, res: Response) => {
   logger.info(
     `${req.user.username} requested refresh of teacher leaderboard for the current and previous academic year`
   )
-  jobMaker.teacherLeaderboard()
+  jobQueue.refreshTeacherLeaderboard()
   res.status(200).json('Teacher leaderboard for the current and previous academic year refreshed')
 })
 
@@ -102,15 +108,13 @@ router.post('/refresh_faculties_v2', async (req: Request, res: Response) => {
 
 router.post('/refresh_language_center_data', (req: Request, res: Response) => {
   logger.info(`${req.user.username} requested refresh of language center data`)
-  if (!languageCenterViewEnabled)
-    res.status(418).json({ error: 'The language center functionality is not activated in your environment.' })
-  jobMaker.languagecenter()
+  jobQueue.refreshLanguagecenter()
   res.status(200).json('Added job for refreshing language center data')
 })
 
 router.post('/refresh-close-to-graduation', (req: Request, res: Response) => {
   logger.info(`${req.user.username} requested refresh of close to graduation data`)
-  jobMaker.closeToGraduation()
+  jobQueue.refreshCloseToGraduation()
   res.status(200).json('Added job for refreshing close to graduation data')
 })
 
@@ -120,13 +124,12 @@ router.get('/abort', async (_req: Request, res: Response) => {
 })
 
 router.get('/jobs', async (_req: Request, res: Response) => {
-  const waiting = await getJobs('waiting')
-  const active = await getJobs('active')
-  res.status(200).json({ waiting, active })
+  const jobs = await jobQueue.getJobs()
+  res.status(200).json(jobs)
 })
 
 router.delete('/jobs', async (_req: Request, res: Response) => {
-  await removeWaitingJobs()
+  await jobQueue.clearWaitingJobs()
   res.status(200).end()
 })
 

services/backend/src/services/banners.ts

@@ -0,0 +1,28 @@
+import { Op } from 'sequelize'
+import { Banner } from '@oodikone/shared/models/kone'
+import { BannerModel } from '../models/kone/banner'
+import { SetOptional } from '../types'
+
+export const getActiveBanners = async () =>
+  await BannerModel.findAll({
+    attributes: ['id', 'text', 'lightness', 'color', 'startDate', 'endDate'],
+    where: {
+      endDate: {
+        [Op.gt]: new Date(),
+      },
+      startDate: {
+        [Op.lt]: new Date(),
+      },
+    },
+    raw: true,
+  })
+
+export const getAllBanners = async () => await BannerModel.findAll({ raw: true })
+
+export const saveBanner = async (payload: SetOptional<Banner, 'id'>) => {
+  if (!payload.id) {
+    await BannerModel.create(payload)
+  } else {
+    await BannerModel.upsert(payload)
+  }
+}

services/backend/src/services/completedCoursesSearch.ts

@@ -1,64 +1,30 @@
 import { col, fn, Op, where } from 'sequelize'
 
-import { CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
-import { omitKeys } from '@oodikone/shared/util'
+import {
+  CompletedCoursesCourse,
+  CompletedCoursesStudent,
+  CreditTypeCode,
+  EnrollmentState,
+  StudentCredits,
+} from '@oodikone/shared/types'
+import { enrollmentTimeDateThreshold, omitKeys } from '@oodikone/shared/util'
 import { CourseModel, CreditModel, EnrollmentModel, StudentModel, StudyplanModel } from '../models'
 
-export type Courses = Array<Pick<CourseModel, 'code' | 'name' | 'substitutions'>>
-
-interface StudentCredit {
-  date: Date
-  courseCode: string
-  creditType: CreditTypeCode
-  substitution: string | null
-}
-
-interface StudentEnrollment {
-  date: Date
-  courseCode: string
-  substitution: string | null
-}
-
-interface StudentInfo {
-  firstnames: string
-  lastname: string
-  email: string
-  sis_person_id: string
-  coursesInStudyPlan: string[]
-  secondaryEmail: string | null
-  credits: StudentCredit[]
-  enrollments: Record<string, StudentEnrollment>
-}
-
-type StudentCredits = Record<
-  string,
-  Omit<StudentInfo, 'credits' | 'enrollments'> & {
-    credits: StudentCredit[]
-    enrollments: StudentEnrollment[]
-  }
->
-
-export interface FormattedStudent extends StudentInfo {
-  studentNumber: string
-  allEnrollments: StudentEnrollment[]
-}
-
-interface StudentWithStudyplanNested
-  extends Pick<
-    StudentModel,
-    'studentnumber' | 'firstnames' | 'lastname' | 'email' | 'sis_person_id' | 'secondary_email'
-  > {
+type StudentWithStudyplanNested = Pick<
+  StudentModel,
+  'studentnumber' | 'firstnames' | 'lastname' | 'email' | 'sis_person_id' | 'secondary_email'
+> & {
   studyplans: { included_courses: string[] }[]
 }
 
-interface StudentWithCourses extends Omit<StudentWithStudyplanNested, 'studyplans'> {
+type StudentWithCourses = Omit<StudentWithStudyplanNested, 'studyplans'> & {
   coursesInStudyPlan: string[]
 }
 
 const getCourses = async (courseCodes: string[]) => {
-  const courses: Courses = (
+  const courses: CompletedCoursesCourse[] = (
     await CourseModel.findAll({
-      attributes: ['code', 'name', 'substitutions'],
+      attributes: ['code', 'name', 'substitution_groups'],
       where: where(fn('LOWER', col('code')), {
         [Op.in]: courseCodes.map(code => code.toLowerCase()),
       }),
@@ -67,44 +33,76 @@ const getCourses = async (courseCodes: string[]) => {
   return courses
 }
 
-const getCredits = async (
-  courses: Courses,
+const getPassedCredits = async (
+  courses: CompletedCoursesCourse[],
   fullCourseCodes: string[],
-  courseCodes: string[],
   studentNumbers: string[]
 ) => {
-  const credits: Array<
-    Pick<CreditModel, 'course_code' | 'student_studentnumber' | 'credittypecode' | 'attainment_date'>
-  > = await CreditModel.findAll({
-    attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'attainment_date'],
-    where: {
-      course_code: {
-        [Op.in]: fullCourseCodes,
+  const credits: Pick<CreditModel, 'course_code' | 'student_studentnumber' | 'credittypecode' | 'attainment_date'>[] =
+    await CreditModel.findAll({
+      raw: true,
+      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'attainment_date'],
+      order: [['attainment_date', 'DESC']],
+      where: {
+        course_code: {
+          [Op.in]: fullCourseCodes,
+        },
+        student_studentnumber: {
+          [Op.in]: studentNumbers,
+        },
+        credittypecode: {
+          [Op.not]: CreditTypeCode.FAILED,
+        },
       },
-      student_studentnumber: {
-        [Op.in]: studentNumbers,
-      },
-    },
-  })
-  const formattedCredits = credits.map(credit => {
-    const originalCode = courseCodes.includes(credit.course_code)
-      ? null
-      : courses.find(course => course.substitutions.includes(credit.course_code))?.code
-    return {
-      courseCode: originalCode ?? credit.course_code,
-      substitution: originalCode ? credit.course_code : null,
-      studentNumber: credit.student_studentnumber,
-      creditType: credit.credittypecode,
-      date: credit.attainment_date,
+    })
+
+  const creditsByStudentNumber = Object.groupBy(credits, ({ student_studentnumber }) => student_studentnumber)
+
+  const formattedCredits: {
+    courseCode: string
+    substitution: string[] | null
+    studentNumber: string
+    creditType: CreditTypeCode
+    date: Date
+  }[] = []
+
+  Object.entries(creditsByStudentNumber).map(([studentNumber, credits]) => {
+    // We know that credits will exist, because of the way we created the previous object
+    const studentCreditCourseCodes = credits!.map(({ course_code }) => course_code)
+    for (const course of courses) {
+      // Also handle the main course code by adding it as a group
+      for (const group of [[course.code]].concat(course.substitution_groups)) {
+        if (group.every(code => studentCreditCourseCodes.includes(code))) {
+          // We just checked that (group) course code exists in credits so .find(...)! is ok
+          const groupCredits = group.map(code => credits!.find(credit => credit.course_code === code)!)
+          if (groupCredits.length) {
+            const groupCreditCodes = groupCredits.map(({ course_code }) => course_code)
+            formattedCredits.push({
+              courseCode: course.code,
+              substitution:
+                groupCreditCodes.includes(course.code) && groupCreditCodes.length === 1 ? null : groupCreditCodes,
+              studentNumber,
+              creditType: groupCredits.length > 1 ? CreditTypeCode.PASSED : groupCredits.at(0)!.credittypecode,
+              date: groupCredits?.at(0)?.attainment_date ?? new Date(), // Credits are sorted by date in desc. order
+            })
+          }
+        }
+      }
     }
   })
+
   return formattedCredits
 }
 
-const getEnrollments = async (courses: Courses, fullCourseCodes: string[], studentNumbers: string[]) => {
+const getEnrollments = async (
+  courses: CompletedCoursesCourse[],
+  fullCourseCodes: string[],
+  studentNumbers: string[]
+) => {
   const enrollments: Array<Pick<EnrollmentModel, 'course_code' | 'enrollment_date_time' | 'studentnumber'>> =
     await EnrollmentModel.findAll({
       attributes: ['course_code', 'enrollment_date_time', 'studentnumber'],
+      order: [['enrollment_date_time', 'DESC']],
       where: {
         course_code: {
           [Op.in]: fullCourseCodes,
@@ -115,17 +113,45 @@ const getEnrollments = async (courses: Courses, fullCourseCodes: string[], stude
         state: {
           [Op.eq]: EnrollmentState.ENROLLED,
         },
+        enrollment_date_time: { [Op.gte]: enrollmentTimeDateThreshold },
       },
     })
-  const formattedEnrollments = enrollments.map(enrollment => {
-    const originalCode = courses.find(course => course.substitutions.includes(enrollment.course_code))?.code
-    return {
-      courseCode: originalCode ?? enrollment.course_code,
-      substitution: originalCode ? enrollment.course_code : null,
-      studentNumber: enrollment.studentnumber,
-      date: enrollment.enrollment_date_time,
+
+  const enrollmentsByStudents = Object.groupBy(enrollments, ({ studentnumber }) => studentnumber)
+
+  const formattedEnrollments: {
+    courseCode: string
+    substitution: string[] | null
+    studentNumber: string
+    date: Date
+  }[] = []
+
+  Object.entries(enrollmentsByStudents).map(([studentNumber, enrollments]) => {
+    // We know that enrollments will exist, because of the way we created the previous object
+    const studentEnrollmentCourseCodes = enrollments!.map(({ course_code }) => course_code)
+    for (const course of courses) {
+      // Also handle the main course code by adding it as a group
+      for (const group of [[course.code]].concat(course.substitution_groups)) {
+        if (group.every(code => studentEnrollmentCourseCodes.includes(code))) {
+          // We just checked that (group) course code exists in enrollments so .find(...)! is ok
+          const groupEnrollments = group.map(code => enrollments!.find(enrollment => enrollment.course_code === code)!)
+          if (groupEnrollments.length) {
+            const groupEnrollmentCodes = groupEnrollments.map(({ course_code }) => course_code)
+            formattedEnrollments.push({
+              courseCode: course.code,
+              substitution:
+                groupEnrollmentCodes.includes(course.code) && groupEnrollmentCodes.length === 1
+                  ? null
+                  : groupEnrollmentCodes,
+              studentNumber,
+              date: groupEnrollments?.at(0)?.enrollment_date_time ?? new Date(), // Enrollments are sorted by date in desc. order
+            })
+          }
+        }
+      }
     }
   })
+
   return formattedEnrollments
 }
 
@@ -156,18 +182,23 @@ const getStudents = async (studentNumbers: string[]): Promise<StudentWithCourses
 export const getCompletedCourses = async (
   studentNumbers: string[],
   courseCodes: string[]
-): Promise<{ students: Omit<FormattedStudent, 'allEnrollments'>[]; courses: Courses }> => {
+): Promise<{ students: Omit<CompletedCoursesStudent, 'allEnrollments'>[]; courses: CompletedCoursesCourse[] }> => {
   const courses = await getCourses(courseCodes)
   const courseCodesSet = new Set(courseCodes)
-  courses.forEach(course => {
+
+  // Get *ALL* courses including any courses in any substitution groups
+  for (const course of courses) {
     courseCodesSet.add(course.code)
-    course.substitutions.forEach(substitution => {
-      courseCodesSet.add(substitution)
-    })
-  })
+    for (const group of course.substitution_groups) {
+      for (const code of group) {
+        courseCodesSet.add(code)
+      }
+    }
+  }
+
   const fullCourseCodes = Array.from(courseCodesSet)
 
-  const credits = await getCredits(courses, fullCourseCodes, courseCodes, studentNumbers)
+  const credits = await getPassedCredits(courses, fullCourseCodes, studentNumbers)
   const enrollments = await getEnrollments(courses, fullCourseCodes, studentNumbers)
   const studentInfo = await getStudents(studentNumbers)
 
@@ -223,27 +254,30 @@ export const getCompletedCourses = async (
     })
   })
 
-  const students = Object.keys(studentCredits).reduce((acc: FormattedStudent[], studentNumber) => {
-    const student = studentCredits[studentNumber]
-    acc.push({
-      studentNumber,
-      sis_person_id: student.sis_person_id,
-      credits: student.credits,
-      enrollments: {},
-      allEnrollments: student.enrollments,
-      firstnames: student.firstnames,
-      lastname: student.lastname,
-      email: student.email,
-      secondaryEmail: student.secondaryEmail,
-      coursesInStudyPlan: student.coursesInStudyPlan,
-    })
-    return acc
-  }, [] as FormattedStudent[])
+  const students = Object.keys(studentCredits).reduce<CompletedCoursesStudent[]>(
+    (acc: CompletedCoursesStudent[], studentNumber) => {
+      const student = studentCredits[studentNumber]
+      acc.push({
+        studentNumber,
+        sis_person_id: student.sis_person_id,
+        credits: student.credits,
+        enrollments: {},
+        allEnrollments: student.enrollments,
+        firstnames: student.firstnames,
+        lastname: student.lastname,
+        email: student.email,
+        secondaryEmail: student.secondaryEmail,
+        coursesInStudyPlan: student.coursesInStudyPlan,
+      })
+      return acc
+    },
+    []
+  )
 
   students.forEach(student => {
     courseCodes.forEach(courseCode => {
       const [latestEnrollment] = student.allEnrollments
-        .filter(enrollment => enrollment.courseCode === courseCode || enrollment.substitution === courseCode)
+        .filter(enrollment => enrollment.courseCode === courseCode)
         .sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime())
       student.enrollments[courseCode] = latestEnrollment
     })

services/backend/src/services/courses/courseFinders.ts

@@ -1,8 +1,6 @@
-import { orderBy } from 'lodash-es'
 import { Op } from 'sequelize'
 
-import { CourseWithSubsId } from '@oodikone/shared/types/course'
-import { getSortRank } from '@oodikone/shared/util/sortRank'
+import { CourseWithSubsDetails } from '@oodikone/shared/types/course'
 import { CourseModel } from '../../models'
 
 const nameLikeTerm = (name: string) => {
@@ -40,7 +38,7 @@ const codeLikeTerm = (code: string) => {
   }
 }
 
-const getRawCourses = async (name: string, code: string, includeSpecial: boolean) => {
+export const getCoursesByNameAndOrCode = async (name: string, code: string, includeSpecial: boolean) => {
   const courses = await CourseModel.findAll({
     where: {
       ...(includeSpecial ? {} : { mainCourseCode: { [Op.ne]: null } }),
@@ -51,7 +49,7 @@ const getRawCourses = async (name: string, code: string, includeSpecial: boolean
   })
 
   const coursesWithSubstitutionGroupDetails = await Promise.all(
-    courses.map(async course => {
+    courses.map<Promise<CourseWithSubsDetails>>(async course => {
       const groupDetails = await Promise.all(
         course?.substitution_groups?.map(async group => {
           return await CourseModel.findAll({
@@ -69,44 +67,6 @@ const getRawCourses = async (name: string, code: string, includeSpecial: boolean
   return coursesWithSubstitutionGroupDetails
 }
 
-const getCourses = async (name: string, code: string, includeSpecial: boolean) => {
-  const rawCourses = await getRawCourses(name, code, includeSpecial)
-  return rawCourses
-    .map(course => ({
-      ...course,
-      substitutions: orderBy(course.substitutions, [
-        substitution => {
-          if (/^A/.exec(substitution.at(0) ?? '')) return 4 // open university codes come last
-          if (/^\d/.exec(substitution.at(0) ?? '')) return 2 // old numeric codes come second
-          if (/^[A-Za-z]/.exec(substitution.at(0) ?? '')) return 1 // new letter based codes come first
-          return 3 // unknown, comes before open uni?
-        },
-      ]),
-    }))
-    .sort((a, b) => getSortRank(b.code) - getSortRank(a.code)) as CourseWithSubsId[]
-}
-
-export const getCoursesByNameAndOrCode = async (name: string, code: string, includeSpecial: boolean) => {
-  const courses = await getCourses(name, code, includeSpecial)
-
-  const visitedCourses = new Set<string>()
-
-  const assignSubstitutionGroup = (course: CourseWithSubsId, index: number) => {
-    const relatedCourses = courses.filter(currentCourse => course.substitutions.includes(currentCourse.code))
-    relatedCourses.unshift(course)
-    relatedCourses.forEach(relatedCourse => {
-      visitedCourses.add(relatedCourse.id)
-      relatedCourse.subsId = index
-    })
-  }
-
-  courses
-    .filter(course => !visitedCourses.has(course.id) && !visitedCourses.has(course.code))
-    .forEach((course, index) => assignSubstitutionGroup(course, index))
-
-  return courses
-}
-
 export const getCoursesByCodes = (codes: string[]) => {
   return CourseModel.findAll({
     where: {

services/backend/src/services/courses/courseYearlyStatsCounter.ts

@@ -124,6 +124,7 @@ export class CourseYearlyStatsCounter {
     this.groups[groupCode].students.grades[studentNumber] = { grade, passed }
   }
 
+  /** NOTE: Duplicates are counted by year eg. you can be both in 2022 and 2023 enrollments */
   public markEnrollmentToGroup(
     studentNumber: string,
     enrollmentDateTime: Date,

services/backend/src/services/courses/creditsAndEnrollmentsOfCourse.ts

@@ -1,7 +1,8 @@
 import { Op } from 'sequelize'
 
 import { Credit, Enrollment } from '@oodikone/shared/models'
-import { EnrollmentState, Unification } from '@oodikone/shared/types'
+import { CreditTypeCode, EnrollmentState, Unification } from '@oodikone/shared/types'
+import { enrollmentTimeDateThreshold } from '@oodikone/shared/util'
 import {
   StudentModel,
   CreditModel,
@@ -13,7 +14,7 @@ import {
 } from '../../models'
 import { getIsOpen } from './helpers'
 
-export const getCreditsForCourses = async (codeGroups: string[][], unification: Unification) => {
+export const getCreditsForCourses = async (codeGroups: string[][], unification: Unification, from: Date, to: Date) => {
   const allCourseCodes = codeGroups.flatMap(group => group)
 
   // We need the credits grouped by student numbers so that we can check if a student has
@@ -39,6 +40,8 @@ export const getCreditsForCourses = async (codeGroups: string[][], unification:
             [Op.in]: allCourseCodes,
           },
           is_open: getIsOpen(unification),
+          credittypecode: { [Op.not]: CreditTypeCode.IMPROVED }, // We do not care about improved grades
+          attainment_date: { [Op.between]: [from, to] },
         },
         order: [['attainment_date', 'ASC']],
         include: [
@@ -132,7 +135,12 @@ export const getStudentNumberToSrElementsMap = async (studentNumbers: string[])
   }, {})
 }
 
-export const getEnrollmentsForCourses = async (codeGroups: string[][], unification: Unification) => {
+export const getEnrollmentsForCourses = async (
+  codeGroups: string[][],
+  unification: Unification,
+  from: Date,
+  to: Date
+) => {
   const allCourseCodes = codeGroups.flatMap(group => group)
 
   const students = await StudentModel.findAll({
@@ -142,7 +150,7 @@ export const getEnrollmentsForCourses = async (codeGroups: string[][], unificati
     include: [
       {
         model: EnrollmentModel,
-        attributes: ['studentnumber', 'enrollment_date_time', 'course_code'],
+        attributes: ['studentnumber', 'enrollment_date_time', 'course_code', 'studyright_id'],
         include: [
           {
             model: SemesterModel,
@@ -158,7 +166,13 @@ export const getEnrollmentsForCourses = async (codeGroups: string[][], unificati
           course_code: {
             [Op.in]: allCourseCodes,
           },
-          enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
+          enrollment_date_time: {
+            [Op.and]: {
+              [Op.between]: [from, to],
+              // Date when OK changed from Oodi to Sisu data, studyright_id is null before that date
+              [Op.gte]: enrollmentTimeDateThreshold,
+            },
+          },
           state: EnrollmentState.ENROLLED,
           is_open: getIsOpen(unification),
         },

services/backend/src/services/courses/index.ts

@@ -3,10 +3,12 @@ import { Op, fn as dbFn, col as dbCol } from 'sequelize'
 
 import { Credit, Enrollment } from '@oodikone/shared/models'
 import { Name, EnrollmentState, Unification } from '@oodikone/shared/types'
+import { enrollmentTimeDateThreshold, getSemesterCodeAt, yearCodeToYear, yearToYearCode } from '@oodikone/shared/util'
 import { dateIsBetween } from '@oodikone/shared/util/datetime'
 import logger from '../../../src/util/logger'
 import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, SISStudyRightElementModel } from '../../models'
 import { isOpenUniCourseCode } from '../../util'
+import { getSemestersAndYears, SemestersAndYears } from '../semesters'
 import { CourseYearlyStatsCounter } from './courseYearlyStatsCounter'
 import {
   getCreditsForCourses,
@@ -52,26 +54,25 @@ const parseCredit = (
   studyRightElements: Array<SISStudyRightElementModel>
 ): FormattedCredit => {
   const singleCourse = isSingleCourse(creditGroup)
-  // Latest attainment for a course
-  const credit =
+
+  // TODO: Calculate substitutions correctly once course_unit_realisation hits
+  const attainment =
     creditGroup
-      .sort((a, b) => b.attainment_date.getTime() - a.attainment_date.getTime())
-      .find(credit => !CreditModel.failed(credit)) ?? creditGroup.at(0)!
+      .sort((a, b) => b.attainment_date.getTime() - a.attainment_date.getTime()) // desc
+      .find(credit => CreditModel.passed(credit)) ??
+    creditGroup.find(credit => CreditModel.failed(credit)) ??
+    creditGroup.at(0)!
 
-  const courseCode = singleCourse ? credit.course_code : mainCourseCode
-  const grade = singleCourse ? credit.grade : 'substituted'
-  const credits = singleCourse ? credit.credits : creditGroup.reduce((acc, credit) => acc + credit.credits, 0)
+  const courseCode = singleCourse ? attainment.course_code : mainCourseCode
+  const grade = singleCourse ? attainment.grade : 'substituted'
+  const credits = singleCourse ? attainment.credits : creditGroup.reduce((acc, credit) => acc + credit.credits, 0)
 
-  // Take the first attainment for the course (see Student statistics info-box)
-  // TODO: Substitution groups should be marked for the semester when
-  // the group was first finished eg. when was the first time the last missing
-  // course of the group was passed
   const {
     semester,
     attainment_date: attainmentDate,
     student_studentnumber: studentNumber,
     studyright_id: studyRightId,
-  } = creditGroup.at(-1)!
+  } = attainment
 
   const { yearcode: yearCode, yearname: yearName, semestercode: semesterCode, name: semesterName } = semester
 
@@ -79,7 +80,7 @@ const parseCredit = (
     studyRightElements.find(studyRightElement => studyRightElement.studyRightId === studyRightId) ??
     studyRightElements
       .filter(({ startDate, endDate }) => dateIsBetween(attainmentDate, startDate, endDate))
-      .sort((a, b) => b.startDate.getTime() - a.startDate.getTime())
+      .sort((a, b) => (b.endDate?.getTime() ?? 0) - (a.endDate?.getTime() ?? 0))
       .at(0) // The newest studyRightElement
 
   const programme = programmeOfCredit
@@ -99,7 +100,7 @@ const parseCredit = (
     attainmentDate,
     courseCode,
     grade,
-    passed: singleCourse ? !CreditModel.failed(credit) : true,
+    passed: singleCourse ? !CreditModel.failed(attainment) : true,
     studentNumber: anonymizationSalt ? anonymizeStudentNumber(studentNumber, anonymizationSalt) : studentNumber,
     programme,
     credits,
@@ -107,31 +108,44 @@ const parseCredit = (
 }
 
 type FormattedEnrollment = {
-  yearCode: number
-  yearName: string
-  semesterCode: number
-  semesterName: Name
   courseCode: string
   enrollmentDateTime: Date
+  programme: FormattedProgramme
   studentNumber: string
 }
 
-const parseEnrollment = (enrollment: Enrollment, anonymizationSalt: string | null): FormattedEnrollment => {
+const parseEnrollment = (
+  enrollment: Enrollment,
+  anonymizationSalt: string | null,
+  studyRightElements: Array<SISStudyRightElementModel>
+): FormattedEnrollment => {
   const {
     studentnumber: studentNumber,
-    semester,
     enrollment_date_time: enrollmentDateTime,
     course_code: courseCode,
+    studyright_id: studyRightId,
   } = enrollment
-  const { yearcode: yearCode, yearname: yearName, semestercode: semesterCode, name: semesterName } = semester
+
+  const programmeOfEnrollment: SISStudyRightElementModel | undefined =
+    studyRightElements.find(studyRightElement => studyRightElement.studyRightId === studyRightId) ??
+    studyRightElements
+      .filter(({ startDate, endDate }) => dateIsBetween(enrollmentDateTime, startDate, endDate))
+      .sort((a, b) => b.startDate?.getTime() - a.startDate?.getTime())
+      .at(0) // The newest studyRightElement
+
+  const programme = programmeOfEnrollment
+    ? formatStudyRightElement(programmeOfEnrollment)
+    : {
+        code: 'OTHER',
+        name: { en: 'Other', fi: 'Muu', sv: 'Andra' },
+        facultyCode: 'OTHER',
+        organization: { name: { en: 'Other', fi: 'Muu', sv: 'Andra' } },
+      }
 
   return {
-    yearCode,
-    yearName,
-    semesterCode,
-    semesterName,
     courseCode,
     enrollmentDateTime,
+    programme,
     studentNumber: anonymizationSalt ? anonymizeStudentNumber(studentNumber, anonymizationSalt) : studentNumber,
   }
 }
@@ -150,14 +164,29 @@ const getSubstitutionGroupDetails = async (codeGroups: string[][]) => {
   )
 }
 
+const getSemesterAndYearByDate = (
+  date: Date,
+  semesters: SemestersAndYears['semesters'],
+  years: SemestersAndYears['years']
+) => {
+  const semesterCode = getSemesterCodeAt(semesters, date) ?? getSemesterCodeAt(semesters, new Date())!
+
+  const semester = semesters[semesterCode]
+  const year = years[semester.yearcode]
+
+  return { year, semester }
+}
+
 const getYearlyStatsOfNew = async (
-  course: Pick<CourseModel, 'code' | 'name' | 'substitutions' | 'substitution_groups'> | null,
+  course: Pick<CourseModel, 'code' | 'name' | 'substitution_groups'> | null,
   courseCode: string,
   separate: boolean,
   unification: Unification,
   anonymizationSalt: string | null,
   combineSubstitutions: boolean,
-  studentNumberToSrElementsMap: Record<string, SISStudyRightElementModel[]>
+  studentNumberToSrElementsMap: Record<string, SISStudyRightElementModel[]>,
+  from: Date,
+  to: Date
 ) => {
   // Includes main course code and substitutions (if enabled)
   const creditGroupCodes =
@@ -179,9 +208,11 @@ const getYearlyStatsOfNew = async (
       break
   }
 
+  const { semesters, years } = await getSemestersAndYears()
+
   const [creditGroups, enrollmentGroups] = await Promise.all([
-    getCreditsForCourses(filteredCreditGroupCodes, unification),
-    getEnrollmentsForCourses(filteredCreditGroupCodes, unification),
+    getCreditsForCourses(filteredCreditGroupCodes, unification, from, to),
+    getEnrollmentsForCourses(filteredCreditGroupCodes, unification, from, to),
   ])
 
   const counter = new CourseYearlyStatsCounter()
@@ -219,6 +250,7 @@ const getYearlyStatsOfNew = async (
       programme.organization
     )
 
+    // Credits/attainments have quaranteed matching attainment_date and semesters/years
     const groupCode = separate ? semesterCode : yearCode
     const groupName = separate ? semesterName : yearName
     counter.markCreditToGroup(studentNumber, passed, grade, groupCode, groupName, creditCourseCode, yearCode)
@@ -231,16 +263,32 @@ const getYearlyStatsOfNew = async (
     for (const enrollment of enrollments) {
       const {
         studentNumber,
-        semesterCode,
-        semesterName,
-        yearCode,
-        yearName,
         courseCode: enrollmentCourseCode,
         enrollmentDateTime,
-      } = parseEnrollment(enrollment, anonymizationSalt)
+        programme,
+      } = parseEnrollment(
+        enrollment,
+        anonymizationSalt,
+        studentNumberToSrElementsMap[enrollment.studentnumber ?? 0] ?? []
+      )
 
-      const groupCode = separate ? semesterCode : yearCode
-      const groupName = separate ? semesterName : yearName
+      // Enrollments can have conflicting enrollment_date_time and semestercode, so we need to manually select
+      // semestercode matching enrollment_date_time
+      const { semester, year } = getSemesterAndYearByDate(enrollment.enrollment_date_time, semesters, years)
+
+      counter.markStudyProgramme(
+        studentNumber,
+        year.yearcode,
+        false, // passed
+        0, // credits
+        programme.code,
+        programme.name,
+        programme.facultyCode,
+        programme.organization
+      )
+
+      const groupCode = separate ? semester.semestercode : year.yearcode
+      const groupName = separate ? semester.name : year.yearname
 
       counter.markEnrollmentToGroup(
         studentNumber,
@@ -248,7 +296,7 @@ const getYearlyStatsOfNew = async (
         groupCode,
         groupName,
         enrollmentCourseCode,
-        yearCode
+        year.yearcode
       )
     }
   }
@@ -268,6 +316,11 @@ const getYearlyStatsOfNew = async (
   }
 }
 
+/**
+  Mostly-legacy function that prevents user from opening too big of a populations.
+  At the moment the threshold to prevent a population is so high it will never trigger.
+
+ */
 export const maxYearsToCreatePopulationFrom = async (courseCodes: string[], unification: Unification) => {
   const lastAttainmentDate = (await CourseModel.findOne({
     attributes: [[dbFn('MAX', dbCol('max_attainment_date')), 'date']],
@@ -279,9 +332,13 @@ export const maxYearsToCreatePopulationFrom = async (courseCodes: string[], unif
 
   if (lastAttainmentDate?.date == null) return 0
 
-  const newestAttainmentDate = lastAttainmentDate?.date.getFullYear()
-  const attainmentDateThreshold = new Date(newestAttainmentDate - 6, 0, 1)
+  /** Amount of years before latest attainment that the attainment count is calcucated for */
+  const yearRange = 6
 
+  const newestAttainmentYear = lastAttainmentDate?.date.getFullYear()
+  const attainmentDateThreshold = new Date(newestAttainmentYear - yearRange, 0, 1)
+
+  /** Amount of attainments between latest-attainment-year - yearRange and latest attainment date */
   const attainmentsWithinThreshold = await CreditModel.count({
     where: {
       course_code: { [Op.in]: courseCodes },
@@ -290,21 +347,30 @@ export const maxYearsToCreatePopulationFrom = async (courseCodes: string[], unif
     },
   })
 
-  // MAGIC NUMBER
-  const maxAllowedAttainments = 1000000 // * Lower this value to get a smaller result if necessary
-  return Math.max(1, maxAllowedAttainments / attainmentsWithinThreshold)
+  /** Limit the allowed attainments to 15_000 during the $yearRange years */
+  const maxAllowedAttainments = 15_000 * yearRange
+  return Math.round(Math.max(1, maxAllowedAttainments / attainmentsWithinThreshold))
 }
 
 export const getCourseYearlyStats = async (
   courseCodes: string[],
   separate: boolean,
   anonymizationSalt: string | null,
-  combineSubstitutions: boolean
+  combineSubstitutions: boolean,
+  fromYearCode = yearToYearCode(1950).toString(),
+  toYearCode: string = yearToYearCode(new Date().getFullYear()).toString()
 ) => {
+  // Default to 1900 - currentYear+1 so that without parameters the api returns stats for all years
+  const from = new Date(`${yearCodeToYear(fromYearCode)}-08-01`) // FALL
+  const to = new Date(`${yearCodeToYear(toYearCode) + 1}-07-31`) // SPRING next year
+
   const [credits, enrollments] = await Promise.all([
     CreditModel.findAll({
       attributes: ['student_studentnumber'],
-      where: { course_code: { [Op.in]: courseCodes } },
+      where: {
+        course_code: { [Op.in]: courseCodes },
+        attainment_date: { [Op.between]: [from, to] },
+      },
     }),
     EnrollmentModel.findAll({
       attributes: ['studentnumber'],
@@ -313,6 +379,12 @@ export const getCourseYearlyStats = async (
           [Op.in]: courseCodes,
         },
         state: EnrollmentState.ENROLLED,
+        enrollment_date_time: {
+          [Op.and]: {
+            [Op.between]: [from, to],
+            [Op.gte]: enrollmentTimeDateThreshold,
+          },
+        },
       },
     }),
   ])
@@ -331,15 +403,15 @@ export const getCourseYearlyStats = async (
 
   const statsRegular = await Promise.all(
     courseCodes.map(async courseCode => {
-      const course: Pick<CourseModel, 'code' | 'name' | 'substitutions' | 'substitution_groups'> | null =
-        await CourseModel.findOne({
-          attributes: ['code', 'name', 'substitutions', 'substitution_groups'],
-          where: { code: courseCode },
-        })
+      const course: Pick<CourseModel, 'code' | 'name' | 'substitution_groups'> | null = await CourseModel.findOne({
+        raw: true,
+        attributes: ['code', 'name', 'substitution_groups'],
+        where: { code: courseCode },
+      })
 
       if (!course) {
         logger.error('Course for course stats not found with code' + courseCode)
-        return []
+        return {}
       }
 
       const [openStats, regularStats, unifyStats] = await Promise.all([
@@ -350,7 +422,9 @@ export const getCourseYearlyStats = async (
           Unification.OPEN,
           anonymizationSalt,
           combineSubstitutions,
-          studentNumberToSrElementsMap
+          studentNumberToSrElementsMap,
+          from,
+          to
         ),
         getYearlyStatsOfNew(
           course,
@@ -359,7 +433,9 @@ export const getCourseYearlyStats = async (
           Unification.REGULAR,
           anonymizationSalt,
           combineSubstitutions,
-          studentNumberToSrElementsMap
+          studentNumberToSrElementsMap,
+          from,
+          to
         ),
         getYearlyStatsOfNew(
           course,
@@ -368,7 +444,9 @@ export const getCourseYearlyStats = async (
           Unification.UNIFY,
           anonymizationSalt,
           combineSubstitutions,
-          studentNumberToSrElementsMap
+          studentNumberToSrElementsMap,
+          from,
+          to
         ),
       ])
 
@@ -379,8 +457,8 @@ export const getCourseYearlyStats = async (
   return statsRegular
 }
 
-export const getCourseProvidersForCourses = async (codes: string[]) => {
-  return (
+export const getCourseProvidersForCourses = async (codes: string[]) =>
+  (
     await OrganizationModel.findAll({
       attributes: ['code'],
       include: {
@@ -394,7 +472,6 @@ export const getCourseProvidersForCourses = async (codes: string[]) => {
       raw: true,
     })
   ).map(({ code }) => code)
-}
 
 export const getCourseDetails = async (codes: string[]) =>
   CourseModel.findAll({

services/backend/src/services/faculty/facultyBasics.ts

@@ -16,7 +16,7 @@ const calculateCombinedStats = async (
     graphStats: [] as BasicStats['graphStats'],
     programmeTableStats: {} as Record<string, BasicStats['tableStats']>,
     tableStats: [] as BasicStats['tableStats'],
-    titles: ['Code', 'Started\nstudying', 'Accepted', 'Graduated'],
+    titles: ['Code', 'Started\nstudying', 'Accepted', 'Graduated', 'Cancelled'],
   }
   if (includeAllSpecials) {
     studentInfo.titles.push('Transferred out\nof programme', 'Transferred into\nprogramme')
@@ -103,7 +103,8 @@ const calculateCombinedStats = async (
     } else {
       for (let i = 0; i < tableStatsForProgramme.length; i++) {
         for (let j = 1; j < tableStatsForProgramme[i].length; j++) {
-          ;(combinedTableStats[i][j] as number) += tableStatsForProgramme[i][j]
+          // @ts-expect-error the only string is in index 0 (the year label) which we skip over
+          combinedTableStats[i][j] += tableStatsForProgramme[i][j]
         }
       }
     }
@@ -126,6 +127,7 @@ const calculateCombinedStats = async (
     } else {
       for (let i = 0; i < stats.tableStats.length; i++) {
         for (let j = 1; j < stats.tableStats[i].length; j++) {
+          // @ts-expect-error the only string is in index 0 (the year label) which we skip over
           studentInfo.tableStats[i][j] += stats.tableStats[i][j]
         }
       }

services/backend/src/services/faculty/facultyGraduationTimes.ts

@@ -1,36 +1,77 @@
-import { cloneDeep } from 'lodash-es'
-
-import { Name, DegreeProgrammeType, Unarray } from '@oodikone/shared/types'
-import { ProgrammeGraduationStats } from '@oodikone/shared/types/studyProgramme'
+import { Name, DegreeProgrammeType, FacultyGraduationStatistics, ClassSizes } from '@oodikone/shared/types'
+import type {
+  GraduationStatistics,
+  GraduationStats,
+  MediansByCategory,
+  MediansByProgrammes,
+  ProgrammeGraduationStats,
+  ProgrammeMedians,
+} from '@oodikone/shared/types'
 import { omitKeys } from '@oodikone/shared/util'
 import { getGraduationStats, getStudyTrackStats, setGraduationStats, setStudyTrackStats } from '../analyticsService'
 import { GraduationTarget } from '../graduationHelpers'
-import { getGraduationStatsForStudyTrack, GraduationTimes } from '../studyProgramme/studyProgrammeGraduations'
-import { getMedian } from '../studyProgramme/studyProgrammeHelpers'
+import { getGraduationStatsForStudyTrack } from '../studyProgramme/studyProgrammeGraduations'
+import { getAverage, getMedian } from '../studyProgramme/studyProgrammeHelpers'
 import { getStudyRightsInProgramme } from '../studyProgramme/studyRightFinders'
 import { getStudyTrackStatsForStudyProgramme } from '../studyProgramme/studyTrackStats'
 import type { ProgrammesOfOrganization } from './faculty'
 
-export type LevelGraduationStats = Omit<Unarray<GraduationTimes['medians']>, 'y'> & { median: number }
-
-type ProgrammeStats = {
-  data: Array<Omit<LevelGraduationStats, 'times'> & { code: string }>
-  programmes: string[]
-}
-
 export const programmeTypes = {
   [DegreeProgrammeType.BACHELOR]: 'bachelor',
   [DegreeProgrammeType.MASTER]: 'master',
   [DegreeProgrammeType.DOCTOR]: 'doctor',
 } as const
 
-const getStatsByGraduationYear = async (facultyProgrammes: ProgrammesOfOrganization) => {
-  const newStats: ProgrammeGraduationStats[] = []
-  const medians: Record<string, LevelGraduationStats[]> = {}
-  const programmes: { medians: Record<string, Record<string, ProgrammeStats>> } = { medians: {} }
+const setYearlyBreakdown = (code: string, name: string, level: ProgrammeMedians, year: GraduationStats) => {
+  if (!year.times.length) return
+
+  level[year.name] ??= { data: [], programmes: [] }
+  level[year.name].programmes.push(code)
+  level[year.name].data.push(omitKeys({ ...structuredClone(year), name, code }, ['classSize']))
+}
+
+const mergeStats = (
+  medians: MediansByCategory,
+  level: keyof MediansByCategory,
+  statsForYear: GraduationStats,
+  classSizes?: ClassSizes
+) => {
+  if (classSizes) {
+    classSizes[level][statsForYear.name] ??= 0
+    classSizes[level][statsForYear.name] += statsForYear.classSize ?? 0
+  }
+
+  const gradStatsForYear = medians[level].find(entry => entry.name === statsForYear.name)
+
+  if (!gradStatsForYear) {
+    medians[level].push(structuredClone(statsForYear))
+  } else {
+    gradStatsForYear.times.push(...statsForYear.times)
+    gradStatsForYear.median = getMedian(gradStatsForYear.times)
+    gradStatsForYear.average = getAverage(gradStatsForYear.times)
+    gradStatsForYear.amount += statsForYear.amount
+    gradStatsForYear.statistics.onTime += statsForYear.statistics.onTime
+    gradStatsForYear.statistics.yearOver += statsForYear.statistics.yearOver
+    gradStatsForYear.statistics.wayOver += statsForYear.statistics.wayOver
+  }
+}
+
+const getStatsByGraduationYear = async (
+  facultyProgrammes: ProgrammesOfOrganization
+): Promise<GraduationStatistics['byGradYear']> => {
+  const newStats: Pick<ProgrammeGraduationStats, 'id' | 'graduationTimes' | 'comboTimes'>[] = []
+  const medians: MediansByCategory = { bachelor: [], master: [], bcMsCombo: [], doctor: [] }
+  const programmes: { medians: MediansByProgrammes } = {
+    medians: {
+      bachelor: {},
+      master: {},
+      bcMsCombo: {},
+      doctor: {},
+    },
+  }
 
   for (const programme of facultyProgrammes) {
-    if (programme.degreeProgrammeType == null || !(programme.degreeProgrammeType in programmeTypes)) {
+    if (!programme.degreeProgrammeType || !(programme.degreeProgrammeType in programmeTypes)) {
       continue
     }
     const statsFromRedis = await getGraduationStats(programme.code, '', 'CALENDAR_YEAR', 'SPECIAL_INCLUDED')
@@ -51,92 +92,44 @@ const getStatsByGraduationYear = async (facultyProgrammes: ProgrammesOfOrganizat
   }
 
   for (const { id: programmeCode, graduationTimes, comboTimes } of newStats) {
-    const { degreeProgrammeType, progId, code } = facultyProgrammes.find(({ code }) => code === programmeCode)!
+    const { degreeProgrammeType, progId } = facultyProgrammes.find(({ code }) => code === programmeCode)!
     const level = programmeTypes[degreeProgrammeType as keyof typeof programmeTypes]
 
-    if (!medians[level]) {
-      medians[level] = cloneDeep(graduationTimes.medians).map(year => ({ ...omitKeys(year, ['y']), median: year.y }))
-    } else {
-      for (const statsForYear of graduationTimes.medians) {
-        const correctYear = medians[level].find(entry => entry.name === statsForYear.name)
-        if (!correctYear) {
-          medians[level].push({ ...omitKeys(cloneDeep(statsForYear), ['y']), median: statsForYear.y })
-          continue
-        }
-        correctYear.times.push(...statsForYear.times)
-        correctYear.median = getMedian(correctYear.times)
-        correctYear.amount += statsForYear.amount
-        correctYear.statistics.onTime += statsForYear.statistics.onTime
-        correctYear.statistics.yearOver += statsForYear.statistics.yearOver
-        correctYear.statistics.wayOver += statsForYear.statistics.wayOver
-      }
-    }
-
-    if (level === 'master' && comboTimes.medians.length) {
-      if (!medians.bcMsCombo) {
-        medians.bcMsCombo = cloneDeep(comboTimes.medians).map(year => ({ ...omitKeys(year, ['y']), median: year.y }))
-      } else {
-        for (const statsForYear of comboTimes.medians) {
-          const correctYear = medians.bcMsCombo.find(entry => entry.name === statsForYear.name)
-          if (!correctYear) {
-            medians.bcMsCombo.push({ ...omitKeys(cloneDeep(statsForYear), ['y']), median: statsForYear.y })
-            continue
-          }
-          correctYear.times.push(...statsForYear.times)
-          correctYear.median = getMedian(correctYear.times)
-          correctYear.amount += statsForYear.amount
-          correctYear.statistics.onTime += statsForYear.statistics.onTime
-          correctYear.statistics.yearOver += statsForYear.statistics.yearOver
-          correctYear.statistics.wayOver += statsForYear.statistics.wayOver
-        }
-      }
-    }
-
-    if (!programmes.medians[level]) {
-      programmes.medians[level] = {}
-    }
-
     for (const year of graduationTimes.medians) {
-      if (year.y === 0) continue
-      if (!programmes.medians[level][year.name]) {
-        programmes.medians[level][year.name] = { data: [], programmes: [] }
-      }
-      programmes.medians[level][year.name].data.push(
-        omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y'])
-      )
-      programmes.medians[level][year.name].programmes.push(code)
+      mergeStats(medians, level, year)
+      setYearlyBreakdown(programmeCode, progId, programmes.medians[level], year)
     }
 
     if (level === 'master' && comboTimes.medians.length) {
-      if (!programmes.medians.bcMsCombo) {
-        programmes.medians.bcMsCombo = {}
-      }
       for (const year of comboTimes.medians) {
-        if (year.y === 0) continue
-        if (!programmes.medians.bcMsCombo[year.name]) {
-          programmes.medians.bcMsCombo[year.name] = { data: [], programmes: [] }
-        }
-        programmes.medians.bcMsCombo[year.name].data.push(
-          omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y'])
-        )
-        programmes.medians.bcMsCombo[year.name].programmes.push(code)
+        mergeStats(medians, 'bcMsCombo', year)
+        setYearlyBreakdown(programmeCode, progId, programmes.medians.bcMsCombo, year)
       }
     }
   }
   return { medians, programmes }
 }
 
-const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization) => {
-  const newStats: Array<Awaited<ReturnType<typeof getStudyTrackStatsForStudyProgramme>>> = []
-  const medians: Record<string, LevelGraduationStats[]> = {}
-  const programmes: { medians: Record<string, Record<string, ProgrammeStats>> } = { medians: {} }
-  const classSizes: Record<string, Record<string, number | Record<string, number>>> = { programmes: {} }
+const getStatsByStartYear = async (
+  facultyProgrammes: ProgrammesOfOrganization
+): Promise<GraduationStatistics['byStartYear'] & Pick<GraduationStatistics, 'classSizes'>> => {
+  const classSizes: ClassSizes = { bachelor: {}, master: {}, bcMsCombo: {}, doctor: {}, programmes: {} }
+  const newStats: Awaited<ReturnType<typeof getStudyTrackStatsForStudyProgramme>>[] = []
+  const medians: MediansByCategory = { bachelor: [], master: [], bcMsCombo: [], doctor: [] }
+  const programmes: { medians: MediansByProgrammes } = {
+    medians: {
+      bachelor: {},
+      master: {},
+      bcMsCombo: {},
+      doctor: {},
+    },
+  }
 
   for (const programme of facultyProgrammes) {
-    if (programme.degreeProgrammeType == null || !(programme.degreeProgrammeType in programmeTypes)) {
+    if (!programme.degreeProgrammeType || !(programme.degreeProgrammeType in programmeTypes)) {
       continue
     }
-    const statsFromRedis = await getStudyTrackStats(programme.code, '', 'SPECIAL_EXCLUDED')
+    const statsFromRedis = await getStudyTrackStats(programme.code, null, 'SPECIAL_EXCLUDED')
     if (statsFromRedis) {
       newStats.push(statsFromRedis)
       continue
@@ -156,9 +149,13 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
   }
 
   for (const { id: programmeCode, graduationTimes } of newStats) {
-    const { degreeProgrammeType, progId, code } = facultyProgrammes.find(({ code }) => code === programmeCode)!
+    const { degreeProgrammeType, progId } = facultyProgrammes.find(({ code }) => code === programmeCode)!
     const level = programmeTypes[degreeProgrammeType as keyof typeof programmeTypes]
-    if (!(programmeCode in graduationTimes)) continue
+
+    if (!(programmeCode in graduationTimes)) {
+      continue
+    }
+
     const statsForProgramme = graduationTimes[programmeCode]
     const basicStats = statsForProgramme.medians.basic
     const comboStats = statsForProgramme.medians.combo
@@ -166,91 +163,31 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
       (acc, year) => ({ ...acc, [year.name]: year.classSize ?? 0 }),
       {}
     )
-    classSizes.programmes[code] = classSizesByYear
+    classSizes.programmes[programmeCode] = classSizesByYear
 
-    if (!medians[level]) {
-      medians[level] = cloneDeep(basicStats).map(year => ({ ...omitKeys(year, ['y', 'classSize']), median: year.y }))
-    }
-    classSizes[level] ??= {}
     for (const statsForYear of basicStats) {
-      classSizes[level][statsForYear.name] ??= 0
-      ;(classSizes[level][statsForYear.name] as number) += statsForYear.classSize ?? 0
-
-      const correctYear = medians[level].find(entry => entry.name === statsForYear.name)
-      if (!correctYear) {
-        medians[level].push({ ...omitKeys(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
-        continue
-      }
-      correctYear.times.push(...statsForYear.times)
-      correctYear.median = getMedian(correctYear.times)
-      correctYear.amount += statsForYear.amount
-      correctYear.statistics.onTime += statsForYear.statistics.onTime
-      correctYear.statistics.yearOver += statsForYear.statistics.yearOver
-      correctYear.statistics.wayOver += statsForYear.statistics.wayOver
+      mergeStats(medians, level, statsForYear, classSizes)
+      setYearlyBreakdown(programmeCode, progId, programmes.medians[level], statsForYear)
     }
 
     if (level === 'master' && comboStats.length) {
-      if (!medians.bcMsCombo) {
-        medians.bcMsCombo = cloneDeep(comboStats).map(year => ({
-          ...omitKeys(year, ['y', 'classSize']),
-          median: year.y,
-        }))
-      }
-      classSizes.bcMsCombo ??= {}
       for (const statsForYear of comboStats) {
-        classSizes.bcMsCombo[statsForYear.name] ??= 0
-        ;(classSizes.bcMsCombo[statsForYear.name] as number) += statsForYear.classSize ?? 0
-        const correctYear = medians.bcMsCombo.find(entry => entry.name === statsForYear.name)
-        if (!correctYear) {
-          medians.bcMsCombo.push({ ...omitKeys(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
-          continue
-        }
-        correctYear.times.push(...statsForYear.times)
-        correctYear.median = getMedian(correctYear.times)
-        correctYear.amount += statsForYear.amount
-        correctYear.statistics.onTime += statsForYear.statistics.onTime
-        correctYear.statistics.yearOver += statsForYear.statistics.yearOver
-        correctYear.statistics.wayOver += statsForYear.statistics.wayOver
-      }
-    }
-
-    if (!programmes.medians[level]) {
-      programmes.medians[level] = {}
-    }
-
-    for (const year of basicStats) {
-      if (year.y === 0) continue
-      if (!programmes.medians[level][year.name]) {
-        programmes.medians[level][year.name] = { data: [], programmes: [] }
-      }
-      programmes.medians[level][year.name].data.push(
-        omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y', 'classSize'])
-      )
-      programmes.medians[level][year.name].programmes.push(code)
-    }
-
-    if (level === 'master' && comboStats.length) {
-      if (!programmes.medians.bcMsCombo) {
-        programmes.medians.bcMsCombo = {}
-      }
-      for (const year of comboStats) {
-        if (year.y === 0) continue
-        if (!programmes.medians.bcMsCombo[year.name]) {
-          programmes.medians.bcMsCombo[year.name] = { data: [], programmes: [] }
-        }
-        programmes.medians.bcMsCombo[year.name].data.push(
-          omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y', 'classSize'])
-        )
-        programmes.medians.bcMsCombo[year.name].programmes.push(code)
+        mergeStats(medians, 'bcMsCombo', statsForYear, classSizes)
+        setYearlyBreakdown(programmeCode, progId, programmes.medians.bcMsCombo, statsForYear)
       }
     }
   }
+
   return { medians, programmes, classSizes }
 }
 
-export const countGraduationTimes = async (faculty: string, programmesOfFaculty: ProgrammesOfOrganization) => {
+export const countGraduationTimes = async (
+  facultyCode: string,
+  programmesOfFaculty: ProgrammesOfOrganization
+): Promise<FacultyGraduationStatistics> => {
   const { medians: mediansByGraduationYear, programmes: programmesByGraduationYear } =
     await getStatsByGraduationYear(programmesOfFaculty)
+
   const {
     medians: mediansByStartYear,
     programmes: programmesByStartYear,
@@ -259,7 +196,7 @@ export const countGraduationTimes = async (faculty: string, programmesOfFaculty:
 
   const goals = {
     bachelor: GraduationTarget.THREE_YEARS,
-    bcMsCombo: faculty === 'H90' ? GraduationTarget.SIX_YEARS : GraduationTarget.FIVE_YEARS,
+    bcMsCombo: facultyCode === 'H90' ? GraduationTarget.SIX_YEARS : GraduationTarget.FIVE_YEARS,
     master: GraduationTarget.TWO_YEARS,
     doctor: GraduationTarget.FOUR_YEARS,
     exceptions: {
@@ -283,7 +220,7 @@ export const countGraduationTimes = async (faculty: string, programmesOfFaculty:
   }, {})
 
   return {
-    id: faculty,
+    id: facultyCode,
     goals,
     byGradYear: {
       medians: mediansByGraduationYear,

services/backend/src/services/faculty/facultyService.ts

@@ -1,4 +1,4 @@
-import { Name, ProgrammeFilter, SpecialGroups, YearType } from '@oodikone/shared/types'
+import { FacultyGraduationStatistics, Name, ProgrammeFilter, SpecialGroups, YearType } from '@oodikone/shared/types'
 import { DegreeProgramme } from '../../routes/faculties'
 import { redisClient } from '../redis'
 import { FacultyProgressData } from './facultyStudentProgress'
@@ -29,16 +29,16 @@ const createRedisKeyForFacultyStudents = (id: string, specialGroups: SpecialGrou
   return `FACULTY_STUDENTS_STATS_${id}_${specialGroups}`
 }
 
-/*
+/**
   Faculty data objects have graduation times left in as arrays, so that
-  university-level evaluation overview can count median-times by itself.
+  university-level evaluation overview can count median-times by itself (in backend).
   Faculties, however, don't need this, and it isn't needed in frontend
-  for universityview either. This removes them from the object.
+  for university view either. This clears the array.
 */
-const removeGraduationTimes = (data: GraduationData) => {
+const removeGraduationTimes = (data: FacultyGraduationStatistics) => {
   Object.values(data.byGradYear.medians).forEach(array =>
     array.forEach(yearStat => {
-      yearStat.times = null
+      yearStat.times = []
     })
   )
 }
@@ -158,7 +158,7 @@ export type GraduationData = {
   classSizes: Record<string, Record<string, number | Record<string, number>>>
 }
 
-export const setGraduationStats = async (data: GraduationData, programmeFilter: ProgrammeFilter) => {
+export const setGraduationStats = async (data: FacultyGraduationStatistics, programmeFilter: ProgrammeFilter) => {
   const { id } = data
   const redisKey = createRedisKeyForGraduationTimeStats(id, programmeFilter)
   await redisClient.set(redisKey, JSON.stringify(data))
@@ -170,7 +170,7 @@ export const getGraduationStats = async (id: string, programmeFilter: ProgrammeF
   if (!dataFromRedis) {
     return null
   }
-  const data = JSON.parse(dataFromRedis) as GraduationData
+  const data = JSON.parse(dataFromRedis) as FacultyGraduationStatistics
   if (!keepGraduationTimes) {
     removeGraduationTimes(data)
   }

services/backend/src/services/faculty/facultyStudentProgress.ts

@@ -112,11 +112,7 @@ export const combineFacultyStudentProgress = async (
 
   for (const { code: studyProgramme } of programmes) {
     const programmeInfo = allDegreeProgrammes.find(programme => programme.code === studyProgramme)
-    if (
-      !programmeInfo ||
-      programmeInfo.degreeProgrammeType == null ||
-      !(programmeInfo.degreeProgrammeType in programmeTypes)
-    ) {
+    if (!programmeInfo?.degreeProgrammeType || !(programmeInfo.degreeProgrammeType in programmeTypes)) {
       continue
     }
     const statsFromRedis = await getStudyTrackStats(studyProgramme, null, specialGroups)

services/backend/src/services/faculty/facultyStudents.ts

@@ -17,9 +17,6 @@ const calculateCombinedStats = (programmeCodes: string[], stats: StudyTrackStats
       programmeCode
     ]
 
-    const programmeOtherCountriesCount = stats.find(programmeStats => programmeStats.id === programmeCode)
-      ?.otherCountriesCount?.[programmeCode]
-
     if (!statsForProgramme) continue
 
     for (const yearStats of statsForProgramme) {
@@ -30,7 +27,8 @@ const calculateCombinedStats = (programmeCodes: string[], stats: StudyTrackStats
         if (facultyTableStats[yearStats[0]][i] == null) {
           facultyTableStats[yearStats[0]].push(yearStats[i])
         } else if (typeof facultyTableStats[yearStats[0]][i] === 'number') {
-          ;(facultyTableStats[yearStats[0]][i] as number) += yearStats[i] as number
+          // @ts-expect-error values at given indices are numbers
+          facultyTableStats[yearStats[0]][i] += yearStats[i]
         } else {
           const allStudentsCount = facultyTableStats[yearStats[0]][1]
           const categoryStudentsCount = facultyTableStats[yearStats[0]][i - 1]
@@ -46,11 +44,12 @@ const calculateCombinedStats = (programmeCodes: string[], stats: StudyTrackStats
       }
     }
 
+    const programmeOtherCountriesCount = stats.find(programmeStats => programmeStats.id === programmeCode)
+      ?.otherCountriesCount?.[programmeCode]
+
     if (!programmeOtherCountriesCount) continue
     for (const [year, countryStats] of Object.entries(programmeOtherCountriesCount)) {
-      if (!(year in facultyTableStatsExtra)) {
-        facultyTableStatsExtra[year] = {}
-      }
+      facultyTableStatsExtra[year] ??= {}
       facultyTableStatsExtra[year][programmeCode] = countryStats
     }
   }

services/backend/src/services/graduationHelpers.ts

@@ -8,7 +8,7 @@ export const GraduationTarget = {
   FOUR_YEARS: 8,
   FIVE_YEARS: 10,
   SIX_YEARS: 12,
-} as const
+}
 
 /** Uses semesters, not months */
 export const countTimeCategories = (times: number[], target: number) => {

services/backend/src/services/languageCenterData.ts

@@ -1,16 +1,18 @@
 import { orderBy } from 'lodash-es'
 import { Op } from 'sequelize'
 
+import { EnrollmentState } from '@oodikone/shared/types'
 import { CourseModel, CreditModel, EnrollmentModel, SISStudyRightModel } from '../models'
 import { redisClient } from './redis'
 
 export const LANGUAGE_CENTER_REDIS_KEY = 'LANGUAGE_CENTER_DATA'
 
-const isBetween = (start, date, end) => {
+const isBetween = (start: Date, date: Date, end: Date) => {
   return new Date(start).getTime() <= new Date(date).getTime() && new Date(date).getTime() <= new Date(end).getTime()
 }
 
-const findStudyRight = (studyRights, date) => {
+/** Assumes studyrights are in correct order */
+const findStudyRight = (studyRights: SISStudyRightModel[], date: Date) => {
   return studyRights?.find(studyRight => isBetween(studyRight.startDate, date, studyRight.endDate))
 }
 
@@ -23,7 +25,7 @@ const getDifference = stats => {
 }
 
 const getLanguageCenterCourses = async () => {
-  const courses = await CourseModel.findAll({
+  const courses: Pick<CourseModel, 'code' | 'name'>[] = await CourseModel.findAll({
     attributes: ['code', 'name'],
     where: {
       [Op.or]: [{ code: { [Op.like]: 'KK%' } }, { code: { [Op.like]: 'AYKK%' } }],
@@ -33,29 +35,37 @@ const getLanguageCenterCourses = async () => {
   return courses
 }
 
-export const createArrayOfCourses = async (attempts, courses) => {
+export const createArrayOfCourses = (attempts: Attempt[], courses: Pick<CourseModel, 'code' | 'name'>[]) => {
   const fields = { completions: 0, enrollments: 0, difference: 0, rejected: 0 }
-  const semesters = {}
-  const faculties = {}
-  const semesterStatsMap = attempts.reduce((obj, cur) => {
+  const semesters: Record<Attempt['semestercode'], boolean> = {}
+  // Remove undefined as a possibility for type
+  const faculties: Record<Exclude<Attempt['faculty'], undefined>, boolean> = {}
+
+  // TODO: Type this properly. Good luck :)
+  const semesterStatsMap = attempts.reduce((acc, cur) => {
     const semester = cur.semestercode
-    faculties[cur.faculty] = true
-    semesters[semester] = true
-    if (!obj[cur.courseCode]) {
-      obj[cur.courseCode] = {}
-    }
-    if (!obj[cur.courseCode][semester]) {
-      obj[cur.courseCode][semester] = { ...fields }
-    }
-    if (cur.faculty && !obj[cur.courseCode][semester][cur.faculty]) {
-      obj[cur.courseCode][semester][cur.faculty] = { ...fields }
-    }
-    const allFacultiesTotal = obj[cur.courseCode][semester]
-    const allStats = [allFacultiesTotal]
     if (cur.faculty) {
-      const semesterFacultyStats = obj[cur.courseCode][semester][cur.faculty]
+      faculties[cur.faculty] = true
+    }
+    semesters[semester] = true
+
+    acc[cur.courseCode] ??= {}
+    // NOTE: This has to be spread because otherwise a ref to fields is saved to
+    // acc[cur.courseCode][semester] which would in the end result in a massive circular object
+    acc[cur.courseCode][semester] ??= { ...fields }
+
+    if (cur.faculty) {
+      acc[cur.courseCode][semester][cur.faculty] ??= { ...fields }
+    }
+
+    const allFacultiesTotal = acc[cur.courseCode][semester]
+    const allStats = [allFacultiesTotal]
+
+    if (cur.faculty) {
+      const semesterFacultyStats = acc[cur.courseCode][semester][cur.faculty]
       allStats.push(semesterFacultyStats)
     }
+
     for (const stats of allStats) {
       if (cur.completed) {
         stats.completions += 1
@@ -65,7 +75,7 @@ export const createArrayOfCourses = async (attempts, courses) => {
         stats.rejected += 1
       }
     }
-    return obj
+    return acc
   }, {})
 
   const courseList = courses
@@ -88,55 +98,60 @@ export const createArrayOfCourses = async (attempts, courses) => {
   return courseList
 }
 
+type Attempt = {
+  studentNumber: string
+  courseCode: string
+  completed: boolean
+  date: Date
+  semestercode: number
+  faculty?: string
+  enrolled?: boolean
+}
+
 export const computeLanguageCenterData = async () => {
   const courses = await getLanguageCenterCourses()
   const autumnSemester2017 = 135
 
   const credits = await CreditModel.findAll({
+    raw: true,
     attributes: ['course_code', 'student_studentnumber', 'semestercode', 'attainment_date', 'studyright_id'],
     where: {
       [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
       semestercode: { [Op.gte]: autumnSemester2017 },
       credittypecode: 4,
     },
-    raw: true,
   })
 
   const enrollments = await EnrollmentModel.findAll({
+    raw: true,
     attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'studyright_id', 'state'],
     where: {
       [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
-      semestercode: { [Op.gte]: autumnSemester2017 },
       state: { [Op.in]: ['ENROLLED', 'REJECTED'] },
+      // EnrollmentDateTimeThreshold need not be used here as we are not counting failed courses
     },
-    raw: true,
   })
 
-  const studyRights = await SISStudyRightModel.findAll({ raw: true })
+  const studyRights = await SISStudyRightModel.findAll({
+    raw: true,
+    order: [['end_date', 'DESC NULLS LAST']],
+  })
 
-  const attemptStudyRightToFacultyMap = studyRights.reduce((obj, cur) => {
-    if (!cur.id) return obj
-    obj[cur.id] = cur.facultyCode
-    return obj
+  const attemptStudyRightToFacultyMap = studyRights.reduce<
+    Record<SISStudyRightModel['id'], SISStudyRightModel['facultyCode']>
+  >((acc, cur) => {
+    if (!cur.id) return acc
+    acc[cur.id] = cur.facultyCode
+    return acc
   }, {})
 
-  credits.forEach(credit => {
-    credit.faculty = attemptStudyRightToFacultyMap[credit.studyright_id]
-  })
-
-  enrollments.forEach(enrollment => {
-    enrollment.faculty = attemptStudyRightToFacultyMap[enrollment.studyright_id]
-  })
-
-  const studentNumbers = new Set()
-  const attemptsByStudents = {}
+  const studentNumbers = new Set<string>()
+  const attemptsByStudents: Record<string, Attempt[]> = {}
 
   credits.forEach(credit => {
     const studentNumber = credit.student_studentnumber
     studentNumbers.add(studentNumber)
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
+    attemptsByStudents[studentNumber] ??= []
     attemptsByStudents[studentNumber].push({
       studentNumber,
       courseCode: credit.course_code,
@@ -149,10 +164,8 @@ export const computeLanguageCenterData = async () => {
 
   enrollments.forEach(enrollment => {
     const studentNumber = enrollment.studentnumber
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
     studentNumbers.add(studentNumber)
+    attemptsByStudents[studentNumber] ??= []
     if (
       attemptsByStudents[studentNumber].find(
         attempt =>
@@ -163,6 +176,7 @@ export const computeLanguageCenterData = async () => {
     ) {
       return
     }
+
     attemptsByStudents[studentNumber].push({
       studentNumber,
       courseCode: enrollment.course_code,
@@ -170,23 +184,25 @@ export const computeLanguageCenterData = async () => {
       date: enrollment.enrollment_date_time,
       faculty: attemptStudyRightToFacultyMap[enrollment.studyright_id],
       semestercode: enrollment.semestercode,
-      enrolled: enrollment.state === 'ENROLLED',
+      enrolled: enrollment.state === EnrollmentState.ENROLLED,
     })
   })
 
-  const studentNumberToStudyRightsMap = studyRights.reduce((obj, cur) => {
-    if (!obj[cur.studentNumber]) obj[cur.studentNumber] = []
-    obj[cur.studentNumber].push(cur)
-    return obj
+  const studentNumberToStudyRightsMap = studyRights.reduce<
+    Record<SISStudyRightModel['studentNumber'], SISStudyRightModel[]>
+  >((acc, cur) => {
+    acc[cur.studentNumber] ??= []
+    acc[cur.studentNumber].push(cur)
+    return acc
   }, {})
 
-  const attempts = []
+  const attempts: Attempt[] = []
   studentNumbers.forEach(studentNumber => attempts.push(...attemptsByStudents[studentNumber]))
   // 93033 Avoin yliopisto, yhteistyöoppilaitokset
   // 93034 Kesäyliopistot
-  const isOpenUni = facultyCode => ['9301', 'H930', '93033', '93034'].includes(facultyCode)
-  const isMisc = facultyCode => ['H906', 'H401'].includes(facultyCode) // Language center, Alexander institute (very few numbers)
-  const isHyFaculty = facultyCode => facultyCode.substring(0, 3).match('^H\\d')
+  const isOpenUni = (facultyCode: string) => ['9301', 'H930', '93033', '93034'].includes(facultyCode)
+  const isMisc = (facultyCode: string) => ['H906', 'H401'].includes(facultyCode) // Language center, Alexander institute (very few numbers)
+  const isHyFaculty = (facultyCode: string) => /^H\d/.exec(facultyCode.substring(0, 3))
 
   attempts.forEach(attempt => {
     if (attempt.faculty) {
@@ -210,7 +226,7 @@ export const computeLanguageCenterData = async () => {
     }
   })
 
-  const unorderedTableData = await createArrayOfCourses(attempts, courses)
+  const unorderedTableData = createArrayOfCourses(attempts, courses)
   const tableData = orderBy(unorderedTableData, 'code')
 
   const faculties = [...new Set(attempts.map(({ faculty }) => faculty))]

services/backend/src/services/openUni/format.ts

@@ -1,33 +0,0 @@
-import { CourseModel, CreditModel, EnrollmentModel, StudentModel } from '../../models'
-
-export const formatCourseInfo = (course: CourseModel) => {
-  return {
-    label: course.code,
-    name: course.name,
-  }
-}
-
-export const formatOpenCredits = (credit: CreditModel) => {
-  return {
-    attainmentCourseCode: credit.course_code,
-    attainmentDate: credit.attainment_date,
-    attainmentGrade: credit.grade,
-    attainmentStudentNumber: credit.student_studentnumber,
-  }
-}
-
-export const formatOpenEnrollments = (enrollment: EnrollmentModel) => {
-  return {
-    enrollmentCourseCode: enrollment.course_code,
-    enrollmentDateTime: enrollment.enrollment_date_time,
-    enrollmentStudentNumber: enrollment.studentnumber,
-  }
-}
-
-export const formatStudentInfo = (student: StudentModel) => {
-  return {
-    studentNumber: student.studentnumber,
-    email: student.email,
-    secondaryEmail: student.secondary_email,
-  }
-}

services/backend/src/services/openUni/openUniSearches.ts

@@ -1,94 +1,4 @@
-import { Op, WhereOptions } from 'sequelize'
-
-import { SISStudyRight } from '@oodikone/shared/models'
-import { EnrollmentState, ExtentCode } from '@oodikone/shared/types'
-import { CourseModel, CreditModel, EnrollmentModel, StudentModel, SISStudyRightModel } from '../../models'
 import { OpenUniPopulationSearchModel } from '../../models/kone'
-import { formatCourseInfo, formatOpenCredits, formatOpenEnrollments, formatStudentInfo } from './format'
-
-export const getCredits = async (courseCodes: string[], startdate: Date) =>
-  (
-    await CreditModel.findAll({
-      attributes: ['attainment_date', 'course_code', 'grade', 'student_studentnumber'],
-      where: {
-        course_code: {
-          [Op.in]: courseCodes,
-        },
-        attainment_date: {
-          [Op.gte]: startdate,
-        },
-      },
-    })
-  ).map(formatOpenCredits)
-
-export const getStudentInfo = async (studentNumbers: string[]) =>
-  (
-    await StudentModel.findAll({
-      attributes: ['studentnumber', 'email', 'secondary_email'],
-      where: {
-        studentnumber: {
-          [Op.in]: studentNumbers,
-        },
-      },
-    })
-  ).map(formatStudentInfo)
-
-export const getEnrollments = async (courseCodes: string[], startDate: Date, endDate: Date) =>
-  (
-    await EnrollmentModel.findAll({
-      attributes: ['course_code', 'enrollment_date_time', 'studentnumber'],
-      where: {
-        course_code: {
-          [Op.in]: courseCodes,
-        },
-        is_open: true,
-        enrollment_date_time: {
-          [Op.and]: {
-            [Op.lte]: endDate,
-            [Op.gte]: startDate,
-          },
-        },
-        state: EnrollmentState.ENROLLED,
-      },
-    })
-  ).map(formatOpenEnrollments)
-
-export const getCourseNames = async (courseCodes: string[]) =>
-  (
-    await CourseModel.findAll({
-      attributes: ['code', 'name'],
-      where: {
-        code: {
-          [Op.in]: courseCodes,
-        },
-      },
-    })
-  ).map(formatCourseInfo)
-
-export const getStudyRights = async (studentNumbers: string[]) => {
-  const where: WhereOptions<SISStudyRight> = {
-    extentCode: {
-      [Op.in]: [
-        ExtentCode.BACHELOR,
-        ExtentCode.BACHELOR_AND_MASTER,
-        ExtentCode.MASTER,
-        ExtentCode.LICENTIATE,
-        ExtentCode.DOCTOR,
-      ],
-    },
-  }
-
-  if (studentNumbers.length > 0) {
-    where.studentNumber = {
-      [Op.in]: studentNumbers,
-    }
-  }
-
-  return await SISStudyRightModel.findAll({
-    attributes: ['startDate', 'endDate', 'studentNumber'],
-    where,
-  })
-}
 
 export const getOpenUniSearchesByUser = async (userId: string) => {
   return await OpenUniPopulationSearchModel.findAll({

services/backend/src/services/populations/getStudentData.ts

@@ -3,6 +3,8 @@ import { Op } from 'sequelize'
 import { Credit, Enrollment, SISStudyRight, SISStudyRightElement, Student, Studyplan } from '@oodikone/shared/models'
 import { Tag, TagStudent } from '@oodikone/shared/models/kone'
 import { EnrollmentState } from '@oodikone/shared/types'
+import { enrollmentTimeDateThreshold } from '@oodikone/shared/util'
+import { dateMaxFromList } from '@oodikone/shared/util/datetime'
 import {
   CreditModel,
   EnrollmentModel,
@@ -48,7 +50,11 @@ export type StudentEnrollment = Pick<
   Enrollment,
   'course_code' | 'state' | 'enrollment_date_time' | 'semestercode' | 'studentnumber' | 'studyright_id'
 >
-export const getEnrollments = (studentNumbers: string[], startDate: string): Promise<Array<StudentEnrollment>> =>
+export const getEnrollments = (
+  studentNumbers: string[],
+  startDate: string,
+  endDate: string
+): Promise<Array<StudentEnrollment>> =>
   EnrollmentModel.findAll({
     attributes: ['course_code', 'state', 'enrollment_date_time', 'semestercode', 'studentnumber', 'studyright_id'],
     where: {
@@ -57,7 +63,7 @@ export const getEnrollments = (studentNumbers: string[], startDate: string): Pro
       },
       state: EnrollmentState.ENROLLED,
       enrollment_date_time: {
-        [Op.gte]: startDate,
+        [Op.between]: [dateMaxFromList(new Date(startDate), enrollmentTimeDateThreshold), new Date(endDate)],
       },
     },
     raw: true,
@@ -192,8 +198,13 @@ export type StudentCredit = Pick<
   | 'studyright_id'
 >
 
-export const getCredits = async (studentNumbers: string[]): Promise<Array<StudentCredit>> =>
+export const getCredits = async (
+  studentNumbers: string[],
+  startDate: string,
+  endDate: string
+): Promise<Array<StudentCredit>> =>
   CreditModel.findAll({
+    raw: true,
     attributes: [
       'grade',
       'credits',
@@ -207,8 +218,8 @@ export const getCredits = async (studentNumbers: string[]): Promise<Array<Studen
     ],
     where: {
       student_studentnumber: { [Op.in]: studentNumbers },
+      attainment_date: { [Op.between]: [startDate, endDate] },
     },
-    raw: true,
   })
 
 export const getStudyRightElementsForStudyRight = async (studentNumbers: string[], code: string) =>

services/backend/src/services/populations/shared.ts

@@ -34,6 +34,7 @@ export const parseDateRangeFromParams = (query: QueryParams): ParsedQueryParams
 
   return { startDate, endDate }
 }
+
 export const getCurriculumVersion = (curriculumPeriodId: string | undefined) => {
   if (!curriculumPeriodId) {
     return null
@@ -78,7 +79,7 @@ export const getOptionsForStudents = (
 
 export const getCourses = (courses: string[]): Promise<Array<CourseStats>> =>
   CourseModel.findAll({
-    attributes: ['code', 'name', 'substitutions', 'substitution_groups', 'is_study_module'],
+    attributes: ['code', 'name', 'substitution_groups', 'is_study_module'],
     where: {
       code: { [Op.in]: courses },
     },

services/backend/src/services/populations/statisticsOf.ts

@@ -30,16 +30,33 @@ export const statisticsOf = async (
   studentNumbers: string[],
   studyRights: string[],
   tagMap: Map<string, StudentTags[]>,
-  startDate?: string
+  filterByDates: boolean,
+  startDate?: string,
+  endDate?: string
 ) => {
+  const defaultStartDate = new Date(1900, 0, 1).toISOString()
+  const defaultEndDate = new Date(new Date().getFullYear() + 1, 0, 1).toISOString()
+
   const code = studyRights[0] ?? ''
-  const mockedStartDate = startDate ?? new Date(1900, 0, 1).toISOString()
+  const mockedStartDate = startDate ?? defaultStartDate
+  const mockedEndDate = endDate ?? defaultEndDate
 
   const [students, enrollments, credits, degreeProgrammeType, criteria, studyRightElementsForStudyRight] =
     await Promise.all([
       getStudents(studentNumbers),
-      getEnrollments(studentNumbers, mockedStartDate),
-      getCredits(studentNumbers),
+
+      // NOTE: If filterByDates is set, use default values for filtering credits and enrollments
+      getEnrollments(
+        studentNumbers,
+        filterByDates ? mockedStartDate : defaultStartDate,
+        filterByDates ? mockedEndDate : defaultEndDate
+      ),
+      getCredits(
+        studentNumbers,
+        filterByDates ? mockedStartDate : defaultStartDate,
+        filterByDates ? mockedEndDate : defaultEndDate
+      ),
+
       getDegreeProgrammeType(code),
       getCriteria(code),
       getStudyRightElementsForStudyRight(studentNumbers, code),

services/backend/src/services/providers.ts

@@ -1,7 +1,9 @@
+import { getOpenUniCourseCode } from '../../src/util'
 import { CourseModel, OrganizationModel } from '../models'
 
 export const getAllProviders = async () => {
   const providers = OrganizationModel.findAll({
+    raw: true,
     attributes: ['code', 'name'],
   })
   return providers
@@ -9,7 +11,8 @@ export const getAllProviders = async () => {
 
 export const getCourseCodesOfProvider = async (provider: string) => {
   const coursesByProvider = await CourseModel.findAll({
-    attributes: ['id', 'code', 'substitutions'],
+    raw: true,
+    attributes: ['id', 'code', 'substitution_groups'],
     include: {
       model: OrganizationModel,
       required: true,
@@ -22,12 +25,13 @@ export const getCourseCodesOfProvider = async (provider: string) => {
     },
   })
 
-  const coursesWithOpenUniSubstitutions = coursesByProvider.map(({ code, substitutions }) => {
-    if (!substitutions?.length) {
+  const coursesWithOpenUniSubstitutions = coursesByProvider.map(({ code, substitution_groups }) => {
+    if (!substitution_groups?.length) {
       return [code]
     }
-    const alternatives = [`AY-${code}`, `AY${code}`, `A-${code}`]
-    return [code].concat(substitutions.filter(sub => alternatives.includes(sub)))
+    // This is ok to flatten, because we only want all credits completed by students, not what courses are
+    // completed and how => we don't care about substitutions
+    return [code].concat(substitution_groups.filter(group => group.some(code => getOpenUniCourseCode(code))).flat())
   })
 
   return coursesWithOpenUniSubstitutions.flat()

services/backend/src/services/semesters.ts

@@ -4,7 +4,7 @@ import { Name } from '@oodikone/shared/types'
 import { dateMinFromList, dateMaxFromList } from '@oodikone/shared/util/datetime'
 import { SemesterModel } from '../models'
 
-type SemestersAndYears = {
+export type SemestersAndYears = {
   years: Record<
     string,
     {

services/backend/src/services/sisUpdaterService.ts

@@ -44,6 +44,11 @@ export const updateSISStudentsByStudentNumber = async (studentnumbers: string[])
   return response.data
 }
 
+export const nukeRedis = async () => {
+  const response = await client.get('/nuke_redis', params)
+  return response.data
+}
+
 export const updateSISStudyPlans = async (days: number) => {
   const limitDate = new Date()
   limitDate.setDate(limitDate.getDate() - days)

services/backend/src/services/students.ts

@@ -1,14 +1,15 @@
-import { Op, QueryTypes } from 'sequelize'
+import { Op } from 'sequelize'
 
 import type { Credit, Student } from '@oodikone/shared/models'
-import { EnrollmentState, UnifyStatus } from '@oodikone/shared/types'
+import { CreditTypeCode, EnrollmentState, UnifyStatus } from '@oodikone/shared/types'
 import { FormattedStudentForSearch, StudentPageStudent } from '@oodikone/shared/types/studentData'
-import { splitByEmptySpace } from '@oodikone/shared/util'
-import { dbConnections } from '../database/connection'
+import { enrollmentTimeDateThresholdAcademicYear, splitByEmptySpace } from '@oodikone/shared/util'
+import { dateMaxFromList } from '@oodikone/shared/util/datetime'
 import {
   StudentModel,
   CreditModel,
   CourseModel,
+  EnrollmentModel,
   ProgrammeModuleModel,
   SemesterModel,
   StudyplanModel,
@@ -17,8 +18,6 @@ import {
 } from '../models'
 import { TagModel, TagStudentModel } from '../models/kone'
 
-const { sequelize } = dbConnections
-
 const byStudentNumber = async (studentNumber: string) => {
   const [student, tags] = await Promise.all([
     StudentModel.findByPk(studentNumber, {
@@ -61,7 +60,14 @@ const byStudentNumber = async (studentNumber: string) => {
         },
         {
           model: StudyplanModel,
-          attributes: ['id', 'included_courses', 'programme_code', 'completed_credits', 'sis_study_right_id'],
+          attributes: [
+            'id',
+            'included_courses',
+            'includedModules',
+            'programme_code',
+            'completed_credits',
+            'sis_study_right_id',
+          ],
         },
       ],
       order: [
@@ -116,19 +122,12 @@ const getUnifyStatus = (unifyCourses: UnifyStatus): [boolean] | [true, false] =>
   }
 }
 
-/* from & to are semestercodes if separate = false, or yearcodes in case separate is true. */
-export const findByCourseAndSemesters = async (
-  codes: string[],
-  from: number,
-  to: number,
-  separate: boolean,
-  unifyCourses: UnifyStatus = 'unifyStats'
-) => {
+export const getStartAndEndDates = async (from: number, to: number, separate: boolean) => {
   const startSemester = await SemesterModel.findOne({
     where: {
       [separate ? 'semestercode' : 'yearcode']: from,
     },
-    order: ['semestercode'],
+    order: [['semestercode', 'ASC']],
     limit: 1,
     raw: true,
   })
@@ -143,64 +142,117 @@ export const findByCourseAndSemesters = async (
   })
 
   if (!startSemester || !endSemester) {
-    return []
+    return {}
   }
 
-  const { startdate, semestercode: fromSemester } = startSemester
-  const { enddate, semestercode: toSemester } = endSemester
+  const { startdate: startDate } = startSemester
+  const { enddate: endDate } = endSemester
+
+  return { startDate, endDate }
+}
+
+/**
+  Returns students who have an attainment/enrollment between to and from.
+ */
+export const findByCourseAndSemesters = async (
+  codes: string[],
+  from: number,
+  to: number,
+  separate: boolean,
+  unifyCourses: UnifyStatus = 'unifyStats'
+) => {
+  /* from & to are semestercodes if separate = false, or yearcodes in case separate is true. */
+
+  const { startDate: startDate, endDate: endDate } = await getStartAndEndDates(from, to, separate)
 
   const unifyStatus = getUnifyStatus(unifyCourses)
 
-  type QueryResult = {
-    studentnumber: string
-  }[]
+  const credits = (await CreditModel.findAll({
+    raw: true,
+    attributes: [
+      ['student_studentnumber', 'studentnumber'],
+      'course_code',
+      ['attainment_date', 'date'],
+      'credittypecode',
+    ],
+    where: {
+      course_code: { [Op.in]: codes },
+      is_open: unifyStatus,
+      attainment_date: { [Op.between]: [startDate, endDate] },
+      credittypecode: { [Op.not]: CreditTypeCode.IMPROVED }, // We do not care about improved grades
+    },
+    order: [
+      ['attainment_date', 'DESC'],
+      ['credittypecode', 'ASC'],
+    ],
+  })) as unknown as Array<{
+    studentnumber: CreditModel['student_studentnumber']
+    course_code: CreditModel['course_code']
+    date: CreditModel['attainment_date']
+    credittypecode: CreditModel['credittypecode']
+  }>
+  // HACK: Sequelize doesn't like renaming attributes so we have to force-cast
 
-  const queryResult: QueryResult = await sequelize.query(
-    `
-    SELECT
-      studentnumber
-    FROM
-      student s
-    WHERE
-      EXISTS (
-        SELECT
-          1
-        FROM
-          credit c
-        WHERE
-          c.student_studentnumber = s.studentnumber
-          AND c.course_code IN (:codes)
-          AND c.is_open IN (:isOpen)
-          AND c.attainment_date BETWEEN '${startdate.toISOString()}' AND '${enddate.toISOString()}'
-      )
-      OR EXISTS (
-        SELECT
-          1
-        FROM
-          enrollment e
-        WHERE
-          e.studentnumber = s.studentnumber
-          AND e.course_code IN (:codes)
-          AND e.semestercode BETWEEN ${fromSemester} AND ${toSemester}
-          AND e.enrollment_date_time >= '2021-05-31'
-          AND e.state = :enrollmentState
-      );
-    `,
-    {
-      replacements: {
-        codes,
-        minYearCode: from,
-        maxYearCode: to,
-        isOpen: unifyStatus,
-        enrollmentState: EnrollmentState.ENROLLED,
+  const enrollments = (await EnrollmentModel.findAll({
+    raw: true,
+    attributes: ['studentnumber', 'course_code', ['enrollment_date_time', 'date']],
+    where: {
+      course_code: { [Op.in]: codes },
+      is_open: unifyStatus,
+      enrollment_date_time: {
+        [Op.between]: [dateMaxFromList(startDate ?? new Date(0), enrollmentTimeDateThresholdAcademicYear), endDate],
       },
-      type: QueryTypes.SELECT,
-      raw: true,
-    }
-  )
+      state: EnrollmentState.ENROLLED,
+    },
+    order: [['enrollment_date_time', 'DESC']],
+  })) as unknown as Array<{
+    studentnumber: EnrollmentModel['studentnumber']
+    course_code: EnrollmentModel['course_code']
+    date: EnrollmentModel['enrollment_date_time']
+  }>
+  // HACK: Sequelize doesn't like renaming attributes so we have to force-cast
 
-  const studentNumbers = queryResult.map(result => result.studentnumber)
-  return studentNumbers
+  const studentCreditsAndEnrollments: Record<
+    string,
+    Array<{
+      studentnumber: string
+      course_code: string
+      date: Date
+      type: 'credit' | 'enrollment'
+      credittypecode?: CreditTypeCode
+    }>
+  > = {}
+
+  for (const credit of credits) {
+    studentCreditsAndEnrollments[credit.studentnumber] ??= []
+    studentCreditsAndEnrollments[credit.studentnumber].push({ ...credit, type: 'credit' })
+  }
+  for (const enrollment of enrollments) {
+    studentCreditsAndEnrollments[enrollment.studentnumber] ??= []
+    studentCreditsAndEnrollments[enrollment.studentnumber].push({ ...enrollment, type: 'enrollment' })
+  }
+
+  const passedCreditTypes = [CreditTypeCode.PASSED, CreditTypeCode.APPROVED]
+
+  const filteredStudentNumbers = new Set<string>()
+  Object.entries(studentCreditsAndEnrollments).map(([studentNumber, creditsOrEnrollments]) => {
+    const latestCredit = creditsOrEnrollments.find(
+      entry => entry.type === 'credit' && codes.includes(entry.course_code)
+    )
+    const latestEnrollment = creditsOrEnrollments.find(
+      entry => entry.type === 'enrollment' && codes.includes(entry.course_code)
+    )
+
+    if (latestCredit && passedCreditTypes.includes(latestCredit.credittypecode!)) {
+      filteredStudentNumbers.add(studentNumber)
+    } else if (latestCredit?.credittypecode === CreditTypeCode.FAILED) {
+      filteredStudentNumbers.add(studentNumber)
+    } else if (latestEnrollment) {
+      filteredStudentNumbers.add(studentNumber)
+    }
+  })
+
+  return [...filteredStudentNumbers]
 }
 
 const formatSharedStudentData = ({

services/backend/src/services/studyProgramme/creditGetters.ts

@@ -39,7 +39,7 @@ export const getCreditsForProvider = async (provider: string, codes: string[], s
   }, {})
 
   const courseIdToShareMap = providers.reduce<Record<string, CourseProviderModel[]>>((obj, provider) => {
-    if (!obj[provider.coursecode]) obj[provider.coursecode] = []
+    obj[provider.coursecode] ??= []
     obj[provider.coursecode].push(provider)
     return obj
   }, {})

services/backend/src/services/studyProgramme/index.ts

@@ -1,14 +1,17 @@
 import { memoize } from 'lodash-es'
 import { Op, QueryTypes } from 'sequelize'
 
-import { Name, CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
+import { Name, EnrollmentState } from '@oodikone/shared/types'
+import { enrollmentTimeDateThresholdAcademicYear } from '@oodikone/shared/util'
+import { dateIsBetween } from '@oodikone/shared/util/datetime'
 import { dbConnections } from '../../database/connection'
 import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, ProgrammeModuleModel } from '../../models'
 import logger from '../../util/logger'
 
 export const getAllProgrammeCourses = async (providerCode: string) => {
-  const courses = await CourseModel.findAll({
-    attributes: ['id', 'code', 'name', 'substitutions'],
+  const courses: Pick<CourseModel, 'id' | 'code' | 'name' | 'substitution_groups'>[] = await CourseModel.findAll({
+    raw: true,
+    attributes: ['id', 'code', 'name', 'substitution_groups'],
     include: [
       {
         model: OrganizationModel,
@@ -22,59 +25,88 @@ export const getAllProgrammeCourses = async (providerCode: string) => {
         },
       },
     ],
-    raw: true,
   })
   return courses
 }
 
-const getCourseCode = (code: string) => {
-  if (code.startsWith('AY')) {
-    return code.replace('AY', '')
-  }
-  if (code.startsWith('A')) {
-    return code.replace('A', '')
-  }
-  return code
-}
-
-export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, programmeCourses: string[]) => {
+export const getNotCompletedForProgrammeCourses = async (
+  from: Date,
+  to: Date,
+  programmeCourses: string[],
+  start: Date,
+  end: Date
+) => {
   try {
-    const enrollmentsCourses = await EnrollmentModel.findAll({
-      attributes: ['studentnumber', 'course_code'],
+    const enrollments: Array<
+      Pick<EnrollmentModel, 'studentnumber' | 'course_code' | 'enrollment_date_time' | 'course'>
+    > = await EnrollmentModel.findAll({
+      raw: true,
+      nest: true,
+      attributes: ['studentnumber', 'course_code', 'enrollment_date_time'],
+      include: {
+        model: CourseModel,
+        attributes: ['is_study_module'],
+      },
       where: {
         course_code: {
           [Op.in]: programmeCourses,
         },
         enrollment_date_time: {
-          [Op.between]: [from, to],
+          [Op.and]: {
+            [Op.between]: [start, end],
+            [Op.gte]: enrollmentTimeDateThresholdAcademicYear, // This has to be Academic year to match with Course statistics which doesn't display enrollments for ...-2021
+          },
         },
         state: EnrollmentState.ENROLLED,
       },
+      order: [['enrollment_date_time', 'DESC']],
     })
 
-    const allEnrollments = new Map<string, Set<string>>()
-    for (const { studentnumber, course_code: courseCode } of enrollmentsCourses) {
-      const code = getCourseCode(courseCode)
-      if (!allEnrollments.has(code)) {
-        allEnrollments.set(code, new Set())
-      }
-      allEnrollments.get(code)!.add(studentnumber)
-    }
-
-    const credits = await CreditModel.findAll({
-      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'isStudyModule'],
+    // TODO: This is being called for each year, could be optimized to call only once
+    const credits: Array<
+      Pick<
+        CreditModel,
+        'course_code' | 'student_studentnumber' | 'credittypecode' | 'isStudyModule' | 'attainment_date'
+      >
+    > = await CreditModel.findAll({
+      raw: true,
+      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'isStudyModule', 'attainment_date'],
       where: {
         course_code: {
           [Op.in]: programmeCourses,
         },
         attainment_date: {
-          [Op.between]: [from, to],
+          [Op.between]: [start, end],
         },
       },
+      order: [
+        ['attainment_date', 'DESC'],
+        ['credittypecode', 'ASC'],
+      ], // Passed credits for a student have to be handled first (only PASSED, and FAILED matter)
     })
 
+    const enrolledStudentsByCourseCode = new Map<string, Set<string>>()
+
+    // Pick only this (between to, from) year's credits
+    const filteredCredits = credits.filter(credit => dateIsBetween(credit.attainment_date, from, to))
+
+    const filteredEnrollments = enrollments
+      // Only include the first enrollment for a course for a student.
+      // This assumes that enrollments are sorted in DESC order by enrollment_date_time
+      .filter(({ course_code: courseCode, studentnumber: studentNumber }) => {
+        const courseEnrollments = enrolledStudentsByCourseCode.get(courseCode)
+        const hasEnrolled = courseEnrollments?.has(studentNumber) ?? false
+
+        if (!courseEnrollments) enrolledStudentsByCourseCode.set(courseCode, new Set<string>())
+        if (!hasEnrolled) enrolledStudentsByCourseCode.get(courseCode)!.add(studentNumber)
+
+        return !hasEnrolled
+      })
+      .filter(enrollment => dateIsBetween(enrollment.enrollment_date_time, from, to))
+
     const courseCodeToName = (
       await CourseModel.findAll({
+        raw: true,
         attributes: ['code', 'name'],
         where: {
           code: programmeCourses,
@@ -85,50 +117,102 @@ export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, p
       return acc
     }, new Map())
 
-    const creditCourses = credits.map(credit => ({
-      code: getCourseCode(credit.course_code),
+    const creditCourses = filteredCredits.map(credit => ({
+      code: credit.course_code,
       studentNumber: credit.student_studentnumber,
       creditTypeCode: credit.credittypecode,
       courseName: courseCodeToName.get(credit.course_code)!,
       isStudyModule: credit.isStudyModule,
+      attainmentDate: credit.attainment_date,
     }))
 
     const passedByCourseCodes = new Map<string, Set<string>>()
+    const failedByCourseCodes = new Map<string, Set<string>>()
     const notCompletedByCourseCodes = new Map<string, Set<string>>()
     const courses = new Map<string, { name: Name; isStudyModule: boolean }>()
 
-    for (const { code, courseName, creditTypeCode, isStudyModule, studentNumber } of creditCourses) {
-      if (!courses.has(code)) {
-        courses.set(code, {
+    const studentHasPassedCourse = (studentNumber: string, courseCode: string): boolean =>
+      (passedByCourseCodes.get(courseCode)?.has(studentNumber) ?? false) ||
+      !!credits.find(
+        credit =>
+          credit.student_studentnumber === studentNumber &&
+          credit.course_code === courseCode &&
+          CreditModel.passed(credit)
+      )
+
+    /** Student has already been counted as failed or there is a newer failed credit */
+    const studentHasFailedCourse = (
+      studentNumber: string,
+      courseCode: string,
+      attainmentDate: Date = new Date(0)
+    ): boolean =>
+      (failedByCourseCodes.get(courseCode)?.has(studentNumber) ?? false) ||
+      !!credits.find(
+        credit =>
+          credit.student_studentnumber === studentNumber &&
+          credit.course_code === courseCode &&
+          CreditModel.failed(credit) &&
+          credit.attainment_date > attainmentDate
+      )
+
+    const studentHasCredit = (studentNumber: string, courseCode: string): boolean =>
+      studentHasPassedCourse(studentNumber, courseCode) || studentHasFailedCourse(studentNumber, courseCode)
+
+    for (const {
+      code: courseCode,
+      courseName,
+      creditTypeCode,
+      isStudyModule,
+      studentNumber,
+      attainmentDate,
+    } of creditCourses) {
+      if (!courses.has(courseCode)) {
+        courses.set(courseCode, {
           isStudyModule,
           name: courseName,
         })
-        passedByCourseCodes.set(code, new Set())
-        notCompletedByCourseCodes.set(code, new Set())
+        passedByCourseCodes.set(courseCode, new Set())
+        failedByCourseCodes.set(courseCode, new Set())
+        notCompletedByCourseCodes.set(courseCode, new Set())
       }
-      if ([CreditTypeCode.PASSED, CreditTypeCode.IMPROVED, CreditTypeCode.APPROVED].includes(creditTypeCode)) {
-        passedByCourseCodes.get(code)!.add(studentNumber)
+
+      if (CreditModel.passed({ credittypecode: creditTypeCode })) {
+        passedByCourseCodes.get(courseCode)!.add(studentNumber)
       }
-      if (creditTypeCode === CreditTypeCode.FAILED && !passedByCourseCodes.get(code)!.has(studentNumber)) {
-        notCompletedByCourseCodes.get(code)!.add(studentNumber)
+      if (
+        CreditModel.failed({ credittypecode: creditTypeCode }) &&
+        !studentHasPassedCourse(studentNumber, courseCode) &&
+        !studentHasFailedCourse(studentNumber, courseCode, attainmentDate)
+      ) {
+        failedByCourseCodes.get(courseCode)!.add(studentNumber)
       }
     }
 
-    // If student has enrollments, but no attainment for a particular course, they have no credit info.
-    programmeCourses.forEach(courseCode => {
-      allEnrollments.get(courseCode)?.forEach(studentnumber => {
-        if (passedByCourseCodes.get(courseCode)?.has(studentnumber)) {
-          notCompletedByCourseCodes.get(courseCode)?.add(studentnumber)
-        }
-      })
-    })
+    // Add course details to courseMap if there are only enrollments
+    for (const { studentnumber: studentNumber, course_code: courseCode, course } of filteredEnrollments) {
+      if (!courses.has(courseCode)) {
+        courses.set(courseCode, {
+          isStudyModule: course.is_study_module,
+          name: courseCodeToName.get(courseCode)!,
+        })
+        passedByCourseCodes.set(courseCode, new Set())
+        failedByCourseCodes.set(courseCode, new Set())
+        notCompletedByCourseCodes.set(courseCode, new Set())
+      }
 
-    return [...courses.entries()].map(([code, { isStudyModule, name }]) => ({
-      code,
-      name,
-      isStudyModule,
-      allNotPassed: notCompletedByCourseCodes.get(code)?.size ?? 0,
-    }))
+      if (!studentHasCredit(studentNumber, courseCode)) {
+        notCompletedByCourseCodes.get(courseCode)!.add(studentNumber)
+      }
+    }
+
+    return [...courses.entries()].map(([code, { isStudyModule, name }]) => {
+      return {
+        code,
+        name,
+        isStudyModule,
+        allNotPassed: (notCompletedByCourseCodes.get(code)?.size ?? 0) + (failedByCourseCodes.get(code)?.size ?? 0),
+      }
+    })
   } catch (error) {
     logger.error(`getNotCompletedForProgrammeCourses failed ${error}`)
     return []

services/backend/src/services/studyProgramme/studentGetters.ts

@@ -95,7 +95,7 @@ export const getProgrammeCourseAggregates = async (params: {
           WHERE
             cr.course_code IN (:courseCodes)
             AND cr.attainment_date BETWEEN :from AND :to
-            AND cr.credittypecode = :creditType
+            AND cr.credittypecode IN (:creditTypes)
         )
         SELECT
           fc."courseCode",
@@ -176,7 +176,7 @@ export const getProgrammeCourseAggregates = async (params: {
           courseCodes,
           from,
           to,
-          creditType: CreditTypeCode.PASSED,
+          creditTypes: [CreditTypeCode.PASSED, CreditTypeCode.APPROVED],
           degreeExtentCodes,
           otherUniversityExtentCodes,
           separateStudyExtentCodes,

services/backend/src/services/studyProgramme/studyProgrammeBasics.ts

@@ -13,6 +13,7 @@ import {
   tableTitles,
 } from './studyProgrammeHelpers'
 import { getStudyRightsInProgramme } from './studyRightFinders'
+import { SISStudyRight } from '@oodikone/shared/models'
 
 export const getDateOfFirstSemesterPresent = (
   semesterEnrollments: SemesterEnrollment[] | null,
@@ -45,7 +46,7 @@ const getStartedStats = async ({
   isAcademicYear,
 }: {
   studyProgramme: string
-  years: number[]
+  years: number[] | string[]
   isAcademicYear: boolean
 }) => {
   const studyRightsOfProgramme = await getStudyRightsInProgramme(studyProgramme, false)
@@ -54,6 +55,7 @@ const getStartedStats = async ({
   const { semesters } = await getSemestersAndYears()
   const { semestercode: currentSemester } = Object.values(semesters).find(semester => semester.enddate >= new Date())!
 
+  const now = new Date()
   for (const studyRight of studyRightsOfProgramme) {
     const studyRightElement = studyRight.studyRightElements.find(element => element.code === studyProgramme)
     if (!studyRightElement) {
@@ -68,7 +70,7 @@ const getStartedStats = async ({
     const startedInProgramme = new Date(studyRightElement.startDate)
     const startedInProgrammeYear = defineYear(startedInProgramme, isAcademicYear)
 
-    if (startedInProgramme >= new Date()) {
+    if (startedInProgramme >= now) {
       continue
     }
 
@@ -101,7 +103,7 @@ export const getGraduatedStats = async ({
   includeAllSpecials,
 }: {
   studyProgramme: string
-  years: number[]
+  years: number[] | string[]
   isAcademicYear: boolean
   includeAllSpecials: boolean
 }) => {
@@ -127,14 +129,14 @@ export const getGraduatedStats = async ({
   return { graphStats, tableStats }
 }
 
-const getTransferredStats = async ({
+const getTransferAndCancellationStats = async ({
   studyProgramme,
   years,
   isAcademicYear,
   combinedProgramme,
 }: {
   studyProgramme: string
-  years: number[]
+  years: number[] | string[]
   isAcademicYear: boolean
   combinedProgramme: string
 }) => {
@@ -142,10 +144,19 @@ const getTransferredStats = async ({
   const secondStudyRights = combinedProgramme ? await getStudyRightsInProgramme(combinedProgramme, false) : []
   const transferredAway = getStatsBasis(years)
   const transferredTo = getStatsBasis(years)
+  const cancelled = getStatsBasis(years)
 
-  const calculateTransferredStats = (studyRights, programme: string) => {
+  const calculateStats = (studyRights: SISStudyRight[], programme: string, includeCancelled = false) => {
     for (const studyRight of studyRights) {
-      const studyRightElement = studyRight.studyRightElements.find(element => element.code === programme)
+      if (includeCancelled && studyRight.cancelled) {
+        // Cancellation is per study right (not element) -> only count it once
+        // endDate is always defined for cancelled study rights
+        const cancellationYear = defineYear(studyRight.endDate, isAcademicYear)
+        cancelled.graphStats[indexOf(years, cancellationYear)]++
+        cancelled.tableStats[cancellationYear]++
+      }
+
+      const studyRightElement = studyRight.studyRightElements.find(element => element.code === programme)!
       const studyRightElementsWithSamePhase = getStudyRightElementsWithPhase(studyRight, studyRightElement.phase)
       if (studyRightElementsWithSamePhase.length === 1) {
         continue
@@ -169,10 +180,10 @@ const getTransferredStats = async ({
     }
   }
 
-  calculateTransferredStats(studyRights, studyProgramme)
-  calculateTransferredStats(secondStudyRights, combinedProgramme)
+  calculateStats(studyRights, studyProgramme, true)
+  calculateStats(secondStudyRights, combinedProgramme)
 
-  return { transferredAway, transferredTo }
+  return { cancelled, transferredAway, transferredTo }
 }
 
 type Stats = {
@@ -188,11 +199,12 @@ const initializeGraphStats = (
   accepted: Stats,
   acceptedSecondProg: Stats,
   graduated: Stats,
+  cancelled: Stats,
   transferredAway: Stats,
   transferredTo: Stats,
   graduatedSecondProg: Stats
 ) => {
-  const basicTable = [] as Array<{ name: string; data: number[] }>
+  const basicTable = [] as { name: string; data: number[] }[]
   if (combinedProgramme) {
     basicTable.push(
       { name: 'Started studying bachelor', data: started.graphStats },
@@ -200,13 +212,15 @@ const initializeGraphStats = (
       { name: 'Accepted bachelor', data: accepted.graphStats },
       { name: 'Accepted licentiate', data: acceptedSecondProg.graphStats },
       { name: 'Graduated bachelor', data: graduated.graphStats },
-      { name: 'Graduated licentiate', data: graduatedSecondProg.graphStats }
+      { name: 'Graduated licentiate', data: graduatedSecondProg.graphStats },
+      { name: 'Cancelled', data: cancelled.graphStats }
     )
   } else {
     basicTable.push(
       { name: 'Started studying', data: started.graphStats },
       { name: 'Accepted', data: accepted.graphStats },
-      { name: 'Graduated', data: graduated.graphStats }
+      { name: 'Graduated', data: graduated.graphStats },
+      { name: 'Cancelled', data: cancelled.graphStats }
     )
   }
   if (includeAllSpecials) {
@@ -219,7 +233,7 @@ const initializeGraphStats = (
 }
 
 const initializeTableStats = (
-  year: number,
+  year: string | number,
   combinedProgramme: string,
   includeAllSpecials: boolean,
   started: Stats,
@@ -228,6 +242,7 @@ const initializeTableStats = (
   acceptedSecondProg: Stats,
   graduated: Stats,
   graduatedSecondProg: Stats,
+  cancelled: Stats,
   transferredAway: Stats,
   transferredTo: Stats
 ) => {
@@ -240,6 +255,7 @@ const initializeTableStats = (
       acceptedSecondProg.tableStats[year],
       graduated.tableStats[year],
       graduatedSecondProg.tableStats[year],
+      cancelled.tableStats[year],
       transferredAway.tableStats[year],
       transferredTo.tableStats[year],
     ]
@@ -253,6 +269,7 @@ const initializeTableStats = (
       acceptedSecondProg.tableStats[year],
       graduated.tableStats[year],
       graduatedSecondProg.tableStats[year],
+      cancelled.tableStats[year],
     ]
   }
   if (includeAllSpecials) {
@@ -261,11 +278,18 @@ const initializeTableStats = (
       started.tableStats[year],
       accepted.tableStats[year],
       graduated.tableStats[year],
+      cancelled.tableStats[year],
       transferredAway.tableStats[year],
       transferredTo.tableStats[year],
     ]
   }
-  return [year, started.tableStats[year], accepted.tableStats[year], graduated.tableStats[year]]
+  return [
+    year,
+    started.tableStats[year],
+    accepted.tableStats[year],
+    graduated.tableStats[year],
+    cancelled.tableStats[year],
+  ]
 }
 
 export const getBasicStatsForStudytrack = async ({
@@ -282,7 +306,7 @@ export const getBasicStatsForStudytrack = async ({
 }): Promise<BasicStats> => {
   const { includeAllSpecials, isAcademicYear } = settings
   const since = getStartDate(isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear) as number[]
+  const years = getYearsArray(since.getFullYear(), isAcademicYear) as string[] | number[]
   const queryParameters = { studyProgramme, years, isAcademicYear, includeAllSpecials, combinedProgramme }
   const queryParametersCombinedProg = { studyProgramme: combinedProgramme, years, isAcademicYear, includeAllSpecials }
   const { startedStudying, accepted } = await getStartedStats(queryParameters)
@@ -291,11 +315,11 @@ export const getBasicStatsForStudytrack = async ({
 
   const graduated = await getGraduatedStats(queryParameters)
   const graduatedSecondProg = await getGraduatedStats(queryParametersCombinedProg)
-  const { transferredAway, transferredTo } = await getTransferredStats(queryParameters)
+  const { cancelled, transferredAway, transferredTo } = await getTransferAndCancellationStats(queryParameters)
 
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse() as number[]
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse() as string[] | number[]
   const key = includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED'
-  const titles = tableTitles.basics[combinedProgramme ? `${key}_COMBINED_PROGRAMME` : key] as string[]
+  const titles: string[] = tableTitles.basics[combinedProgramme ? `${key}_COMBINED_PROGRAMME` : key]
   const tableStats = reversedYears.map(year =>
     initializeTableStats(
       year,
@@ -307,6 +331,7 @@ export const getBasicStatsForStudytrack = async ({
       acceptedSecondProg,
       graduated,
       graduatedSecondProg,
+      cancelled,
       transferredAway,
       transferredTo
     )
@@ -323,6 +348,7 @@ export const getBasicStatsForStudytrack = async ({
       accepted,
       acceptedSecondProg,
       graduated,
+      cancelled,
       transferredAway,
       transferredTo,
       graduatedSecondProg