[Docker][Dev] fix permissions with node_modules...................

We use xlsx which is distributed by their own cdn rather than npm. Npm
v12 by defaults blocks this installation.

.github/actions/build/backend/action.yaml

@@ -0,0 +1,41 @@
+name: 'Build backend'
+description: 'Build backend in staging or production version'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+
+outputs:
+  image:
+    description: 'Image name provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.image }}
+  tags:
+    description: 'Tag names provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@v6
+
+    - name: Build backend image
+      id: build-image-action
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: oodikone-backend
+        tags: ${{ inputs.environment }} ${{ github.sha }}
+        context: services
+        containerfiles: |
+          services/backend/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          BASE_PATH=${{ case(inputs.environment == 'staging', '/oodikone', '') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}

.github/actions/build/frontend/action.yaml

@@ -0,0 +1,50 @@
+name: 'Build frontend'
+description: 'Build frontend in staging or production versions'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+  sentry_dns:
+    required: false
+    description: 'Sentry url to send error logs to (only production, enforced)'
+    default: ''
+
+outputs:
+  image:
+    description: 'Image name provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.image }}
+  tags:
+    description: 'Tag names provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - name: Get current time
+      id: get_current_time
+      shell: bash
+      run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
+
+    - name: Build frontend image
+      id: build-image-action
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: oodikone-frontend
+        tags: ${{ inputs.environment }} ${{ github.sha }}
+        context: services
+        containerfiles: |
+          services/frontend/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          BASE_PATH=${{ case(inputs.environment == 'staging', '/oodikone', '') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}
+          BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+          SENTRY_DSN=${{ case(inputs.environment == 'production', inputs.sentry_dns, '') }}

.github/actions/build/scheduler/action.yaml

@@ -0,0 +1,40 @@
+name: 'Build updater scheduler'
+description: 'Build updater scheduler in staging or production version'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+
+outputs:
+  image:
+    description: 'Image name provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.image }}
+  tags:
+    description: 'Tag names provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@v6
+
+    - name: Build scheduler image
+      id: build-image-action
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: updater-scheduler
+        tags: ${{ inputs.environment }} ${{ github.sha }}
+        context: updater/sis-updater-scheduler
+        containerfiles: |
+          updater/sis-updater-scheduler/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}

.github/actions/build/worker/action.yaml

@@ -0,0 +1,40 @@
+name: 'Build updater worker'
+description: 'Build updater worker in staging or production version'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+
+outputs:
+  image:
+    description: 'Image name provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.image }}
+  tags:
+    description: 'Tag names provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@v6
+
+    - name: Build worker image
+      id: build-image-action
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: updater-worker
+        tags: ${{ inputs.environment }} ${{ github.sha }}
+        context: updater/sis-updater-worker
+        containerfiles: |
+          updater/sis-updater-worker/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}

.github/actions/test/api/action.yaml

@@ -0,0 +1,37 @@
+name: 'Run api tests'
+description: 'Runs api tests in backend'
+
+inputs:
+  username:
+    required: true
+    description: 'Toska registry username'
+  password:
+    required: true
+    description: 'Toska registry password'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Login to toska docker registry
+      uses: docker/login-action@v4
+      with:
+        registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+        username: ${{ inputs.username }}
+        password: ${{ inputs.password }}
+
+    - name: Start oodikone (without frontend and backend)
+      shell: bash
+      run: docker compose --file docker-compose.ci.yml up --quiet-pull --detach --wait
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/backend
+
+    - name: Run api-tests
+      shell: bash
+      run: npm run test:api --prefix ./services/backend

.github/actions/test/component/action.yaml

@@ -0,0 +1,31 @@
+name: 'Run component tests'
+description: 'Runs frontend component tests'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/shared && npm ci --prefix ./services/frontend
+
+    - name: Install Playwright Browsers
+      shell: bash
+      run: npm exec -- playwright install --with-deps
+      working-directory: ./services/frontend
+
+    - name: Run Playwright tests
+      shell: bash
+      run: npm run test:component --prefix ./services/frontend
+
+    - uses: actions/upload-artifact@v4
+      if: |
+        !cancelled()
+      with:
+        name: playwright-report
+        path: playwright-report/
+        retention-days: 7

.github/actions/test/e2e/action.yaml

@@ -19,7 +19,7 @@ runs:
   using: composite
   steps:
     - name: 'Login to toska Docker registry'
-      uses: docker/login-action@v3
+      uses: docker/login-action@v4
       with:
         registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
         username: ${{ inputs.username }}
@@ -27,24 +27,25 @@ runs:
 
     - name: 'Run services, pull rest of the needed images'
       shell: bash
-      run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull
+      run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull
 
     - name: 'Run cypress'
-      uses: cypress-io/github-action@v6
+      uses: cypress-io/github-action@v7
       with:
-        browser: chrome
+        browser: firefox
         spec: ${{ inputs.spec }}
       env:
         CYPRESS_baseUrl: http://localhost:1337
 
     - name: 'Upload video if tests fail'
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v7
       if: failure()
       with:
         name: 'Cypress videos - ${{ inputs.test_name }}'
         path: ./cypress/videos/*
+        retention-days: 7
 
     - name: 'Print logs on failure'
       shell: bash
       if: failure()
-      run: docker compose --file docker-compose.ci.yml logs
+      run: npm run logs:ci && sleep 15

.github/actions/test/services/action.yaml

@@ -0,0 +1,37 @@
+name: 'Run service tests'
+description: 'Runs service tests in backend'
+
+inputs:
+  username:
+    required: true
+    description: 'Toska registry username'
+  password:
+    required: true
+    description: 'Toska registry password'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Login to toska docker registry
+      uses: docker/login-action@v4
+      with:
+        registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+        username: ${{ inputs.username }}
+        password: ${{ inputs.password }}
+
+    - name: Start oodikone (without frontend and backend)
+      shell: bash
+      run: docker compose --file docker-compose.ci.yml up --quiet-pull --detach --wait
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/backend
+
+    - name: Run service-tests
+      shell: bash
+      run: npm run test:services --prefix ./services/backend

.github/actions/test/unit/action.yaml

@@ -0,0 +1,22 @@
+name: 'Run unit tests'
+description: 'Runs unit tests in backend and frontend'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/shared && npm ci --prefix ./services/backend && npm ci --prefix ./services/frontend
+
+    - name: Run frontend unit tests
+      shell: bash
+      run: npm run test:unit --prefix ./services/frontend
+
+    - name: Run backend unit tests
+      shell: bash
+      run: npm run test:unit --prefix ./services/backend

.github/workflows/production.yaml

@@ -5,35 +5,23 @@ on:
     types: [published]
 
 jobs:
-  release_frontend:
-    name: 'Build and release frontend'
+  release_backend:
+    name: 'Build and release backend'
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v5
-
-      - name: 'Get current time'
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
+      - uses: actions/checkout@v6
 
       - name: Extract release version
         id: extract_version
         run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
 
-      - name: Build frontend image
+      - name: 'Build backend'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/backend
         with:
-          image: oodikone-frontend
-          tags: production ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/frontend/Dockerfile
-          build-args: |
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
-            SENTRY_DSN=${{ secrets.OODIKONE_FRONTEND_SENTRY_DSN }}
+          environment: 'production'
+          release_version: ${{ steps.extract_version.outputs.VERSION }}
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2
@@ -44,29 +32,24 @@ jobs:
           username: toska+github
           password: ${{ secrets.QUAY_IO_TOKEN }}
 
-  release_backend:
-    name: 'Build and release backend'
+  release_frontend:
+    name: 'Build and release frontend'
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Extract release version
         id: extract_version
         run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
 
-      - name: Build backend image
+      - name: 'Build frontend'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/frontend
         with:
-          image: oodikone-backend
-          tags: production ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/backend/Dockerfile
-          build-args: |
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
+          environment: 'production'
+          release_version: ${{ steps.extract_version.outputs.VERSION }}
+          sentry_dns: ${{ secrets.OODIKONE_FRONTEND_SENTRY_DSN }} # Used only in production
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2
@@ -85,7 +68,7 @@ jobs:
       - release_backend
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Extract release version
         id: extract_version
@@ -119,7 +102,7 @@ jobs:
       - release_backend
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Extract release version
         id: extract_version
@@ -157,13 +140,13 @@ jobs:
     if: success()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
-          softa-url: https://oodikone.cs.helsinki.fi/
+          softa-url: https://oodikone.helsinki.fi/
           deployment-target: production
 
   report_failure:
@@ -178,7 +161,7 @@ jobs:
     if: failure()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:

.github/workflows/pull_request.yaml

@@ -5,28 +5,60 @@ on:
     branches:
       - master
     paths:
+      - '.github/actions/**'
+      - '.github/workflows/pull_request.yaml'
+      - '.github/workflows/test.yaml'
+      - 'services/**'
+      - 'updater/**'
+      - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
+      - 'package*.json'
       - 'cypress/**'
       - 'cypress.config.json'
-      - 'services/**'
-      - '!services/backend/src/rapodiff/**'
-      - 'docker-compose.ci.yml'
-      - 'package*.json'
 
 jobs:
   lint_packages:
     name: 'Lint packages'
     runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
+
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Lint backend, frontend and shared
         uses: ./.github/actions/lint
 
-  test_end_to_end:
+  test_unit:
+    name: 'Run unit tests'
     runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
     needs:
       - lint_packages
+
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/test/unit
+
+  test_api:
+    name: 'Test backend api'
+    runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
+    needs:
+      - lint_packages
+
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/test/api
+
+  test_end_to_end:
+    name: ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
+    needs:
+      - lint_packages
+      - test_api
     strategy:
       fail-fast: false
+
       matrix:
         include:
           - name: 'Test course statistics'
@@ -49,19 +81,19 @@ jobs:
               cypress/e2e/Users.js
           - name: 'Test faculty statistics'
             spec: cypress/e2e/Faculty_statistics.js
-          - name: 'Test open uni search'
-            spec: cypress/e2e/Openuni.js
           - name: 'Test language center view'
             spec: cypress/e2e/Language_center.js
           - name: 'Test study guidance groups'
             spec: cypress/e2e/Study_guidance_groups.js
           - name: 'Test university'
             spec: cypress/e2e/University.js
-    name: ${{ matrix.name }}
+          - name: 'Test numbers match'
+            spec: cypress/e2e/Numbers_match.js
+
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Run tests'
-        uses: ./.github/actions/run-tests
+        uses: ./.github/actions/test/e2e
         with:
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}

.github/workflows/staging.yaml

@@ -8,13 +8,13 @@ on:
       - '.github/actions/**'
       - '.github/workflows/staging.yaml'
       - '.github/workflows/test.yaml'
+      - 'services/**'
+      - 'updater/**'
+      - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
+      - 'package*.json'
       - 'cypress/**'
       - 'cypress.config.json'
-      - 'services/**'
-      - '!services/backend/src/rapodiff/**'
-      - 'updater/sis-updater-worker/src/db/migrations/**'
-      - 'docker-compose.ci.yml'
-      - 'package*.json'
 
 jobs:
   test:
@@ -23,31 +23,20 @@ jobs:
       username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
       password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
 
-  release_frontend:
-    name: 'Build and release frontend'
+  release_backend:
+    name: 'Build and release backend'
     runs-on: ubuntu-latest
     needs: test
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
-      - name: Get current time
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-
-      - name: Build frontend image
+      - name: 'Build backend'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/backend
         with:
-          image: oodikone-frontend
-          tags: staging ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/frontend/Dockerfile
-          build-args: |
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-            BASE_PATH=/oodikone
-            STAGING=true
+          environment: 'staging'
+          release_version: ${{ github.sha }}
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2
@@ -58,27 +47,68 @@ jobs:
           username: toska+github
           password: ${{ secrets.QUAY_IO_TOKEN }}
 
-  release_backend:
-    name: 'Build and release backend'
+  release_frontend:
+    name: 'Build and release frontend'
     runs-on: ubuntu-latest
     needs: test
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
-      - name: Build backend image
+      - name: 'Build frontend'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/frontend
         with:
-          image: oodikone-backend
-          tags: staging ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/backend/Dockerfile
-          build-args: |
-            BASE_PATH=/oodikone
-            GIT_SHA=${{ github.sha }}
-            STAGING=true
+          environment: 'staging'
+          release_version: ${{ github.sha }}
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_updater_worker:
+    name: 'Build and release updater worker'
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: 'Build worker'
+        id: build-image
+        uses: ./.github/actions/build/worker
+        with:
+          environment: 'staging'
+          release_version: ${{ github.sha }}
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_updater_scheduler:
+    name: 'Build and release updater scheduler'
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: 'Build scheduler'
+        id: build-image
+        uses: ./.github/actions/build/scheduler
+        with:
+          environment: 'staging'
+          release_version: ${{ github.sha }}
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2
@@ -92,14 +122,15 @@ jobs:
   report_success:
     name: 'Report deployment success to the "oodikone_notifications" channel'
     runs-on: ubuntu-latest
+    if: success()
     needs:
       - release_frontend
       - release_backend
-
-    if: success()
+      - release_updater_worker
+      - release_updater_scheduler
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
@@ -111,14 +142,15 @@ jobs:
   report_failure:
     name: 'Report deployment failure to the "oodikone_notifications" channel'
     runs-on: ubuntu-latest
+    if: failure()
     needs:
       - release_frontend
       - release_backend
-
-    if: failure()
+      - release_updater_worker
+      - release_updater_scheduler
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:

.github/workflows/test.yaml

@@ -16,7 +16,7 @@ jobs:
     name: 'Lint packages'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Lint backend, frontend and shared
         uses: ./.github/actions/lint
 
@@ -25,49 +25,25 @@ jobs:
     name: 'Build backend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: docker/setup-buildx-action@v3
-      - name: 'Build image and export it'
-        uses: docker/build-push-action@v6
+      - uses: actions/checkout@v6
+      - name: Build backend
+        uses: ./.github/actions/build/backend
         with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          context: services
-          file: services/backend/Dockerfile
-          push: false
-          tags: oodikone-backend:staging
-          outputs: type=docker,dest=/tmp/backend.tar
-        env:
-          DOCKER_BUILD_SUMMARY: false
-      - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v5
-        with:
-          name: backend-image
-          path: /tmp/backend.tar
+          environment: 'staging'
+          release_version: ${{ github.sha }}
 
   build_frontend:
     name: 'Build frontend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - name: Get current time
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-      - name: Build frontend image
-        id: build-image
-        uses: redhat-actions/buildah-build@v2
+      - uses: actions/checkout@v6
+      - name: Build frontend
+        uses: ./.github/actions/build/frontend
         with:
-          image: oodikone-frontend
-          tags: staging ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/frontend/Dockerfile
-          build-args: |
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-            BASE_PATH=/oodikone
-            STAGING=true
+          environment: 'staging'
+          release_version: ${{ github.sha }}
 
+  # === Update databases ===
   check_migration_files:
     name: Check if new migrations have been added
     runs-on: ubuntu-latest
@@ -75,11 +51,12 @@ jobs:
       kone_db_migrations: ${{ steps.check_migrations.outputs.kone_db_migrations }}
       user_db_migrations: ${{ steps.check_migrations.outputs.user_db_migrations }}
       sis_db_migrations: ${{ steps.check_migrations.outputs.sis_db_migrations }}
+
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Check if new migrations have been added
         id: check_migrations
-        uses: dorny/paths-filter@v3
+        uses: dorny/paths-filter@v4
         with:
           filters: |
             kone_db_migrations:
@@ -94,94 +71,231 @@ jobs:
 
   build_and_push_anon_sis_db:
     name: 'Build anon sis-db and push to toska hub if new migrations have been added'
-    needs: check_migration_files
     runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.sis_db_migrations == 'true'
+    if: |
+      !cancelled() && !failure() &&
+      needs.check_migration_files.outputs.sis_db_migrations == 'true'
+    needs:
+      - check_migration_files
+      - build_backend
+      - build_frontend
+      - lint_packages
+
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
+
       - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
         run: |
-          docker compose --file docker-compose.ci.yml up --detach sis-updater-worker
+          docker compose --file docker-compose.ci.yml up --detach --quiet-pull sis-updater-worker
           sleep 30 # wait for sis-updater-worker to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
 
   build_and_push_anon_kone_db:
     name: 'Build anon kone-db and push to toska hub if new migrations have been added'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure() &&
+      needs.check_migration_files.outputs.kone_db_migrations == 'true'
     needs:
+      - check_migration_files
       - build_backend
       - build_frontend
-      - check_migration_files
-    runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.kone_db_migrations == 'true'
+      - lint_packages
+
     steps:
-      - uses: actions/checkout@v5
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v6
-        with:
-          name: backend-image
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
+      - uses: actions/checkout@v6
+
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
+
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         run: |
-          docker compose --file docker-compose.ci.yml up --detach backend
+          docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
           sleep 30 # wait for backend to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
 
   build_and_push_anon_user_db:
     name: 'Build anon user-db and push to toska hub if new migrations have been added'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure() &&
+      needs.check_migration_files.outputs.user_db_migrations == 'true'
     needs:
+      - check_migration_files
       - build_backend
       - build_frontend
-      - check_migration_files
-    runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.user_db_migrations == 'true'
+      - lint_packages
+
     steps:
-      - uses: actions/checkout@v5
-      - uses: dorny/paths-filter@v3
-        id: check_migrations
-        with:
-          filters: |
-            migrations:
-              - 'services/backend/src/database/migrations_user/**'
-              - '.github/workflows/test.yaml'
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v6
-        with:
-          name: backend-image
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
+      - uses: actions/checkout@v6
+
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
+
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         run: |
-          docker compose --file docker-compose.ci.yml up --detach backend
+          docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
           sleep 30 # wait for backend to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
 
-  # === End-to-end tests ===
-  test_end_to_end:
+  # === Update sis-db with updater ===
+  check_updater_changed:
+    name: 'Check if updater has been updated'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    outputs:
+      has_updater_changed: ${{ steps.check_updater_changed.outputs.has_updater_changed }}
+
+    steps:
+      - uses: actions/checkout@v6
+      - uses: dorny/paths-filter@v4
+        id: check_updater_changed
+        with:
+          filters: |
+            has_updater_changed:
+              - 'updater/**'
+              - '.github/workflows/test.yaml'
+              - '.github/workflows/staging.yaml'
+              - 'docker-compose.ci.yml'
+
+  run_updater_updates:
+    name: 'Run updater tests and updates'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure() &&
+      (needs.check_updater_changed.outputs.has_updater_changed == 'true' ||
+      needs.check_migration_files.outputs.kone_db_migrations == 'true' ||
+      needs.check_migration_files.outputs.user_db_migrations == 'true' ||
+      needs.check_migration_files.outputs.sis_db_migrations == 'true')
     needs:
+      - check_migration_files
+      - check_updater_changed
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
-    # Always run the tests, even if some of the needed jobs failed or were skipped
-    if: always()
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v4
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+
+      - name: Start worker containers
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-worker --quiet-pull --detach --wait
+
+      - name: Run hourly jobs
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
+        env:
+          SCHEDULE_IMMEDIATE: hourly
+          EXIT_AFTER_IMMEDIATES: yes
+
+      - name: Run weekly jobs
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
+        env:
+          SCHEDULE_IMMEDIATE: weekly
+          EXIT_AFTER_IMMEDIATES: yes
+
+      # TODO: Push the updated sis-db to toska-hub when updater is deterministic with test-data
+      # - name: Push newly build db to toska hub
+      #   if: success()
+      #   run: ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
+
+      - name: Dump logs on failure
+        if: failure()
+        run: npm run logs:ci
+
+  # === Backend api-tests ===
+  test_api:
+    name: 'Run backend api tests'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - run_updater_updates
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Run api tests
+        uses: ./.github/actions/test/api
+        with:
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+  #
+  # === Backend service-tests ===
+  test_services:
+    name: 'Run backend service-tests'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - run_updater_updates
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Run service-tests
+        uses: ./.github/actions/test/services
+        with:
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+
+  # === Unit testing ===
+  test_unit:
+    name: 'Run unit tests'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - run_updater_updates
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Run unit tests (frontend and backend)
+        uses: ./.github/actions/test/unit
+
+  # === Frontend component testing ===
+  test_components:
+    name: 'Test frontend components'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - run_updater_updates
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Run frontend component tests
+        uses: ./.github/actions/test/component
+
+  # === End-to-end tests ===
+  test_end_to_end:
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - run_updater_updates
     strategy:
       fail-fast: false
       matrix:
@@ -206,23 +320,28 @@ jobs:
               cypress/e2e/Users.js
           - name: 'Test faculty statistics'
             spec: cypress/e2e/Faculty_statistics.js
-          - name: 'Test open uni search'
-            spec: cypress/e2e/Openuni.js
           - name: 'Test language center view'
             spec: cypress/e2e/Language_center.js
           - name: 'Test study guidance groups'
             spec: cypress/e2e/Study_guidance_groups.js
           - name: 'Test university'
             spec: cypress/e2e/University.js
-
+          - name: 'Test numbers match'
+            spec: cypress/e2e/Numbers_match.js
+          - name: 'Test access rights'
+            spec: cypress/e2e/Routes.js
     name: ${{ matrix.name }}
-    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
+
+      - name: Install node-24 for cypress
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
 
       - name: 'Run tests'
-        uses: ./.github/actions/run-tests
+        uses: ./.github/actions/test/e2e
         with:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
@@ -232,23 +351,23 @@ jobs:
   # === Other tests ===
   test_service_exit_statuses:
     name: 'Check services run correctly'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    if: always()
     runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - run_updater_updates
+
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: 'Run services, pull rest of the needed images'
-        run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull
+      - name: 'Run services, build rest of the needed images'
+        run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull --wait
 
       - name: 'Check that exit statuses are not errors'
         run: |
@@ -260,4 +379,4 @@ jobs:
 
       - name: 'Print logs on failure'
         if: failure()
-        run: docker compose --file docker-compose.ci.yml logs
+        run: npm run logs:ci && sleep 15

.github/workflows/updater.yaml

@@ -1,29 +1,22 @@
 name: 'Test (and release) Updater'
 
 on:
-  push:
-    branches:
-      - master
+  release:
+    types: [published]
     paths:
       - '.github/workflows/updater.yaml'
-      - 'updater/sis-updater-worker/**'
-      - 'updater/sis-updater-scheduler/**'
-  pull_request:
-    branches:
-      - master
-    paths:
-      - 'updater/sis-updater-worker/**'
-      - 'updater/sis-updater-scheduler/**'
+      - 'updater/**'
 
 jobs:
   lint_scheduler_and_worker:
     name: 'Lint scheduler and worker'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: actions/setup-node@v6
         with:
           node-version: 24
+
       - name: 'Install dependencies'
         run: npm ci && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker
       - name: 'Run linter'
@@ -32,11 +25,12 @@ jobs:
   build_worker:
     name: 'Build Updater worker'
     runs-on: ubuntu-latest
+
     steps:
-      - uses: actions/checkout@v5
-      - uses: docker/setup-buildx-action@v3
+      - uses: actions/checkout@v6
+      - uses: docker/setup-buildx-action@v4
       - name: 'Build the image'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: updater/sis-updater-worker/
           push: false
@@ -44,20 +38,23 @@ jobs:
           tags: toska/sis-updater-worker:latest
         env:
           DOCKER_BUILD_SUMMARY: false
+
       - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v7
         with:
           name: updater-worker-image
           path: /tmp/worker.tar
+          retention-days: 7
 
   build_scheduler:
     name: 'Build Updater scheduler'
     runs-on: ubuntu-latest
+
     steps:
-      - uses: actions/checkout@v5
-      - uses: docker/setup-buildx-action@v3
+      - uses: actions/checkout@v6
+      - uses: docker/setup-buildx-action@v4
       - name: 'Build the image'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: updater/sis-updater-scheduler/
           push: false
@@ -65,11 +62,13 @@ jobs:
           tags: toska/sis-updater-scheduler:latest
         env:
           DOCKER_BUILD_SUMMARY: false
+
       - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v7
         with:
           name: updater-scheduler-image
           path: /tmp/scheduler.tar
+          retention-days: 7
 
   test_updater:
     name: 'Test Updater'
@@ -78,42 +77,50 @@ jobs:
       - lint_scheduler_and_worker
       - build_worker
       - build_scheduler
+
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+
       - name: 'Download built images'
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v8
         with:
           pattern: 'updater-{worker,scheduler}-image'
           merge-multiple: true
+
       - name: 'Load worker image'
         run: docker load --input worker.tar
       - name: 'Load scheduler image'
         run: docker load --input scheduler.tar
+
       - name: 'Start worker'
-        run: docker compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
+        run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull --no-recreate --wait sis-updater-worker
+
       - name: 'Run hourly jobs'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: hourly
           EXIT_AFTER_IMMEDIATES: yes
+
       - name: 'Run weekly jobs'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: weekly
           EXIT_AFTER_IMMEDIATES: yes
+
       - name: 'Run prepurge'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: prepurge
           EXIT_AFTER_IMMEDIATES: yes
+
       - name: 'Run purge'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: purge
           EXIT_AFTER_IMMEDIATES: yes
@@ -122,22 +129,15 @@ jobs:
     name: 'Build and release Updater worker'
     runs-on: ubuntu-latest
     needs: test_updater
-    if: github.event_name == 'push'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
-      - name: Build worker image
+      - name: 'Build updater worker'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/worker
         with:
-          image: updater-worker
-          tags: staging production ${{ github.sha }}
-          context: updater/sis-updater-worker
-          containerfiles: |
-            updater/sis-updater-worker/Dockerfile
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=production
+          environment: 'production'
+          release_version: ${{ github.sha }}
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2
@@ -152,22 +152,15 @@ jobs:
     name: 'Build and release Updater scheduler'
     runs-on: ubuntu-latest
     needs: test_updater
-    if: github.event_name == 'push'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
-      - name: Build scheduler image
+      - name: 'Build udpater scheduler'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/scheduler
         with:
-          image: updater-scheduler
-          tags: staging production ${{ github.sha }}
-          context: updater/sis-updater-scheduler
-          containerfiles: |
-            updater/sis-updater-scheduler/Dockerfile
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=production
+          environment: 'production'
+          release_version: ${{ github.sha }}
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2

.gitignore

@@ -56,7 +56,7 @@ cypress/videos/
 coverage/
 instrumented/
 
-# Pycharm 
+# Pycharm
 .idea
 
 # VS Code
@@ -65,9 +65,8 @@ instrumented/
 .vscode/launch.json
 .vscode/terminals.json
 
-# Funidata specific
-environments/
-
 # Target of shared symlink
 services/**/src/shared
 
+
+!services/backend/.env.test

.npmrc

@@ -0,0 +1,3 @@
+ignore-scripts=true  
+min-release-age=7  
+save-prefix=  

.nycrc

@@ -1,5 +0,0 @@
-{
-  "all": true,
-  "include": ["services/frontend/src/**/*.js", "services/frontend/src/**/*.jsx"],
-  "extends": "./nyc-config.js"
-}

README.md

@@ -1,4 +1,5 @@
 # Oodikone
+
 [![Maintainability](https://qlty.sh/gh/UniversityOfHelsinkiCS/projects/oodikone/maintainability.svg)](https://qlty.sh/gh/UniversityOfHelsinkiCS/projects/oodikone)
 
 An application for analyzing university data, running at [https://oodikone.helsinki.fi/](https://oodikone.helsinki.fi/).
@@ -122,6 +123,8 @@ Some useful commands are defined in `package.json` and can be run with `npm run
 
 - `npm run oodikone`: starts Oodikone with anonymized data
 - `npm run oodikone:real`: starts Oodikone with real data
+- `npm run oodikone:test`: starts Oodikone with non-persistent database (wipes after every reset)
+- `npm run oodikone:ci`: starts Oodikone in CI mode
 - `npm run updater`: starts updater with anonymized data
 - `npm run updater:real`: starts updater with real data
 - `npm run both`: starts Oodikone and updater with anonymized data
@@ -129,7 +132,6 @@ Some useful commands are defined in `package.json` and can be run with `npm run
 - `npm run docker:down`: stops the whole environment
 - `npm run testupdater`: runs tests for updater
 - `npm run flushredis`: clears Redis, which forces big calculations to be redone in e.g. studyprogramme overview and faculty views
-- `npm run rapodiff`: runs [Rapodiff](./services/backend/src/rapodiff/)
 - `npm run install:local`: installs `node_modules` for each package (backend, frontend, sis-updater-scheduler and sis-updater-worker) locally
 - `npm run install:docker`: installs `node_modules` for each package inside containers
 - `npm run install:both`: runs both `npm run install:local` and `npm run install:docker`
@@ -149,8 +151,8 @@ It is recommended to spend some time to become familiar with `docker` and `docke
 - `docker compose ps`: view the containers in the running environment
 - `docker compose logs frontend`: print logs for just frontend
 - `docker compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
-- `docker exec -it backend sh`: open bash terminal inside backend container
-- `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database
+- `docker exec -it oodikone-backend sh`: open bash terminal inside backend container
+- `docker exec -it oodikone-sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database
 
 ### User types for development
 
@@ -187,13 +189,11 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 - Updater is deployed to production when pushed if tests pass
 - Test databases can be updated, instructions in [anonyymioodi README](anonyymioodi/README.md)
 
-[Rapodiff](./services/backend/src/rapodiff/) can be used to check differences between Rapo and Oodikone data.
-
 ## ❓FAQ
 
 ### Modules are missing after updating package.json
 
-You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container or `npm run install:docker` to install modules inside all packages (backend, frontend, sis-updater-scheduler and sis-updater-worker).
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Prefer `npm run ci:docker` or `npm run install:docker`, which also fix `node_modules` ownership for the non-root runtime user. If you run installs manually, use root inside the container and restore ownership afterwards (`chown -R 1001:0 node_modules`) to avoid permission issues.
 
 ### Study guidance groups don't work on my machine
 
@@ -240,6 +240,20 @@ npm run cypress:run
 
 Remember to revert the `baseUrl` in [cypress.config.js](./cypress.config.js) to its original value when you're done.
 
+## How to read clients minds
+
+Search for client queries in grafana using a variation of this query:
+
+```
+{app="oodikone"} | regexp `\s(?P<payload>\{.*\}$)` | line_format `{{.payload}}` | json userId, mockedBy | userId = `client` | mockedBy != `admin`
+```
+
+For quick access to _any and all params_ under certain path:
+
+```
+{app="oodikone"} |= `path` | regexp `\s(?P<payload>\{.*\}$)` | line_format `{{.payload}}` | json userId, mockedBy | userId = `client` | mockedBy != `admin`
+```
+
 ## ✌🏼 Maintainers and contribution
 
 [Toska](https://toska.dev) - University of Helsinki

anonyymioodi/README.md

@@ -6,14 +6,14 @@ The quick guide should be enough for most purposes: updating schema, data, or th
 
 ## Quick guide to updating a test database
 
-1. Ensure you have the latest test-db. Just pulling images might not replace your existing volume, so you may need to remove it manually. This command might work: `docker volume rm oodikone_sis-db-data` (replace `oodikone_sis-db-data` with the correct volume name). The new image, including data, should be pulled the next time you launch Oodikone normally.
+1. Ensure you have the latest test-db. Just pulling images might not replace your existing volume, so you may need to remove it manually. This command might work: `docker volume rm oodikone-sis-db-data` (replace `oodikone-sis-db-data` with the correct volume name). The new image, including data, should be pulled the next time you launch Oodikone normally.
 
 2. Do your dev work as usual with the test database, for example, using `npm run both`. Edit data as needed, (for example, run the updater via the frontend), and then run tests to see if they pass (follow the instructions in the [README](../README.md) to run the tests more quickly).
 
 3. If you're updating `sis-db`:
 
    - Modify the scheduler limit in [config.js](../updater/sis-updater-scheduler/src/config.js) by setting `module.exports.DEV_SCHEDULE_COUNT = null`.
-   - Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone_sis-updater-worker_1`).
+   - Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone-sis-updater-worker`).
 
 4. When you're satisfied with the database, create a dump:
 
@@ -67,6 +67,6 @@ The quick guide should be enough for most purposes: updating schema, data, or th
 
 5. Navigate to <http://localhost:3000/updater> and click the three buttons (Update meta, Update students, Update curriculums) one by one. This will populate `sis-db` with data from the new `sis-importer-db`. You might want to do each update a couple of times, as the database tables affect each other, and not all necessary data might be generated on the first try.
 
-6. Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone_sis-updater-worker_1`). If you see any errors, try clicking the buttons again.
+6. Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone-sis-updater-worker`). If you see any errors, try clicking the buttons again.
 
 7. Run all the tests to ensure everything is working as expected (follow the instructions in the [README](../README.md) to run the tests more quickly). If any tests fail, update the test files accordingly.

anonyymioodi/commit_and_push_to_toska_hub.sh

@@ -43,6 +43,6 @@ fi
 
 infomsg "Committing and pushing to registry"
 database=${args[0]}
-docker commit "$database" "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
+docker commit "oodikone-$database" "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
 docker push "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
 successmsg "Succesfully pushed"

anonyymioodi/create_container_from_dump.sh

@@ -52,13 +52,13 @@ fi
 
 
 infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
-database=${args[0]}
-docker-compose down --remove-orphans
-docker-compose up -d "$database"
+database="${args[0]}"
+docker compose down --remove-orphans
+docker compose up -d "$database"
 
 database_dump="$database.sqz"
 infomsg "Attempting to restore database $database from dump $database_dump"
-retry docker-compose exec "$database" pg_isready --dbname="$database"
-docker exec -i "$database" /bin/bash -c "pg_restore --username=postgres \
+retry docker compose exec "$database" pg_isready --dbname="$database"
+docker exec -i "oodikone-$database" /bin/bash -c "pg_restore --username=postgres \
 --format=custom --dbname=$database --no-owner" < "$database_dump"
 successmsg "Database $database succesfully created"

anonyymioodi/docker-compose.yml

@@ -1,40 +1,57 @@
 x-defaults: &defaults
   restart: unless-stopped
 
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
 services:
   kone-db:
     <<: *defaults
-    container_name: kone-db
+    container_name: oodikone-kone-db
     environment:
       PGDATA: /data
       POSTGRES_DB: kone-db
       POSTGRES_HOST_AUTH_METHOD: trust
     image: postgres:15.10
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
+
+  user-db:
+    <<: *defaults
+    container_name: oodikone-user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: postgres:15.10
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-db:
     <<: *defaults
-    container_name: sis-db
+    container_name: oodikone-sis-db
     environment:
       PGDATA: /data
       POSTGRES_DB: sis-db
       POSTGRES_HOST_AUTH_METHOD: trust
     image: postgres:15.10
     shm_size: 2gb
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-importer-db:
     <<: *defaults
-    container_name: sis-importer-db
+    container_name: oodikone-sis-importer-db
     environment:
       PGDATA: /data
       POSTGRES_DB: sis-importer-db
       POSTGRES_HOST_AUTH_METHOD: trust
     image: postgres:15.10
-
-  user-db:
-    <<: *defaults
-    container_name: user-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: user-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults

anonyymioodi/update_staging_db.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+
+# This script will first make db dumps of the test environment and upload the dumps to staging
+
+
+. ../scripts/utils.sh
+set -euo pipefail
+
+main() {
+  declare -r db_names="sis-db kone-db user-db"
+  declare -A db_url_names
+  db_url_names["sis-db"]="SIS_DB_URL"
+  db_url_names["kone-db"]="DB_URL_KONE"
+  db_url_names["user-db"]="DB_URL_USER"
+
+  declare -r dump_directory="dumps"
+
+  # Takes the first db-tools pod
+  check_oc_logged_staging
+  db_tools_name=$(oc get pods -l app=db-tools --output jsonpath='{.items[0].metadata.name}')
+  readonly db_tools_name
+
+
+  create_dumps
+  upload_and_migrate_dumps
+}
+
+
+
+function create_dumps() {
+  echo "Starting test/anon containers..."
+  (npm run docker:down && npm run both) || die "Failed to start containers"
+  echo
+  echo "Waiting for containers to start..."
+  sleep 10 # wait for containers to start
+
+  # Create dumps for all dbs
+  echo
+  echo "Creating dumps for $db_names"
+  echo
+
+  # Create dir if not exists
+  if [ ! -d $dump_directory ]; then mkdir -p $dump_directory; fi
+
+  for db in $db_names
+  do
+    create_dump_from_db "$db"
+    if [ ! -f "$dump_directory/$db.sqz" ]; then die "Db dump not created"; fi
+  done
+
+  echo "Dumps can be found in $PWD/\*.sqz"
+}
+
+
+# $1 db-name
+function create_dump_from_db() {
+  echo "Dumping db $1..."
+
+  docker exec "oodikone-$1" pg_dump -Fc -U postgres "$1" > "$dump_directory/$1.sqz" || die "Failed to dump db $1"
+}
+
+
+function check_oc_logged_staging() {
+  echo -n "Checking if oc is logged into toska staging..."
+  [[ $(oc config current-context) == "toska/api-ocp-test-0-k8s-it-helsinki-fi"* ]] || die "\nNot logged in to toska staging environment: $(oc config current-context)"
+  echo " OK"
+}
+
+
+function upload_and_migrate_dumps() {
+
+  local target_dump_directory="$db_tools_name:/tmp/oodikone-staging-dumps"
+  echo -n "Dumping $dump_directory/ to $target_dump_directory..."
+  oc rsync "$dump_directory/" "$target_dump_directory" 1> /dev/null # show errors
+  echo " OK"
+  echo
+
+  for db in $db_names
+  do
+    migrate_dumps "$db"
+  done
+}
+
+
+function migrate_dumps() {
+  local db_url_b64
+  local db_url_raw
+  local db_url
+
+  db_url_b64=$(oc get secrets -l app.kubernetes.io/instance=oodikone --output jsonpath="{.items[0].data.${db_url_names[$1]}}")
+  db_url_raw=$(echo "$db_url_b64" | base64 --decode)
+  db_url="${db_url_raw%\?*}"
+
+  [[ $db_url != *"oodikone_*_test"* ]] || die "Trying to use non-test database: $db_url"
+
+  echo "Migrating $1 to url: $db_url"
+
+  oc exec "$db_tools_name" -- pg_restore --clean --no-owner -d "$db_url" "/tmp/oodikone-staging-dumps/$1.sqz"
+}
+
+
+# Run script
+main

cli.sh

@@ -18,11 +18,11 @@ source "$PROJECT_ROOT"/scripts/functions_for_setup.sh
 # Try to run cleanup function if things fail.
 trap cleanup SIGINT SIGTERM ERR EXIT
 
-# Run docker-compose down on cleanup
+# Run docker compose down on cleanup
 cleanup() {
   trap - SIGINT SIGTERM ERR EXIT
-  warningmsg "Trying to run docker-compose down and remove orphans"
-  docker-compose --profile oodikone --profile updater down --remove-orphans
+  warningmsg "Trying to run docker compose down and remove orphans"
+  docker compose --profile oodikone --profile updater down --remove-orphans
 }
 
 # === CLI ===
@@ -49,7 +49,7 @@ set_custom_select_prompt() {
 
   options=(
     "Set up Oodikone from scratch"
-    "Reset all real data"
+    "Reset all data"
     "Reset single database"
     "Restore data from dumps"
     "Docker system prune"
@@ -68,8 +68,8 @@ while true; do
     case $opt in
       "Set up Oodikone from scratch")
         set_up_oodikone_from_scratch;;
-      "Reset all real data")
-        reset_all_real_data;;
+      "Reset all data")
+        reset_all_data;;
       "Reset single database")
         reset_single_database
         set_custom_select_prompt # Set prompt back to initial values

cypress.config.js

@@ -3,7 +3,6 @@
 const { defineConfig } = require('cypress')
 
 module.exports = defineConfig({
-  experimentalStudio: process.env.NODE_ENV === 'development',
   projectId: 'c3jsph',
   defaultCommandTimeout: 30000,
   requestTimeout: 30000,
@@ -14,7 +13,6 @@ module.exports = defineConfig({
     // We've imported your old cypress plugins here.
     // You may want to clean this up later by importing these.
     setupNodeEvents(on, config) {
-      require('@cypress/code-coverage/task')(on, config)
       require('./cypress/plugins/index.js')(on, config)
       return config
     },

cypress/e2e/Course_statistics.js

@@ -33,6 +33,7 @@ const checkGradeTable = gradesTableContents => {
 }
 
 const checkTableContents = contents => {
+  cy.get('table tbody').scrollIntoView()
   cy.get('table tbody').within(() => {
     contents.forEach((values, trIndex) => {
       cy.get('tr')
@@ -91,17 +92,17 @@ const clickAway = () => {
 
 const getCourseSearchHooksForSingleYear = code => {
   cy.intercept({
-    pathname: '/api/v2/coursesmulti',
+    pathname: '/api/coursesmulti',
     query: { code },
   }).as('courses')
 
   cy.intercept({
-    pathname: '/api/v3/courseyearlystats',
+    pathname: '/api/courseyearlystats',
     query: { 'codes[]': code },
   }).as('yearlystats')
 
   cy.intercept({
-    pathname: '/api/v3/populationstatisticsbycourse',
+    pathname: '/api/populationstatisticsbycourse',
   }).as('coursestats')
 }
 
@@ -184,13 +185,16 @@ describe('Basic user', () => {
       cy.wait('@yearlystats').its('response.statusCode').should('be.oneOf', [200, 304])
       cy.url().should('contain', `courseCodes=%5B%22${coursecode}%22%5D`)
 
-      cy.contains('50131 • Pro gradu -tutkielma tietojenkäsittelytieteessä')
+      cy.contains('50131') // Pro gradu -tutkielma tietojenkäsittelytieteessä
       cy.contains('Show population').should('be.enabled')
       cy.contains('Show population').click()
 
       cy.wait('@coursestats').its('response.statusCode').should('be.oneOf', [200, 304])
 
-      cy.contains('Population of course Pro gradu -tutkielma tietojenkäsittelytieteessä 2007-2020 (open and normal)')
+      cy.contains('Pro gradu -tutkielma tietojenkäsittelytieteessä')
+      cy.contains('Class of 2007-2020, 10 students')
+      cy.contains('Include substitutions, Open and normal')
+
       cy.contains('Showing 10 out of 10 students')
 
       const gradesTableContents = [
@@ -213,17 +217,15 @@ describe('Basic user', () => {
       cy.wait('@yearlystats').its('response.statusCode').should('be.oneOf', [200, 304])
       cy.url().should('contain', `courseCodes=%5B%22${coursecode}%22%5D`)
 
-      cy.contains(
-        'ON-310 • Tieteellisen kirjoittamisen seminaarin alkuopetus: Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot'
-      )
-      cy.contains('200012 • Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot')
+      cy.contains('ON-310') // Tieteellisen kirjoittamisen seminaarin alkuopetus: Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot
+      cy.contains('200012') // Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot
       cy.contains('Show population').should('be.enabled')
       cy.contains('Show population').click()
 
       cy.wait('@coursestats').its('response.statusCode').should('be.oneOf', [200, 304])
-      cy.contains(
-        'Population of course Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot 2011-2018 (open and normal)'
-      )
+      cy.contains('Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot')
+      cy.contains('Class of 2011-2018, 4 students')
+      cy.contains('Include substitutions, Open and normal')
       cy.contains('Showing 4 out of 4 students')
 
       const gradesTableContents = [
@@ -262,17 +264,17 @@ describe('Basic user', () => {
 
     it('Searching multiple courses having substitution mappings shows course statistics', () => {
       cy.intercept({
-        pathname: '/api/v2/coursesmulti',
+        pathname: '/api/coursesmulti',
         query: { code: 'TKT' },
       }).as('courses')
 
       cy.intercept({
-        pathname: '/api/v3/courseyearlystats',
+        pathname: '/api/courseyearlystats',
         query: { 'codes[]': 'TKT20001', 'codes[]': 'TKT10002' },
       }).as('yearlystats')
 
       cy.intercept({
-        pathname: '/api/v3/populationstatisticsbycourse',
+        pathname: '/api/populationstatisticsbycourse',
       }).as('coursestats')
 
       cy.cs('select-multiple-courses-toggle').should('not.have.class', 'Mui-checked')
@@ -308,17 +310,17 @@ describe('Basic user', () => {
 
   it('On searches with multiple courses, has correct links on the Course tab', () => {
     cy.intercept({
-      pathname: '/api/v2/coursesmulti',
+      pathname: '/api/coursesmulti',
       query: { code: 'BSCS' },
     }).as('courses')
 
     cy.intercept({
-      pathname: '/api/v3/courseyearlystats',
+      pathname: '/api/courseyearlystats',
       query: { 'codes[]': 'BSCS1003', 'codes[]': 'BSCS1001' },
     }).as('yearlystats')
 
     cy.intercept({
-      pathname: '/api/v3/populationstatisticsbycourse',
+      pathname: '/api/populationstatisticsbycourse',
     }).as('coursestats')
 
     cy.cs('select-multiple-courses-toggle').should('not.have.class', 'Mui-checked')
@@ -337,7 +339,9 @@ describe('Basic user', () => {
     cy.url().should('contain', 'courseCodes=%5B%22BSCS1001%22%2C%22BSCS1003%22%5D')
 
     cy.cs('course-population-for-2023-2024').click()
-    cy.contains('Population of course Introduction to Programming 2023-2024 (open and normal)')
+    cy.contains('Introduction to Programming')
+    cy.contains('Class of 2023-2024')
+    cy.contains('Include substitutions, Open and normal')
 
     // NOTE: These should also have `cy.intercept`s.
     //       The calls should be either cached or cheap so this is only
@@ -348,7 +352,9 @@ describe('Basic user', () => {
     // Check the link has updated correctly
     cy.cs('course-population-for-2022-2023').invoke('attr', 'href').should('include', 'BSCS1003')
     cy.cs('course-population-for-2022-2023').click()
-    cy.contains('Population of course Data Structures and Algorithms 2022-2023 (open and normal)')
+    cy.contains('Data Structures and Algorithms')
+    cy.contains('Class of 2022-2023')
+    cy.contains('Include substitutions, Open and normal')
   })
 
   it('Searching course by name displays right courses', () => {
@@ -360,11 +366,11 @@ describe('Basic user', () => {
 
     cy.contains('Search for courses').should('not.exist')
 
-    cy.contains('TKT10004 • Tietokantojen perusteet')
-    cy.contains('AYTKT10004 • Avoin yo: Tietokantojen perusteet')
-    cy.contains('BSCS2001 • Introduction to Databases')
-    cy.contains('581328 • Tietokantojen perusteet')
-    cy.contains('A581328 • Avoin yo: Tietokantojen perusteet')
+    cy.contains('TKT10004') // Tietokantojen perusteet
+    cy.contains('AYTKT10004') // Avoin yo: Tietokantojen perusteet
+    cy.contains('BSCS2001') // Introduction to Databases
+    cy.contains('581328') // Tietokantojen perusteet
+    cy.contains('A581328') //  Avoin yo: Tietokantojen perusteet
     clickNewQueryButton()
     cy.contains('Please enter at least 5 characters for course name or 2 characters for course code.')
   })
@@ -376,27 +382,26 @@ describe('Basic user', () => {
     cy.cs(`course-TKT20001`).click()
     cy.contains('Search for courses').should('not.exist')
 
-    cy.contains('TKT20001 • Tietorakenteet ja algoritmit')
-    cy.contains('AYTKT20001 • Avoin yo: Tietorakenteet ja algoritmit')
-    cy.contains('BSCS1003 • Data Structures and Algorithms')
-    cy.contains('58131 • Tietorakenteet')
+    cy.contains('TKT20001') // Tietorakenteet ja algoritmit
+    cy.contains('AYTKT20001') // Avoin yo: Tietorakenteet ja algoritmit
+    cy.contains('BSCS1003') // Data Structures and Algorithms
+    cy.contains('58131') // Tietorakenteet
     clickNewQueryButton()
     cy.contains('Please enter at least 5 characters for course name or 2 characters for course code.')
     searchByCourseName('tietorakenteet ja algoritmit')
     cy.cs(`course-TKT20001`).click()
 
     cy.contains('Search for courses').should('not.exist')
-    cy.contains('TKT20001 • Tietorakenteet ja algoritmit')
-    cy.contains('AYTKT20001 • Avoin yo: Tietorakenteet ja algoritmit')
-    cy.contains('BSCS1003 • Data Structures and Algorithms')
-    cy.contains('58131 • Tietorakenteet')
+    cy.contains('TKT20001') // Tietorakenteet ja algoritmit
+    cy.contains('AYTKT20001') // Avoin yo: Tietorakenteet ja algoritmit
+    cy.contains('BSCS1003') // Data Structures and Algorithms
+    cy.contains('58131') // Tietorakenteet
   })
 
   it('"Select all search results" button is not showing unless "Select multiple courses" toggle is on', () => {
     cy.contains('Search for courses')
     searchByCourseCode('TKT')
     cy.cs('select-multiple-courses-toggle').should('not.have.class', 'Mui-checked')
-    cy.contains('TKT10004, BSCS2001, 581328, A581328, AYTKT10004')
     cy.contains('Select all search results').should('not.exist')
     cy.cs('select-multiple-courses-toggle').click()
     cy.cs('select-multiple-courses-toggle').should('have.class', 'Mui-checked')
@@ -410,11 +415,11 @@ describe('Basic user', () => {
     cy.cs(`course-TKT10004`).click()
     cy.contains('Search for courses').should('not.exist')
 
-    cy.contains('TKT10004 • Tietokantojen perusteet')
-    cy.contains('AYTKT10004 • Avoin yo: Tietokantojen perusteet')
-    cy.contains('BSCS2001 • Introduction to Databases')
-    cy.contains('581328 • Tietokantojen perusteet')
-    cy.contains('A581328 • Avoin yo: Tietokantojen perusteet')
+    cy.contains('TKT10004') // Tietokantojen perusteet
+    cy.contains('AYTKT10004') // Avoin yo: Tietokantojen perusteet
+    cy.contains('BSCS2001') // Introduction to Databases
+    cy.contains('581328') // Tietokantojen perusteet
+    cy.contains('A581328') // Avoin yo: Tietokantojen perusteet
 
     cy.cs('ProviderOrganizationSelect').click()
     cy.cs('ProviderOrganizationSelectOptionBoth').should('have.class', 'Mui-selected')
@@ -438,10 +443,14 @@ describe('Basic user', () => {
       cy.contains('Search for courses')
       searchByCourseCode('TKT20003')
       cy.contains('tr', 'TKT20003').click()
-      cy.contains('TKT20003 • Käyttöjärjestelmät')
-      cy.contains('582219 • Käyttöjärjestelmät')
+      cy.contains('TKT20003') // Käyttöjärjestelmät
+      cy.contains('582219') //  Käyttöjärjestelmät
       cy.cs('course-population-for-2020-2021').click()
-      cy.contains('Population of course Käyttöjärjestelmät 2020-2021 (open and normal')
+
+      cy.contains('Käyttöjärjestelmät')
+      cy.contains('Class of 2020-2021, 19 students')
+      cy.contains('Include substitutions, Open and normal')
+
       cy.contains('TKT20003')
 
       cy.contains('Students (19)').click()
@@ -452,13 +461,17 @@ describe('Basic user', () => {
     it('Population of course shows grades for each student', () => {
       searchByCourseCode('TKT20001')
       cy.cs(`course-TKT20001`).click()
-      cy.contains('TKT20001 • Tietorakenteet ja algoritmit')
-      cy.contains('AYTKT20001 • Avoin yo: Tietorakenteet ja algoritmit')
-      cy.contains('BSCS1003 • Data Structures and Algorithms')
-      cy.contains('58131 • Tietorakenteet')
+      cy.contains('TKT20001') // Tietorakenteet ja algoritmit
+      cy.contains('AYTKT20001') // Avoin yo: Tietorakenteet ja algoritmit
+      cy.contains('BSCS1003') // Data Structures and Algorithms
+      cy.contains('58131') // Tietorakenteet
       cy.cs('course-population-for-2019-2020').click()
-      cy.contains('Population of course Tietorakenteet ja algoritmit 2019-2020 (open and normal)')
-      cy.contains('Students (33)').click()
+
+      cy.contains('Tietorakenteet ja algoritmit')
+      cy.contains('Class of 2019-2020, 32 students')
+      cy.contains('Include substitutions, Open and normal')
+
+      cy.contains('Students (32)').click()
       cy.contains('td', '394776').siblings().eq(2).contains('3')
       cy.contains('td', '497388').siblings().eq(2).contains('2')
     })
@@ -466,13 +479,17 @@ describe('Basic user', () => {
     it("In 'Course population' view, student numbers of students that the user isn't allowed to see are hidden", () => {
       searchByCourseCode('TKT20001')
       cy.cs(`course-TKT20001`).click()
-      cy.contains('TKT20001 • Tietorakenteet ja algoritmit')
-      cy.contains('AYTKT20001 • Avoin yo: Tietorakenteet ja algoritmit')
-      cy.contains('BSCS1003 • Data Structures and Algorithms')
-      cy.contains('58131 • Tietorakenteet')
+      cy.contains('TKT20001') //  Tietorakenteet ja algoritmit
+      cy.contains('AYTKT20001') // Avoin yo: Tietorakenteet ja algoritmit
+      cy.contains('BSCS1003') // Data Structures and Algorithms
+      cy.contains('58131') // Tietorakenteet
       cy.cs('course-population-for-2019-2020').click()
-      cy.contains('Population of course Tietorakenteet ja algoritmit 2019-2020 (open and normal)')
-      cy.contains('Students (33)').click()
+
+      cy.contains('Tietorakenteet ja algoritmit')
+      cy.contains('Class of 2019-2020, 32 students')
+      cy.contains('Include substitutions, Open and normal')
+
+      cy.contains('Students (32)').click()
       cy.get('table tbody td').filter(':contains("Hidden")').should('have.length', 9)
     })
   })
@@ -480,10 +497,14 @@ describe('Basic user', () => {
   it('Language distribution is correct', () => {
     searchByCourseCode('TKT20003')
     cy.cs(`course-TKT20003`).click()
-    cy.contains('TKT20003 • Käyttöjärjestelmät')
-    cy.contains('582219 • Käyttöjärjestelmät')
+    cy.contains('TKT20003') // Käyttöjärjestelmät
+    cy.contains('582219') // Käyttöjärjestelmät
     cy.cs('course-population-for-2021-2022').click()
-    cy.contains('Population of course Käyttöjärjestelmät 2021-2022 (open and normal)')
+
+    cy.contains('Käyttöjärjestelmät')
+    cy.contains('Class of 2021-2022')
+    cy.contains('Include substitutions, Open and normal')
+
     cy.contains('Language distribution').click()
     cy.contains('td', 'finnish').siblings().eq(0).contains('5')
     cy.contains('td', 'english').siblings().eq(0).contains('2')
@@ -499,7 +520,7 @@ describe('Basic user', () => {
         searchByCourseCode('TKT10002')
         cy.cs(`course-TKT10002`).click()
         cy.contains('Search for courses').should('not.exist')
-        cy.contains('TKT10002 • Ohjelmoinnin perusteet')
+        cy.contains('TKT10002') // Ohjelmoinnin perusteet
       })
 
       it('Time range', () => {
@@ -516,7 +537,8 @@ describe('Basic user', () => {
         cy.contains('Show population').should('be.enabled')
       })
 
-      describe('Students tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Students tab', () => {
         describe('Info boxes', () => {
           it('Student statistics table', () => {
             cy.cs('StudentStatistics-info-box-button').click()
@@ -538,14 +560,14 @@ describe('Basic user', () => {
         it('Show grades off, Separate by semesters off', () => {
           const tableContents = [
             // [Time, Total students, Passed, Failed, Enrolled no grade, Pass rate, Fail rate]
-            ['Total', 187, 140, 11, 36, '74.87 %', '25.13 %'],
+            ['Total', 183, 140, 6, 37, '76.50 %', '23.50 %'],
             ['2023-2024', 7, 1, 0, 6, '14.29 %', '85.71 %'],
-            ['2022-2023', 33, 27, 0, 6, '81.82 %', '18.18 %'],
-            ['2021-2022', 39, 15, 0, 24, '38.46 %', '61.54 %'],
-            ['2020-2021', 23, 23, 0, null, '100.00 %', '0.00 %'],
+            ['2022-2023', 32, 26, 0, 6, '81.25 %', '18.75 %'],
+            ['2021-2022', 39, 14, 0, 25, '35.90 %', '64.10 %'],
+            ['2020-2021', 21, 21, 0, null, '100.00 %', '0.00 %'],
             ['2019-2020', 28, 28, 0, null, '100.00 %', '0.00 %'],
-            ['2018-2019', 30, 26, 4, null, '86.67 %', '13.33 %'],
-            ['2017-2018', 26, 19, 7, null, '73.08 %', '26.92 %'],
+            ['2018-2019', 29, 27, 2, null, '93.10 %', '6.90 %'],
+            ['2017-2018', 26, 22, 4, null, '84.62 %', '15.38 %'],
             ['2016-2017', 1, 1, 0, null, '100.00 %', '0.00 %'],
           ]
           checkTableContents(tableContents)
@@ -554,21 +576,21 @@ describe('Basic user', () => {
         it('Show grades off, Separate by semesters on', () => {
           const tableContents = [
             // [Time, Total students, Passed, Failed, Enrolled no grade, Pass rate, Fail rate]
-            ['Total', 191, 140, 13, 38, '73.30 %', '26.70 %'],
+            ['Total', 185, 140, 6, 39, '75.68 %', '24.32 %'],
             ['Syksy 2023', 7, 1, 0, 6, '14.29 %', '85.71 %'],
             ['Kevät 2023', 10, 9, 0, 1, '90.00 %', '10.00 %'],
-            ['Syksy 2022', 23, 18, 0, 5, '78.26 %', '21.74 %'],
+            ['Syksy 2022', 22, 17, 0, 5, '77.27 %', '22.73 %'],
             ['Kevät 2022', 26, 9, 0, 17, '34.62 %', '65.38 %'],
-            ['Syksy 2021', 15, 6, 0, 9, '40.00 %', '60.00 %'],
-            ['Kevät 2021', 4, 4, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2020', 19, 19, 0, null, '100.00 %', '0.00 %'],
-            ['Kevät 2020', 4, 4, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2019', 25, 24, 1, null, '96.00 %', '4.00 %'],
-            ['Kevät 2019', 8, 8, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2018', 22, 18, 4, null, '81.82 %', '18.18 %'],
-            ['Kevät 2018', 19, 12, 7, null, '63.16 %', '36.84 %'],
-            ['Syksy 2017', 8, 7, 1, null, '87.50 %', '12.50 %'],
-            ['Kevät 2017', 0, 0, 0, null, '–', '–'],
+            ['Syksy 2021', 15, 5, 0, 10, '33.33 %', '66.67 %'],
+            ['Kevät 2021', 3, 3, 0, null, '100.00 %', '0.00 %'],
+            ['Syksy 2020', 18, 18, 0, null, '100.00 %', '0.00 %'],
+            ['Kevät 2020', 3, 3, 0, null, '100.00 %', '0.00 %'],
+            ['Syksy 2019', 25, 25, 0, null, '100.00 %', '0.00 %'],
+            ['Kevät 2019', 7, 7, 0, null, '100.00 %', '0.00 %'],
+            ['Syksy 2018', 22, 20, 2, null, '90.91 %', '9.09 %'],
+            ['Kevät 2018', 18, 14, 4, null, '77.78 %', '22.22 %'],
+            ['Syksy 2017', 8, 8, 0, null, '100.00 %', '0.00 %'],
+            ['Kevät 2017', 0, 0, 0, 0, '–', '–'],
             ['Syksy 2016', 1, 1, 0, null, '100.00 %', '0.00 %'],
           ]
           toggleSeparateBySemesters()
@@ -578,14 +600,14 @@ describe('Basic user', () => {
         it('Show grades on, Separate by semesters off', () => {
           const tableContents = [
             // [Time, Total students, Failed, 1, 2, 3, 4, 5, Other passed, Enrolled no grade, Pass rate, Fail rate]
-            ['Total', 187, 11, 3, 8, 5, 24, 99, 1, 36, '74.87 %', '25.13 %'],
+            ['Total', 183, 6, 3, 8, 5, 24, 99, 1, 37, '76.50 %', '23.50 %'],
             ['2023-2024', 7, 0, 0, 0, 0, 0, 1, 0, 6, '14.29 %', '85.71 %'],
-            ['2022-2023', 33, 0, 0, 0, 0, 6, 21, 0, 6, '81.82 %', '18.18 %'],
-            ['2021-2022', 39, 0, 0, 0, 0, 2, 13, 0, 24, '38.46 %', '61.54 %'],
-            ['2020-2021', 23, 0, 0, 2, 0, 2, 18, 1, null, '100.00 %', '0.00 %'],
+            ['2022-2023', 32, 0, 0, 0, 0, 6, 20, 0, 6, '81.25 %', '18.75 %'],
+            ['2021-2022', 39, 0, 0, 0, 0, 2, 12, 0, 25, '35.90 %', '64.10 %'],
+            ['2020-2021', 21, 0, 0, 1, 0, 2, 17, 1, null, '100.00 %', '0.00 %'],
             ['2019-2020', 28, 0, 1, 4, 1, 5, 17, 0, null, '100.00 %', '0.00 %'],
-            ['2018-2019', 30, 4, 2, 1, 2, 3, 18, 0, null, '86.67 %', '13.33 %'],
-            ['2017-2018', 26, 7, 0, 1, 2, 5, 11, 0, null, '73.08 %', '26.92 %'],
+            ['2018-2019', 29, 2, 2, 1, 2, 3, 19, 0, null, '93.10 %', '6.90 %'],
+            ['2017-2018', 26, 4, 0, 2, 2, 5, 13, 0, null, '84.62 %', '15.38 %'],
             ['2016-2017', 1, 0, 0, 0, 0, 1, 0, 0, null, '100.00 %', '0.00 %'],
           ]
           toggleShowGrades()
@@ -595,21 +617,21 @@ describe('Basic user', () => {
         it('Show grades on, Separate by semesters on', () => {
           const tableContents = [
             // [Time, Total students, Failed, 1, 2, 3, 4, 5, Other passed, Enrolled no grade, Pass rate, Fail rate]
-            ['Total', 191, 13, 3, 8, 5, 24, 99, 1, 38, '73.30 %', '26.70 %'],
+            ['Total', 185, 6, 3, 8, 5, 24, 99, 1, 39, '75.68 %', '24.32 %'],
             ['Syksy 2023', 7, 0, 0, 0, 0, 0, 1, 0, 6, '14.29 %', '85.71 %'],
             ['Kevät 2023', 10, 0, 0, 0, 0, 3, 6, 0, 1, '90.00 %', '10.00 %'],
-            ['Syksy 2022', 23, 0, 0, 0, 0, 3, 15, 0, 5, '78.26 %', '21.74 %'],
+            ['Syksy 2022', 22, 0, 0, 0, 0, 3, 14, 0, 5, '77.27 %', '22.73 %'],
             ['Kevät 2022', 26, 0, 0, 0, 0, 2, 7, 0, 17, '34.62 %', '65.38 %'],
-            ['Syksy 2021', 15, 0, 0, 0, 0, 0, 6, 0, 9, '40.00 %', '60.00 %'],
-            ['Kevät 2021', 4, 0, 0, 0, 0, 0, 4, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2020', 19, 0, 0, 2, 0, 2, 14, 1, null, '100.00 %', '0.00 %'],
-            ['Kevät 2020', 4, 0, 1, 0, 1, 0, 2, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2019', 25, 1, 0, 4, 0, 5, 15, 0, null, '96.00 %', '4.00 %'],
-            ['Kevät 2019', 8, 0, 0, 0, 1, 1, 6, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2018', 22, 4, 2, 1, 1, 2, 12, 0, null, '81.82 %', '18.18 %'],
-            ['Kevät 2018', 19, 7, 0, 1, 0, 3, 8, 0, null, '63.16 %', '36.84 %'],
-            ['Syksy 2017', 8, 1, 0, 0, 2, 2, 3, 0, null, '87.50 %', '12.50 %'],
-            ['Kevät 2017', 0, 0, 0, 0, 0, 0, 0, 0, null, '–', '–'],
+            ['Syksy 2021', 15, 0, 0, 0, 0, 0, 5, 0, 10, '33.33 %', '66.67 %'],
+            ['Kevät 2021', 3, 0, 0, 0, 0, 0, 3, 0, null, '100.00 %', '0.00 %'],
+            ['Syksy 2020', 18, 0, 0, 1, 0, 2, 14, 1, null, '100.00 %', '0.00 %'],
+            ['Kevät 2020', 3, 0, 0, 0, 1, 0, 2, 0, null, '100.00 %', '0.00 %'],
+            ['Syksy 2019', 25, 0, 1, 4, 0, 5, 15, 0, null, '100.00 %', '0.00 %'],
+            ['Kevät 2019', 7, 0, 0, 0, 1, 1, 5, 0, null, '100.00 %', '0.00 %'],
+            ['Syksy 2018', 22, 2, 2, 1, 1, 2, 14, 0, null, '90.91 %', '9.09 %'],
+            ['Kevät 2018', 18, 4, 0, 2, 0, 3, 9, 0, null, '77.78 %', '22.22 %'],
+            ['Syksy 2017', 8, 0, 0, 0, 2, 2, 4, 0, null, '100.00 %', '0.00 %'],
+            ['Kevät 2017', 0, 0, 0, 0, 0, 0, 0, 0, 0, '–', '–'],
             ['Syksy 2016', 1, 0, 0, 0, 0, 1, 0, 0, null, '100.00 %', '0.00 %'],
           ]
           toggleShowGrades()
@@ -618,7 +640,8 @@ describe('Basic user', () => {
         })
       })
 
-      describe('Attempts tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Attempts tab', () => {
         beforeEach(() => {
           openAttemptsTab()
         })
@@ -644,15 +667,15 @@ describe('Basic user', () => {
         it('Show grades off, Separate by semesters off', () => {
           const tableContents = [
             // [Time, Total attempts, Passed, Failed, Pass rate, Enrollments]
-            ['Total', 186, 140, 16, '89.74 %', 73],
-            ['2023-2024', 6, 1, 0, '16.67 %', 6],
-            ['2022-2023', 26, 27, 0, '100.00 %', 26],
-            ['2021-2022', 41, 15, 0, '36.59 %', 41],
-            ['2020-2021', 23, 23, 0, '100.00 %', null],
-            ['2019-2020', 29, 28, 1, '96.55 %', null],
-            ['2018-2019', 32, 26, 6, '81.25 %', null],
-            ['2017-2018', 28, 19, 9, '67.86 %', null],
-            ['2016-2017', 1, 1, 0, '100.00 %', null],
+            ['Total', 178, 140, 6, '95.89 %', 73],
+            ['2023-2024', 6, 1, null, '16.67 %', 6],
+            ['2022-2023', 26, 26, null, '100.00 %', 26],
+            ['2021-2022', 41, 14, null, '34.15 %', 41],
+            ['2020-2021', 21, 21, null, '100.00 %', null],
+            ['2019-2020', 28, 28, null, '100.00 %', null],
+            ['2018-2019', 29, 27, 2, '93.10 %', null],
+            ['2017-2018', 26, 22, 4, '84.62 %', null],
+            ['2016-2017', 1, 1, null, '100.00 %', null],
           ]
           checkTableContents(tableContents)
         })
@@ -660,20 +683,20 @@ describe('Basic user', () => {
         it('Show grades off, Separate by semesters on', () => {
           const tableContents = [
             // [Time, Total attempts, Passed, Failed, Pass rate, Enrollments]
-            ['Total', 186, 140, 16, '89.74 %', 73],
+            ['Total', 178, 140, 6, '95.89 %', 73],
             ['Syksy 2023', 6, 1, 0, '16.67 %', 6],
             ['Kevät 2023', 11, 9, 0, '81.82 %', 11],
-            ['Syksy 2022', 15, 18, 0, '100.00 %', 15],
+            ['Syksy 2022', 15, 17, 0, '100.00 %', 15],
             ['Kevät 2022', 29, 9, 0, '31.03 %', 29],
-            ['Syksy 2021', 12, 6, 0, '50.00 %', 12],
-            ['Kevät 2021', 4, 4, 0, '100.00 %', null],
-            ['Syksy 2020', 19, 19, 0, '100.00 %', null],
-            ['Kevät 2020', 4, 4, 0, '100.00 %', null],
-            ['Syksy 2019', 25, 24, 1, '96.00 %', null],
-            ['Kevät 2019', 8, 8, 0, '100.00 %', null],
-            ['Syksy 2018', 24, 18, 6, '75.00 %', null],
-            ['Kevät 2018', 20, 12, 8, '60.00 %', null],
-            ['Syksy 2017', 8, 7, 1, '87.50 %', null],
+            ['Syksy 2021', 12, 5, 0, '41.67 %', 12],
+            ['Kevät 2021', 3, 3, 0, '100.00 %', null],
+            ['Syksy 2020', 18, 18, 0, '100.00 %', null],
+            ['Kevät 2020', 3, 3, 0, '100.00 %', null],
+            ['Syksy 2019', 25, 25, 0, '100.00 %', null],
+            ['Kevät 2019', 7, 7, 0, '100.00 %', null],
+            ['Syksy 2018', 22, 20, 2, '90.91 %', null],
+            ['Kevät 2018', 18, 14, 4, '77.78 %', null],
+            ['Syksy 2017', 8, 8, 0, '100.00 %', null],
             ['Kevät 2017', 0, 0, 0, '–', 0],
             ['Syksy 2016', 1, 1, 0, '100.00 %', null],
           ]
@@ -684,14 +707,14 @@ describe('Basic user', () => {
         it('Show grades on, Separate by semesters off', () => {
           const tableContents = [
             // [Time, Total attempts, 0, 1, 2, 3, 4, 5, Other passed]
-            ['Total', 186, 16, 3, 8, 5, 24, 99, 1],
+            ['Total', 178, 6, 3, 8, 5, 24, 99, 1],
             ['2023-2024', 6, 0, 0, 0, 0, 0, 1, 0],
-            ['2022-2023', 26, 0, 0, 0, 0, 6, 21, 0],
-            ['2021-2022', 41, 0, 0, 0, 0, 2, 13, 0],
-            ['2020-2021', 23, 0, 0, 2, 0, 2, 18, 1],
-            ['2019-2020', 29, 1, 1, 4, 1, 5, 17, 0],
-            ['2018-2019', 32, 6, 2, 1, 2, 3, 18, 0],
-            ['2017-2018', 28, 9, 0, 1, 2, 5, 11, 0],
+            ['2022-2023', 26, 0, 0, 0, 0, 6, 20, 0],
+            ['2021-2022', 41, 0, 0, 0, 0, 2, 12, 0],
+            ['2020-2021', 21, 0, 0, 1, 0, 2, 17, 1],
+            ['2019-2020', 28, 0, 1, 4, 1, 5, 17, 0],
+            ['2018-2019', 29, 2, 2, 1, 2, 3, 19, 0],
+            ['2017-2018', 26, 4, 0, 2, 2, 5, 13, 0],
             ['2016-2017', 1, 0, 0, 0, 0, 1, 0, 0],
           ]
           toggleShowGrades()
@@ -701,20 +724,20 @@ describe('Basic user', () => {
         it('Show grades on, Separate by semesters on', () => {
           const tableContents = [
             // [Time, Total attempts, 0, 1, 2, 3, 4, 5, Other passed]
-            ['Total', 186, 16, 3, 8, 5, 24, 99, 1],
+            ['Total', 178, 6, 3, 8, 5, 24, 99, 1],
             ['Syksy 2023', 6, 0, 0, 0, 0, 0, 1, 0],
             ['Kevät 2023', 11, 0, 0, 0, 0, 3, 6, 0],
-            ['Syksy 2022', 15, 0, 0, 0, 0, 3, 15, 0],
+            ['Syksy 2022', 15, 0, 0, 0, 0, 3, 14, 0],
             ['Kevät 2022', 29, 0, 0, 0, 0, 2, 7, 0],
-            ['Syksy 2021', 12, 0, 0, 0, 0, 0, 6, 0],
-            ['Kevät 2021', 4, 0, 0, 0, 0, 0, 4, 0],
-            ['Syksy 2020', 19, 0, 0, 2, 0, 2, 14, 1],
-            ['Kevät 2020', 4, 0, 1, 0, 1, 0, 2, 0],
-            ['Syksy 2019', 25, 1, 0, 4, 0, 5, 15, 0],
-            ['Kevät 2019', 8, 0, 0, 0, 1, 1, 6, 0],
-            ['Syksy 2018', 24, 6, 2, 1, 1, 2, 12, 0],
-            ['Kevät 2018', 20, 8, 0, 1, 0, 3, 8, 0],
-            ['Syksy 2017', 8, 1, 0, 0, 2, 2, 3, 0],
+            ['Syksy 2021', 12, 0, 0, 0, 0, 0, 5, 0],
+            ['Kevät 2021', 3, 0, 0, 0, 0, 0, 3, 0],
+            ['Syksy 2020', 18, 0, 0, 1, 0, 2, 14, 1],
+            ['Kevät 2020', 3, 0, 0, 0, 1, 0, 2, 0],
+            ['Syksy 2019', 25, 0, 1, 4, 0, 5, 15, 0],
+            ['Kevät 2019', 7, 0, 0, 0, 1, 1, 5, 0],
+            ['Syksy 2018', 22, 2, 2, 1, 1, 2, 14, 0],
+            ['Kevät 2018', 18, 4, 0, 2, 0, 3, 9, 0],
+            ['Syksy 2017', 8, 0, 0, 0, 2, 2, 4, 0],
             ['Kevät 2017', 0, 0, 0, 0, 0, 0, 0, 0],
             ['Syksy 2016', 1, 0, 0, 0, 0, 1, 0, 0],
           ]
@@ -732,11 +755,11 @@ describe('Basic user', () => {
         searchByCourseCode('TKT10002')
         cy.cs(`course-TKT10002`).click()
         cy.contains('Search for courses').should('not.exist')
-        cy.contains('TKT10002 • Ohjelmoinnin perusteet')
-        cy.contains('AYTKT10002 • Avoin yo: Ohjelmoinnin perusteet')
-        cy.contains('BSCS1001 • Introduction to Programming')
-        cy.contains('581325 • Ohjelmoinnin perusteet')
-        cy.contains('A581325 • Avoin yo: Ohjelmoinnin perusteet')
+        cy.contains('TKT10002') // Ohjelmoinnin perusteet
+        cy.contains('AYTKT10002') // Avoin yo: Ohjelmoinnin perusteet
+        cy.contains('BSCS1001') // Introduction to Programming
+        cy.contains('581325') // Ohjelmoinnin perusteet
+        cy.contains('A581325') //  Avoin yo: Ohjelmoinnin perusteet
       })
 
       it('Time range', () => {
@@ -753,22 +776,23 @@ describe('Basic user', () => {
         cy.contains('Show population').should('be.enabled')
       })
 
-      describe('Students tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Students tab', () => {
         it('Show grades off, Separate by semesters off', () => {
           const tableContents = [
             // [Time, Total students, Passed, Failed, Enrolled no grade, Pass rate, Fail rate]
-            ['Total', 284, 237, 16, 31, '83.45 %', '16.55 %'],
+            ['Total', 277, 237, 8, 32, '85.56 %', '14.44 %'],
             ['2023-2024', 8, 2, 0, 6, '25.00 %', '75.00 %'],
-            ['2022-2023', 35, 28, 0, 7, '80.00 %', '20.00 %'],
-            ['2021-2022', 45, 27, 0, 18, '60.00 %', '40.00 %'],
-            ['2020-2021', 33, 33, 0, null, '100.00 %', '0.00 %'],
-            ['2019-2020', 57, 56, 1, null, '98.25 %', '1.75 %'],
-            ['2018-2019', 42, 38, 4, null, '90.48 %', '9.52 %'],
-            ['2017-2018', 32, 25, 7, null, '78.13 %', '21.88 %'],
-            ['2016-2017', 7, 6, 1, null, '85.71 %', '14.29 %'],
-            ['2015-2016', 4, 3, 1, null, '75.00 %', '25.00 %'],
-            ['2014-2015', 6, 6, 0, null, '100.00 %', '0.00 %'],
-            ['2013-2014', 2, 1, 1, null, '50.00 %', '50.00 %'],
+            ['2022-2023', 34, 27, 0, 7, '79.41 %', '20.59 %'],
+            ['2021-2022', 45, 26, 0, 19, '57.78 %', '42.22 %'],
+            ['2020-2021', 31, 31, 0, null, '100.00 %', '0.00 %'],
+            ['2019-2020', 56, 56, 0, null, '100.00 %', '0.00 %'],
+            ['2018-2019', 41, 39, 2, null, '95.12 %', '4.88 %'],
+            ['2017-2018', 32, 28, 4, null, '87.50 %', '12.50 %'],
+            ['2016-2017', 6, 5, 1, null, '83.33 %', '16.67 %'],
+            ['2015-2016', 4, 4, 0, null, '100.00 %', '0.00 %'],
+            ['2014-2015', 5, 5, 0, null, '100.00 %', '0.00 %'],
+            ['2013-2014', 2, 2, 0, null, '100.00 %', '0.00 %'],
             ['2012-2013', 4, 4, 0, null, '100.00 %', '0.00 %'],
             ['2011-2012', 1, 1, 0, null, '100.00 %', '0.00 %'],
             ['2010-2011', 1, 1, 0, null, '100.00 %', '0.00 %'],
@@ -790,28 +814,28 @@ describe('Basic user', () => {
         it('Show grades off, Separate by semesters on', () => {
           const tableContents = [
             // [Time, Total students, Passed, Failed, Enrolled no grade, Pass rate, Fail rate]
-            ['Total', 293, 239, 19, 35, '81.57 %', '18.43 %'],
+            ['Total', 282, 237, 9, 36, '84.04 %', '15.96 %'],
             ['Syksy 2023', 8, 2, 0, 6, '25.00 %', '75.00 %'],
             ['Kevät 2023', 12, 10, 0, 2, '83.33 %', '16.67 %'],
-            ['Syksy 2022', 23, 18, 0, 5, '78.26 %', '21.74 %'],
+            ['Syksy 2022', 22, 17, 0, 5, '77.27 %', '22.73 %'],
             ['Kevät 2022', 28, 11, 0, 17, '39.29 %', '60.71 %'],
-            ['Syksy 2021', 21, 16, 0, 5, '76.19 %', '23.81 %'],
-            ['Kevät 2021', 8, 8, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2020', 25, 25, 0, null, '100.00 %', '0.00 %'],
-            ['Kevät 2020', 26, 26, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2019', 33, 31, 2, null, '93.94 %', '6.06 %'],
-            ['Kevät 2019', 17, 16, 1, null, '94.12 %', '5.88 %'],
-            ['Syksy 2018', 27, 23, 4, null, '85.19 %', '14.81 %'],
-            ['Kevät 2018', 24, 17, 7, null, '70.83 %', '29.17 %'],
-            ['Syksy 2017', 9, 8, 1, null, '88.89 %', '11.11 %'],
+            ['Syksy 2021', 21, 15, 0, 6, '71.43 %', '28.57 %'],
+            ['Kevät 2021', 7, 7, 0, null, '100.00 %', '0.00 %'],
+            ['Syksy 2020', 24, 24, 0, null, '100.00 %', '0.00 %'],
+            ['Kevät 2020', 24, 24, 0, null, '100.00 %', '0.00 %'],
+            ['Syksy 2019', 32, 32, 0, null, '100.00 %', '0.00 %'],
+            ['Kevät 2019', 15, 14, 1, null, '93.33 %', '6.67 %'],
+            ['Syksy 2018', 27, 25, 2, null, '92.59 %', '7.41 %'],
+            ['Kevät 2018', 23, 19, 4, null, '82.61 %', '17.39 %'],
+            ['Syksy 2017', 9, 9, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2017', 4, 3, 1, null, '75.00 %', '25.00 %'],
-            ['Syksy 2016', 3, 3, 0, null, '100.00 %', '0.00 %'],
-            ['Kevät 2016', 3, 2, 1, null, '66.67 %', '33.33 %'],
+            ['Syksy 2016', 2, 2, 0, null, '100.00 %', '0.00 %'],
+            ['Kevät 2016', 3, 3, 0, null, '100.00 %', '0.00 %'],
             ['Syksy 2015', 1, 1, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2015', 3, 3, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2014', 3, 3, 0, null, '100.00 %', '0.00 %'],
+            ['Syksy 2014', 2, 2, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2014', 1, 1, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2013', 1, 0, 1, null, '0.00 %', '100.00 %'],
+            ['Syksy 2013', 1, 1, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2013', 0, 0, 0, 0, '–', '–'],
             ['Syksy 2012', 4, 4, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2012', 1, 1, 0, null, '100.00 %', '0.00 %'],
@@ -848,18 +872,18 @@ describe('Basic user', () => {
         it('Show grades on, Separate by semesters off', () => {
           const tableContents = [
             // [Time, Total students, Failed, 1, 2, 3, 4, 5, Other passed, Enrolled no grade, Pass rate, Fail rate]
-            ['Total', 284, 16, 4, 14, 9, 37, 168, 5, 31, '83.45 %', '16.55 %'],
+            ['Total', 277, 8, 4, 14, 9, 37, 168, 5, 32, '85.56 %', '14.44 %'],
             ['2023-2024', 8, 0, 0, 0, 0, 0, 2, 0, 6, '25.00 %', '75.00 %'],
-            ['2022-2023', 35, 0, 0, 0, 0, 6, 22, 0, 7, '80.00 %', '20.00 %'],
-            ['2021-2022', 45, 0, 0, 1, 0, 2, 24, 0, 18, '60.00 %', '40.00 %'],
-            ['2020-2021', 33, 0, 0, 2, 1, 2, 27, 1, null, '100.00 %', '0.00 %'],
-            ['2019-2020', 57, 1, 1, 5, 3, 9, 38, 0, null, '98.25 %', '1.75 %'],
-            ['2018-2019', 42, 4, 3, 2, 2, 6, 25, 0, null, '90.48 %', '9.52 %'],
-            ['2017-2018', 32, 7, 0, 2, 2, 6, 15, 0, null, '78.13 %', '21.88 %'],
-            ['2016-2017', 7, 1, 0, 0, 1, 1, 4, 0, null, '85.71 %', '14.29 %'],
-            ['2015-2016', 4, 1, 0, 1, 0, 0, 1, 1, null, '75.00 %', '25.00 %'],
-            ['2014-2015', 6, 0, 0, 1, 0, 0, 2, 3, null, '100.00 %', '0.00 %'],
-            ['2013-2014', 2, 1, 0, 0, 0, 0, 1, 0, null, '50.00 %', '50.00 %'],
+            ['2022-2023', 34, 0, 0, 0, 0, 6, 21, 0, 7, '79.41 %', '20.59 %'],
+            ['2021-2022', 45, 0, 0, 1, 0, 2, 23, 0, 19, '57.78 %', '42.22 %'],
+            ['2020-2021', 31, 0, 0, 1, 1, 2, 26, 1, null, '100.00 %', '0.00 %'],
+            ['2019-2020', 56, 0, 1, 5, 3, 9, 38, 0, null, '100.00 %', '0.00 %'],
+            ['2018-2019', 41, 2, 3, 2, 2, 6, 26, 0, null, '95.12 %', '4.88 %'],
+            ['2017-2018', 32, 4, 0, 3, 2, 6, 17, 0, null, '87.50 %', '12.50 %'],
+            ['2016-2017', 6, 1, 0, 0, 1, 1, 3, 0, null, '83.33 %', '16.67 %'],
+            ['2015-2016', 4, 0, 0, 1, 0, 0, 2, 1, null, '100.00 %', '0.00 %'],
+            ['2014-2015', 5, 0, 0, 1, 0, 0, 1, 3, null, '100.00 %', '0.00 %'],
+            ['2013-2014', 2, 0, 0, 0, 0, 0, 2, 0, null, '100.00 %', '0.00 %'],
             ['2012-2013', 4, 0, 0, 0, 0, 3, 1, 0, null, '100.00 %', '0.00 %'],
             ['2011-2012', 1, 0, 0, 0, 0, 0, 1, 0, null, '100.00 %', '0.00 %'],
             ['2010-2011', 1, 0, 0, 0, 0, 1, 0, 0, null, '100.00 %', '0.00 %'],
@@ -882,28 +906,28 @@ describe('Basic user', () => {
         it('Show grades on, Separate by semesters on', () => {
           const tableContents = [
             // [Time, Total students, Failed, 1, 2, 3, 4, 5, Other passed, Enrolled no grade, Pass rate, Fail rate]
-            ['Total', 293, 19, 5, 15, 9, 37, 168, 5, 35, '81.57 %', '18.43 %'],
+            ['Total', 282, 9, 4, 14, 9, 37, 168, 5, 36, '84.04 %', '15.96 %'],
             ['Syksy 2023', 8, 0, 0, 0, 0, 0, 2, 0, 6, '25.00 %', '75.00 %'],
             ['Kevät 2023', 12, 0, 0, 0, 0, 3, 7, 0, 2, '83.33 %', '16.67 %'],
-            ['Syksy 2022', 23, 0, 0, 0, 0, 3, 15, 0, 5, '78.26 %', '21.74 %'],
+            ['Syksy 2022', 22, 0, 0, 0, 0, 3, 14, 0, 5, '77.27 %', '22.73 %'],
             ['Kevät 2022', 28, 0, 0, 0, 0, 2, 9, 0, 17, '39.29 %', '60.71 %'],
-            ['Syksy 2021', 21, 0, 0, 1, 0, 0, 15, 0, 5, '76.19 %', '23.81 %'],
-            ['Kevät 2021', 8, 0, 0, 0, 0, 0, 8, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2020', 25, 0, 0, 2, 1, 2, 19, 1, null, '100.00 %', '0.00 %'],
-            ['Kevät 2020', 26, 0, 1, 1, 2, 4, 18, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2019', 33, 2, 1, 4, 1, 5, 20, 0, null, '93.94 %', '6.06 %'],
-            ['Kevät 2019', 17, 1, 1, 1, 1, 2, 11, 0, null, '94.12 %', '5.88 %'],
-            ['Syksy 2018', 27, 4, 2, 2, 1, 4, 14, 0, null, '85.19 %', '14.81 %'],
-            ['Kevät 2018', 24, 7, 0, 2, 0, 4, 11, 0, null, '70.83 %', '29.17 %'],
-            ['Syksy 2017', 9, 1, 0, 0, 2, 2, 4, 0, null, '88.89 %', '11.11 %'],
+            ['Syksy 2021', 21, 0, 0, 1, 0, 0, 14, 0, 6, '71.43 %', '28.57 %'],
+            ['Kevät 2021', 7, 0, 0, 0, 0, 0, 7, 0, null, '100.00 %', '0.00 %'],
+            ['Syksy 2020', 24, 0, 0, 1, 1, 2, 19, 1, null, '100.00 %', '0.00 %'],
+            ['Kevät 2020', 24, 0, 0, 1, 2, 4, 17, 0, null, '100.00 %', '0.00 %'],
+            ['Syksy 2019', 32, 0, 1, 4, 1, 5, 21, 0, null, '100.00 %', '0.00 %'],
+            ['Kevät 2019', 15, 1, 1, 1, 1, 2, 9, 0, null, '93.33 %', '6.67 %'],
+            ['Syksy 2018', 27, 2, 2, 1, 1, 4, 17, 0, null, '92.59 %', '7.41 %'],
+            ['Kevät 2018', 23, 4, 0, 3, 0, 4, 12, 0, null, '82.61 %', '17.39 %'],
+            ['Syksy 2017', 9, 0, 0, 0, 2, 2, 5, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2017', 4, 1, 0, 0, 1, 0, 2, 0, null, '75.00 %', '25.00 %'],
-            ['Syksy 2016', 3, 0, 0, 0, 0, 1, 2, 0, null, '100.00 %', '0.00 %'],
-            ['Kevät 2016', 3, 1, 0, 1, 0, 0, 1, 0, null, '66.67 %', '33.33 %'],
+            ['Syksy 2016', 2, 0, 0, 0, 0, 1, 1, 0, null, '100.00 %', '0.00 %'],
+            ['Kevät 2016', 3, 0, 0, 1, 0, 0, 2, 0, null, '100.00 %', '0.00 %'],
             ['Syksy 2015', 1, 0, 0, 0, 0, 0, 0, 1, null, '100.00 %', '0.00 %'],
             ['Kevät 2015', 3, 0, 0, 0, 0, 0, 0, 3, null, '100.00 %', '0.00 %'],
-            ['Syksy 2014', 3, 0, 0, 1, 0, 0, 2, 0, null, '100.00 %', '0.00 %'],
+            ['Syksy 2014', 2, 0, 0, 1, 0, 0, 1, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2014', 1, 0, 0, 0, 0, 0, 1, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2013', 1, 1, 0, 0, 0, 0, 0, 0, null, '0.00 %', '100.00 %'],
+            ['Syksy 2013', 1, 0, 0, 0, 0, 0, 1, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2013', 0, 0, 0, 0, 0, 0, 0, 0, null, '–', '–'],
             ['Syksy 2012', 4, 0, 0, 0, 0, 3, 1, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2012', 1, 0, 0, 0, 0, 0, 1, 0, null, '100.00 %', '0.00 %'],
@@ -939,7 +963,8 @@ describe('Basic user', () => {
         })
       })
 
-      describe('Attempts tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Attempts tab', () => {
         beforeEach(() => {
           openAttemptsTab()
         })
@@ -947,18 +972,18 @@ describe('Basic user', () => {
         it('Show grades off, Separate by semesters off', () => {
           const tableContents = [
             // [Time, Total attempts, Passed, Failed, Pass rate, Enrollments]
-            ['Total', 288, 241, 25, '90.60 %', 79],
+            ['Total', 270, 237, 9, '96.34 %', 79],
             ['2023-2024', 6, 2, 0, '33.33 %', 6],
-            ['2022-2023', 28, 28, 0, '100.00 %', 28],
-            ['2021-2022', 45, 27, 0, '60.00 %', 45],
-            ['2020-2021', 34, 34, 0, '100.00 %', null],
-            ['2019-2020', 60, 57, 3, '95.00 %', null],
-            ['2018-2019', 46, 39, 7, '84.78 %', null],
-            ['2017-2018', 35, 26, 9, '74.29 %', null],
-            ['2016-2017', 7, 6, 1, '85.71 %', null],
-            ['2015-2016', 5, 3, 2, '60.00 %', null],
-            ['2014-2015', 7, 6, 1, '85.71 %', null],
-            ['2013-2014', 2, 1, 1, '50.00 %', null],
+            ['2022-2023', 28, 27, 0, '96.43 %', 28],
+            ['2021-2022', 45, 26, 0, '57.78 %', 45],
+            ['2020-2021', 31, 31, 0, '100.00 %', null],
+            ['2019-2020', 56, 56, 0, '100.00 %', null],
+            ['2018-2019', 42, 39, 3, '92.86 %', null],
+            ['2017-2018', 32, 28, 4, '87.50 %', null],
+            ['2016-2017', 6, 5, 1, '83.33 %', null],
+            ['2015-2016', 4, 4, 0, '100.00 %', null],
+            ['2014-2015', 5, 5, 0, '100.00 %', null],
+            ['2013-2014', 2, 2, 0, '100.00 %', null],
             ['2012-2013', 4, 4, 0, '100.00 %', null],
             ['2011-2012', 1, 1, 0, '100.00 %', null],
             ['2010-2011', 1, 1, 0, '100.00 %', null],
@@ -980,55 +1005,55 @@ describe('Basic user', () => {
         it('Show grades off, Separate by semesters on', () => {
           const tableContents = [
             // [Time, Total attempts, Passed, Failed, Pass rate, Enrollments]
-            ['Total', 288, 241, 25, '90.60 %', 79],
+            ['Total', 270, 237, 9, '96.34 %', 79],
             ['Syksy 2023', 6, 2, 0, '33.33 %', 6],
             ['Kevät 2023', 13, 10, 0, '76.92 %', 13],
-            ['Syksy 2022', 15, 18, 0, '100.00 %', 15],
+            ['Syksy 2022', 15, 17, 0, '100.00 %', 15],
             ['Kevät 2022', 30, 11, 0, '36.67 %', 30],
-            ['Syksy 2021', 15, 16, 0, '100.00 %', 15],
-            ['Kevät 2021', 8, 8, 0, '100.00 %', null],
-            ['Syksy 2020', 26, 26, 0, '100.00 %', null],
-            ['Kevät 2020', 26, 26, 0, '100.00 %', null],
-            ['Syksy 2019', 34, 31, 3, '91.18 %', null],
-            ['Kevät 2019', 17, 16, 1, '94.12 %', null],
-            ['Syksy 2018', 29, 23, 6, '79.31 %', null],
-            ['Kevät 2018', 26, 18, 8, '69.23 %', null],
-            ['Syksy 2017', 9, 8, 1, '88.89 %', null],
+            ['Syksy 2021', 15, 15, 0, '100.00 %', 15],
+            ['Kevät 2021', 7, 7, 0, '100.00 %', null],
+            ['Syksy 2020', 24, 24, 0, '100.00 %', null],
+            ['Kevät 2020', 24, 24, 0, '100.00 %', null],
+            ['Syksy 2019', 32, 32, 0, '100.00 %', null],
+            ['Kevät 2019', 15, 14, 1, '93.33 %', null],
+            ['Syksy 2018', 27, 25, 2, '92.59 %', null],
+            ['Kevät 2018', 23, 19, 4, '82.61 %', null],
+            ['Syksy 2017', 9, 9, 0, '100.00 %', null],
             ['Kevät 2017', 4, 3, 1, '75.00 %', null],
-            ['Syksy 2016', 3, 3, 0, '100.00 %', null],
-            ['Kevät 2016', 4, 2, 2, '50.00 %', null],
+            ['Syksy 2016', 2, 2, 0, '100.00 %', null],
+            ['Kevät 2016', 3, 3, 0, '100.00 %', null],
             ['Syksy 2015', 1, 1, 0, '100.00 %', null],
-            ['Kevät 2015', 4, 3, 1, '75.00 %', null],
-            ['Syksy 2014', 3, 3, 0, '100.00 %', null],
+            ['Kevät 2015', 3, 3, 0, '100.00 %', null],
+            ['Syksy 2014', 2, 2, 0, '100.00 %', null],
             ['Kevät 2014', 1, 1, 0, '100.00 %', null],
-            ['Syksy 2013', 1, 0, 1, '0.00 %', null],
-            ['Kevät 2013', 0, 0, 0, '–', 0],
+            ['Syksy 2013', 1, 1, 0, '100.00 %', null],
+            ['Kevät 2013', 0, 0, 0, '–', null],
             ['Syksy 2012', 4, 4, 0, '100.00 %', null],
             ['Kevät 2012', 1, 1, 0, '100.00 %', null],
-            ['Syksy 2011', 0, 0, 0, '–', 0],
+            ['Syksy 2011', 0, 0, 0, '–', null],
             ['Kevät 2011', 1, 1, 0, '100.00 %', null],
-            ['Syksy 2010', 0, 0, 0, '–', 0],
-            ['Kevät 2010', 0, 0, 0, '–', 0],
-            ['Syksy 2009', 0, 0, 0, '–', 0],
-            ['Kevät 2009', 0, 0, 0, '–', 0],
+            ['Syksy 2010', 0, 0, 0, '–', null],
+            ['Kevät 2010', 0, 0, 0, '–', null],
+            ['Syksy 2009', 0, 0, 0, '–', null],
+            ['Kevät 2009', 0, 0, 0, '–', null],
             ['Syksy 2008', 1, 0, 1, '0.00 %', null],
             ['Kevät 2008', 1, 1, 0, '100.00 %', null],
-            ['Syksy 2007', 0, 0, 0, '–', 0],
+            ['Syksy 2007', 0, 0, 0, '–', null],
             ['Kevät 2007', 1, 1, 0, '100.00 %', null],
-            ['Syksy 2006', 0, 0, 0, '–', 0],
-            ['Kevät 2006', 0, 0, 0, '–', 0],
+            ['Syksy 2006', 0, 0, 0, '–', null],
+            ['Kevät 2006', 0, 0, 0, '–', null],
             ['Syksy 2005', 1, 1, 0, '100.00 %', null],
-            ['Kevät 2005', 0, 0, 0, '–', 0],
-            ['Syksy 2004', 0, 0, 0, '–', 0],
+            ['Kevät 2005', 0, 0, 0, '–', null],
+            ['Syksy 2004', 0, 0, 0, '–', null],
             ['Kevät 2004', 1, 1, 0, '100.00 %', null],
-            ['Syksy 2003', 0, 0, 0, '–', 0],
-            ['Kevät 2003', 0, 0, 0, '–', 0],
-            ['Syksy 2002', 0, 0, 0, '–', 0],
-            ['Kevät 2002', 0, 0, 0, '–', 0],
-            ['Syksy 2001', 0, 0, 0, '–', 0],
-            ['Kevät 2001', 0, 0, 0, '–', 0],
-            ['Syksy 2000', 0, 0, 0, '–', 0],
-            ['Kevät 2000', 0, 0, 0, '–', 0],
+            ['Syksy 2003', 0, 0, 0, '–', null],
+            ['Kevät 2003', 0, 0, 0, '–', null],
+            ['Syksy 2002', 0, 0, 0, '–', null],
+            ['Kevät 2002', 0, 0, 0, '–', null],
+            ['Syksy 2001', 0, 0, 0, '–', null],
+            ['Kevät 2001', 0, 0, 0, '–', null],
+            ['Syksy 2000', 0, 0, 0, '–', null],
+            ['Kevät 2000', 0, 0, 0, '–', null],
             ['Syksy 1999', 2, 2, 0, '100.00 %', null],
           ]
           toggleSeparateBySemesters()
@@ -1038,18 +1063,18 @@ describe('Basic user', () => {
         it('Show grades on, Separate by semesters off', () => {
           const tableContents = [
             // [Time, Total attempts, 0, 1, 2, 3, 4, 5, Other passed]
-            ['Total', 288, 25, 6, 15, 9, 37, 169, 5],
+            ['Total', 270, 9, 4, 14, 9, 37, 168, 5],
             ['2023-2024', 6, 0, 0, 0, 0, 0, 2, 0],
-            ['2022-2023', 28, 0, 0, 0, 0, 6, 22, 0],
-            ['2021-2022', 45, 0, 0, 1, 0, 2, 24, 0],
-            ['2020-2021', 34, 0, 0, 2, 1, 2, 28, 1],
-            ['2019-2020', 60, 3, 2, 5, 3, 9, 38, 0],
-            ['2018-2019', 46, 7, 3, 3, 2, 6, 25, 0],
-            ['2017-2018', 35, 9, 1, 2, 2, 6, 15, 0],
-            ['2016-2017', 7, 1, 0, 0, 1, 1, 4, 0],
-            ['2015-2016', 5, 2, 0, 1, 0, 0, 1, 1],
-            ['2014-2015', 7, 1, 0, 1, 0, 0, 2, 3],
-            ['2013-2014', 2, 1, 0, 0, 0, 0, 1, 0],
+            ['2022-2023', 28, 0, 0, 0, 0, 6, 21, 0],
+            ['2021-2022', 45, 0, 0, 1, 0, 2, 23, 0],
+            ['2020-2021', 31, 0, 0, 1, 1, 2, 26, 1],
+            ['2019-2020', 56, 0, 1, 5, 3, 9, 38, 0],
+            ['2018-2019', 42, 3, 3, 2, 2, 6, 26, 0],
+            ['2017-2018', 32, 4, 0, 3, 2, 6, 17, 0],
+            ['2016-2017', 6, 1, 0, 0, 1, 1, 3, 0],
+            ['2015-2016', 4, 0, 0, 1, 0, 0, 2, 1],
+            ['2014-2015', 5, 0, 0, 1, 0, 0, 1, 3],
+            ['2013-2014', 2, 0, 0, 0, 0, 0, 2, 0],
             ['2012-2013', 4, 0, 0, 0, 0, 3, 1, 0],
             ['2011-2012', 1, 0, 0, 0, 0, 0, 1, 0],
             ['2010-2011', 1, 0, 0, 0, 0, 1, 0, 0],
@@ -1072,28 +1097,28 @@ describe('Basic user', () => {
         it('Show grades on, Separate by semesters on', () => {
           const tableContents = [
             // [Time, Total attempts, 0, 1, 2, 3, 4, 5, Other passed]
-            ['Total', 288, 25, 6, 15, 9, 37, 169, 5],
+            ['Total', 270, 9, 4, 14, 9, 37, 168, 5],
             ['Syksy 2023', 6, 0, 0, 0, 0, 0, 2, 0],
             ['Kevät 2023', 13, 0, 0, 0, 0, 3, 7, 0],
-            ['Syksy 2022', 15, 0, 0, 0, 0, 3, 15, 0],
+            ['Syksy 2022', 15, 0, 0, 0, 0, 3, 14, 0],
             ['Kevät 2022', 30, 0, 0, 0, 0, 2, 9, 0],
-            ['Syksy 2021', 15, 0, 0, 1, 0, 0, 15, 0],
-            ['Kevät 2021', 8, 0, 0, 0, 0, 0, 8, 0],
-            ['Syksy 2020', 26, 0, 0, 2, 1, 2, 20, 1],
-            ['Kevät 2020', 26, 0, 1, 1, 2, 4, 18, 0],
-            ['Syksy 2019', 34, 3, 1, 4, 1, 5, 20, 0],
-            ['Kevät 2019', 17, 1, 1, 1, 1, 2, 11, 0],
-            ['Syksy 2018', 29, 6, 2, 2, 1, 4, 14, 0],
-            ['Kevät 2018', 26, 8, 1, 2, 0, 4, 11, 0],
-            ['Syksy 2017', 9, 1, 0, 0, 2, 2, 4, 0],
+            ['Syksy 2021', 15, 0, 0, 1, 0, 0, 14, 0],
+            ['Kevät 2021', 7, 0, 0, 0, 0, 0, 7, 0],
+            ['Syksy 2020', 24, 0, 0, 1, 1, 2, 19, 1],
+            ['Kevät 2020', 24, 0, 0, 1, 2, 4, 17, 0],
+            ['Syksy 2019', 32, 0, 1, 4, 1, 5, 21, 0],
+            ['Kevät 2019', 15, 1, 1, 1, 1, 2, 9, 0],
+            ['Syksy 2018', 27, 2, 2, 1, 1, 4, 17, 0],
+            ['Kevät 2018', 23, 4, 0, 3, 0, 4, 12, 0],
+            ['Syksy 2017', 9, 0, 0, 0, 2, 2, 5, 0],
             ['Kevät 2017', 4, 1, 0, 0, 1, 0, 2, 0],
-            ['Syksy 2016', 3, 0, 0, 0, 0, 1, 2, 0],
-            ['Kevät 2016', 4, 2, 0, 1, 0, 0, 1, 0],
+            ['Syksy 2016', 2, 0, 0, 0, 0, 1, 1, 0],
+            ['Kevät 2016', 3, 0, 0, 1, 0, 0, 2, 0],
             ['Syksy 2015', 1, 0, 0, 0, 0, 0, 0, 1],
-            ['Kevät 2015', 4, 1, 0, 0, 0, 0, 0, 3],
-            ['Syksy 2014', 3, 0, 0, 1, 0, 0, 2, 0],
+            ['Kevät 2015', 3, 0, 0, 0, 0, 0, 0, 3],
+            ['Syksy 2014', 2, 0, 0, 1, 0, 0, 1, 0],
             ['Kevät 2014', 1, 0, 0, 0, 0, 0, 1, 0],
-            ['Syksy 2013', 1, 1, 0, 0, 0, 0, 0, 0],
+            ['Syksy 2013', 1, 0, 0, 0, 0, 0, 1, 0],
             ['Kevät 2013', 0, 0, 0, 0, 0, 0, 0, 0],
             ['Syksy 2012', 4, 0, 0, 0, 0, 3, 1, 0],
             ['Kevät 2012', 1, 0, 0, 0, 0, 0, 1, 0],
@@ -1156,9 +1181,9 @@ describe('Basic user', () => {
       searchByCourseCode('TKT20014')
       cy.contains('TKT20014').click()
 
-      cy.contains('TKT20014 • Kypsyysnäyte LuK')
-      cy.contains('50037 • Ruotsinkielinen kypsyysnäyte LuK')
-      cy.contains('50036 • Suomenkielinen kypsyysnäyte LuK')
+      cy.contains('TKT20014') //  Kypsyysnäyte LuK
+      cy.contains('50037') // Ruotsinkielinen kypsyysnäyte LuK
+      cy.contains('50036') // Suomenkielinen kypsyysnäyte LuK
 
       cy.cs('ProviderOrganizationSelect').click()
 
@@ -1180,14 +1205,13 @@ describe('Basic user', () => {
     it('Has right to see all the students, because course rovider is TKT', () => {
       cy.visit('coursestatistics?courseCodes=%5B%22TKT10004%22%5D&separate=false')
       cy.cs('course-population-for-2021-2022').click()
-      cy.contains('Students (29)').click()
-      cy.contains('509781')
-      cy.contains('529866')
+      // cy.contains('Students (29)').click()
     })
   })
 })
 
-describe('Only course statistics', () => {
+// TODO: Writing these as backend api tests
+describe.skip('Only course statistics', () => {
   beforeEach(() => {
     cy.init('/coursestatistics', 'onlycoursestatistics')
     cy.url().should('include', '/coursestatistics')
@@ -1213,15 +1237,15 @@ describe('Only course statistics', () => {
 
     const attemptsTableContents = [
       // [Time, Total attempts, Passed, Failed, Pass rate, Enrollments]
-      ['Total *', 255, 217, 20, '91.56 %', 73],
+      ['Total *', 240, 212, 8, '96.36 %', 73],
       emptyYear('2023-2024'),
-      ['2022-2023', 28, 28, 0, '100.00 %', 28],
-      ['2021-2022', 45, 27, 0, '60.00 %', 45],
-      ['2020-2021', 34, 34, 0, '100.00 %'],
-      ['2019-2020', 60, 57, 3, '95.00 %'],
-      ['2018-2019', 46, 39, 7, '84.78 %'],
-      ['2017-2018', 35, 26, 9, '74.29 %'],
-      ['2016-2017', 7, 6, 1, '85.71 %'],
+      ['2022-2023', 28, 27, 0, '96.43 %', 28],
+      ['2021-2022', 45, 26, 0, '57.78 %', 45],
+      ['2020-2021', 31, 31, 0, '100.00 %'],
+      ['2019-2020', 56, 56, 0, '100.00 %'],
+      ['2018-2019', 42, 39, 3, '92.86 %'],
+      ['2017-2018', 32, 28, 4, '87.50 %'],
+      ['2016-2017', 6, 5, 1, '83.33 %'],
       emptyYear('2015-2016'),
       emptyYear('2014-2015'),
       emptyYear('2013-2014'),

cypress/e2e/Custom_population.js

@@ -12,7 +12,7 @@ const save = () => {
 }
 
 const selectSavedPopulation = name => {
-  cy.cs('history-search').click().type(`${name}{enter}`)
+  cy.cs('history-search').type(`${name}{enter}`)
 }
 
 const deleteAllSearches = () => {

cypress/e2e/Degree_programme_overview.js

@@ -32,36 +32,21 @@ const deleteTag = tagName => {
 
 describe('Degree programme overview', () => {
   describe('Degree programme selector', () => {
-    beforeEach(() => {
-      cy.init('/study-programme', 'admin')
-    })
-
-    it('Correct categories', () => {
-      cy.contains('Bachelor programmes')
-      cy.contains('Master programmes')
-      cy.contains('Combined programmes')
-      cy.contains('Doctoral programmes')
-    })
-
     it('Degree programme search filter', () => {
+      cy.init('/study-programme', 'admin')
       cy.contains('Tietojenkäsittelytieteen kandiohjelma').should('exist')
       cy.contains('Matemaattisten tieteiden kandiohjelma').should('exist')
       cy.cs('study-programme-filter').type('Tietojenkäsittelytieteen')
       cy.contains('Tietojenkäsittelytieteen kandiohjelma').should('exist')
       cy.contains('Matemaattisten tieteiden kandiohjelma').should('not.exist')
     })
-
-    it('Old and specialized programmes filter', () => {
-      cy.contains('Biotekniikka (ylempi)').should('not.exist')
-      cy.cs('filter-old-programmes-toggle').click()
-      cy.contains('Biotekniikka (ylempi)').should('exist')
-    })
   })
 
   describe('Basic information tab works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
       cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click({ force: true })
+      cy.cs('year-toggle').click() // NOTE: Tests are written for calendar years
     })
 
     // If the backend breaks for one of the sections, the section header is not rendered and this will fail
@@ -79,11 +64,11 @@ describe('Degree programme overview', () => {
         // [Year, Started studying, Accepted, Graduated, Transferred Away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
         [2023, 8, 8, 26, 0, 0],
-        [2022, 25, 26, 48, 1, 4],
-        [2021, 29, 32, 47, 0, 2],
+        [2022, 25, 26, 47, 1, 3],
+        [2021, 29, 32, 48, 0, 2],
         [2020, 26, 27, 12, 1, 3],
         [2019, 28, 34, 1, 0, 1],
-        [2018, 40, 45, 0, 0, 0],
+        [2018, 40, 45, 0, 0, 1],
         [2017, 41, 47, 0, 0, 0],
       ]
 
@@ -93,31 +78,14 @@ describe('Degree programme overview', () => {
     it('Basic information contains correct credits', () => {
       const years = getEmptyYears()
       const tableContents = [
-        ...years.map(year => [year, 0, 0, 0, 0, 0]),
-        [2023, 1519, 1519, 0, 0, 222],
-        [2022, 3235, 3205, 0, 30, 209],
-        [2021, 5133, 5108, 0, 25, 428],
-        [2020, 5801, 5796, 0, 5, 94],
-        [2019, 5305, 5305, 0, 0, 162],
-        [2018, 3442, 3432, 0, 10, 21],
-        [2017, 1211, 1211, 0, 0, 189],
-      ]
-
-      cy.checkTableStats(tableContents, 'credits-produced-by-the-study-programme')
-    })
-
-    it("Toggling 'Show special categories' on displays additional information", () => {
-      cy.cs('special-categories-toggle').click()
-      const years = getEmptyYears()
-      const tableContents = [
-        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0]),
-        [2023, 1519, 1519, 0, 0, 0, 0, 222],
-        [2022, 3235, 3205, 0, 30, 0, 0, 209],
-        [2021, 5133, 5108, 0, 25, 0, 0, 428],
-        [2020, 5801, 5796, 0, 5, 0, 0, 94],
-        [2019, 5305, 5305, 0, 0, 0, 0, 162],
-        [2018, 3442, 3432, 0, 10, 0, 0, 21],
-        [2017, 1211, 1211, 0, 0, 0, 0, 189],
+        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+        [2023, 1519, 1519, 0, 0, 0, 0, 0, 222, 0],
+        [2022, 3235, 3205, 0, 0, 30, 0, 0, 209, 0],
+        [2021, 5133, 5108, 0, 0, 25, 0, 0, 428, 25],
+        [2020, 5801, 5796, 0, 0, 5, 0, 0, 94, 0],
+        [2019, 5305, 5305, 0, 0, 0, 0, 0, 162, 0],
+        [2018, 3442, 3432, 0, 0, 10, 0, 0, 21, 0],
+        [2017, 1211, 1211, 0, 0, 0, 0, 0, 189, 0],
       ]
 
       cy.checkTableStats(tableContents, 'credits-produced-by-the-study-programme')
@@ -129,8 +97,8 @@ describe('Degree programme overview', () => {
         // [Year, Graduated, Wrote thesis]
         ...years.map(year => [year, 0, 0]),
         [2023, 26, 5],
-        [2022, 48, 13],
-        [2021, 47, 76],
+        [2022, 47, 13],
+        [2021, 48, 76],
         [2020, 12, 23],
         [2019, 1, 19],
         [2018, 0, 0],
@@ -147,8 +115,8 @@ describe('Degree programme overview', () => {
         // [Year, Started studying, Accepted, Graduated]
         ...years.map(year => [year, 0, 0, 0]),
         [2023, 8, 8, 24],
-        [2022, 25, 26, 44],
-        [2021, 29, 32, 46],
+        [2022, 25, 26, 43],
+        [2021, 29, 32, 47],
         [2020, 26, 27, 11],
         [2019, 28, 34, 1],
         [2018, 40, 45, 0],
@@ -161,8 +129,8 @@ describe('Degree programme overview', () => {
         // [Year, Graduated, Wrote thesis]
         ...years.map(year => [year, 0, 0]),
         [2023, 24, 5],
-        [2022, 44, 12],
-        [2021, 46, 69],
+        [2022, 43, 12],
+        [2021, 47, 69],
         [2020, 11, 22],
         [2019, 1, 18],
         [2018, 0, 0],
@@ -180,24 +148,24 @@ describe('Degree programme overview', () => {
         // [Year, Started studying, Accepted, Graduated, Transferred away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
         ['2023 - 2024', 8, 8, 4, 0, 0],
-        ['2022 - 2023', 25, 26, 36, 0, 0],
-        ['2021 - 2022', 29, 32, 57, 1, 6],
+        ['2022 - 2023', 25, 26, 35, 0, 0],
+        ['2021 - 2022', 29, 32, 58, 1, 5],
         ['2020 - 2021', 26, 27, 30, 0, 3],
         ['2019 - 2020', 28, 34, 7, 1, 1],
-        ['2018 - 2019', 40, 45, 0, 0, 0],
+        ['2018 - 2019', 40, 45, 0, 0, 1],
         ['2017 - 2018', 41, 47, 0, 0, 0],
       ]
       cy.checkTableStats(studentTableContents, 'students-of-the-study-programme')
 
       const creditTableContents = [
-        ...years.map(year => [year, 0, 0, 0, 0, 0]),
-        ['2023 - 2024', 160, 160, 0, 0, 67],
-        ['2022 - 2023', 2725, 2720, 0, 5, 337],
-        ['2021 - 2022', 4092, 4042, 0, 50, 198],
-        ['2020 - 2021', 5420, 5415, 0, 5, 321],
-        ['2019 - 2020', 6043, 6043, 0, 0, 101],
-        ['2018 - 2019', 4856, 4851, 0, 5, 107],
-        ['2017 - 2018', 2350, 2345, 0, 5, 26],
+        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+        ['2023 - 2024', 160, 160, 0, 0, 0, 0, 0, 67, 0],
+        ['2022 - 2023', 2725, 2720, 0, 0, 5, 0, 0, 337, 0],
+        ['2021 - 2022', 4092, 4042, 0, 0, 50, 0, 0, 198, 0],
+        ['2020 - 2021', 5420, 5415, 0, 0, 5, 0, 0, 321, 25],
+        ['2019 - 2020', 6043, 6043, 0, 0, 0, 0, 0, 101, 0],
+        ['2018 - 2019', 4856, 4851, 0, 0, 5, 0, 0, 107, 0],
+        ['2017 - 2018', 2350, 2345, 0, 0, 5, 0, 0, 26, 0],
       ]
 
       cy.checkTableStats(creditTableContents, 'credits-produced-by-the-study-programme')
@@ -224,18 +192,20 @@ describe('Degree programme overview', () => {
         .should('contain', 47)
         .should('contain', 76)
 
-      cy.cs('bachelor-breakdown-bar-chart-section')
-      cy.cs('graduation-time-toggle').click()
-      cy.cs('bachelor-median-time-bar-chart-section').within(() => {
-        cy.contains('Graduation year')
-        cy.contains('2022')
-        cy.contains('48 graduated').trigger('mouseover')
-        cy.contains('48 students graduated in year 2022')
-        cy.contains('median study time: 44.5 months')
-        cy.contains('22 graduated on time')
-        cy.contains('12 graduated max year overtime')
-        cy.contains('14 graduated over year late')
+      cy.cs('graduation-mode-selector').within(() => {
+        cy.cs('select-median').click()
       })
+      cy.cs('unset-median-bar-chart').within(() => {
+        cy.contains('47 graduated').should('be.visible')
+        cy.contains('47 graduated').hover()
+      })
+
+      cy.get('.grad-vals').contains('47 students graduated in year 2022').should('be.visible')
+
+      cy.contains('median study time: 6 semesters')
+      cy.contains('24 graduated on time')
+      cy.contains('12 graduated max year overtime')
+      cy.contains('11 graduated over year late')
 
       cy.cs('programmes-before-or-after-stacked-bar-chart-section')
         .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
@@ -248,37 +218,43 @@ describe('Degree programme overview', () => {
   })
 
   describe('Graduation times of master programmes', () => {
-    it('are split into two graphs', () => {
+    it('are split into three graphs', () => {
       cy.init('/study-programme')
       cy.contains('a', 'Matematiikan ja tilastotieteen maisteriohjelma').click({ force: true })
+      cy.cs('year-toggle').click() // Tests are written for calendar year
 
-      cy.cs('bachelor-breakdown-bar-chart-section')
-      cy.cs('master-breakdown-bar-chart-section')
+      cy.cs('unset-breakdown-bar-chart')
 
-      cy.cs('graduation-time-toggle').click()
-      cy.cs('master-median-time-bar-chart-section').within(() => {
-        cy.contains('Master study right')
-        cy.contains('Graduation year')
-        cy.contains('2021')
-        cy.contains('2 graduated').trigger('mouseover')
-        cy.contains('2 students graduated in year 2021')
-        cy.contains('median study time: 25 months')
-        cy.contains('1 graduated on time')
-        cy.contains('1 graduated max year overtime')
-        cy.contains('0 graduated over year late')
+      cy.cs('graduation-mode-selector').within(() => {
+        cy.cs('select-average').click()
       })
+      cy.cs('unset-graduation-times-section')
+        .eq(1)
+        .within(() => {
+          cy.contains('Master study right')
+          cy.contains('2 graduated').should('be.visible')
+          cy.contains('2 graduated').hover()
+        })
 
-      cy.cs('bachelor-median-time-bar-chart-section').within(() => {
-        cy.contains('Bachelor + master study right')
-        cy.contains('Graduation year')
-        cy.contains('2023')
-        cy.contains('11 graduated').trigger('mouseover')
-        cy.contains('11 students graduated in year 2023')
-        cy.contains('median study time: 69 months')
-        cy.contains('3 graduated on time')
-        cy.contains('7 graduated max year overtime')
-        cy.contains('1 graduated over year late')
-      })
+      cy.get('.grad-vals').contains('2 students graduated in year 2021').should('be.visible')
+      cy.contains('average study time: 4.5 semesters')
+      cy.contains('1 graduated on time')
+      cy.contains('1 graduated max year overtime')
+      cy.contains('0 graduated over year late')
+
+      cy.cs('unset-graduation-times-section')
+        .eq(0)
+        .within(() => {
+          cy.contains('Bachelor + master study right')
+          cy.contains('11 graduated').should('be.visible')
+          cy.contains('11 graduated').hover()
+        })
+
+      cy.get('.grad-vals').contains('11 students graduated in year 2023').should('be.visible')
+      cy.contains('average study time: 11.18 semesters')
+      cy.contains('4 graduated on time')
+      cy.contains('6 graduated max year overtime')
+      cy.contains('1 graduated over year late')
     })
   })
 
@@ -286,7 +262,7 @@ describe('Degree programme overview', () => {
     beforeEach(() => {
       cy.intercept(
         'GET',
-        'api/v2/studyprogrammes/KH50_001/studytrackstats?graduated=GRADUATED_INCLUDED&special_groups=SPECIAL_INCLUDED&combined_programme='
+        'api/studyprogrammes/KH50_001/studytrackstats?special_groups=SPECIAL_INCLUDED&combined_programme='
       ).as('stQuery')
       cy.init('/study-programme')
       cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click()
@@ -319,15 +295,15 @@ describe('Degree programme overview', () => {
 
     it('Students of the degree programme are shown correctly', () => {
       const tableContents = [
-        // [Year, All, Started studying, Present, Absent, Inactive, Graduated, Men, Women, Other/Unknown, Finland, Other]
-        ['2023 - 2024', 8, 8, 0, 0, 8, 0, 5, 3, 0, 8, 1],
-        ['2022 - 2023', 26, 25, 0, 0, 24, 2, 19, 7, 0, 25, 1],
-        ['2021 - 2022', 38, 29, 0, 0, 33, 5, 29, 9, 0, 36, 4],
-        ['2020 - 2021', 30, 26, 0, 0, 11, 19, 15, 15, 0, 29, 3],
-        ['2019 - 2020', 35, 28, 0, 0, 8, 27, 22, 13, 0, 35, 2],
-        ['2018 - 2019', 45, 40, 0, 0, 6, 39, 26, 19, 0, 44, 1],
-        ['2017 - 2018', 47, 41, 0, 0, 5, 42, 31, 16, 0, 47, 0],
-        ['Total', 229, 197, 0, 0, 95, 134, 147, 82, 0, 224, 12],
+        // [Year, All, Started studying, Present, Absent, Passive, Graduated, Has recent attainments, Men, Women, Other/Unknown, Finland, Other]
+        ['2023 - 2024', 8, 8, 0, 0, 8, 0, 0, 5, 3, 0, 8, 1],
+        ['2022 - 2023', 26, 25, 0, 0, 24, 2, 0, 19, 7, 0, 25, 1],
+        ['2021 - 2022', 37, 29, 0, 0, 33, 4, 0, 29, 8, 0, 35, 4],
+        ['2020 - 2021', 30, 26, 0, 0, 11, 19, 0, 15, 15, 0, 29, 3],
+        ['2019 - 2020', 35, 28, 0, 0, 8, 27, 0, 22, 13, 0, 35, 2],
+        ['2018 - 2019', 46, 40, 0, 0, 6, 40, 0, 26, 20, 0, 45, 1],
+        ['2017 - 2018', 47, 41, 0, 0, 5, 42, 0, 31, 16, 0, 47, 0],
+        ['Total', 229, 197, 0, 0, 95, 134, 0, 147, 82, 0, 224, 12],
       ]
 
       cy.checkTableStats(tableContents, 'study-tracks-and-class-statistics')
@@ -372,14 +348,14 @@ describe('Degree programme overview', () => {
     describe('Population link button works for', () => {
       it('a single year', () => {
         cy.cs('2023-population-link-button').first().click()
-        cy.contains('Matemaattisten tieteiden kandiohjelma 2023 - 2024')
-        cy.contains('Class size 8 students')
+        cy.contains('Matemaattisten tieteiden kandiohjelma')
+        cy.contains('Class of 2023 - 2024, 8 students')
       })
 
       it('total', () => {
         cy.cs('total-population-link-button').click()
         cy.contains('Matemaattisten tieteiden kandiohjelma')
-        cy.contains('Class size 227 students')
+        cy.contains('Class of 2017 - 2026, 227 students')
       })
 
       it('Links to class statistics page with study track info included work', () => {
@@ -388,9 +364,9 @@ describe('Degree programme overview', () => {
           cy.get('a').click()
         })
 
-        cy.contains('Matemaattisten tieteiden kandiohjelma 2022 - 2023')
+        cy.contains('Matemaattisten tieteiden kandiohjelma')
         cy.contains('Studytrack MAT-MAT')
-        cy.contains('Class size 26 students')
+        cy.contains('Class of 2022 - 2023, 26 students')
         cy.contains('Showing 3 out of 26 students')
       })
     })
@@ -398,55 +374,68 @@ describe('Degree programme overview', () => {
     it('Student progress data is shown correctly', () => {
       const years = getEmptyYears(true)
       const tableContents = [
-        // [Year, All, < 30 credits, 30–60 credits, 60–90 credits, 90–120 credits, 120–150 credits, 150–180 credits, ≥ 180 credits]
-        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0]),
-        ['2023 - 2024', 8, 8, 0, 0, 0, 0, 0, 0],
-        ['2022 - 2023', 26, 9, 9, 4, 3, 0, 1, 0],
-        ['2021 - 2022', 38, 8, 6, 11, 8, 5, 0, 0],
-        ['2020 - 2021', 30, 2, 1, 3, 7, 4, 4, 9],
-        ['2019 - 2020', 35, 1, 0, 2, 1, 0, 4, 27],
-        ['2018 - 2019', 45, 0, 1, 1, 2, 0, 3, 38],
-        ['2017 - 2018', 47, 0, 1, 3, 0, 1, 2, 40],
-        ['Total', 229, 28, 18, 24, 21, 10, 14, 114],
+        // [Year, All, < 30 credits, 30–60 credits, 60–90 credits, 90–120 credits, 120–150 credits, 150–180 credits, ≥ 180 credits, Graduated]
+        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+        ['2023 - 2024', 8, 8, 0, 0, 0, 0, 0, 0, 0],
+        ['2022 - 2023', 26, 9, 8, 4, 3, 0, 0, 0, 2],
+        ['2021 - 2022', 37, 8, 4, 9, 8, 4, 0, 0, 4],
+        ['2020 - 2021', 30, 2, 1, 2, 5, 0, 0, 1, 19],
+        ['2019 - 2020', 35, 1, 0, 2, 1, 0, 0, 4, 27],
+        ['2018 - 2019', 46, 0, 1, 1, 2, 0, 0, 2, 40],
+        ['2017 - 2018', 47, 0, 1, 3, 0, 0, 1, 0, 42],
+        ['Total', 229, 28, 15, 21, 19, 4, 1, 7, 134],
       ]
 
       cy.checkTableStats(tableContents, 'study-programme-progress')
     })
 
     it('Progress section', () => {
+      cy.cs('programme-progress-bar-chart-section').scrollIntoView()
+
       cy.cs('programme-progress-bar-chart-section')
         .should('contain', 'Less than 30 credits')
         .should('contain', '30–60 credits')
         .should('contain', 'At least 180 credits')
-        .should('contain', '49.8%') // The percentage for total, at least 180 credits, to check that the graph renders
+        .should('contain', '58.5%') // The percentage for total graduated, to check that the graph renders
 
-      cy.cs('programme-progress-bar-chart-section').contains('49.8%').trigger('mouseover', { force: true })
-      cy.contains('At least 180 credits: 114')
+      cy.cs('programme-progress-bar-chart-section').contains('58.5%').hover()
+      cy.contains('Graduated: 134')
     })
 
-    it('Average graduation times section', () => {
+    // CI contains different data than locally
+    it.skip('Average graduation times section', () => {
+      cy.cs('average-graduation-times-section').scrollIntoView()
+
       cy.cs('average-graduation-times-section').within(() => {
-        cy.contains('2020 - 2021')
-        cy.get('[aria-label="2020 - 2021, 15. On time."]').trigger('mouseover')
-        cy.contains('On time: 15')
-        cy.contains("Click a bar to view that year's study track level breakdown")
-        cy.get('[aria-label="2019 - 2020, 22. On time."]').click()
+        cy.cs('unset-graduation-times-section').within(() => {
+          cy.contains('2020 - 2021')
+        })
+
+        cy.get('text').contains('16').hover()
+        cy.contains('On time: 16')
+
+        cy.cs('unset-graduation-times-section').within(() => {
+          cy.contains("Click a bar to view that year's study track level breakdown")
+          cy.get('text').contains('22').click()
+        })
+
         cy.contains('Year 2019 - 2020 by start year')
-        cy.get('[aria-label="MAT-MAT, 14. On time."]').trigger('mouseover')
+
+        cy.get('text').contains('13').hover()
         cy.contains('Matematiikka')
         cy.contains('MAT-MAT')
-        cy.contains('On time: 14')
+        cy.contains('On time: 13')
       })
 
       cy.cs('graduation-time-toggle').click()
 
       cy.cs('average-graduation-times-section').within(() => {
         cy.contains('2020 - 2021')
-        cy.contains('19 graduated').trigger('mouseover')
+        cy.contains('19 graduated').hover()
         cy.contains('From class of 2020 - 2021, 19/30 students have graduated')
-        cy.contains('median study time: 33 months')
-        cy.contains('15 graduated on time')
-        cy.contains('3 graduated max year overtime')
+        cy.contains('median study time: 6 semesters')
+        cy.contains('16 graduated on time')
+        cy.contains('2 graduated max year overtime')
         cy.contains('1 graduated over year late')
       })
     })
@@ -465,14 +454,14 @@ describe('Degree programme overview', () => {
       it('Students of the study track are shown correctly', () => {
         cy.cs('study-track-overview-section').contains('Students of the study track MAT-MAT by starting year')
         const tableContents = [
-          // [Year, All, Started studying, Present, Absent, Inactive, Graduated, Men, Women, Other/Unknown, Finland, Other]
-          ['2022 - 2023', 3, 3, 0, 0, 1, 2, 2, 1, 0, 3, 0],
-          ['2021 - 2022', 5, 1, 0, 0, 1, 4, 3, 2, 0, 5, 0],
-          ['2020 - 2021', 13, 10, 0, 0, 1, 12, 7, 6, 0, 12, 3],
-          ['2019 - 2020', 17, 14, 0, 0, 0, 17, 10, 7, 0, 17, 1],
-          ['2018 - 2019', 23, 21, 0, 0, 2, 21, 11, 12, 0, 23, 0],
-          ['2017 - 2018', 28, 24, 0, 0, 1, 27, 15, 13, 0, 28, 0],
-          ['Total', 89, 73, 0, 0, 6, 83, 48, 41, 0, 88, 4],
+          // [Year, All, Started studying, Present, Absent, Passive, Graduated, Has recent attainment, Men, Women, Other/Unknown, Finland, Other]
+          ['2022 - 2023', 3, 3, 0, 0, 1, 2, 0, 2, 1, 0, 3, 0],
+          ['2021 - 2022', 4, 1, 0, 0, 1, 3, 0, 3, 1, 0, 4, 0],
+          ['2020 - 2021', 13, 10, 0, 0, 1, 12, 0, 7, 6, 0, 12, 3],
+          ['2019 - 2020', 17, 14, 0, 0, 0, 17, 0, 10, 7, 0, 17, 1],
+          ['2018 - 2019', 24, 21, 0, 0, 2, 22, 0, 11, 13, 0, 24, 0],
+          ['2017 - 2018', 28, 24, 0, 0, 1, 27, 0, 15, 13, 0, 28, 0],
+          ['Total', 89, 73, 0, 0, 6, 83, 0, 48, 41, 0, 88, 4],
         ]
         cy.checkTableStats(tableContents, 'study-tracks-and-class-statistics')
       })
@@ -480,13 +469,14 @@ describe('Degree programme overview', () => {
       it('Links to class statistics page with study track info included work', () => {
         cy.cs('2020-population-link-button').first().click()
 
-        cy.contains('Matemaattisten tieteiden kandiohjelma 2020 - 2021')
+        cy.contains('Matemaattisten tieteiden kandiohjelma')
         cy.contains('Studytrack MAT-MAT')
-        cy.contains('Class size 30 students')
+        cy.contains('Class of 2020 - 2021, 30 students')
         cy.contains('Showing 10 out of 30 students')
       })
 
-      it('Info message about missing progress stats is displayed', () => {
+      // FIXME: This feature was implemented. The data is no-good, so maybe reimplement this when it's clean.
+      it.skip('Info message about missing progress stats is displayed', () => {
         cy.contains('Progress of students of the study track MAT-MAT by starting year')
         cy.contains('Progress data is currently only available for all students of the degree programme.')
       })
@@ -496,7 +486,7 @@ describe('Degree programme overview', () => {
         cy.get("[data-cy='graduation-times-graph-breakdownBachelor']").within(() => {
           cy.contains('Start year')
           cy.contains('2020 - 2021')
-          cy.get('[aria-label="2020 - 2021, 9. On time."]').trigger('mouseover')
+          cy.get('[aria-label="2020 - 2021, 9. On time."]').hover()
           cy.contains('Graduated On time: 9 students')
         })
 
@@ -504,7 +494,7 @@ describe('Degree programme overview', () => {
         cy.cs('bachelor-median-time-bar-chart').within(() => {
           cy.contains('Start year')
           cy.contains('2020 - 2021')
-          cy.contains('12 graduated').trigger('mouseover')
+          cy.contains('12 graduated').hover()
           cy.contains('From class of 2020 - 2021, 12/13 students have graduated')
           cy.contains('median study time: 34 months')
           cy.contains('9 graduated on time')
@@ -528,14 +518,23 @@ describe('Degree programme overview', () => {
 
     describe('By credit type tab', () => {
       beforeEach(() => {
-        cy.get('tbody').within(() => {
+        cy.cs('show-credits-students-toggle').click()
+        cy.cs('year-toggle').click()
+        cy.get('thead').within(() => {
           cy.get('tr')
             .eq(0)
+            .within(() => {
+              cy.get('th').eq(3).click()
+            })
+        })
+        cy.get('tbody').within(() => {
+          cy.get('tr')
+            .eq(8)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('341')
+              cy.get('td').eq(1).contains('MAT11004')
+              cy.get('td').eq(2).contains('Differentiaalilaskenta')
+              cy.get('td').eq(3).contains('1240')
             })
         })
       })
@@ -548,18 +547,18 @@ describe('Degree programme overview', () => {
 
         cy.get('tbody').within(() => {
           cy.get('tr')
-            .eq(0)
+            .eq(9)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('121')
+              cy.get('td').eq(1).contains('MAT21009')
+              cy.get('td').eq(2).contains('Kandidaatintutkielma')
+              cy.get('td').eq(3).contains('414')
             })
         })
       })
 
       it('year toggle works', () => {
-        cy.cs('year-toggle').click()
+        cy.cs('year-toggle').click() // Default view now Academic years
 
         cy.cs('from-year-select').click()
         cy.cs('from-year-select-option-2017').click()
@@ -568,12 +567,12 @@ describe('Degree programme overview', () => {
 
         cy.get('tbody').within(() => {
           cy.get('tr')
-            .eq(0)
+            .eq(5)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('311')
+              cy.get('td').eq(1).contains('MAT11001')
+              cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
+              cy.get('td').eq(3).contains('1265')
             })
         })
       })
@@ -632,31 +631,69 @@ describe('Degree programme overview', () => {
       })
 
       it("'Show credits/Show students' toggle works", () => {
-        cy.get('thead tr').within(() => {
-          cy.get('th').should('have.length', 8)
-          const headers = ['Type', 'Code', 'Name', 'Total', 'Major', 'Non-major', 'Non-degree', 'Transferred']
-          headers.forEach((header, index) => {
-            cy.get('th').eq(index).contains(header)
+        cy.cs('year-toggle').click()
+        cy.wait(500) // So that the toggle has time to load
+        cy.get('thead tr')
+          .eq(0)
+          .within(() => {
+            cy.get('th').should('have.length', 6)
+            const headers = ['Type', 'Code', 'Name', 'Total credits', 'Credits produced by', 'Transferred credits']
+            headers.forEach((header, index) => {
+              cy.get('th').eq(index).contains(header)
+            })
+          })
+        cy.get('thead tr')
+          .eq(1)
+          .within(() => {
+            cy.get('th').should('have.length', 7)
+            const headers = [
+              'Degree students',
+              'Open university (hetu)',
+              'Open university (no hetu)',
+              'Exchange students',
+              'Other universities',
+              'Separate studies',
+              'Other',
+            ]
+            headers.forEach((header, index) => {
+              cy.get('th').eq(index).contains(header)
+            })
           })
-        })
 
         cy.cs('show-credits-students-toggle').click()
 
+        cy.get('thead tr')
+          .eq(0)
+          .within(() => {
+            cy.get('th').should('have.length', 7)
+            const headers = [
+              'Type',
+              'Code',
+              'Name',
+              'Total students',
+              'Breakdown of total',
+              'Breakdown of passed',
+              'Not included in total or passed',
+            ]
+            headers.forEach((header, index) => {
+              cy.get('th').eq(index).contains(header)
+            })
+          })
         cy.get('thead tr')
           .eq(1)
           .within(() => {
             cy.get('th').should('have.length', 10)
             const headers = [
-              'Type',
-              'Code',
-              'Name',
-              'Total',
               'Passed',
-              'Not',
-              'Major',
-              'Non-major',
-              'Non-degree',
-              'Transferred',
+              'Not completed',
+              'Degree students',
+              'Open university students (with hetu)',
+              'Open university students (without hetu)',
+              'Exchange students',
+              'Other university students',
+              'Separate studies',
+              'Other students',
+              'Students with transferred credits',
             ]
             headers.forEach((header, index) => {
               cy.get('th').eq(index).contains(header)
@@ -665,12 +702,12 @@ describe('Degree programme overview', () => {
 
         cy.get('tbody').within(() => {
           cy.get('tr')
-            .eq(0)
+            .eq(5)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('341')
+              cy.get('td').eq(1).contains('MAT11001')
+              cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
+              cy.get('td').eq(3).contains('272')
             })
         })
       })
@@ -682,7 +719,7 @@ describe('Degree programme overview', () => {
       })
 
       // FIXME: Flaky
-      it('test', () => {
+      it.skip('test', () => {
         // TODO: Implement some real tests for this tab
         cy.contains('From')
       })
@@ -797,7 +834,8 @@ describe('Degree programme overview', () => {
         cy.contains('Students added to tag')
 
         cy.cs('population-link-button').click()
-        cy.contains('Matemaattisten tieteiden kandiohjelma 2022 - 2023')
+        cy.contains('Matemaattisten tieteiden kandiohjelma')
+        cy.contains('Class of 2022 - 2023')
         cy.contains(`Tagged with: ${tagName}`)
         cy.contains('Students (5)')
           .parent()

cypress/e2e/Faculty_statistics.js

@@ -110,15 +110,15 @@ describe('Faculty statistics', () => {
       })
 
       it('Toggle years works', () => {
-        cy.cs('credits-produced-by-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('thesis-writers-of-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('students-of-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('graduated-of-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('year-toggle').click()
         cy.cs('credits-produced-by-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
         cy.cs('thesis-writers-of-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
         cy.cs('students-of-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
         cy.cs('graduated-of-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
+        cy.cs('year-toggle').click()
+        cy.cs('credits-produced-by-the-faculty-interactive-data-table').should('contain', '2022')
+        cy.cs('thesis-writers-of-the-faculty-interactive-data-table').should('contain', '2022')
+        cy.cs('students-of-the-faculty-interactive-data-table').should('contain', '2022')
+        cy.cs('graduated-of-the-faculty-interactive-data-table').should('contain', '2022')
       })
 
       it('Toggle programmes works', () => {

cypress/e2e/Filters.js

@@ -19,7 +19,7 @@ describe("Population statistics with a master's programme", { testIsolation: fal
 
   it('Study track filter works', () => {
     cy.cs('studyTrackFilter-header').click()
-    cy.selectFromDropdown('studyTrackFilter', 0, true)
+    cy.selectFromDropdown('studyTrackFilter', 'Matematiikka ja soveltava matematiikka (MAST-MSM)', true)
     checkFilteredStudentCount(15)
 
     cy.cs('studyTrackFilter-clear').click()
@@ -184,7 +184,7 @@ describe.skip('Population Statistics', { testIsolation: false }, () => {
     cy.selectFromDropdown(`courseFilter-${courses[0].code}`, 1)
     checkFilteredStudentCount(7)
     cy.selectFromAutocomplete('courseFilter', `${courses[1].code} - ${courses[1].name}`)
-    checkFilteredStudentCount(5)
+    checkFilteredStudentCount(6)
     cy.selectFromDropdown(`courseFilter-${courses[1].code}`, 1)
     checkFilteredStudentCount(4)
 
@@ -273,12 +273,12 @@ describe('Population Statistics with Bachelor + Master', { testIsolation: false
     checkFilteredStudentCount(0)
 
     // Non-Bachelor
-    cy.get('input[name="studyRightStatusFilter"][value="Inactive Bachelor study right"]').click()
-    cy.get('input[name="studyRightStatusFilter"][value="Inactive Bachelor study right"]').should('be.checked')
+    cy.get('input[name="studyRightStatusFilter"][value="Passive Bachelor study right"]').click()
+    cy.get('input[name="studyRightStatusFilter"][value="Passive Bachelor study right"]').should('be.checked')
     checkFilteredStudentCount(5)
     // Non-Master (combined)
-    cy.get('input[name="studyRightStatusFilter"][value="Inactive Master study right"]').click()
-    cy.get('input[name="studyRightStatusFilter"][value="Inactive Master study right"]').should('be.checked')
+    cy.get('input[name="studyRightStatusFilter"][value="Passive Master study right"]').click()
+    cy.get('input[name="studyRightStatusFilter"][value="Passive Master study right"]').should('be.checked')
     checkFilteredStudentCount(24)
 
     cy.cs('studyRightStatusFilter-clear').click()
@@ -453,9 +453,9 @@ describe('Custom Population Statistics', { testIsolation: false }, () => {
     cy.selectFromDropdown(`courseFilter-${courses[0].code}`, 1)
     checkFilteredStudentCount(6)
     cy.selectFromAutocomplete('courseFilter', `${courses[1].code} - ${courses[1].name}`)
-    checkFilteredStudentCount(5)
+    checkFilteredStudentCount(6)
     cy.selectFromDropdown(`courseFilter-${courses[1].code}`, 3)
-    checkFilteredStudentCount(2)
+    checkFilteredStudentCount(6)
 
     cy.cs(`courseFilter-${courses[1].code}-clear`).click()
     checkFilteredStudentCount(6)

cypress/e2e/Language_center.js

@@ -44,7 +44,7 @@ describe('When language center is opened', () => {
         cy.get('table > tbody > tr:first').within(() => {
           cy.get('td').should('have.length', 15)
           cy.get('td').eq(0).contains('All courses total')
-          const numbers = [2076, 36, 9, 33, 756, 15, 40, 6, 48, 1, 24, 1059, 49, 0]
+          const numbers = [2076, 36, 9, 34, 759, 15, 40, 6, 51, 1, 24, 1059, 42, 0]
           numbers.forEach((number, index) => {
             cy.get('td')
               .eq(index + 1)
@@ -55,12 +55,12 @@ describe('When language center is opened', () => {
 
       it('Faculties tab "exceeding" button works', () => {
         cy.cs('difference-button').click()
-        checkNumbers([65, 2, 0, 3, 13, 0, 1, 1, 2, 0, 1, 30, 11, 1], 15, 'faculties')
+        checkNumbers([66, 3, 0, 4, 18, 0, 1, 1, 4, 0, 0, 31, 3, 1], 15, 'faculties')
       })
 
       it('Faculties tab semester selector changes numbers', () => {
         chooseSemester('Syksy 2020', 'from')
-        checkNumbers([1184, 28, 6, 27, 496, 4, 10, 2, 33, 0, 9, 535, 34, 0], 15, 'faculties')
+        checkNumbers([1184, 28, 6, 28, 499, 4, 10, 2, 36, 0, 9, 535, 27, 0], 15, 'faculties')
       })
     })
 

cypress/e2e/Numbers_match.js

@@ -0,0 +1,122 @@
+/// <reference types="cypress" />
+
+const selectCourseStatus = (courseCode, status) => {
+  cy.cs(`courseFilter-${courseCode}-selector`).click()
+  cy.get(`[role="listbox"][aria-labelledby=courseFilter-${courseCode}]`).within(() => {
+    cy.get('li').contains(status).click()
+  })
+}
+
+describe('Numbers should match between', () => {
+  describe('Programme courses, Course statistics and Course population (no substitutions)', () => {
+    const test = [
+      // Code, name, total students, passed, not completed, failed*, enrolled no grade* (not used by Programme courses)
+      ['Course', 'MAT11001', 'Johdatus yliopistomatematiikkaan', '272', '253', (1 + 18).toString(), '1', '18'],
+      ['Course', 'MAT11003', 'Raja-arvot', '270', '249', (1 + 20).toString(), '1', '20'],
+      ['Course', 'MAT11004', 'Differentiaalilaskenta', '262', '248', (1 + 13).toString(), '1', '13'],
+      ['Course', 'MAT21002', 'Sarjat', '256', '250', (0 + 6).toString(), '0', '6'],
+      ['Course', 'MAT11005', 'Integraalilaskenta', '251', '234', (0 + 17).toString(), '0', '17'],
+      ['Course', 'MAT11002', 'Lineaarialgebra ja matriisilaskenta I', '247', '242', (1 + 4).toString(), '1', '4'],
+      ['Course', 'MAT12003', 'Todennäköisyyslaskenta I', '234', '227', (3 + 4).toString(), '3', '4'],
+      ['Course', 'MAT21001', 'Lineaarialgebra ja matriisilaskenta II', '230', '216', (4 + 10).toString(), '4', '10'],
+      ['Course', 'MAT21003', 'Vektorianalyysi I', '228', '202', (3 + 23).toString(), '3', '23'],
+      ['Course', 'MAT20005', 'Akateemiset taidot', '201', '178', (0 + 23).toString(), '0', '23'],
+      ['Module', 'MAT110', 'Matematiikka, perusopinnot', '195', '195', (0 + 0).toString(), '0', '0'],
+      ['Course', 'MAT21014', 'Johdatus logiikkaan I', '188', '170', (0 + 18).toString(), '0', '18'],
+    ]
+
+    it('in Programme courses', () => {
+      // Move to Math bachelor programme's programme courses
+      cy.init('/study-programme')
+      cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click()
+      cy.cs('ProgrammeCoursesTab').click()
+
+      // Sort by Total students
+      cy.get('thead').within(() => {
+        cy.get('tr')
+          .eq(0)
+          .within(() => {
+            cy.get('th').eq(3).click()
+          })
+      })
+
+      cy.get('tbody').within(() => {
+        test.forEach((course, index) => {
+          cy.get('tr')
+            .eq(index)
+            .within(() => {
+              // Check a course's fields
+              course.forEach((field, fieldIndex) => {
+                if (fieldIndex <= 5) {
+                  // Failed, Enrolled no grade values not used
+                  cy.get('td').eq(fieldIndex).contains(field)
+                }
+              })
+
+              // Course link button, (name is second entry)
+              // cy.cs(`${course.at(1).toLocaleLowerCase()}-course-statistics-link-button`).click()
+            })
+        })
+      })
+    })
+
+    it('in Course statistics', () => {
+      test.forEach(course => {
+        // Move to Course statistics
+        cy.init(`/coursestatistics?courseCodes=%5B%22${course.at(1)}%22%5D&combineSubstitutions=false`, 'basic')
+        cy.url().should('include', `/coursestatistics?courseCodes=%5B%22${course.at(1)}%22%5D`)
+        cy.contains('Course statistics')
+        cy.contains(course.at(2)) // course name
+
+        cy.cs('FromYearSelector').click()
+        cy.cs('FromYearSelectorOption2017-2018').click()
+
+        cy.get('tbody').within(() => {
+          cy.get('tr')
+            .eq(0)
+            .within(() => {
+              cy.get('td').eq(1).contains(course.at(3))
+              cy.get('td').eq(2).contains(course.at(4))
+              cy.get('td').eq(3).contains(course.at(6)) // jump over combined Not completed
+              cy.get('td').eq(4).contains(course.at(7))
+            })
+        })
+      })
+    })
+
+    it('in Course population', () => {
+      test.forEach(course => {
+        // Academic years between 2017-2023
+        cy.init(
+          `/coursepopulation?from=68&to=74&coursecodes=%5B%22${course.at(1)}%22%5D&includeSubstitutions=false`,
+          'basic'
+        )
+        cy.contains(course.at(2)) // Course name
+
+        // Select course from Course-filter
+        cy.cs('courseFilter-filter-card').click()
+        cy.cs('courseFilter-selector').click()
+        cy.cs('courseFilter-popper').within(() => {
+          cy.contains('li', `${course.at(1)} - ${course.at(2)}`).click() // "code - name"
+        })
+
+        selectCourseStatus(course.at(1), 'All')
+        cy.contains(`Students (${course.at(3)})`)
+
+        selectCourseStatus(course.at(1), 'Passed')
+        cy.contains(`Students (${course.at(4)})`)
+
+        selectCourseStatus(course.at(1), 'Failed')
+        cy.contains(`Students (${course.at(6)})`)
+
+        selectCourseStatus(course.at(1), 'Enrolled, No Grade')
+        cy.contains(`Students (${course.at(7)})`)
+      })
+    })
+  })
+
+  describe('Course statistics and Course population', () => {
+    it.skip('when selecting one year')
+    it.skip('when selecting the all years')
+  })
+})

cypress/e2e/Openuni.js

@@ -1,126 +0,0 @@
-/// <reference types="cypress" />
-
-const deleteAllSearches = () => {
-  cy.cs('open-uni-search-button').click()
-  cy.contains('Saved populations').parent().parent().parent().parent().get('.search').as('search')
-
-  cy.get('@search').then($el => {
-    const searchItems = $el.find('div[role=option] > span[class=text]')
-    for (let i = searchItems.length - 1; 0 <= i; i--) {
-      if (searchItems[i].textContent.includes('TEST-')) {
-        cy.get('[data-cy="history-search"]').children().eq(0).type(`${searchItems[i].textContent}{enter}`)
-        cy.get('button').contains('Delete').click()
-      }
-    }
-  })
-}
-
-// Skipping the whole test suite as it is unclear whether the feature has any use or will be
-// removed alltogether.
-describe.skip('Open uni population tests', () => {
-  describe('Open uni form can be used', () => {
-    // When openUniUser is created, change it here
-    beforeEach(() => {
-      cy.init('/openunipopulation', 'admin')
-    })
-    after(() => {
-      cy.init('/openunipopulation', 'admin')
-      cy.contains('Open uni student population')
-      deleteAllSearches()
-    })
-
-    // TODO: Rewrite the whole component
-    it.skip('Finds a proper population', () => {
-      cy.fixture('openUniPopulation').then(({ courseCodesSet1 }) => {
-        cy.get('[data-cy="open-uni-search-button"]').click()
-        cy.contains('Insert course code(s)').siblings().get('textarea').as('textarea')
-        cy.get('@textarea').type(courseCodesSet1.join('\n'))
-        cy.cs('begin-of-search').click()
-        cy.cs('begin-of-search').clear()
-        cy.cs('begin-of-search').type('01.01.2015')
-        cy.cs('end-of-search').click()
-        cy.cs('end-of-search').clear()
-        cy.cs('end-of-search').type('01.01.2020')
-        cy.get('button').contains('Search population').click()
-        cy.contains('Beginning of the search for all fields:')
-        cy.contains('01.01.2015')
-        cy.contains('End of the search for enrollments:')
-        cy.contains('01.01.2020')
-        cy.location('pathname').should('eq', '/openunipopulation')
-        cy.location('search').then(search => {
-          const params = new URLSearchParams(search)
-
-          const courseCodes = params.getAll('courseCode')
-          expect(courseCodes).to.include.members(['TKT10001', 'TKT10002'])
-
-          expect(params.get('startdate')).to.eq('01-01-2015')
-          expect(params.get('enddate')).to.eq('01-01-2020')
-        })
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[0])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[1])
-      })
-    })
-
-    it('Textarea can handle different kinds of input', () => {
-      cy.get('[data-cy="open-uni-search-button"]').click()
-      cy.contains('Insert course code(s)')
-        .siblings()
-        .get('textarea')
-        .type('tkt10001    tkt10002,tkt10003\n TKT10004,,TKt10005\nTKT-1234')
-      cy.get('button').contains('Search population').click()
-      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10001')
-      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10002')
-      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10003')
-      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10004')
-      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10005')
-      cy.get('[data-cy="open-uni-table-div"]').should('not.contain', 'TKT-1234')
-    })
-
-    it('Saves a custom population search', () => {
-      cy.fixture('openUniPopulation').then(({ courseCodesSet1 }) => {
-        cy.get('[data-cy="open-uni-search-button"]').click()
-        cy.get('[data-cy="search-name"]').type('TEST-Avoimen-populaatio')
-        cy.contains('Insert course code(s)').siblings().get('textarea').type(courseCodesSet1.join('\n'))
-        cy.get('[data-cy="save-search"]').click()
-        cy.get('[data-cy="history-search"]').children().eq(0).type('TEST-Avoimen-populaatio{enter}')
-        cy.contains('TEST-Avoimen-populaatio')
-        cy.get('button').contains('Search population').as('btn')
-        cy.get('@btn').click()
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[0])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[1])
-      })
-    })
-
-    it('Updates a custom population search', () => {
-      cy.fixture('openUniPopulation').then(({ courseCodesSet1, courseCodesSet2 }) => {
-        cy.get('[data-cy="open-uni-search-button"]').click()
-        cy.get('[data-cy="search-name"]').type('TEST-Avoimen-populaatio')
-        cy.contains('Insert course code(s)').siblings().get('textarea').type(courseCodesSet1.join('\n'))
-        cy.get('[data-cy="save-search"]').click()
-        cy.get('button').contains('Search population').click()
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[0])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[1])
-
-        cy.get('[data-cy="open-uni-search-button"]').click()
-        cy.get('[data-cy="history-search"]').children().eq(0).type('TEST-Avoimen-populaatio{enter}')
-        cy.contains('Insert course code(s)').siblings().get('textarea').should('contain', courseCodesSet1.join(', '))
-        cy.contains('Insert course code(s)').siblings().get('textarea').type('\n')
-        cy.contains('Insert course code(s)').siblings().get('textarea').type(courseCodesSet2.join('\n'))
-        cy.get('[data-cy="save-search"]').click()
-        cy.get('button').contains('Search population').click()
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[0])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[1])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet2[0])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet2[1])
-        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet2[2])
-      })
-    })
-
-    it('View does not crash with non-existing courses', () => {
-      cy.get('[data-cy="open-uni-search-button"]').click()
-      cy.contains('Insert course code(s)').siblings().get('textarea').type('cypress-fun')
-      cy.get('button').contains('Search population').click()
-      cy.get('[data-cy="open-uni-table-div"]').should('not.contain', 'cypress-fun')
-    })
-  })
-})

cypress/e2e/Population_statistics.js

@@ -27,14 +27,14 @@ describe('Population statistics tests', () => {
   describe('When using basic user', () => {
     beforeEach(() => {
       cy.init('/populations')
-      cy.intercept('/api/v3/populationstatistics/studyprogrammes').as('studyprogrammes')
-      cy.intercept('/api/v2/studyprogrammes/**/studytracks').as('studytracks')
+      cy.intercept('/api/populationstatistics/studyprogrammes').as('studyprogrammes')
+      cy.intercept('/api/studyprogrammes/**/studytracks').as('studytracks')
     })
 
     describe('Population search', () => {
       it('Info box works', () => {
         cy.cs('PopulationSearch-info-box-content').should('not.exist')
-        cy.cs('PopulationSearch-info-box-button').trigger('mouseover')
+        cy.cs('PopulationSearch-info-box-button').hover()
         cy.cs('PopulationSearch-info-box-content').should('be.visible')
         cy.cs('PopulationSearch-info-box-content').contains('Tässä osiossa voi tarkastella')
         cy.cs('PopulationSearch-info-box-button').trigger('mouseout')
@@ -74,8 +74,8 @@ describe('Population statistics tests', () => {
           selectStudyProgramme('Matemaattisten tieteiden kandiohjelma')
           cy.wait('@studytracks').its('response.statusCode').should('be.oneOf', [200, 304])
           cy.contains('See class').click()
-          cy.contains('Matemaattisten tieteiden kandiohjelma 2017 - 2018')
-          cy.contains('Class size 47 students')
+          cy.contains('Matemaattisten tieteiden kandiohjelma')
+          cy.contains('Class of 2017 - 2018, 47 students')
         })
 
         // TODO: Delete MUI
@@ -95,7 +95,7 @@ describe('Population statistics tests', () => {
       cy.visit(pathToMathBSc2020)
       cy.cs('filtered-students')
       cy.contains('Matemaattisten tieteiden kandiohjelma')
-      cy.contains('Class size 30 students')
+      cy.contains('Class of 2020 - 2021, 30 students')
       cy.contains('Excludes exchange students')
       cy.contains('Excludes students with non-degree study right')
       cy.contains('Excludes students who have transferred out of this programme')
@@ -251,15 +251,15 @@ describe('Population statistics tests', () => {
       it('Is displayed and link to individual course stats page works', () => {
         cy.visit(pathToMathBSc2020)
         cy.contains('Courses of class').click()
-        cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
+        cy.intercept('/api/courseyearlystats**').as('coursePage')
         cy.cs('toggle-group-module-MAT110').click()
         cy.contains('td', 'MAT11001').siblings().find('[data-testid="NorthEastIcon"]').click()
         cy.wait('@coursePage')
         cy.url().should('include', '/coursestatistics')
-        cy.contains('MAT11001 • Johdatus yliopistomatematiikkaan')
-        cy.contains('AYMAT11001 • Avoin yo: Johdatus yliopistomatematiikkaan')
-        cy.contains('57033 • Johdatus yliopistomatematiikkaan')
-        cy.contains('A57033 • Avoin yo: Johdatus yliopistomatematiikkaan')
+        cy.contains('MAT11001') // Johdatus yliopistomatematiikkaan
+        cy.contains('AYMAT11001') // Avoin yo: Johdatus yliopistomatematiikkaan
+        cy.contains('57033') // Johdatus yliopistomatematiikkaan
+        cy.contains('A57033') // Avoin yo: Johdatus yliopistomatematiikkaan
       })
 
       it('Curriculum selection works', () => {
@@ -383,7 +383,7 @@ describe('Population statistics tests', () => {
       cy.contains(nonExisting).should('not.exist')
       cy.contains('button', 'Check student numbers').click()
       cy.contains('Check for student numbers')
-      cy.cs('check-student-numbers').click().type(`${existing}{enter}${nonExisting}`)
+      cy.cs('check-student-numbers').type(`${existing}{enter}${nonExisting}`)
       cy.contains('button', 'Check students').click()
       cy.get('#checkstudentsresults').within(() => {
         cy.cs('found-title').click()
@@ -401,8 +401,8 @@ describe('Population statistics tests', () => {
   describe('When using IAM user', () => {
     beforeEach(() => {
       cy.init(pathToMathBSc2020, 'onlyiamrights')
-      cy.contains('Matemaattisten tieteiden kandiohjelma 2020 - 2021')
-      cy.contains('Class size 30 students')
+      cy.contains('Matemaattisten tieteiden kandiohjelma')
+      cy.contains('Class of 2020 - 2021, 30 students')
     })
 
     it('Population statistics is visible', () => {

cypress/e2e/Routes.js

@@ -0,0 +1,85 @@
+import { userHeaders } from '../support/commands.js'
+
+const getBaseRoutes = [
+  ['/', {}],
+  ['/changelog', {}],
+  ['/completedcoursessearch', {}],
+  ['/university', {}],
+  ['/feedback', {}],
+  ['/populations', { roles: ['fullSisuAccess'], programmeRights: [] }],
+  ['/study-programme', { roles: ['fullSisuAccess'], programmeRights: [] }],
+  ['/study-programme/KH50_001', { roles: ['fullSisuAccess'], programmeRights: ['KH50_001'] }],
+  ['/coursepopulation', { roles: ['fullSisuAccess'], programmeRights: [] }],
+  ['/users', { roles: ['admin'] }],
+  ['/users/334', { roles: ['admin'] }],
+  ['/updater', { roles: ['admin'] }],
+  ['/coursestatistics', { roles: ['fullSisuAccess', 'courseStatistics'], programmeRights: [] }],
+  ['/faculties', { roles: ['fullSisuAccess', 'facultyStatistics'] }],
+  ['/faculties/hy-org-1000000911', { roles: ['fullSisuAccess', 'facultyStatistics'] }],
+  ['/students', { roles: ['fullSisuAccess', 'studyGuidanceGroups'], programmeRights: [] }],
+  ['/students/545400', { roles: ['fullSisuAccess', 'studyGuidanceGroups'], programmeRights: [] }],
+  ['/custompopulation', { roles: ['fullSisuAccess', 'studyGuidanceGroups'], programmeRights: [] }],
+  ['/teachers', { roles: ['teachers'] }],
+  ['/teachers/hy-hlo-1463447', { roles: ['teachers'] }],
+  ['/studyguidancegroups', { roles: ['studyGuidanceGroups'] }],
+  ['/studyguidancegroups/otm-b3b52ff5-88f6-4d52-bd18-0588023b6322', { userId: '', roles: ['studyGuidanceGroups'] }],
+  ['/close-to-graduation', { roles: ['fullSisuAccess', 'studyGuidanceGroups'] }],
+  ['/languagecenterview', { iamGroups: ['grp-kielikeskus-esihenkilot'] }],
+]
+
+for (const user of userHeaders) {
+  if (user.uid === 'admin') continue
+  describe(`${user.displayname} has access only to views that were intended`, () => {
+    let userInfo
+    before(() => {
+      cy.request({ method: 'GET', url: '/api/login', headers: user }).then(({ body }) => (userInfo = body?.user))
+    })
+
+    beforeEach(() => cy.wrap(userInfo).should('not.be.undefined'))
+    beforeEach(() => {
+      cy.intercept('**/api/**', req => {
+        req.on('response', res => {
+          expect(res.statusCode, `${req.method} ${req.url}`).to.be.lessThan(500)
+        })
+      })
+    })
+
+    for (const [route, requirements] of getBaseRoutes) {
+      it(`Checking access ${route}`, () => {
+        cy.init(route, user.uid)
+
+        const requiredUserId = !requirements.userId || requirements.userId === userInfo.userId
+
+        const requiredRoles = !requirements.roles || requirements.roles?.some(role => userInfo.roles?.includes(role))
+
+        const requiredRights =
+          !requirements.requireProgrammeRights ||
+          userInfo.roles?.includes('fullSisuAccess') ||
+          requirements.programmeRights.every(role => userInfo.programmeRights?.includes(role))
+
+        // This is defined in frontend ProtectedRoute component.
+        const specialNeedsRoutes = [
+          'populations',
+          'study-programme',
+          'coursepopulation',
+          'coursestatistics',
+          'students',
+          'custompopulation',
+        ]
+
+        const mankeli = specialNeedsRoutes.some(path => route.includes(path))
+          ? requiredRoles || requiredRights
+          : requiredRoles && requiredRights
+
+        const shouldHaveAccess = mankeli && requiredUserId
+
+        if (shouldHaveAccess) {
+          cy.contains('Something broke').should('not.exist')
+          cy.contains('Access denied').should('not.exist')
+        } else {
+          cy.contains('Access denied')
+        }
+      })
+    }
+  })
+}

cypress/e2e/Students.js

@@ -114,9 +114,9 @@ describe('Students tests', () => {
           cy.get('a').click()
         })
       cy.url().should('include', '/coursestatistics')
-      cy.contains('MAT12004 • Tilastollinen päättely I')
-      cy.contains('AYMAT12004 • Avoin yo: Tilastollinen päättely I')
-      cy.contains('57046 • Johdatus tilastolliseen päättelyyn')
+      cy.contains('MAT12004') // Tilastollinen päättely I
+      cy.contains('AYMAT12004') // Avoin yo: Tilastollinen päättely I
+      cy.contains('57046') // Johdatus tilastolliseen päättelyyn
     })
 
     it('Has correct Sisu link', () => {
@@ -162,8 +162,8 @@ describe('Students tests', () => {
           .parent()
           .within(() => cy.get('a').click())
       })
-      cy.contains('Matemaattisten tieteiden kandiohjelma 2020 - 2021')
-      cy.contains('Class size 30 students')
+      cy.contains('Matemaattisten tieteiden kandiohjelma')
+      cy.contains('Class of 2020 - 2021, 30 students')
     })
 
     it('Grade graph works in all three different modes', () => {
@@ -174,18 +174,13 @@ describe('Students tests', () => {
         cy.contains('label', 'Group size')
         cy.get('input').should('have.value', '5')
       })
-      cy.contains('.highcharts-container text', "Nov '22")
-      cy.contains('.highcharts-container text', "Jul '22").should('not.exist')
       cy.cs('group-size-input').within(() => {
         cy.contains('label', 'Group size')
         cy.get('input').clear()
         cy.get('input').type('10')
         cy.get('input').should('have.value', '10')
       })
-      cy.contains('.highcharts-container text', "Jul '22")
-      cy.contains('.highcharts-container text', "Nov '22").should('not.exist')
       cy.contains('button', 'Show semester mean').click()
-      cy.contains('.highcharts-container text', "Jan '22")
     })
 
     describe('Bachelor Honours section', () => {
@@ -264,8 +259,10 @@ describe('Students tests', () => {
 
     it('When a study plan is selected, courses included in the study plan are highlighted with a blue background', () => {
       cy.visit('/students/550789')
-      cy.contains('table tbody tr', 'Kulttuurien tutkimuksen kandiohjelma (01.08.2020–30.06.2023)').within(() => {
-        cy.get('td').eq(0).click()
+      cy.contains('Study rights')
+      cy.get('[data-cy=KH40_005-0]').contains('Kulttuurien tutkimuksen kandiohjelma')
+      cy.get('[data-cy=KH40_005-0]').within(() => {
+        cy.get('td').eq(1).click()
       })
       cy.contains('table tbody tr', 'Kandidaatintutkielma (KUKA-LIS222)').should(
         'have.css',
@@ -274,31 +271,23 @@ describe('Students tests', () => {
       )
     })
 
-    it('When a study plan is selected, the time frame of the credit graph is updated', () => {
-      cy.clock(new Date('2024-08-30').getTime(), ['Date'])
-      cy.visit('/students/550789')
-      cy.contains('.highcharts-container text', '1 Aug 2020')
-      cy.contains('.highcharts-container text', '30 Aug 2024')
-      cy.contains('table tbody tr', 'Kulttuurien tutkimuksen kandiohjelma (01.08.2020–30.06.2023)').within(() => {
-        cy.get('td').eq(0).click()
-      })
-      cy.contains('.highcharts-container text', '1 Aug 2020')
-      cy.contains('.highcharts-container text', '19 Aug 2023')
-    })
-
     it("If there's a study plan corresponding to the degree programme, completed credits are displayed", () => {
       cy.visit('/students/550789')
-      cy.contains('table tbody tr', 'Kulttuurien tutkimuksen kandiohjelma (01.08.2020–30.06.2023)').within(() => {
+      cy.contains('Study rights')
+      cy.get('[data-cy=KH40_005-0]').contains('Kulttuurien tutkimuksen kandiohjelma')
+      cy.get('[data-cy=KH40_005-0]').within(() => {
         cy.get('td')
-          .eq(-1)
+          .eq(4)
           .contains(/^185 cr$/)
       })
     })
 
     it("If there's a study plan corresponding to the degree programme and the student hasn't graduated, percentage of completion is also displayed", () => {
       cy.visit('/students/458723')
-      cy.contains('table tbody tr', 'Oikeusnotaarin koulutusohjelma (01.08.2020–31.07.2027)').within(() => {
-        cy.get('td').eq(-1).contains('69% (125 cr)')
+      cy.contains('Study rights')
+      cy.get('[data-cy=KH20_001-0]').contains('Oikeusnotaarin koulutusohjelma')
+      cy.get('[data-cy=KH20_001-0]').within(() => {
+        cy.get('td').eq(4).contains('69% (125 cr)')
       })
     })
   })

cypress/e2e/Teachers.js

@@ -35,7 +35,7 @@ describe('Teachers page tests', () => {
     cy.cs('Search').click()
     cy.cs('teacher-search').type(teacher1.split(' ')[0])
     cy.get('table tbody tr').should('have.length', 4)
-    cy.get('table tbody tr').eq(3).contains('td', teacher1)
+    cy.get('table tbody tr').eq(2).contains('td', teacher1)
   })
 
   it('Can check teacher page', () => {

cypress/e2e/University.js

@@ -10,21 +10,18 @@ const checkProgressTables = () => {
   progressLevels.forEach(level => cy.cs(`${level}-faculty-progress-table`))
 }
 
-const graduationTimesLevels = ['bachelor', 'master', 'doctor'] // ? Bachelor + master seems to be missing in test data
+const graduationTimesLevels = ['bachelor', 'bcMsCombo', 'master', 'doctor']
 
-const checkAverageGraduationTimesBreakdownBarCharts = () => {
+const checkGraduationCharts = mode => {
   graduationTimesLevels.forEach(level => {
-    cy.cs(`${level}-breakdown-bar-chart`)
+    cy.cs(`${level}-${mode}-bar-chart`).within(() => {
+      // Check that bar has loaded
+      cy.contains('Loading content').should('not.exist')
+    })
   })
 }
 
-const checkAverageGraduationTimesMedianBarCharts = () => {
-  graduationTimesLevels.forEach(level => {
-    cy.cs(`${level}-median-bar-chart`)
-  })
-}
-
-describe('University view', () => {
+describe('University view', { defaultCommandTimeout: 60 * 1000 }, () => {
   beforeEach(() => {
     cy.init('/university')
     cy.contains('University')
@@ -39,12 +36,6 @@ describe('University view', () => {
       checkProgressTables()
     })
 
-    it("'Graduated included / Graduated excluded' toggle works", () => {
-      cy.cs('graduated-toggle').click()
-      checkProgressBarCharts()
-      checkProgressTables()
-    })
-
     it("'All study rights / Special study rights excluded' toggle works", () => {
       cy.cs('study-right-toggle').click()
       checkProgressBarCharts()
@@ -55,7 +46,7 @@ describe('University view', () => {
     it.skip('years in the tables can be clicked to show faculty level breakdown', () => {
       cy.cs('study-right-toggle').click()
       cy.cs('bachelors-faculty-progress-table-show-button3').click()
-      cy.contains('29.5%').trigger('mouseover', { force: true })
+      cy.contains('29.5%').hover()
       cy.contains('Matemaattis-luonnontieteellinen tiedekunta')
       cy.contains('H50')
       cy.contains('0 Credits: 0')
@@ -79,17 +70,31 @@ describe('University view', () => {
       cy.cs('faculty-graduations-tab').click()
     })
 
-    it('has all the correct median time bar charts', () => {
-      checkAverageGraduationTimesBreakdownBarCharts()
-    })
+    describe('Different modes work', () => {
+      it('Breakdown', () => {
+        cy.cs('graduation-mode-selector').within(() => {
+          cy.cs('select-breakdown').click()
+        })
+        checkGraduationCharts('breakdown')
+      })
 
-    it("'Breakdown/Median times' toggle works", () => {
-      cy.cs('graduation-time-toggle').click()
-      checkAverageGraduationTimesMedianBarCharts()
+      it('Median', () => {
+        cy.cs('graduation-mode-selector').within(() => {
+          cy.cs('select-median').click()
+        })
+        checkGraduationCharts('median')
+      })
+
+      it('Average', () => {
+        cy.cs('graduation-mode-selector').within(() => {
+          cy.cs('select-average').click()
+        })
+        checkGraduationCharts('average')
+      })
     })
 
     it('info boxes contain correct information', () => {
-      cy.cs('average-graduation-times-info-box-button').click()
+      cy.cs('average-graduation-times-info-box-button').hover()
       cy.cs('average-graduation-times-info-box-content').contains('Opiskelijoiden keskimääräiset valmistumisajat')
     })
   })

cypress/e2e/Users.js

@@ -72,7 +72,7 @@ describe('Users tests', () => {
       })
 
       it("only the mocked user's programmes are visible", () => {
-        cy.intercept('/api/v3/populationstatistics/studyprogrammes').as('studyprogrammes')
+        cy.intercept('/api/populationstatistics/studyprogrammes').as('studyprogrammes')
         cy.visit('/populations')
         cy.wait('@studyprogrammes').its('response.statusCode').should('be.oneOf', [200, 304])
 

cypress/support/commands.js

@@ -58,18 +58,7 @@ const onlyStudyGuidanceGroupsUser = {
   hypersonsisuid: 'hy-hlo-2222222',
 }
 
-// TODO add user into anon-DB (check the need for changing hypersonid)
-// REMEMBER TO ADD ROLE openUniSearch
-// const openUniUser = {
-//   uid: 'openuniuser',
-//   displayname: 'Open Uni User',
-//   'shib-session-id': 'mock-cypress-session',
-//   hygroupcn: 'grp-oodikone-basics-users;hy-ypa-opa-dojo',
-//   mail: 'grp-toska+mockopenuniuser@helsinki.fi',
-//   hypersonsisuid: 'hy-hlo-9999999',
-// }
-
-const userHeaders = [
+export const userHeaders = [
   adminUserHeaders,
   basicUserHeaders,
   onlycoursestatisticsUserHeaders,
@@ -190,3 +179,20 @@ Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {
     cy.get('tr').filter(':visible').should('have.length', correctStats.length)
   })
 })
+
+/**
+ * Hover on the selected element.
+ */
+Cypress.Commands.add('hover', { prevSubject: 'element' }, (subject, options = {}) => {
+  cy.wrap(subject).trigger('mouseover', {
+    bubbles: true,
+    cancelable: true,
+    ...options,
+  })
+
+  cy.wrap(subject).trigger('mousemove', {
+    bubbles: true,
+    cancelable: true,
+    ...options,
+  })
+})

cypress/support/e2e.js

@@ -13,7 +13,6 @@
 // https://on.cypress.io/configuration
 // ***********************************************************
 
-require('@cypress/code-coverage/support')
 require('./commands')
 
 Cypress.on('uncaught:exception', error => {

docker-compose.ci.yml

@@ -1,15 +1,30 @@
+name: oodikone
+
 x-defaults: &defaults
   restart: unless-stopped
 
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
 services:
   backend:
     <<: *defaults
-    container_name: backend
+    container_name: oodikone-backend
+    image: oodikone-backend:staging
+    build:
+      context: services
+      dockerfile: backend/Dockerfile
     depends_on:
-      - kone-db
-      - redis
-      - sis-db
-      - user-db
+      kone-db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      user-db:
+        condition: service_healthy
     environment:
       CI: 'true'
       DB_URL_USER: postgres://postgres@user-db:5432/user-db
@@ -22,122 +37,152 @@ services:
       IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
       JAMI_URL: 'http://jami:3003/'
       CRYPT_KEY: 'e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'
-    build:
-      context: services
-      dockerfile: backend/Dockerfile
+    ports:
+      - 8080:8080
+    profiles:
+      - oodikone
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8080/health'] # In CI/prod the routing happens at a different level, hence no /api/ needed
+      <<: *healthcheck-defaults
 
   frontend:
     <<: *defaults
-    container_name: frontend
+    container_name: oodikone-frontend
+    image: oodikone-frontend:staging
     build:
       context: services
       dockerfile: frontend/Dockerfile
+    depends_on:
+      backend:
+        condition: service_healthy
+    ports:
+      - 5000:5000
+    profiles:
+      - oodikone
 
   kone-db:
     <<: *defaults
-    container_name: kone-db
+    container_name: oodikone-kone-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
     environment:
       PGDATA: /data
       POSTGRES_DB: kone-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
+    ports:
+      - 4321:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
+
+  user-db:
+    <<: *defaults
+    container_name: oodikone-user-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    ports:
+      - 4320:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   redis:
     <<: *defaults
+    container_name: oodikone-redis
+    image: redis:7.2.6
     command: >
       sh -c 'redis-server --appendonly yes && redis-cli flushall'
-    image: redis:7.2.6
     ports:
       - 6379:6379
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping', '|', 'grep', 'PONG']
+      <<: *healthcheck-defaults
 
   sis-db:
     <<: *defaults
-    container_name: sis-db
+    container_name: oodikone-sis-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
     environment:
       PGDATA: /data
       POSTGRES_DB: sis-db
       POSTGRES_HOST_AUTH_METHOD: trust
     shm_size: 2gb
+    ports:
+      - 12345:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
-  user-db:
+  sis-importer-db:
     <<: *defaults
-    container_name: user-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: user-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    container_name: oodikone-sis-importer-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
+    profiles:
+      - updater
+    ports:
+      - 12346:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   jami:
     <<: *defaults
+    container_name: oodikone-jami
     image: quay.io/toska/jami:production
+    depends_on:
+      jami-db:
+        condition: service_healthy
     environment:
       - npm_config_cache=/tmp
       - PORT=3003
-      - NODE_ENV=dev
+      - NODE_ENV=test # previously was development which borks CI
       - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
     ports:
       - 3003:3003
-    container_name: jami
 
   jami-db:
     <<: *defaults
+    container_name: oodikone-jami-db
     image: postgres:15.10
     environment:
       - PGDATA=/data
       - POSTGRES_PASSWORD=postgres
-    container_name: jami-db
     ports:
       - 5433:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
+  # This is still needed to make backend calls correctly route from localhost:5000/api to localhost:8080
   nginx:
+    container_name: oodikone-nginx
     image: nginx:latest
-    container_name: nginx
     depends_on:
-      - frontend
-      - backend
+      frontend:
+        condition: service_started
+      backend:
+        condition: service_healthy
     ports:
       - 1337:80
+    profiles:
+      - oodikone
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf
 
-  # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
-  sis-importer-db:
-    <<: *defaults
-    container_name: sis-importer-db
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
-    profiles:
-      - updater
-
-  sis-updater-worker:
-    <<: *defaults
-    build:
-      context: updater/sis-updater-worker
-    depends_on:
-      - redis
-      - sis-db
-      - sis-importer-db
-    environment:
-      CI: 'true'
-      DB_URL: postgres://postgres@sis-db:5432/sis-db
-      REDIS_HOST: redis
-      SIS_IMPORTER_DATABASE: sis-importer-db
-      SIS_IMPORTER_HOST: sis-importer-db
-      SIS_IMPORTER_PASSWORD: postgres
-      SIS_IMPORTER_USER: postgres
-      SIS_PASSWORD: postgres
-    profiles:
-      - updater
-
+  # Following services + sis-importer-db are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
   sis-updater-scheduler:
-    <<: *defaults
+    restart: no
+    container_name: oodikone-updater-scheduler
     build:
       context: updater/sis-updater-scheduler
-    container_name: sis-updater-scheduler
+      dockerfile: ./Dockerfile
     depends_on:
-      - redis
-      - sis-importer-db
+      redis:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
     environment:
       CI: 'true'
       REDIS_HOST: redis
@@ -152,3 +197,31 @@ services:
       - 8082:8082
     profiles:
       - updater
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8082/health']
+      <<: *healthcheck-defaults
+
+  sis-updater-worker:
+    <<: *defaults
+    container_name: oodikone-updater-worker
+    build:
+      context: updater/sis-updater-worker
+      dockerfile: ./Dockerfile
+    depends_on:
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
+    environment:
+      CI: 'true'
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_HOST: redis
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_PASSWORD: postgres
+    profiles:
+      - updater

docker-compose.fd.ci.yml

@@ -1,47 +0,0 @@
-services:
-  oodikone-backend:
-    build:
-      context: ${OODIKONE_REPO:-.}/services
-      dockerfile: backend/Dockerfile
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        SERVICE_PROVIDER: fd
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/oodikone-backend:${DOCKER_IMAGE_TAG:-dev}
-
-  oodikone-frontend:
-    build:
-      context: ${OODIKONE_REPO:-.}/services
-      dockerfile: frontend/Dockerfile
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        SERVICE_PROVIDER: fd
-        LANGUAGE_CENTER_VIEW_ENABLED: 'false' # currently no intention to be enabled
-        SIS_URL: 'https://sis-qa.funidata.fi'
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/oodikone-frontend:${DOCKER_IMAGE_TAG:-dev}
-
-  updater-scheduler:
-    build:
-      context: ${OODIKONE_REPO:-.}/updater/sis-updater-scheduler
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/updater-scheduler:${DOCKER_IMAGE_TAG:-dev}
-
-  updater-worker:
-    build:
-      context: ${OODIKONE_REPO:-.}/updater/sis-updater-worker
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        SERVICE_PROVIDER: fd
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/updater-worker:${DOCKER_IMAGE_TAG:-dev}

docker-compose.fd.yml

@@ -1,190 +0,0 @@
-x-defaults: &defaults
-  restart: unless-stopped
-
-x-build_defaults: &build_defaults
-  dockerfile: dev.Dockerfile
-
-services:
-  adminer:
-    <<: *defaults
-    container_name: adminer
-    environment:
-      ADMINER_DESIGN: pepa-linha
-    image: adminer:4.8.1
-    ports:
-      - 5050:8080
-    volumes:
-      - ./adminer-plugins:/var/www/html/plugins-enabled
-
-  backend:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/backend
-    depends_on:
-      - kone-db
-      - redis
-      - sis-db
-      - user-db
-    env_file:
-      - .env
-    environment:
-      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
-      DB_URL_USER: postgres://postgres@user-db:5432/user-db
-      FRONT_URL: http://localhost:3000
-      REDIS: redis
-      SECRET_TOKEN: dev
-      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
-      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      IMPORTER_DB_API_URL: 'https://localhost:3005'
-      JAMI_URL: 'http://jami:3003/'
-      SERVICE_PROVIDER: fd
-      SIS_HOST: ${SIS_HOST}
-      SIS_GRAPHQL_APPLICATION_ACCOUNT: ${SIS_GRAPHQL_APPLICATION_ACCOUNT}
-      SIS_GRAPHQL_APPLICATION_KEY: ${SIS_GRAPHQL_APPLICATION_KEY}
-      IMPORTER_DB_API_USER: dev_user
-      IMPORTER_DB_API_PASSWORD: dev_password
-      ROOT_ORG_ID: ${ROOT_ORG_ID}
-      LANGUAGE_CENTER_VIEW_ENABLED: 'false'
-    ports:
-      - 8080:8080
-    volumes:
-      - backend-nodemod:/opt/app-root/backend/node_modules
-      - ./environments/testu:/opt/app-root/backend/src/environment
-      - ./services/backend/:/opt/app-root/backend
-      - ./services/shared/:/opt/app-root/shared
-
-  frontend:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/frontend
-    container_name: frontend
-    ports:
-      - 3000:3000
-    environment:
-      REACT_APP_SERVICE_PROVIDER: fd
-      REACT_APP_LANGUAGE_CENTER_VIEW_ENABLED: 'false'
-      REACT_APP_SIS_URL: 'https://sis-qa.funidata.fi'
-    volumes:
-      - frontend-nodemod:/opt/app-root/frontend/node_modules
-      - ./services/frontend/:/opt/app-root/frontend
-      - ./services/shared/:/opt/app-root/shared
-
-  kone-db:
-    <<: *defaults
-    container_name: kone-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: kone-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
-    ports:
-      - 12349:5432
-    volumes:
-      - kone-db-data:/data
-
-  redis:
-    <<: *defaults
-    command: 'redis-server --appendonly yes'
-    container_name: redis
-    image: redis:6.2.16
-    ports:
-      - '6379:6379'
-    volumes:
-      - redis-data:/data
-
-  sis-db:
-    <<: *defaults
-    container_name: sis-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: sis-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
-    volumes:
-      - sis-db-data:/data
-    shm_size: 2gb
-    ports:
-      - 12345:5432
-
-  sis-updater-scheduler:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: updater/sis-updater-scheduler
-    container_name: sis-updater-scheduler
-    extra_hosts:
-      - 'host.docker.internal:host-gateway'
-    depends_on:
-      - redis
-    environment:
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      SECRET_TOKEN: dev
-      SIS_IMPORTER_HOST: ${SIS_IMPORTER_DB_HOST:-host.docker.internal}
-      SIS_IMPORTER_PORT: ${SIS_IMPORTER_DB_PORT:-15234}
-      SIS_IMPORTER_DATABASE: importer-db
-      SIS_IMPORTER_PASSWORD: dev
-      SIS_IMPORTER_USER: dev
-      SERVICE_PROVIDER: fd
-    ports:
-      - 8082:8082
-    volumes:
-      - ./updater/sis-updater-scheduler/:/opt/app-root/src
-
-  sis-updater-worker:
-    <<: *defaults
-    deploy:
-      replicas: 2
-      mode: replicated
-    build:
-      <<: *build_defaults
-      context: updater/sis-updater-worker
-    depends_on:
-      - redis
-      - sis-db
-    extra_hosts:
-      - 'host.docker.internal:host-gateway'
-    environment:
-      DB_URL: postgres://postgres@sis-db:5432/sis-db
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      SIS_IMPORTER_HOST: ${SIS_IMPORTER_DB_HOST:-host.docker.internal}
-      SIS_IMPORTER_PORT: ${SIS_IMPORTER_DB_PORT:-15234}
-      SIS_IMPORTER_DATABASE: importer-db
-      SIS_IMPORTER_PASSWORD: dev
-      SIS_IMPORTER_USER: dev
-      SIS_PASSWORD: postgres
-      ROOT_ORG_ID: ${ROOT_ORG_ID}
-      SERVICE_PROVIDER: fd
-    volumes:
-      - ./updater/sis-updater-worker/:/opt/app-root/src
-
-  user-db:
-    <<: *defaults
-    container_name: user-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: user-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
-    volumes:
-      - user-db-data:/data
-    ports:
-      - 12347:5432
-
-volumes:
-  backend-nodemod:
-  frontend-nodemod:
-  kone-db-data:
-  redis-data:
-  sis-db-data:
-  user-db-data:
-
-networks:
-  default:
-    name: oodikone
-  importer:
-    name: importer_network
-    external: true

docker-compose.real.yml

@@ -1,3 +1,5 @@
+name: oodikone
+
 services:
   backend:
     environment:
@@ -20,3 +22,4 @@ services:
 
 volumes:
   redis-data-real:
+    name: oodikone-redis-data-real

docker-compose.test.yml

@@ -1,5 +1,27 @@
+name: oodikone
+
+x-defaults: &defaults
+  restart: unless-stopped
+
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
 services:
-  frontend:
-    volumes:
-      - frontend-nodemod:/opt/app-root/frontend/node_modules
-      - ./services/frontend/instrumented:/opt/app-root/frontend/src
+  sis-db:
+    <<: *defaults
+    container_name: oodikone-sis-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
+    pull_policy: always
+    volumes: !reset [] # Remove previous volume definitions
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    shm_size: 2gb
+    ports:
+      - 12345:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults

docker-compose.yml

@@ -1,13 +1,20 @@
+name: oodikone
+
 x-defaults: &defaults
   restart: unless-stopped
 
 x-build_defaults: &build_defaults
   dockerfile: dev.Dockerfile
 
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
 services:
   adminer:
     <<: *defaults
-    container_name: adminer
+    container_name: oodikone-adminer
     environment:
       ADMINER_DESIGN: pepa-linha
     image: adminer:4.8.1
@@ -18,14 +25,19 @@ services:
 
   backend:
     <<: *defaults
+    container_name: oodikone-backend
     build:
-      <<: *build_defaults
-      context: services/backend
+      dockerfile: backend/dev.Dockerfile
+      context: services
     depends_on:
-      - kone-db
-      - redis
-      - sis-db
-      - user-db
+      kone-db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      user-db:
+        condition: service_healthy
     env_file:
       - .env
     tty: true
@@ -44,57 +56,90 @@ services:
     profiles:
       - oodikone
     volumes:
-      - backend-nodemod:/opt/app-root/backend/node_modules
       - ./services/backend/:/opt/app-root/backend
       - ./services/shared/:/opt/app-root/shared
+      - shared-nodemod:/opt/app-root/shared/node_modules
+      - backend-nodemod:/opt/app-root/backend/node_modules
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8080/api/health']
+      <<: *healthcheck-defaults
 
   frontend:
     <<: *defaults
+    container_name: oodikone-frontend
     build:
-      <<: *build_defaults
-      context: services/frontend
-    container_name: frontend
+      dockerfile: frontend/dev.Dockerfile
+      context: services
+    depends_on:
+      backend:
+        condition: service_healthy
     ports:
       - 3000:3000
     profiles:
       - oodikone
     volumes:
-      - frontend-nodemod:/opt/app-root/frontend/node_modules
       - ./services/frontend/:/opt/app-root/frontend
       - ./services/shared/:/opt/app-root/shared
+      - shared-nodemod:/opt/app-root/shared/node_modules
+      - frontend-nodemod:/opt/app-root/frontend/node_modules
 
   kone-db:
     <<: *defaults
-    container_name: kone-db
+    container_name: oodikone-kone-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
     profiles:
       - oodikone
+    ports:
+      - 4321:5432
     volumes:
       - kone-db-data:/data
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
+
+  user-db:
+    <<: *defaults
+    container_name: oodikone-user-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    profiles:
+      - oodikone
+    ports:
+      - 4320:5432
+    volumes:
+      - user-db-data:/data
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   redis:
     <<: *defaults
+    container_name: oodikone-redis
     command: 'redis-server --appendonly yes'
-    container_name: redis
     image: redis:7.2.6
     ports:
-      - '6379:6379'
+      - 6379:6379
     volumes:
       - redis-data:/data
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping', '|', 'grep', 'PONG']
+      <<: *healthcheck-defaults
 
   sis-db:
     <<: *defaults
-    container_name: sis-db
+    container_name: oodikone-sis-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
     volumes:
       - sis-db-data:/data
     shm_size: 2gb
     ports:
       - 12345:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-importer-db:
     <<: *defaults
-    container_name: sis-importer-db
+    container_name: oodikone-sis-importer-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
     profiles:
       - updater
@@ -102,40 +147,55 @@ services:
       - sis-importer-db-data:/data
     ports:
       - 12346:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
-  sis-updater-scheduler:
+  jami:
     <<: *defaults
-    build:
-      <<: *build_defaults
-      context: updater/sis-updater-scheduler
-    container_name: sis-updater-scheduler
+    container_name: oodikone-jami
+    image: quay.io/toska/jami:production
     depends_on:
-      - redis
-      - sis-importer-db
+      jami-db:
+        condition: service_healthy
+    env_file:
+      - .env
     environment:
-      REDIS_HOST: redis
-      SECRET_TOKEN: dev
-      SIS_IMPORTER_DATABASE: sis-importer-db
-      SIS_IMPORTER_HOST: sis-importer-db
-      SIS_IMPORTER_PASSWORD: postgres
-      SIS_IMPORTER_USER: postgres
+      - npm_config_cache=/tmp
+      - PORT=3003
+      - NODE_ENV=development
+      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/jami-db-real
     ports:
-      - 8082:8082
-    profiles:
-      - updater
+      - 3003:3003
+
+  jami-db:
+    <<: *defaults
+    container_name: oodikone-jami-db
+    image: postgres:15.10
+    environment:
+      - PGDATA=/data
+      - POSTGRES_PASSWORD=postgres
+    ports:
+      - 5433:5432
     volumes:
-      - sis-updater-scheduler-nodemod:/opt/app-root/src/node_modules
-      - ./updater/sis-updater-scheduler/:/opt/app-root/src
+      - jami-data:/data
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-updater-worker:
     <<: *defaults
+    container_name: oodikone-updater-worker
     build:
       <<: *build_defaults
       context: updater/sis-updater-worker
     depends_on:
-      - redis
-      - sis-db
-      - sis-importer-db
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
     environment:
       DB_URL: postgres://postgres@sis-db:5432/sis-db
       REDIS_HOST: redis
@@ -147,52 +207,60 @@ services:
     profiles:
       - updater
     volumes:
-      - sis-updater-worker-nodemod:/opt/app-root/src/node_modules
       - ./updater/sis-updater-worker/:/opt/app-root/src
+      - sis-updater-worker-nodemod:/opt/app-root/src/node_modules
 
-  user-db:
+  sis-updater-scheduler:
     <<: *defaults
-    container_name: user-db
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    container_name: oodikone-updater-scheduler
+    build:
+      <<: *build_defaults
+      context: updater/sis-updater-scheduler
+    depends_on:
+      redis:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
+    environment:
+      REDIS_HOST: redis
+      SECRET_TOKEN: dev
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
+    ports:
+      - 8082:8082
     profiles:
-      - oodikone
+      - updater
     volumes:
-      - user-db-data:/data
-
-  jami:
-    <<: *defaults
-    image: quay.io/toska/jami:production
-    env_file:
-      - .env
-    environment:
-      - npm_config_cache=/tmp
-      - PORT=3003
-      - NODE_ENV=dev
-      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
-    ports:
-      - 3003:3003
-    container_name: jami
-
-  jami-db:
-    <<: *defaults
-    image: postgres:15.10
-    environment:
-      - PGDATA=/data
-      - POSTGRES_PASSWORD=postgres
-    container_name: jami-db
-    ports:
-      - 5433:5432
-    volumes:
-      - jami-data:/data
+      - ./updater/sis-updater-scheduler/:/opt/app-root/src
+      - sis-updater-scheduler-nodemod:/opt/app-root/src/node_modules
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8082/health']
+      <<: *healthcheck-defaults
 
 volumes:
   backend-nodemod:
+    name: oodikone-backend-nodemod
   frontend-nodemod:
+    name: oodikone-frontend-nodemod
+  shared-nodemod:
+    name: oodikone-shared-nodemod
   kone-db-data:
+    name: oodikone-kone-db-data
   redis-data:
+    name: oodikone-redis-data
   sis-db-data:
+    name: oodikone-sis-db-data
   sis-importer-db-data:
+    name: oodikone-sis-importer-db-data
   sis-updater-scheduler-nodemod:
+    name: oodikone-updater-scheduler-nodemod
   sis-updater-worker-nodemod:
+    name: oodikone-updater-worker-nodemod
   user-db-data:
+    name: oodikone-user-db-data
   jami-data:
+    name: oodikone-jami-data

documentation/access_rights.md

@@ -8,11 +8,10 @@ Roles are defined [here](../services/backend/src/config/roles.ts). Criteria for
 - courseStatistics
 - facultyStatistics
 - fullSisuAccess
-- openUniSearch
 - studyGuidanceGroups
 - teachers
 
-The roles `facultyStatistics`, `openUniSearch`, and `studyGuidanceGroups` simply enable certain views (_Faculties_, _Open uni student population_, and _Guidance groups_ respectively).
+The roles `facultyStatistics` and `studyGuidanceGroups` simply enable certain views (_Faculties_ and _Guidance groups_ respectively).
 
 ### courseStatistics
 
@@ -43,7 +42,6 @@ The roles `facultyStatistics`, `openUniSearch`, and `studyGuidanceGroups` simply
 | Custom population    |             |  ✅   |       ✅       |
 | Completed courses    |     ✅      |  ✅   |       ✅       |
 | Close to graduation  |             |  ✅   |       ✅       |
-| Open uni population  |             |  ✅   |                |
 | Courses              |             |  ✅   |       ✅       |
 
 \* = anyone who can log in to Oodikone, check the [IAM groups](#iam-groups) section for more information
@@ -63,7 +61,6 @@ A user must be a member of either the `grp-oodikone-basic-users` or `grp-oodikon
 | grp-oodikone-basic-users | courseStatistics              |
 | grp-oodikone-users       | facultyStatistics             |
 | grp-toska                | admin                         |
-| hy-ypa-opa-dojo          | openUniSearch                 |
 
 `grp-oodikone-basic-users`: basically the whole staff (hy-\*-allstaff)
 

documentation/combined_programme.md

@@ -36,7 +36,7 @@ All things listed below should work automatically, if new combinations are done.
 
 - _Students of the studyprogramme by starting year_
   - Graduated field separated for both programmes
-  - All = currently enrolled + absent + inactive + graduated licentiate
+  - All = currently enrolled + absent + passive + graduated licentiate
 - _Progress of students of the studyprogramme by starting year_
   - computed for whole bachelor + master/licentiate studyright period
 - _Average graduation times_
@@ -85,12 +85,12 @@ All things listed below should work automatically, if new combinations are done.
 
 - **Graduated from programme** - possibility to filter out students based on graduation either in bachelor's or master's.
 
-- **Studyright status** - active or inactive based on the chosen degree
+- **Studyright status** - active or passive based on the chosen degree
 
 - **Studytrack filter** (needs programme code) is not touched at the moment, thus **may need future manual work**.
 
 ## Study guidance groups
 
-- Backend: student data is fetched by bachelor degree programme code in `/v3/populationstatisticsbystudentnumbers`. Separation from code 'KHxx_xxx+MHxx_xxx' is done in backend.
+- Backend: student data is fetched by bachelor degree programme code in `/populationstatisticsbystudentnumbers`. Separation from code 'KHxx_xxx+MHxx_xxx' is done in backend.
 - Frontend: things are done in the same way as with class stats.
 - **Need manual work**: `./oodikone/services/frontend/src/redux/elementdetails.js` add combined programme codes (both bachelor and master) to `combinedProgrammeCodes` and modify `combinedOptions` list accordingly in `useFilterAndFormattedElementDetails()` function.

documentation/migrations.md

@@ -8,7 +8,7 @@ These instructions explain how to update the schemas for the following databases
 
 The steps to **add, remove, or modify columns** are mostly the same for each database, but the file paths and container names differ. Always **double-check** that you have the correct function name, column name, data type, and other relevant details in your migration file.
 
-> [!NOTE]  
+> [!NOTE]
 > If you need to change the `sis-importer-db` schema, see the instructions in the [sis-importer repo](https://github.com/UniversityOfHelsinkiCS/sis-importer/blob/master/README.md#tricks--tips).
 
 ## General guidelines
@@ -25,7 +25,7 @@ The steps to **add, remove, or modify columns** are mostly the same for each dat
 - Use **snake_case** in the actual database columns.
 - If a model has `underscored: true`, Sequelize automatically converts camelCase to snake_case in all queries. Otherwise, specify the column name explicitly.
 
-> [!IMPORTANT]  
+> [!IMPORTANT]
 > The column name **in the database** (snake_case) must be used in the migration file, not the camelCase name from your model.
 
 ### Verifying the migration locally
@@ -73,7 +73,7 @@ If you need to **revert a migration locally**, follow these steps:
 
 ## `sis-db`
 
-> [!NOTE]  
+> [!NOTE]
 > These instructions assume that needed data is already in the `sis-importer-db`. If not, you must add the data there first. See the [sis-importer repo](https://github.com/UniversityOfHelsinkiCS/sis-importer/blob/master/README.md#tricks--tips) for more details.
 
 1. **Update the updater model:**

documentation/numeric_types.md

@@ -55,11 +55,11 @@ The `semester_enrollments` column is an array of JSON objects. Each object conta
 
 The `type` field represents one of the following enrollment types. The `semester` field refers to a semester code, defined as `semestercode` in the model Semester.
 
-| value | meaning  | notes                            |
-| ----: | :------- | :------------------------------- |
-|     1 | Present  |                                  |
-|     2 | Absent   |                                  |
-|     3 | Inactive | neither 1 or 2 has been declared |
+| value | meaning | notes                            |
+| ----: | :------ | :------------------------------- |
+|     1 | Present |                                  |
+|     2 | Absent  |                                  |
+|     3 | Passive | neither 1 or 2 has been declared |
 
 ## Credit type code
 

documentation/resources.md

@@ -2,26 +2,19 @@
 
 A collection of useful tools, documentation and other resources.
 
-## AI tools
-
-- [ChatGPT](https://chatgpt.com)
-- [CurreChat](https://curre.helsinki.fi/chat)
-- [GitHub Copilot](https://github.com/features/copilot)
-  - Free for students!
-- [Microsoft Copilot](https://copilot.microsoft.com)
-
 ## Code style and maintainability
 
-- [CodeClimate documentation](https://docs.codeclimate.com/docs)
-  - [Advanced configuration](https://docs.codeclimate.com/docs/advanced-configuration)
 - [ESLint documentation](https://eslint.org/docs/v8.x)
 - [Prettier documentation](https://prettier.io/docs/en)
+- [qlty.sh](https://qlty.sh/gh/UniversityOfHelsinkiCS/projects/oodikone)
+  - Qlty.sh replaced Code climate which was introduced by Riku
+  - The usefulness of qlty.sh is sometimes questionable
 
 ## Database
 
 - [pgcli](https://www.pgcli.com)
   - A useful commandline tool to interact with the database
-  - Install: `$ pip install pgcli`
+  - Can be installed from some package managers or optionally via `$ pip install pgcli`
 - [Postgres documentation](https://www.postgresql.org/docs/15/index.html)
 - [Sequelize documentation](https://sequelize.org/docs/v6/)
 
@@ -29,8 +22,6 @@ A collection of useful tools, documentation and other resources.
 
 - [GitHub documentation](https://docs.github.com/en)
   - [GitHub Actions](https://docs.github.com/en/actions)
-- [GitHub Education](https://github.com/education/students)
-  - Free benefits for students!
 - [Mermaid.js documentation](https://mermaid.js.org/intro/syntax-reference.html)
 
 ## TypeScript

eslint.config.mjs

@@ -25,7 +25,7 @@ export default tseslint.config(
       'nyc-config.cjs',
       '.lintstagedrc.mjs',
       'cypress.config.js',
-    ]
+    ],
   },
 
   // Global options (excl. e2e)
@@ -44,8 +44,8 @@ export default tseslint.config(
       parser: tseslint.parser,
       parserOptions: {
         projectService: {
-          allowDefaultProject: ['*.{js,cjs}']
-        }
+          allowDefaultProject: ['*.{js,cjs}'],
+        },
       },
       globals: {
         ...globals.browser,
@@ -53,16 +53,16 @@ export default tseslint.config(
       },
     },
     plugins: {
-      'import-x': eslintPluginImportX
+      'import-x': eslintPluginImportX,
     },
     linterOptions: {
       reportUnusedDisableDirectives: 'error',
     },
     rules: {
-      'camelcase': 'off', // TODO: Enable. old db/backend logic is still using snake_case every now and then and needs a big refactor
+      camelcase: 'off', // TODO: Enable. old db/backend logic is still using snake_case every now and then and needs a big refactor
       'class-methods-use-this': 'off', // Disabled in favor of @typescript-eslint
       'consistent-return': 'off', // Disabled in favor of @typescript-eslint
-      'id-denylist': ['error', 'c', 'd', 'e', 'err', 't'],
+      'id-denylist': ['error', 'err'],
       'no-alert': 'error',
       'no-await-in-loop': 'error',
       'no-console': 'error',
@@ -75,24 +75,42 @@ export default tseslint.config(
       'prefer-const': 'error',
       'prefer-destructuring': 'off', // Disabled in favor of @typescript-eslint
       'dot-notation': 'off', // Disabled in favor of @typescript-eslint
-      'no-restricted-imports': ['error', { 'patterns': [{ 'regex': '^@mui/[^/]+$' }] }],
-      'quotes': ['error', 'single',
-        { avoidEscape: true, allowTemplateLiterals: false }
+      'no-restricted-imports': [
+        'error',
+        {
+          patterns: [
+            {
+              regex: '^@mui/[^/]+$',
+              message:
+                'Import only the needed module. This has no effect in prod due to tree-shake, but speeds up development builds.',
+            },
+            {
+              regex: '^src/.*',
+              message: 'Root imports will not resolve in production build (instead silently crash).',
+            },
+          ],
+        },
       ],
+      quotes: ['error', 'single', { avoidEscape: true, allowTemplateLiterals: false }],
 
       'import-x/no-commonjs': 'error',
       'import-x/no-default-export': 'error',
       'import-x/no-extraneous-dependencies': ['error', { devDependencies: false }],
-      'import-x/no-unused-modules': ['error',
+      'import-x/no-unused-modules': [
+        'error',
         {
           missingExports: true,
           unusedExports: true,
-          src: ['{services,updater}/**/*.{js,jsx,ts,tsx}']
-        }],
-      'import-x/order': ['error', {
-        alphabetize: { order: 'asc', caseInsensitive: true },
-        groups: [['builtin', 'external'], ['internal'], ['parent'], ['sibling', 'index']],
-      }],
+          src: ['{services,updater}/**/*.{js,jsx,ts,tsx}'],
+        },
+      ],
+      'import-x/order': [
+        'error',
+        {
+          alphabetize: { order: 'asc', caseInsensitive: true },
+          groups: [['builtin', 'external'], ['internal'], ['parent'], ['sibling', 'index']],
+        },
+      ],
 
       // TypeScript
       '@typescript-eslint/no-unused-vars': [
@@ -105,9 +123,7 @@ export default tseslint.config(
       '@typescript-eslint/consistent-return': 'error', // NOTE: This is weaker than tsconfig.json noImplicitReturns, and should be replaced once ts migration complete
       '@typescript-eslint/consistent-type-definitions': 'off',
       '@typescript-eslint/dot-notation': 'error',
-      '@typescript-eslint/prefer-destructuring': ['error',
-        { VariableDeclarator: { object: true } }
-      ],
+      '@typescript-eslint/prefer-destructuring': ['error', { VariableDeclarator: { object: true } }],
       '@typescript-eslint/no-empty-object-type': ['error', { allowInterfaces: 'with-single-extends' }],
       '@typescript-eslint/no-explicit-any': 'off', // TODO: enable when feeling brave
       '@typescript-eslint/no-implied-eval': 'error',
@@ -127,12 +143,12 @@ export default tseslint.config(
             'services/frontend/tsconfig.json',
             'services/frontend/tsconfig.node.json',
             'services/backend/tsconfig.json',
-            'services/shared/tsconfig.json'
+            'services/shared/tsconfig.json',
           ],
         }),
         createNodeResolver({
           extensions: ['.js', '.jsx', 'cjs', 'mjs', '.ts', '.tsx'],
-        })
+        }),
       ],
     },
   },
@@ -140,22 +156,22 @@ export default tseslint.config(
   {
     files: ['services/backend/src/routes/*.{ts,js}'],
     rules: {
-      '@typescript-eslint/no-misused-promises': 'off', // Errors come mainly from Express route handlers that are handled correctly by express-async-errors      
+      '@typescript-eslint/no-misused-promises': 'off', // Errors come mainly from Express route handlers that are handled correctly by express-async-errors
       '@typescript-eslint/consistent-return': 'off',
-    }
+    },
   },
 
   // Disable type-aware linting for javascript files
   {
     files: ['{updater,services}/**/*.{js,cjs}'],
-    extends: [tseslint.configs.disableTypeChecked]
+    extends: [tseslint.configs.disableTypeChecked],
   },
 
   // Frontend camelCase and react
   {
     files: ['services/frontend/**/*.{js,jsx,mjs,cjs,ts,tsx}'],
     plugins: {
-      'react': eslintPluginReact,
+      react: eslintPluginReact,
     },
     extends: [
       eslintPluginReact.configs.flat.recommended,
@@ -166,14 +182,15 @@ export default tseslint.config(
       parserOptions: {
         ecmaFeatures: {
           jsx: true,
-        }
-      }
+        },
+      },
     },
     rules: {
-      'camelcase': 'warn',
+      camelcase: 'warn',
       'react/prop-types': 'off', // Typescript but worse
       'react/no-unescaped-entities': 'off',
-      'react/function-component-definition': ['error',
+      'react/function-component-definition': [
+        'error',
         { namedComponents: 'arrow-function', unnamedComponents: 'arrow-function' },
       ],
       'react/no-array-index-key': 'error',
@@ -181,9 +198,7 @@ export default tseslint.config(
       'react/no-this-in-sfc': 'error',
       'react/prefer-stateless-function': 'error',
       'react/jsx-boolean-value': ['error', 'never'],
-      'react/jsx-filename-extension': ['error',
-        { allow: 'as-needed', extensions: ['.jsx', '.tsx'] }
-      ],
+      'react/jsx-filename-extension': ['error', { allow: 'as-needed', extensions: ['.jsx', '.tsx'] }],
       'react/jsx-no-leaked-render': 'warn', // TODO: enable, tho not that useful after full ts migration
       'react/jsx-no-useless-fragment': 'error',
       'react/jsx-props-no-spreading': 'warn', // TODO: enable, improves readability
@@ -191,18 +206,25 @@ export default tseslint.config(
       'react/jsx-tag-spacing': 'warn',
     },
     settings: {
-      'react': {
-        'version': 'detect'
-      }
-    }
+      react: {
+        version: 'detect',
+      },
+    },
   },
 
   // Cypress should only use rules provided by the plugin below
   {
     files: ['cypress/**/*.js'],
-    extends: [
-      eslintPluginCypress.configs.recommended
-    ]
+    extends: [eslintPluginCypress.configs.recommended],
+  },
+
+  // Override for backend api tests
+  {
+    files: ['services/backend/tests/**/*.ts'],
+    rules: {
+      'import-x/no-extraneous-dependencies': 'off',
+      'no-console': 'off',
+    },
   },
 
   // Overrides for CommonJS
@@ -210,7 +232,7 @@ export default tseslint.config(
     files: [
       'services/backend/**/*.js',
       'updater/sis-updater-worker/src/db/migrations/*.cjs',
-      'updater/sis-updater-scheduler/**/*.js'
+      'updater/sis-updater-scheduler/**/*.js',
     ],
     languageOptions: {
       sourceType: 'commonjs',
@@ -221,20 +243,9 @@ export default tseslint.config(
     },
   },
 
-  // Rapodiff
-  {
-    files: ['services/backend/src/rapodiff/**/*'],
-    rules: {
-      'no-console': 'off',
-    },
-  },
-
   // Migration files
   {
-    files: [
-      'updater/sis-updater-worker/src/db/migrations/*.cjs',
-      'services/backend/src/database/**/*.js'
-    ],
+    files: ['updater/sis-updater-worker/src/db/migrations/*.cjs', 'services/backend/src/database/**/*.js'],
     rules: {
       'import-x/no-unused-modules': 'off',
       '@typescript-eslint/no-empty-function': 'off',
@@ -252,5 +263,26 @@ export default tseslint.config(
       'no-promise-executor-return': 'off',
     },
   },
-  eslintPluginPrettierRecommended, // This must be the last plugin so it can override other configs
+
+
+  // Backend tests
+  {
+    files: ["services/backend/tests/**/*.ts", "services/backend/vitest.config.ts"],
+    rules: {
+      'import-x/no-extraneous-dependencies': 'off',
+    }
+  },
+
+  // Frontend tests
+  {
+    files: ['services/frontend/tests/**/*.{ts,tsx}', "services/frontend/playwright/**/*.{ts,tsx}", 'services/frontend/playwright-ct.config.ts'],
+    rules: {
+      'import-x/no-extraneous-dependencies': 'off',
+      'import-x/no-unused-modules': 'off',
+      'import-x/no-default-export': 'off',
+      'react/jsx-filename-extension': 'off',
+    },
+  },
+
+  eslintPluginPrettierRecommended // This must be the last plugin so it can override other configs
 )

nyc-config.cjs

@@ -1,6 +0,0 @@
-const { parserPlugins } = require('@istanbuljs/schema').defaults.nyc
-
-module.exports = {
-  cache: false,
-  parserPlugins: parserPlugins.concat('typescript', 'jsx'),
-}

package.json

@@ -6,31 +6,32 @@
   },
   "scripts": {
     "cypress": "cypress",
-    "cypress:open": "cypress open",
-    "cypress:run": "cypress run --browser chrome",
-    "instrument": "./scripts/instrument.sh",
+    "cypress:run": "cypress run --browser firefox",
     "eslint": "eslint",
     "lint:all": "eslint services/ updater/",
     "stylelint": "stylelint",
     "prettier": "prettier",
     "cli": "./cli.sh",
-    "oodikone": "./run.sh oodikone anon up --detach",
+    "logs": "docker compose --file docker-compose.yml logs backend frontend sis-updater-worker sis-updater-scheduler jami -f",
+    "logs:all": "docker compose --file docker-compose.yml --profile oodikone --profile updater logs -f",
+    "logs:ci": "docker compose --file docker-compose.ci.yml logs backend frontend sis-updater-worker sis-updater-scheduler jami",
+    "oodikone": "npm run docker:down && ./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
-    "oodikone:test": "./run.sh oodikone test up --detach",
+    "oodikone:test": "npm run docker:down && docker compose --profile oodikone up --detach && docker compose --file docker-compose.test.yml up --detach",
+    "oodikone:ci": "npm run docker:down && docker compose --file docker-compose.ci.yml --profile oodikone up",
     "updater": "./run.sh updater anon up --detach",
     "updater:real": "./run.sh updater real up --detach",
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
     "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
-    "testupdater": "docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start",
-    "flushredis": "docker exec -it redis redis-cli flushall",
+    "testupdater": "docker compose --file docker-compose.ci.yml up --detach --no-recreate --wait sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker compose --file docker-compose.ci.yml up sis-updater-scheduler --exit-code-from sis-updater-scheduler",
+    "flushredis": "docker exec -it oodikone-redis redis-cli flushall",
     "prepare": "husky",
-    "rapodiff": "docker exec -t oodikone_backend_1 /opt/app-root/backend/node_modules/.bin/tsx /opt/app-root/backend/src/rapodiff/rapodiff.js",
     "install:local": "npm i && npm i --prefix services/shared && npm i --prefix services/frontend && npm i --prefix services/backend && npm i --prefix updater/sis-updater-scheduler && npm i --prefix updater/sis-updater-worker",
-    "ci:local": "npm ci && npm ci --prefix services/shared && npm ci --prefix services/frontend && npm ci --prefix services/backend && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker",
-    "install:docker": "docker exec frontend npm i && docker exec oodikone_backend_1 npm i && docker exec sis-updater-scheduler npm i && docker exec oodikone_sis-updater-worker_1 npm i",
-    "ci:docker": "docker exec frontend npm ci && docker exec oodikone_backend_1 npm ci && docker exec sis-updater-scheduler npm ci && docker exec oodikone_sis-updater-worker_1 npm ci",
+    "install:docker": "docker exec --user 0 oodikone-frontend sh -lc \"npm i --prefix /opt/app-root/shared && npm i --prefix /opt/app-root/frontend && chown -R 1001:0 /opt/app-root/shared/node_modules /opt/app-root/frontend/node_modules\" && docker exec --user 0 oodikone-backend sh -lc \"npm i --prefix /opt/app-root/backend && chown -R 1001:0 /opt/app-root/backend/node_modules\" && docker exec --user 0 oodikone-updater-scheduler sh -lc \"npm i --prefix /opt/app-root/src && chown -R 1001:0 /opt/app-root/src/node_modules\" && docker exec --user 0 oodikone-updater-worker sh -lc \"npm i --prefix /opt/app-root/src && chown -R 1001:0 /opt/app-root/src/node_modules\"",
     "install:both": "npm run install:local && npm run install:docker",
+    "ci:local": "npm ci && npm ci --prefix services/shared && npm ci --prefix services/frontend && npm ci --prefix services/backend && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker",
+    "ci:docker": "docker exec --user 0 oodikone-frontend sh -lc \"npm ci --prefix /opt/app-root/shared && npm ci --prefix /opt/app-root/frontend && chown -R 1001:0 /opt/app-root/shared/node_modules /opt/app-root/frontend/node_modules\" && docker exec --user 0 oodikone-backend sh -lc \"npm ci --prefix /opt/app-root/backend && chown -R 1001:0 /opt/app-root/backend/node_modules\" && docker exec --user 0 oodikone-updater-scheduler sh -lc \"npm ci --prefix /opt/app-root/src && chown -R 1001:0 /opt/app-root/src/node_modules\" && docker exec --user 0 oodikone-updater-worker sh -lc \"npm ci --prefix /opt/app-root/src && chown -R 1001:0 /opt/app-root/src/node_modules\"",
     "ci:both": "npm run ci:local && npm run ci:docker"
   },
   "repository": {
@@ -43,28 +44,22 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@cypress/code-coverage": "^3.14.4",
     "@eslint/js": "^9.28.0",
-    "@istanbuljs/schema": "^0.1.3",
-    "cypress": "^15.0.0",
+    "cypress": "^15.13.0",
     "eslint": "^9.28.0",
     "eslint-config-prettier": "^10.1.5",
     "eslint-import-resolver-typescript": "^4.4.3",
-    "eslint-plugin-cypress": "^5.1.1",
+    "eslint-plugin-cypress": "^6.2.3",
     "eslint-plugin-import-x": "^4.16.1",
     "eslint-plugin-prettier": "^5.4.1",
     "eslint-plugin-react": "^7.37.5",
     "eslint-plugin-react-hooks": "^5.2.0",
-    "globals": "^16.2.0",
+    "globals": "^17.4.0",
     "husky": "^9.1.7",
     "lint-staged": "^16.1.0",
-    "nyc": "^17.1.0",
     "prettier": "3.5.3",
-    "stylelint": "^16.20.0",
-    "stylelint-config-standard": "^39.0.0",
+    "stylelint": "^17.6.0",
+    "stylelint-config-standard": "^40.0.0",
     "typescript-eslint": "^8.34.0"
-  },
-  "dependencies": {
-    "lodash": "^4.17.21"
   }
 }

run.sh

@@ -23,7 +23,7 @@ Usage: $(basename "${BASH_SOURCE[0]}") option [version] command --flag
 Parameters:
 * Option: oodikone/updater/both
 * Version: anon/real. Not necessary when running down/logs/pull.
-* Command: will be passed to docker-compose.
+* Command: will be passed to docker compose.
 EOF
 }
 
@@ -35,7 +35,7 @@ parse_params() {
   option=${args[0]}
 
   #  parse arguments
-  [[ ("$option" != "oodikone" && "$option" != "updater" && "$option" != "both") ]] && \
+  [[ ("$option" != "oodikone" && "$option" != "updater" && "$option" != "both" && "$option" != "tests") ]] && \
     usage && die "Wrong option: $option"
 
   [[ ${#args[@]} -eq 1 ]] && usage && die "Wrong number of arguments"
@@ -47,7 +47,7 @@ parse_params() {
     compose_command=${args[*]:1}
   else
     version=${args[1]}
-    [[ "$version" != "anon" && "$version" != "real" && "$version" != "test" ]] && usage && \
+    [[ "$version" != "anon" && "$version" != "real" && "$version" != "api" ]] && usage && \
       die "Wrong version $version"
     compose_command=${args[*]:2}
   fi
@@ -70,8 +70,8 @@ parse_env() {
   if [[ "$version" == "real" ]]; then
     env="--file docker-compose.yml --file docker-compose.real.yml"
   fi
-  if [[ "$version" == "test" ]]; then
-    env="--file docker-compose.yml --file docker-compose.test.yml"
+  if [[ "$version" == "api" ]]; then
+    env="--file docker-compose.ci.yml"
   fi
   return 0
 }
@@ -85,9 +85,9 @@ parse_params "$@"
 parse_profiles
 parse_env
 if [[ "$compose_command" == "" ]]; then
-  final_command="docker-compose ${env}"
+  final_command="docker compose ${env}"
 else
-  final_command="docker-compose ${env} ${profiles} ${compose_command}"
+  final_command="docker compose ${env} ${profiles} ${compose_command}"
 fi
 
 msg "${BLUE}Running: ${final_command}${NOFORMAT}"

scripts/functions_for_setup.sh

@@ -11,9 +11,8 @@ PROJECT_ROOT="$(git rev-parse --show-toplevel)"
 DUMP_DIR="$PROJECT_ROOT/.databasedumps"
 S3_CONFIG_FILE=~/.s3cfg
 S3_BUCKET="s3://psyduck"
-DOCKER_COMPOSE=$PROJECT_ROOT/docker-compose.yml
 
-## Following the naming convention in docker-compose, these are names for database
+## Following the naming convention in docker compose, these are names for database
 ## services. The real data databases inside service will have names suffixed by "-real".
 KONE_DB_NAME="kone-db"
 SIS_DB_NAME="sis-db"
@@ -113,11 +112,11 @@ reset_databases() {
   infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
 
   npm run docker:down
-  docker-compose up -d "${databases[@]}"
+  docker compose up -d "${databases[@]}"
 
   for database in "${databases[@]}"; do
     local database_dump="$database_dump_dir/$database.sql.gz"
-    local database_container="$database"
+    local database_container="oodikone-$database"
     local database_name="$database-real"
 
     infomsg "Attempting to create database $database_name from dump $database_dump inside container $database_container"
@@ -139,30 +138,7 @@ reset_databases() {
   successmsg "Database setup finished"
 }
 
-reset_jami_data() {
-  local database="jami-db"
-  local dump_destination="$DUMP_DIR/$database.sql.gz"
-  local s3_path="jami"
-
-  infomsg "Removing old data"
-
-
-
-  infomsg "Removing database and related volume"
-  docker volume rm oodikone_jami-data || warningmsg "This is okay, continuing"
-  docker-compose -f "$DOCKER_COMPOSE" down "$database" || warningmsg "This is okay, continuing"
-
-  infomsg "Starting postgres in the background"
-  docker-compose -f "$DOCKER_COMPOSE" up -d "$database"
-  retry docker-compose -f "$DOCKER_COMPOSE" exec "$database" pg_isready --dbname="postgres"
-
-  infomsg "Populating Jami"
-  docker exec -i "$database" /bin/bash -c "gunzip | psql -U postgres" < "$dump_destination" 2> /dev/null
-  docker-compose stop "$database"
-  msg ""
-}
-
-reset_all_real_data() {
+reset_all_data() {
   infomsg "Downloading real data dumps"
   for i in ${!DATABASES[*]}; do
     local database="${DATABASES[$i]}"
@@ -239,6 +215,7 @@ set_up_oodikone_from_scratch() {
     "$PROJECT_ROOT"
     "$PROJECT_ROOT/services/frontend"
     "$PROJECT_ROOT/services/backend"
+    "$PROJECT_ROOT/services/shared"
     "$PROJECT_ROOT/updater/sis-updater-scheduler"
     "$PROJECT_ROOT/updater/sis-updater-worker"
   )
@@ -247,7 +224,7 @@ set_up_oodikone_from_scratch() {
   "$PROJECT_ROOT"/run.sh both down --remove-orphans --volumes --rmi all || infomsg "Cleaning errored, but will continue"
   for folder in "${folders_to_set_up[@]}"; do
     cd "$folder" || die "Couldn't change directory to folder $folder"
-    rm -rf node_modules|| die "Couldn't remove node_modules in folder $folder"
+    rm -rf node_modules || die "Couldn't remove node_modules in folder $folder"
   done
 
   infomsg "Installing npm packages locally to root and each subfolder to enable linting and formatting"

scripts/instrument.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-
-# This script is used for instrumenting the frontend code
-# Counters are added for calculating the code coverage
-# Instrumented code is used for generating code coverage reports
-
-# Ensure the script exits if any command fails
-set -euo pipefail
-
-# Define the directories
-SOURCE_DIR="services/frontend/src"
-INSTRU_DIR="services/frontend/instrumented"
-
-# Print status
-echo -n "Instrumenting frontend code... "
-
-# Create the instrumented directory if it does not exist
-mkdir -p $INSTRU_DIR
-
-# Instrument the code
-nyc instrument --all --compact=false $SOURCE_DIR $INSTRU_DIR
-
-# Copy remaining files without overwriting instrumented files
-# 
-# Images, CSS and other static files are not instrumended
-# but they are still needed to run the application
-rsync -av --quiet --ignore-existing $SOURCE_DIR/ $INSTRU_DIR/
-
-# Print status
-echo "Done!"

services/.dockerignore

@@ -0,0 +1,116 @@
+# disallow directory named "private", it can be used to store things that should not end up in version control or images
+# NOTE: needs to be at the root of the project
+private/
+gitignored/
+
+# disallow version control, package managment, IDEs, tools etc
+**/node_modules
+**/.venv
+**/.git
+**/.github
+**/.vscode
+**/.idea
+**/.zed
+**/coverage/
+**/.eslintcache
+**/.DS_Store
+
+# do not include any data by default (except json)
+**/*.csv
+**/*.tsv
+
+**/*.db
+**/*.sqlite
+**/*.sql
+**/*.psql
+**/*.sqlz
+**/*.sqz
+**/*.sql.gz
+**/*.bson
+**/*.jsonl
+
+**/*.bin
+**/*.dat
+**/*.data
+
+**/*.swp
+**/*.tmp
+**/.~lock*
+**/*.pid
+**/*.seed
+**/*.lock
+
+**/*.gz
+**/*.tgz
+**/*.xz
+**/*.zip
+**/*.gzip
+**/*.tar
+**/*.xz
+**/*.7z
+**/*.rar
+
+**/*.docx
+**/*.doc
+**/*.xlsx
+**/*.xls
+**/*.pptx
+**/*.ppt
+**/*.odt
+**/*.odp
+**/*.ods
+**/*.pdf
+
+
+# disallow logs
+**/*.log
+**/*.log*
+**/logs/
+**/log/
+**/*.cpuprofile
+**/*.heapprofile
+**/*.heapsnapshot
+
+# disallow configuration
+**/*.yml
+**/*.yaml
+**/*.toml
+**/*.conf
+**/*.ini
+**/toska_login/
+**/toska_login/*
+**/cypress.env.json
+cypress/
+
+
+# disallow secrects
+**/.env
+**/.env*
+**/*.env.*
+**/.secrects
+**/*.s3cfg
+
+# disallow backups
+**/backups/*
+**/backups
+**/backup/*
+**/backup
+**/dump/
+**/dump
+**/dumps/
+**/dumps
+**/databasedumps/
+**/privateData
+**/temp
+**/minio-data/*
+
+
+# additional inclusions here
+
+# exclusions here
+**/.dockerignore
+**/.gitignore
+**/.eslintrc.json
+**/*Dockerfile
+**/build
+**/dist

services/backend/.dockerignore

@@ -1,3 +0,0 @@
-.dockerignore
-*Dockerfile
-node_modules/

services/backend/.env.test

@@ -0,0 +1,12 @@
+CI='true'
+NODE_ENV='test'
+DB_URL_KONE=postgres://postgres@localhost:4321/kone-db
+DB_URL_USER=postgres://postgres@localhost:4320/user-db
+FRONT_URL=http://localhost:3000
+REDIS=localhost
+SECRET_TOKEN=test
+SIS_DB_URL=postgres://postgres@localhost:12345/sis-db
+SIS_UPDATER_URL=http://localhost:8082
+JAMI_URL='http://localhost:3003/'
+IMPORTER_DB_API_URL='https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
+CRYPT_KEY='e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'

services/backend/.npmrc

@@ -0,0 +1,3 @@
+ignore-scripts=true  
+min-release-age=7  
+save-prefix=  

services/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi9/nodejs-22-minimal AS build
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal AS build
 
 ENV TZ="Europe/Helsinki"
 
@@ -29,7 +29,7 @@ RUN npm ci --include=dev --prefix shared && npm ci --include=dev --prefix backen
 RUN npm run build --prefix shared && npm run build --prefix backend
 
 
-FROM registry.access.redhat.com/ubi9/nodejs-22-minimal
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 
@@ -50,9 +50,13 @@ ENV SENTRY_RELEASE=$SENTRY_RELEASE
 
 WORKDIR /opt/app-root/shared
 COPY --from=build /opt/app-root/shared/dist .
+COPY --from=build /opt/app-root/shared/.npmrc .
+COPY --from=build /opt/app-root/shared/package*.json ./
+RUN npm ci --omit=dev
 
 WORKDIR /opt/app-root/backend
-COPY --from=build /opt/app-root/backend/package*.json .
+COPY --from=build /opt/app-root/backend/.npmrc .
+COPY --from=build /opt/app-root/backend/package*.json ./
 COPY --from=build /opt/app-root/backend/dist ./dist
 
 RUN npm ci --omit=dev

services/backend/dev.Dockerfile

@@ -1,13 +1,21 @@
-FROM node:22-alpine
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 
 ENV NODE_ENV=development
 ENV NODE_OPTIONS=--max-old-space-size=4096
 
+USER 0
+
+WORKDIR /opt/app-root/shared
+COPY --chown=1001:0 ./shared .
+RUN npm ci && chown -R 1001:0 /opt/app-root/shared
+
 WORKDIR /opt/app-root/backend
-COPY . .
-RUN npm ci
+COPY --chown=1001:0 ./backend .
+RUN npm ci && chown -R 1001:0 /opt/app-root/backend
 
 EXPOSE 8080
-CMD ["node_modules/.bin/tsx", "watch", "--clear-screen=false", "index.js"]
+
+USER 1001
+CMD ["node_modules/.bin/nodemon", "--exec", "node --import tsx index.ts", "--ext", "ts,js,json", "--watch", "src", "--watch", "index.ts", "--legacy-watch"]

services/backend/package.json

@@ -2,49 +2,50 @@
   "name": "backend",
   "private": true,
   "scripts": {
-    "build": "tsc"
+    "build": "tsc",
+    "test:all": "vitest",
+    "test:api": "vitest ./tests/api",
+    "test:services": "vitest ./tests/services",
+    "test:unit": "vitest ./tests/unit",
+    "test:silent": "SILENT_TESTING=true vitest"
   },
   "dependencies": {
     "@oodikone/shared": "file:../shared",
-    "@sentry/node": "^10.5.0",
-    "@sentry/profiling-node": "^10.5.0",
+    "@sentry/node": "^10.43.0",
+    "@sentry/profiling-node": "^10.43.0",
     "axios": "^1.9.0",
     "bullmq": "^5.53.2",
     "compression": "^1.8.0",
     "cors": "^2.8.5",
     "cron": "^4.3.1",
     "dayjs": "^1.11.13",
-    "express": "^4.21.2",
-    "express-async-errors": "^3.1.1",
-    "lodash": "^4.17.21",
+    "express": "^5.2.1",
+    "lodash-es": "^4.18.1",
     "lru-cache": "^11.1.0",
     "morgan": "^1.10.0",
     "papaparse": "^5.5.3",
-    "pg": "^8.16.0",
+    "pg": "^8.20.0",
     "redis": "^4.7.1",
-    "reflect-metadata": "^0.2.2",
-    "sequelize": "^6.37.7",
+    "sequelize": "^6.37.8",
     "sequelize-typescript": "^2.1.6",
     "umzug": "^3.8.2",
     "unfuck-utf8-headers-middleware": "^1.0.1",
-    "vite": "^7.1.12",
     "winston": "^3.17.0",
-    "winston-gelf-transporter": "^1.0.2",
     "winston-loki": "^6.1.3"
   },
   "devDependencies": {
     "@types/compression": "^1.8.1",
     "@types/cors": "^2.8.19",
-    "@types/express": "^4.17.23",
-    "@types/lodash": "^4.17.17",
+    "@types/express": "^5.0.6",
+    "@types/lodash-es": "^4.17.12",
     "@types/morgan": "^1.9.10",
     "@types/node": "^24.10.1",
+    "@types/pg": "^8.20.0",
+    "@types/supertest": "^6.0.3",
+    "nodemon": "^3.1.11",
+    "supertest": "^7.2.2",
     "tsx": "^4.19.4",
-    "typescript": "^5.9.2"
-  },
-  "overrides": {
-    "winston-gelf-transporter": {
-      "gelf-pro": "^1.4.0"
-    }
+    "typescript": "^5.9.2",
+    "vitest": "4.1.9"
   }
 }

services/backend/src/app.ts

@@ -1,4 +1,3 @@
-import 'express-async-errors'
 import express from 'express'
 
 import { baseUrl, backendPort } from './config'

services/backend/src/config/index.ts

@@ -1,14 +1,15 @@
 // Node env to use
 export const isDev = process.env.NODE_ENV === 'development'
+export const isTest = process.env.NODE_ENV === 'test'
 export const isStaging = process.env.REACT_APP_STAGING === 'true'
 export const isProduction = !isStaging && process.env.NODE_ENV === 'production'
 
 export const runningInCI = process.env.CI === 'true'
 
+export const silentTesting = process.env.SILENT_TESTING === 'true'
+
 // IAM group
-export const requiredGroup = isStaging
-  ? ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
-  : ['grp-oodikone-users', 'grp-oodikone-basic-users']
+export const requiredGroup = ['grp-oodikone-users', 'grp-oodikone-basic-users']
 
 // Pate
 export const pateToken = process.env.PATE_API_TOKEN ?? ''
@@ -32,20 +33,11 @@ export const baseUrl = isDev ? '/api' : ''
 export const frontUrl = process.env.FRONT_URL
 export const backendPort = 8080
 
-// System run in whose environment
-export const serviceProvider = process.env.SERVICE_PROVIDER?.toLowerCase() ?? 'toska'
-
-// Optional logout-url configuration
-export const configLogoutUrl = serviceProvider === 'toska' ? undefined : process.env.LOGOUT_URL
-
 // Other stuff
 export const { DB_URL_KONE, DB_URL_USER, SECRET_TOKEN, SIS_DB_URL, SIS_UPDATER_URL, CRYPT_KEY } = process.env
 
 export const rootOrgId = process.env.ROOT_ORG_ID ?? 'hy-university-root-id'
 
-export const languageCenterViewEnabled =
-  process.env.LANGUAGE_CENTER_VIEW_ENABLED == null ? true : process.env.LANGUAGE_CENTER_VIEW_ENABLED === 'true'
-
 export const concurrentWorkers = parseInt(process.env.CONCURRENT_WORKERS!, 10) || 2
 
 export const DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS!, 10) || 5 // sequelize's default

services/backend/src/config/roles.ts

@@ -5,7 +5,6 @@ export const roles = [
   'courseStatistics',
   'facultyStatistics',
   'fullSisuAccess',
-  'openUniSearch',
   'studyGuidanceGroups',
   'teachers',
 ] as const

services/backend/src/database/connection.ts

@@ -24,6 +24,7 @@ import {
   TeacherModel,
 } from '../models'
 import {
+  BannerModel,
   CustomPopulationSearchModel,
   ExcludedCourseModel,
   OpenUniPopulationSearchModel,
@@ -107,6 +108,7 @@ const sequelizeKone = new Sequelize(conf.DB_URL_KONE!, {
   logging: false,
   password: conf.KONE_PASSWORD,
   models: [
+    BannerModel,
     CustomPopulationSearchModel,
     ExcludedCourseModel,
     OpenUniPopulationSearchModel,

services/backend/src/database/migrations_kone/20260709_01_create_table_banners.js

@@ -0,0 +1,46 @@
+const { INTEGER, STRING, DATE } = require('sequelize')
+
+module.exports = {
+  up: async ({ context: queryInterface }) => {
+    await queryInterface.createTable('banners', {
+      id: {
+        type: INTEGER,
+        primaryKey: true,
+        autoIncrement: true,
+      },
+      text: {
+        type: STRING,
+        allowNull: false,
+      },
+      color: {
+        type: STRING,
+        allowNull: false,
+      },
+      lightness: {
+        type: STRING,
+        allowNull: false,
+      },
+      start_date: {
+        type: DATE,
+        allowNull: false,
+      },
+      end_date: {
+        type: DATE,
+        allowNull: false,
+      },
+      last_modified_by: {
+        type: STRING,
+        allowNull: false,
+      },
+      created_at: {
+        type: DATE,
+      },
+      updated_at: {
+        type: DATE,
+      },
+    })
+  },
+  down: async ({ context: queryInterface }) => {
+    await queryInterface.dropTable('banners')
+  },
+}

services/backend/src/events.ts

@@ -1,6 +1,6 @@
 import { CronJob } from 'cron'
 
-import { isProduction, runningInCI, languageCenterViewEnabled, CRON_SCHEDULE } from './config'
+import { isProduction, runningInCI, CRON_SCHEDULE } from './config'
 import { getDegreeProgrammesOfFaculty } from './services/faculty/faculty'
 import { getFaculties } from './services/faculty/facultyHelpers'
 import { updateFacultyOverview, updateFacultyProgressOverview } from './services/faculty/facultyUpdates'
@@ -13,7 +13,7 @@ import { updateBasicView, updateStudyTrackView } from './services/studyProgramme
 import { findAndSaveTeachers } from './services/teachers/top'
 import { deleteOutdatedUsers } from './services/userService'
 import logger from './util/logger'
-import { jobMaker, addToFlow } from './worker/queue'
+import { jobQueue } from './worker/queue'
 
 const schedule = (cronTime: string, onTick: () => void) => {
   const onComplete = null
@@ -41,7 +41,7 @@ export const refreshFaculties = async () => {
   logger.info('Adding jobs to refresh all faculties')
   const faculties = await getFaculties()
   for (const faculty of faculties) {
-    jobMaker.faculty(faculty.code)
+    jobQueue.refreshFaculty(faculty.code)
   }
 }
 
@@ -53,12 +53,13 @@ export const refreshLanguageCenterData = async () => {
 }
 
 const refreshProgrammesAndFaculties = async () => {
+  logger.info('Refreshing all degree programmes and faculties')
   const facultyCodes = (await getFaculties()).map(faculty => faculty.code)
   for (const faculty of facultyCodes) {
     const programmeCodes = (await getDegreeProgrammesOfFaculty(faculty, true))
       .map(programme => programme.code)
       .filter(code => isRelevantProgramme(code))
-    await addToFlow(faculty, programmeCodes)
+    await jobQueue.addToFlow(faculty, programmeCodes)
   }
 }
 
@@ -82,7 +83,7 @@ export const refreshProgrammes = async () => {
   }
   for (const code of programmeCodes) {
     // If combined programme is given, this updates only the bachelor programme
-    jobMaker.programme(code)
+    jobQueue.refreshProgramme(code)
   }
 }
 
@@ -96,8 +97,9 @@ const dailyJobs = async () => {
   try {
     await refreshTeacherLeaderboard()
     await refreshProgrammesAndFaculties()
-    if (languageCenterViewEnabled) jobMaker.languagecenter()
-    jobMaker.closeToGraduation()
+
+    jobQueue.refreshLanguagecenter()
+    jobQueue.refreshCloseToGraduation()
   } catch (error) {
     logger.error('Daily jobs failed', error)
   }
@@ -118,11 +120,11 @@ export const startCron = () => {
     })
     schedule('0 19 * * 1', () => {
       logger.info('Updating students whose studyplans have not been updated recently')
-      jobMaker.studyplansUpdate(4)
+      jobQueue.refreshStudyplans(4)
     })
     schedule('0 10 * * 2', () => {
       logger.info('Updating students whose studyplans have not been updated recently')
-      jobMaker.studyplansUpdate(5)
+      jobQueue.refreshStudyplans(5)
     })
   }
 }

services/backend/src/middleware/accessLogger.ts

@@ -1,8 +1,8 @@
 import { Request, Response } from 'express'
-import { omit } from 'lodash'
 import morgan from 'morgan'
 
-import { FormattedUser } from '../types'
+import { FormattedUser } from '@oodikone/shared/types'
+import { omitKeys } from '@oodikone/shared/util'
 import { getFullStudyProgrammeRights } from '../util'
 import logger from '../util/logger'
 
@@ -38,6 +38,11 @@ const accessLogger = morgan((tokens, req: Request, res: Response): undefined =>
     meta['req-route'] = req.route.path
   }
 
+  // Ignore admin route that is frequently pinged for updates.
+  if (/.*\/updater\/jobs$/.test(meta.url ?? '')) {
+    return
+  }
+
   const message = [
     `${user.mockedBy ? '(mocking) ' : ''}${user.name}`,
     `${(tokens['response-time'](req, res) ?? '0').split('.')[0]} ms`.padEnd(8, ' '),
@@ -52,7 +57,7 @@ const accessLogger = morgan((tokens, req: Request, res: Response): undefined =>
   const onlyIamRights = !user.isAdmin && fullStudyProgrammeRights.length === 0
 
   logger.info(message, {
-    ...omit(meta, ['studentsUserCanAccess']), // Don't log student list which might be huge
+    ...omitKeys(meta, ['studentsUserCanAccess']), // Don't log student list which might be huge
     isUsageStats: true, // Pass this as a custom field so we can filter by it in Graylog
     isIamRights: onlyIamRights || usingIamRights, // Needed for Grafana IAM users panel
   })

services/backend/src/middleware/currentUser.ts

@@ -1,10 +1,9 @@
 import * as Sentry from '@sentry/node'
 import { NextFunction, Request, Response } from 'express'
-import { intersection } from 'lodash'
+import { intersection } from 'lodash-es'
 
-import { configLogoutUrl, isDev, requiredGroup, serviceProvider } from '../config'
-import { getMockedUser, getMockedUserFd, getOrganizationAccess, getUserFd, getUserToska } from '../services/userService'
-import { FormattedUser } from '../types'
+import { requiredGroup, baseUrl } from '../config'
+import { getMockedUser, getOrganizationAccess, getUser } from '../services/userService'
 import { ApplicationError } from '../util/customErrors'
 import logger from '../util/logger'
 
@@ -25,7 +24,7 @@ type Headers = {
   'x-show-as-user'?: string
 }
 
-const toskaUserMiddleware = async (req: Request, _: Response, next: NextFunction) => {
+export default async (req: Request, _: Response, next: NextFunction) => {
   const {
     'shib-session-id': sessionId,
     'x-show-as-user': showAsUser,
@@ -37,6 +36,11 @@ const toskaUserMiddleware = async (req: Request, _: Response, next: NextFunction
     uid: username,
   } = req.headers as Headers
 
+  if (req.path === `${baseUrl}/health`) {
+    next()
+    return
+  }
+
   if (!username || !sessionId)
     throw new ApplicationError(
       `${username ?? 'Anonymous user'} requested ${req.url} without authorization headers.`,
@@ -61,7 +65,7 @@ const toskaUserMiddleware = async (req: Request, _: Response, next: NextFunction
   const user =
     showAsUser && specialGroup?.superAdmin
       ? await getMockedUser({ userToMock: showAsUser, mockedBy: username })
-      : await getUserToska({
+      : await getUser({
           username,
           name: name!,
           email: email!,
@@ -80,34 +84,3 @@ const toskaUserMiddleware = async (req: Request, _: Response, next: NextFunction
 
   next()
 }
-
-const fdUserMiddleware = async (req: Request, _res: Response, next: NextFunction) => {
-  const { remote_user: remoteUser, 'x-show-as-user': showAsUser } = req.headers
-
-  if (!remoteUser) {
-    throw new ApplicationError('Not enough data in request headers, remote_user was missing', 403, { configLogoutUrl })
-  }
-
-  let user: FormattedUser | null
-
-  // getMockedUser in production requires the superAdmin role, which is only available
-  // via iamGroups, so it's now only implemented for the dev environment
-  if (showAsUser && isDev) {
-    user = await getMockedUserFd({ userToMock: showAsUser as string, mockedBy: remoteUser as string })
-  } else {
-    user = await getUserFd({ username: remoteUser as string })
-  }
-
-  if (!user) {
-    throw new ApplicationError(`Could not grant access with the eppn ${remoteUser}.`, 403, { configLogoutUrl })
-  }
-
-  req.user = user
-  req.logoutUrl = configLogoutUrl
-
-  next()
-}
-
-const currentUserMiddleware = serviceProvider === 'toska' ? toskaUserMiddleware : fdUserMiddleware
-
-export default currentUserMiddleware

services/backend/src/middleware/headers.ts

@@ -1,10 +1,4 @@
 import headersMiddleware from 'unfuck-utf8-headers-middleware'
 
-import { serviceProvider } from '../config'
-
-const headersToska = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
-const headersOther = ['remote_user']
-
-const headers = serviceProvider === 'toska' ? headersToska : headersOther
-
+const headers = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
 export default headersMiddleware(headers)

services/backend/src/models/SISStudyRight.ts

@@ -71,6 +71,9 @@ export class SISStudyRightModel extends Model implements SISStudyRight {
   @Column(DataType.JSONB)
   declare semesterEnrollments: SISStudyRight['semesterEnrollments']
 
+  @Column(DataType.JSONB)
+  declare transferInfo: SISStudyRight['transferInfo']
+
   @HasMany(() => CreditModel, { foreignKey: 'studyright_id', sourceKey: 'id' })
   declare credits: SISStudyRight['credits']
 

services/backend/src/models/course.ts

@@ -73,6 +73,9 @@ export class CourseModel extends Model implements Course {
   @Column(DataType.JSONB)
   declare substitutions: Course['substitutions']
 
+  @Column(DataType.JSONB)
+  declare substitution_groups: Course['substitution_groups']
+
   @Column(DataType.STRING)
   declare course_unit_type: Course['course_unit_type']
 

services/backend/src/models/kone/banner.ts

@@ -0,0 +1,33 @@
+import { Column, DataType, Model, PrimaryKey, Table, AutoIncrement } from 'sequelize-typescript'
+
+import type { Banner } from '@oodikone/shared/models/kone'
+
+@Table({
+  underscored: true,
+  modelName: 'banner',
+  tableName: 'banners',
+})
+export class BannerModel extends Model implements Banner {
+  @PrimaryKey
+  @AutoIncrement
+  @Column(DataType.INTEGER)
+  declare id: number
+
+  @Column(DataType.DATE)
+  declare startDate: Date
+
+  @Column(DataType.DATE)
+  declare endDate: Date
+
+  @Column(DataType.STRING)
+  declare text: string
+
+  @Column(DataType.STRING)
+  declare color: string
+
+  @Column(DataType.STRING)
+  declare lightness: 'light' | 'main' | 'dark'
+
+  @Column(DataType.STRING)
+  declare lastModifiedBy: string
+}

services/backend/src/models/kone/index.ts

@@ -1,3 +1,4 @@
+export { BannerModel } from './banner'
 export { CustomPopulationSearchModel } from './customPopulationSearch'
 export { ExcludedCourseModel } from './excludedCourse'
 export { OpenUniPopulationSearchModel } from './openUniPopulationSearch'

services/backend/src/rapodiff/.gitignore

@@ -1,2 +0,0 @@
-*.csv
-*.csv#

services/backend/src/rapodiff/README.md

@@ -1,22 +0,0 @@
-# Rapodiff
-
-Use this to check differences between oodikone and rapo.
-
-## How to load the CSV file from Rapo
-
-1. Go to Rapo and open ”Koulutusohjelmien vuosiseuranta”
-2. Navigate to ”Opintopisteet ja opetus / Credits and teaching”
-3. From the faculty dropdown menu, select all the faculties
-4. From the bottom of the page, click ”Vie -> Tiedot -> CSV”
-5. Rename the file to `data.csv` and copy it to this directory
-
-## Usage
-
-1. Have real oodikone running. Then run rapodiff **in the root directory**: `npm run rapodiff -- args`
-2. place args with a mode. omit args to print possible modes.
-
-## Known differences in APIs
-
-- rapo does not show students that have graduated (it however takes some days for rapo to notice a graduation)
-- rapo does not handle the starting year of transfers properly (shows in old staring population)
-- rapo show some students incorrectly passive, see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3521

services/backend/src/rapodiff/creditsDiff.js

@@ -1,188 +0,0 @@
-const { orderBy } = require('lodash')
-
-// dbConnections must be imported to avoid Sequelize errors even if it's not used
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-const { dbConnections } = require('../database/connection')
-const { computeCreditsProduced } = require('../services/providerCredits')
-const { parseCsv, printProgressBar } = require('./helpers')
-
-const diffs = []
-let noDiffCounter = 0
-
-// Change weird rapo programmecodes to oodikone format, example: 300-M003 => MH30_003
-const transformProgrammeCode = oldCode => `${oldCode[4]}H${oldCode.slice(0, 2)}_${oldCode.slice(5, 8)}`
-
-const formatData = data =>
-  data.reduce((obj, cur) => {
-    const parseNum = str => (str === '' ? 0 : parseFloat(str.replace(',', '.')))
-    const [
-      year,
-      facultyCode,
-      programmeInfo,
-      basic,
-      exchange,
-      otherUni,
-      openUni,
-      special,
-      total,
-      transferredFromAbroad,
-      otherTransferred,
-    ] = cur
-    const programmeCode = transformProgrammeCode(programmeInfo.split(' ')[0])
-    if (!obj[programmeCode]) obj[programmeCode] = {}
-    obj[programmeCode][year] = {
-      year: parseNum(year),
-      facultyCode,
-      programmeInfo,
-      basic: parseNum(basic),
-      'incoming-exchange': parseNum(exchange),
-      agreement: parseNum(otherUni),
-      'open-uni': parseNum(openUni),
-      separate: parseNum(special),
-      total: parseNum(total),
-      transferred: parseNum(transferredFromAbroad) + parseNum(otherTransferred),
-    }
-    return obj
-  }, {})
-
-const diff = (rapoData, okData, code) => {
-  for (const year of Object.keys(okData)) {
-    if (year !== '2022' && year !== '2023') continue
-    const rapo = rapoData[year]
-    const ok = okData[year]
-    if (!rapo) {
-      if (!ok) console.log(`Year ${year} not in rapo for code ${code}, skipping`)
-      continue
-    }
-    for (const field of ['basic', 'incoming-exchange', 'open-uni', 'agreement', 'separate', 'transferred']) {
-      const okValue = Math.round(ok[field] || 0)
-      const rapoValue = Math.round(rapo[field] || 0)
-      const diff = Math.abs(okValue - rapoValue)
-      if (diff === 0) {
-        noDiffCounter++
-        continue
-      }
-      const percentage = rapoValue !== 0 ? Number(((diff / rapoValue) * 100).toFixed(2)) : 100
-      diffs.push({ field, year, code, okValue, rapoValue, diff, 'diff %': percentage })
-    }
-  }
-}
-
-const processProgrammes = async data => {
-  let programmeCounter = 0
-  const rapoProgrammeData = formatData(data.slice(1))
-  const allProgrammeCodes = [...new Set(Object.keys(rapoProgrammeData))]
-  for (const programmeCode of allProgrammeCodes) {
-    const okProgrammeData = await computeCreditsProduced(programmeCode, false, true)
-    diff(rapoProgrammeData[programmeCode], okProgrammeData.stats, programmeCode)
-    printProgressBar(allProgrammeCodes.length, ++programmeCounter)
-  }
-  const fieldToSortBy = 'diff' // 'diff %'
-  const orderedDiffs = orderBy(diffs, fieldToSortBy, 'asc')
-  console.table(orderedDiffs)
-  console.log(`${diffs.length} diffs found with current settings. Numbers with no diff: ${noDiffCounter}`)
-  const totalDiff = orderedDiffs.reduce((sum, cur) => cur.diff + sum, 0)
-  console.log('Total difference: ', totalDiff)
-}
-
-const programmeCreditsDiff = async fileName => {
-  console.log('Running studyprogramme credits diff')
-  await parseCsv(fileName, processProgrammes)
-  console.log('Diff completed.')
-}
-
-/* 
-separate
-Suoritus on erillisellä opiskeluoikeudella opettajankoulutuksen opintoja suorittavan opiskelijan suorittama
-Suoritus on erillisellä opiskeluoikeudella opiskelevan suorittama
-
-agreement
-Suoritus on korkeakoulujen välisillä yhteistyösopimuksilla opiskelevan suorittama
-
-
-open-uni
-Suoritus on avoimessa korkeakouluopetuksessa suoritettu
-
-incoming-exchange
-Suoritus on tulevan kansainvälisen vaihto-opiskelijan suorittama
-
-basic
-perustutkinto-opiskelija = 1
-ja
-Tuntematon
-tai (ehkä):
-Suoritus on kotimainen harjoittelujakso
-?? Opintosuoritus on täydennyskoulutuksessa suoritettu ??
-
-other
-perustutkinto-opiskelija = 0 && tuntematon
-
-*/
-
-const getCategory = (typeName, isBasic, hyvaksiluettu) => {
-  if (hyvaksiluettu) return 'transferred' // ?? unsure
-  if (isBasic || ['Suoritus on kotimainen harjoittelujakso', 'Tuntematon'].includes(typeName)) return 'basic'
-  if (typeName === 'Suoritus on avoimessa korkeakouluopetuksessa suoritettu') return 'open-uni'
-  if (typeName === 'Suoritus on tulevan kansainvälisen vaihto-opiskelijan suorittama') return 'incoming-exchange'
-  if (typeName.startsWith('Suoritus on erillisellä opiskeluoikeudella')) return 'separate'
-  if (typeName === 'Suoritus on korkeakoulujen välisillä yhteistyösopimuksilla opiskelevan suorittama')
-    return 'agreement'
-  return 'other'
-}
-
-const processIds = async (rawData, code, field) => {
-  const data = rawData.slice(0, 21628) // 11476 = KTM viimeinen
-  const rapoData = data
-    .map(row => ({
-      id: row[0].slice(11),
-      type: getCategory(row[1], row[3] === '1', row[6] === '1'),
-      typeCode: row[8],
-      isModule: row[9] !== '0',
-      credits: parseFloat(row[10]),
-      included: row[4] === '1',
-    }))
-    .filter(course => !course.isModule && (field === 'transferred' || course.included))
-  const rapoIds = rapoData.filter(course => course.credits > 0 && course.type === field).map(row => row.id)
-  const rapoCredits = rapoData.reduce(
-    (stats, cur) => {
-      stats[cur.type] += cur.credits
-      return stats
-    },
-    {
-      transferred: 0,
-      total: '_',
-      basic: 0,
-      'open-uni': 0,
-      separate: 0,
-      'incoming-exchange': 0,
-      agreement: 0,
-      other: 0,
-    }
-  )
-  const stats = await computeCreditsProduced(code, false, true, field)
-  if (!stats.stats.ids) {
-    console.log({ rapoCredits })
-    console.log('Edit getCreditsProduced code so that it saves a list of the relevant ids you want to compare.')
-    return
-  }
-  const okIdMap = stats.stats.ids
-  const rapoIdMap = rapoIds.reduce((obj, cur) => {
-    obj[cur] = true
-    return obj
-  }, {})
-  const onlyInRapo = Object.keys(rapoIdMap).filter(id => !okIdMap[id])
-  const onlyInOk = Object.keys(okIdMap).filter(id => !rapoIdMap[id])
-  console.log('\n onlyInRapo: ', onlyInRapo.length)
-  console.dir(onlyInRapo.slice(0, 30))
-  console.log('\n onlyInOk: ', onlyInOk.length)
-  console.dir(onlyInOk.slice(0, 30))
-  delete stats.ids
-  console.log(stats.stats['2022'])
-  console.log('rapoCredits: ', rapoCredits)
-}
-
-const testNewCalc = async (fileName, code, field) => {
-  await parseCsv(fileName, async data => processIds(data, code, field))
-}
-
-module.exports = { programmeCreditsDiff, testNewCalc }

services/backend/src/rapodiff/helpers.js

@@ -1,36 +0,0 @@
-const fs = require('fs/promises')
-const Papa = require('papaparse')
-
-const parseCsv = async (fileName, fun) => {
-  try {
-    const file = await fs.readFile(`${__dirname}/${fileName}`, 'utf8')
-    Papa.parsePromise = file => {
-      return new Promise((complete, error) => {
-        Papa.parse(file, { complete, error })
-      })
-    }
-    const data = await Papa.parsePromise(file)
-    await fun(data?.data)
-  } catch (error) {
-    console.log(error)
-  }
-}
-
-const printProgressBar = (total, current) => {
-  const barLength = 30
-  const progress = Math.floor((current / total) * barLength)
-  const empty = barLength - progress
-
-  const progressBar = `[${'#'.repeat(progress)}${'-'.repeat(empty)}]`
-  const percentage = Math.floor((current / total) * 100)
-
-  process.stdout.clearLine()
-  process.stdout.cursorTo(0)
-  process.stdout.write(`${progressBar} ${percentage} %`)
-
-  if (current >= total) {
-    console.log()
-  }
-}
-
-module.exports = { parseCsv, printProgressBar }

services/backend/src/rapodiff/rapodiff.js

@@ -1,31 +0,0 @@
-const { programmeCreditsDiff, testNewCalc } = require('./creditsDiff')
-
-// ** To add new mode, just type key and function here **
-const modes = {
-  // Check credits with current local oodikone data against provided data.csv in rapodiff folder.
-  credits: () => programmeCreditsDiff('data.csv'),
-  // Provide programme code as argument. Parses credits from credits.csv in rapodiff folder. This must be obtained from Rapo people. This tells which credit ids are found in oodikone but not rapo, and vice versa.
-  // notice: check mode requires writing ids in to the stats object in credits calculating function.
-  // Add something like this there:
-  // if (category === 'open-uni' && credits > 0 && new Date(attainmentDate).getFullYear() === 2022) stats.ids[id] = true
-  check: (fileName, code, field) => testNewCalc(fileName, code, field),
-}
-
-const main = async () => {
-  console.log('**RAPODIFF**\n')
-  const mode = process.argv[2]
-  const modeFunction = modes[mode]
-  const arg = process.argv[3]
-  const arg2 = process.argv[4]
-  const arg3 = process.argv[5]
-  if (!modeFunction) {
-    console.log(
-      'Unknown or missing mode. Supply the mode as an argument (e.g. `npm run rapodiff -- credits`). Supported modes:\n'
-    )
-    console.log(Object.keys(modes).join('\n'))
-    return
-  }
-  await modeFunction(arg, arg2, arg3)
-}
-
-main().then(() => process.exit(0))

services/backend/src/routes.ts

@@ -3,22 +3,23 @@ import compression from 'compression'
 import cors from 'cors'
 import { Express, json as jsonExpress } from 'express'
 
-import { frontUrl, languageCenterViewEnabled, serviceProvider } from './config'
+import { frontUrl } from './config'
 import accessLogger from './middleware/accessLogger'
 import * as auth from './middleware/auth'
 import currentUserMiddleware from './middleware/currentUser'
 import errorHandler from './middleware/errorHandler'
 import headersMiddleware from './middleware/headers'
+import banners from './routes/banners'
 import changelog from './routes/changelog'
 import closeToGraduation from './routes/closeToGraduation'
 import completedCoursesSearch from './routes/completedCoursesSearch'
 import courseExclusions from './routes/courseExclusions'
 import courses from './routes/courses'
 import curriculumPeriods from './routes/curriculumPeriods'
-import customOpenUniSearch from './routes/customOpenUniSearch'
 import customPopulationSearch from './routes/customPopulationSearch'
 import faculties from './routes/faculties'
 import feedback from './routes/feedback'
+import health from './routes/health'
 import languageCenterData from './routes/languageCenterData'
 import login from './routes/login'
 import population from './routes/population'
@@ -34,8 +35,7 @@ import tags from './routes/tags'
 import teachers from './routes/teachers'
 import university from './routes/university'
 import updater from './routes/updater'
-import usersToska from './routes/users'
-import usersFd from './routes/usersFd'
+import users from './routes/users'
 
 const routes = (app: Express, url: string) => {
   app.use(cors({ credentials: true, origin: frontUrl }))
@@ -43,9 +43,10 @@ const routes = (app: Express, url: string) => {
   app.use(compression())
 
   app.use(headersMiddleware)
-  app.use(currentUserMiddleware) // eslint-disable-line @typescript-eslint/no-misused-promises
+  app.use(currentUserMiddleware)
   app.use(accessLogger)
   app.use(`${url}/login`, login)
+  app.use(`${url}/banners`, banners)
   app.use(url, courses)
   app.use(`${url}/students`, students)
   app.use(url, population)
@@ -53,32 +54,25 @@ const routes = (app: Express, url: string) => {
   app.use(`${url}/semesters/codes`, semesters)
   app.use(url, tags)
   app.use(url, programmeModules)
-  app.use(`${url}/v3/course-exclusions`, courseExclusions)
-  app.use(`${url}/v2/studyprogrammes`, studyProgramme)
+  app.use(`${url}/course-exclusions`, courseExclusions)
+  app.use(`${url}/studyprogrammes`, studyProgramme)
   app.use(`${url}/programme-criteria`, studyProgrammeCriteria)
-  app.use(`${url}/openunisearch`, auth.roles(['openUniSearch']), customOpenUniSearch)
   app.use(`${url}/changelog`, changelog)
   app.use(`${url}/completedcoursessearch`, completedCoursesSearch)
-  if (languageCenterViewEnabled) {
-    app.use(`${url}/languagecenterdata`, languageCenterData)
-  }
+  app.use(`${url}/languagecenterdata`, languageCenterData)
   app.use(`${url}/faculties`, faculties)
   app.use(`${url}/university`, university)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, teachers)
-  if (serviceProvider === 'toska') {
-    app.use(`${url}/users`, usersToska)
-  } else {
-    app.use(`${url}/users`, usersToska)
-    app.use(`${url}/users`, usersFd)
-  }
+  app.use(`${url}/users`, users)
   app.use(`${url}/feedback`, feedback)
   app.use(`${url}/custom-population-search`, customPopulationSearch)
   app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
   app.use(`${url}/close-to-graduation`, auth.roles(['fullSisuAccess', 'studyGuidanceGroups']), closeToGraduation)
   app.use(`${url}/study-programme-pins`, studyProgrammePins)
   app.use(`${url}/curriculum-periods`, curriculumPeriods)
-  app.get('*', (_, res) => {
+  app.use(`${url}/health`, health)
+  app.get('', (_, res) => {
     res.status(404).json({ error: 'unknown endpoint' })
   })
   Sentry.setupExpressErrorHandler(app)

services/backend/src/routes/banners.ts

@@ -0,0 +1,53 @@
+import { Request, Response, Router } from 'express'
+
+import { Banner } from '@oodikone/shared/models/kone'
+import { saveBanner, getActiveBanners, getAllBanners } from '../services/banners'
+
+const router = Router()
+
+router.get<never, Banner[]>('/', async (_req: Request, res: Response) => {
+  const banners = await getActiveBanners()
+  res.json(banners).end()
+})
+
+router.get<never, Banner[]>('/all', async (req: Request, res: Response) => {
+  const { user } = req
+  if (!user.isAdmin || !user.iamGroups.includes('grp-toska')) {
+    return res.status(403).end()
+  }
+
+  const banners = await getAllBanners()
+  res.json(banners)
+})
+
+router.post('/new', async (req: Request, res: Response) => {
+  const { user } = req
+  if (!user.isAdmin || !user.iamGroups.includes('grp-toska')) {
+    return res.status(403).end()
+  }
+
+  const payload = {
+    ...req.body,
+    lastModifiedBy: user.username,
+  }
+
+  await saveBanner(payload)
+  res.status(201).end()
+})
+
+router.put('/update', async (req: Request, res: Response) => {
+  const { user } = req
+  if (!user.isAdmin || !user.iamGroups.includes('grp-toska')) {
+    return res.status(403).end()
+  }
+
+  const payload = {
+    ...req.body,
+    lastModifiedBy: user.username,
+  }
+
+  await saveBanner(payload)
+  res.status(201).end()
+})
+
+export default router

services/backend/src/routes/changelog.ts

@@ -2,7 +2,7 @@ import axios from 'axios'
 import { Router } from 'express'
 
 import { Release } from '@oodikone/shared/types'
-import { isDev } from '../config'
+import { isDev, isTest } from '../config'
 
 const router = Router()
 
@@ -13,7 +13,7 @@ router.get<never, Release[]>('/', async (_, res) => {
     return res.status(200).send(changelog.data)
   }
 
-  if (isDev) {
+  if (isDev || isTest) {
     const fakeRelease: Release[] = [
       {
         description: '**Feature 1**\n- Added a fancy new feature \n\n**Feature 2**\n- Fixed a bug\n- Fixed another bug',

services/backend/src/routes/closeToGraduation.ts

@@ -14,7 +14,7 @@ router.get<never, CloseToGraduationResBody>('/', async (req, res) => {
     const studentNumbers = await getAllStudentsUserHasInGroups(user.sisPersonId)
     const result = await getCloseToGraduationData(studentNumbers)
 
-    res.json(result)
+    return res.json(result)
   }
 
   const result = await getCloseToGraduationData()

services/backend/src/routes/completedCoursesSearch.ts

@@ -1,14 +1,15 @@
 import { Router } from 'express'
-import { difference, intersection } from 'lodash'
+import { difference, intersection } from 'lodash-es'
 
 import { CanError } from '@oodikone/shared/routes'
+import type { CompletedCoursesStudent, CompletedCoursesCourse } from '@oodikone/shared/types'
 import { tryCatch } from '@oodikone/shared/util'
-import { Courses, FormattedStudent, getCompletedCourses } from '../services/completedCoursesSearch'
+import { getCompletedCourses } from '../services/completedCoursesSearch'
 import {
-  getOpenUniSearches,
-  createNewSearch,
-  deleteSearch,
-  updateSearch,
+  getUserSearches,
+  createNewUserSearch,
+  deleteUserSearch,
+  updateUserSearch,
   FoundSearch,
 } from '../services/openUni/openUniManageSearches'
 import { hasFullAccessToStudentData, safeJSONParse } from '../util'
@@ -19,11 +20,14 @@ const router = Router()
 
 const importerClient = getImporterClient()
 
-export type SearchResBody = CanError<
-  { discardedStudentNumbers: string[] } & { students: Omit<FormattedStudent, 'allEnrollments'>[]; courses: Courses }
+type SearchResBody = CanError<
+  { discardedStudentNumbers: string[] } & {
+    students: Omit<CompletedCoursesStudent, 'allEnrollments'>[]
+    courses: CompletedCoursesCourse[]
+  }
 >
-export type SearchReqBody = never
-export type SearchQuery = {
+type SearchReqBody = never
+type SearchQuery = {
   studentlist: string
   courselist: string
 }
@@ -72,7 +76,7 @@ export type GetSearchResBody = FoundSearch[]
 
 router.get<never, GetSearchResBody>('/searches', async (req, res) => {
   const userId = req.user.id
-  const foundSearches = await getOpenUniSearches(userId)
+  const foundSearches = await getUserSearches(userId)
   return res.json(foundSearches)
 })
 
@@ -98,7 +102,7 @@ router.post<never, CreateSearchResBody, CreateSearchReqBody>('/searches', async
   if (courseCodes && !Array.isArray(courseCodes)) {
     return res.status(400).json({ error: 'Courselist must be type of array' })
   }
-  const createdSearch = await createNewSearch(userId, name, courseCodes)
+  const createdSearch = await createNewUserSearch(userId, name, courseCodes)
   if (!createdSearch) {
     return res.status(400).json({ error: 'Failed to create courselist' })
   }
@@ -130,7 +134,7 @@ router.put<UpdateSearchParams, UpdateSearchResBody, UpdateSearchReqBody>('/searc
     return res.status(422).end()
   }
 
-  const updatedSearch = await updateSearch(userId, id, courseCodes)
+  const updatedSearch = await updateUserSearch(userId, id, courseCodes)
   if (!updatedSearch) {
     return res.status(404).json({ error: 'Courselist could not be found' })
   }
@@ -155,7 +159,7 @@ router.delete<DeleteSearchParams, DeleteSearchResBody>('/searches/:id', async (r
     return res.status(422).end()
   }
 
-  const deletedSearch = await deleteSearch(userId, id)
+  const deletedSearch = await deleteUserSearch(userId, id)
   if (!deletedSearch) {
     return res.status(404).json({ error: 'Courselist could not be found' })
   }

services/backend/src/routes/courses.ts

@@ -9,71 +9,58 @@ import {
   CourseYearlyStatsReqBody,
   CourseYearlyStatsQuery,
 } from '@oodikone/shared/routes/courses'
-import type { CourseWithSubsId } from '@oodikone/shared/types/course'
-import { uniq } from '@oodikone/shared/util'
-import { getCourseYearlyStats } from '../services/courses'
-import { getCoursesByNameAndOrCode, getCoursesByCodes } from '../services/courses/courseFinders'
-import { getFullStudyProgrammeRights, hasFullAccessToStudentData, validateParamLength } from '../util'
+import type { CourseWithSubsDetails } from '@oodikone/shared/types/course'
+import { getCourseDetails, getCourseYearlyStats } from '../services/courses'
+import { getCoursesByNameAndOrCode } from '../services/courses/courseFinders'
+import {
+  getFullStudyProgrammeRights,
+  handleQueryArrays,
+  hasFullAccessToStudentData,
+  validateParamLength,
+} from '../util'
 
 const router = Router()
 
 router.get<never, CanError<CoursesMultiResBody>, CoursesMultiReqBody, CoursesMultiQuery>(
-  '/v2/coursesmulti',
+  '/coursesmulti',
   async (req, res) => {
-    const { name, code, combineSubstitutions, includeSpecial } = req.query
+    const { name, code, includeSpecial } = req.query
     if (!(validateParamLength(name, 5) || validateParamLength(code, 2))) {
       return res.status(400).json({ error: 'Query parameter name or code is invalid' })
     }
 
     const courses = await getCoursesByNameAndOrCode(name, code, includeSpecial === 'true')
 
-    if (combineSubstitutions === 'false') {
-      const courseCodes = courses.map(course => course.code)
-      const substitutions = uniq(
-        courses.flatMap(course => course.substitutions).filter(code => !courseCodes.includes(code))
-      )
-      const substitutionDetails = await getCoursesByCodes(substitutions)
-      for (const substitution of substitutionDetails) {
-        courses.push(substitution.toJSON())
-      }
-    }
-
-    const mergedCourses: Record<string, CourseWithSubsId> = {}
+    const mergedCourses = new Map<string, CourseWithSubsDetails>()
 
     for (const course of courses) {
       if (!(course.max_attainment_date && course.min_attainment_date)) continue
 
-      const groupId = combineSubstitutions === 'true' ? course.subsId?.toString() : course.code
-      if (!groupId) continue
+      if (!mergedCourses.has(course.code)) mergedCourses.set(course.code, structuredClone(course))
+      const mergedCourse = mergedCourses.get(course.code)!
 
-      if (!mergedCourses[groupId]) {
-        mergedCourses[groupId] = {
-          ...course,
-          substitutions: combineSubstitutions === 'true' ? course.substitutions : [],
-        }
-      } else {
-        const mergedCourse = mergedCourses[groupId]
-        if (mergedCourse.max_attainment_date < course.max_attainment_date) {
-          mergedCourse.max_attainment_date = course.max_attainment_date
-        }
-        if (mergedCourse.min_attainment_date > course.min_attainment_date) {
-          mergedCourse.min_attainment_date = course.min_attainment_date
-        }
+      if (mergedCourse.max_attainment_date < course.max_attainment_date) {
+        mergedCourse.max_attainment_date = course.max_attainment_date
+      }
+      if (mergedCourse.min_attainment_date > course.min_attainment_date) {
+        mergedCourse.min_attainment_date = course.min_attainment_date
       }
     }
 
-    res.json({ courses: Object.values(mergedCourses) })
+    res.json({ courses: Array.from(mergedCourses.values()) })
   }
 )
 
-type CourseYearlyStatsResBody = Awaited<ReturnType<typeof getCourseYearlyStats>>
+export type CourseYearlyStatsResBody = Awaited<ReturnType<typeof getCourseYearlyStats>>
 
 router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody, CourseYearlyStatsQuery>(
-  '/v3/courseyearlystats',
+  '/courseyearlystats',
   async (req, res) => {
     const { codes, combineSubstitutions, separate } = req.query
 
-    if (!codes) {
+    const courseCodes = handleQueryArrays(codes)
+
+    if (!courseCodes?.length) {
       return res.status(422).send({ error: 'Missing required query parameters' })
     }
 
@@ -95,9 +82,19 @@ router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody,
     const useCombineSubstitutions = combineSubstitutions !== 'false'
     const useSeparate = separate === 'true'
 
-    const results = await getCourseYearlyStats(codes, useSeparate, anonymizationSalt, useCombineSubstitutions)
-    res.json(results)
+    const results = await getCourseYearlyStats(courseCodes, useSeparate, anonymizationSalt, useCombineSubstitutions)
+    return res.json(results)
   }
 )
 
+router.get<never, CanError<unknown>, never, { codes: string | string[] }>('/coursedetails', async (req, res) => {
+  const { codes: coursecodes } = req.query
+  const codes = handleQueryArrays(coursecodes)
+
+  if (!codes?.length) return res.status(500).end()
+  const details = await getCourseDetails(codes)
+
+  return res.json(details)
+})
+
 export default router

services/backend/src/routes/customOpenUniSearch.ts

@@ -1,148 +0,0 @@
-import { Router } from 'express'
-
-import { CanError } from '@oodikone/shared/routes'
-import {
-  getOpenUniSearches,
-  createNewSearch,
-  deleteSearch,
-  updateSearch,
-} from '../services/openUni/openUniManageSearches'
-import { getCustomOpenUniCourses } from '../services/openUni/openUniStats'
-import { safeJSONParse } from '../util'
-
-const router = Router()
-
-type SearchResBody = CanError<Awaited<ReturnType<typeof getCustomOpenUniCourses>>>
-type SearchReqBody = never
-type SearchQuery = {
-  courselist: string
-  startdate: string
-  enddate: string
-}
-
-router.get<never, SearchResBody, SearchReqBody, SearchQuery>('/', async (req, res) => {
-  const { courselist, startdate, enddate } = req.query
-
-  const courseCodes = (await safeJSONParse(courselist)) ?? []
-  if (!Array.isArray(courseCodes)) {
-    return res.status(400).json({ error: 'Courses must be of type array' })
-  }
-
-  const startDate = startdate !== 'null' ? new Date(startdate) : new Date(2017, 7, 1, 0, 0, 0)
-  const endDate = enddate !== 'null' ? new Date(enddate) : new Date()
-  if (enddate === 'null') {
-    endDate.setHours(23, 59, 59, 999)
-  }
-
-  const customOpenUniSearches = await getCustomOpenUniCourses(courseCodes, startDate, endDate)
-  return res.json(customOpenUniSearches)
-})
-
-type GetSearchResBody = Awaited<ReturnType<typeof getOpenUniSearches>>
-router.get<never, GetSearchResBody>('/searches', async (req, res) => {
-  const { id: userId } = req.user
-  const foundSearches = await getOpenUniSearches(userId)
-  return res.json(foundSearches)
-})
-
-type CreateSearchResBody = CanError<{
-  id: string
-  userId: string
-  name: string
-  courseList: string[]
-  updatedAt: string
-}>
-type CreateSearchReqBody = {
-  courselist: string[]
-  name: string
-}
-
-router.post<never, CreateSearchResBody, CreateSearchReqBody>('/searches', async (req, res) => {
-  const { courselist, name } = req.body
-  const { id: userId } = req.user
-
-  const courseCodes = courselist || []
-
-  if (!name) {
-    return res.status(400).json({ error: 'Name missing' })
-  }
-
-  if (courseCodes && !Array.isArray(courseCodes)) {
-    return res.status(400).json({ error: 'Course codes must be type of array' })
-  }
-
-  const createdSearch = await createNewSearch(userId, name, courseCodes)
-  if (!createdSearch) {
-    return res.status(400).json({ error: 'Failed to create search' })
-  }
-
-  return res.status(201).json({
-    id: createdSearch.id,
-    userId: createdSearch.userId,
-    name: createdSearch.name,
-    courseList: createdSearch.courseCodes,
-    updatedAt: createdSearch.updatedAt,
-  })
-})
-
-type UpdateSearchParams = {
-  id: string
-}
-type UpdateSearchResBody = CanError<{
-  id: string
-  userId: string
-  name: string
-  courseList: string[]
-  updatedAt: string
-}>
-type UpdateSearchReqBody = {
-  courselist: string[]
-}
-
-router.put<UpdateSearchParams, UpdateSearchResBody, UpdateSearchReqBody>('/searches/:id', async (req, res) => {
-  const { id } = req.params
-  const { courselist } = req.body
-  const { id: userId } = req.user
-
-  const courseCodes = courselist || []
-
-  if (!id || !userId) {
-    return res.status(422).end()
-  }
-
-  const updatedSearch = await updateSearch(userId, id, courseCodes)
-  if (!updatedSearch) {
-    return res.status(404).json({ error: 'Open uni search could not be found' })
-  }
-
-  return res.json({
-    id: updatedSearch.id,
-    userId: updatedSearch.userId,
-    name: updatedSearch.name,
-    courseList: updatedSearch.courseCodes,
-    updatedAt: updatedSearch.updatedAt,
-  })
-})
-
-type DeleteSearchParams = {
-  id: string
-}
-type DeleteSearchResBody = CanError<string>
-
-router.delete<DeleteSearchParams, DeleteSearchResBody>('/searches/:id', async (req, res) => {
-  const { id } = req.params
-  const { id: userId } = req.user
-
-  if (!id || !userId) {
-    return res.status(422).end()
-  }
-
-  const deletedSearch = await deleteSearch(userId, id)
-  if (!deletedSearch) {
-    return res.status(404).json({ error: 'Open uni search could not be found' })
-  }
-
-  return res.json(id)
-})
-
-export default router

services/backend/src/routes/faculties.ts

@@ -1,8 +1,19 @@
-import { Request, Response, Router } from 'express'
+import { Router } from 'express'
 
-import { Graduated, ProgrammeFilter, SpecialGroups, StatsType, YearType } from '@oodikone/shared/types'
+import type { CanError, GenericApplicationError } from '@oodikone/shared/routes'
+import type {
+  FacultyGraduationStatistics,
+  Graduated,
+  Name,
+  NameWithCode,
+  ProgrammeFilter,
+  SpecialGroups,
+  StatsType,
+  Unarray,
+  YearType,
+} from '@oodikone/shared/types'
 import * as auth from '../middleware/auth'
-import { getDegreeProgrammesOfFaculty, getFacultyCodeById } from '../services/faculty/faculty'
+import { getDegreeProgrammesOfFaculty, getFacultyCodeById, ProgrammesOfOrganization } from '../services/faculty/faculty'
 import { combineFacultyBasics } from '../services/faculty/facultyBasics'
 import { getFacultyCredits } from '../services/faculty/facultyCredits'
 import { countGraduationTimes } from '../services/faculty/facultyGraduationTimes'
@@ -12,248 +23,277 @@ import {
   setBasicStats,
   getThesisWritersStats,
   setThesisWritersStats,
-  getGraduationStats,
   setGraduationStats,
   getFacultyStudentStats,
   setFacultyStudentStats,
   getFacultyProgressStats,
   setFacultyProgressStats,
+  type BasicData,
+  type ThesisWriterData,
+  getGraduationStats,
 } from '../services/faculty/facultyService'
 import { combineFacultyStudentProgress } from '../services/faculty/facultyStudentProgress'
 import { combineFacultyStudents } from '../services/faculty/facultyStudents'
 import { combineFacultyThesisWriters } from '../services/faculty/facultyThesisWriters'
 import { updateFacultyOverview, updateFacultyProgressOverview } from '../services/faculty/facultyUpdates'
-import logger from '../util/logger'
+import { objectKeyAnnihilator } from '../util'
+import { ApplicationError } from '../util/customErrors'
 
 // Faculty uses a lot of tools designed for Degree programme.
 // Some of them have been copied here and slightly edited for faculty purpose.
 
 const router = Router()
 
-router.get('/', async (_req: Request, res: Response) => {
-  const faculties = await getFacultiesForFacultyList()
-  res.json(faculties)
-})
+type GetFacultyListResBody = {
+  id: string
+  code: string
+  name: Name
+}[]
 
-interface GetStatsRequest extends Request {
-  query: {
-    programme_filter: ProgrammeFilter
-    special_groups: SpecialGroups
-    year_type: YearType
-  }
+router.get<never, GetFacultyListResBody>('/', async (_req, res) => res.json(await getFacultiesForFacultyList()))
+
+type GetStatsParams = { id: string }
+type GetStatsReqBody = never
+type GetStatsResBody = BasicData
+type GetStatsQuery = {
+  programme_filter: ProgrammeFilter
+  special_groups: SpecialGroups
+  year_type: YearType
 }
 
-router.get('/:id/basicstats', auth.roles(['facultyStatistics']), async (req: GetStatsRequest, res: Response) => {
-  const facultyId = req.params.id
-  const code = await getFacultyCodeById(facultyId)
-  if (!code) {
-    return res.status(422).end()
-  }
+router.get<GetStatsParams, CanError<GetStatsResBody, GenericApplicationError>, GetStatsReqBody, GetStatsQuery>(
+  '/:id/basicstats',
+  auth.roles(['facultyStatistics']),
+  async (req, res) => {
+    const { id: facultyId } = req.params
+    const { year_type: yearType, programme_filter: programmeFilter, special_groups: specialGroups } = req.query
 
-  const { year_type: yearType, programme_filter: programmeFilter, special_groups: specialGroups } = req.query
-  const data = await getBasicStats(code, yearType, programmeFilter, specialGroups)
-  if (data) {
-    return res.json(data)
-  }
+    const faculty = await getFacultyCodeById(facultyId)
+    if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
-  const programmes = await getDegreeProgrammesOfFaculty(code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
-  if (!programmes.length) {
-    return res.status(422).end()
-  }
+    const data = await getBasicStats(faculty.code, yearType, programmeFilter, specialGroups)
+    if (data) return res.json(data)
 
-  // TODO: The types could be handled better. Possible null value is causing issues with TS.
-  let updatedStats: any = await combineFacultyBasics(code, programmes, yearType, specialGroups)
-  if (updatedStats) {
-    updatedStats = await setBasicStats(updatedStats, yearType, programmeFilter, specialGroups)
-  }
-  return res.json(updatedStats)
-})
+    const programmes = await getDegreeProgrammesOfFaculty(faculty.code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
+    if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
 
-interface GetCreditStatsRequest extends Request {
-  query: {
-    year_type: YearType
+    const updatedStats = await combineFacultyBasics(faculty.code, programmes, yearType, specialGroups)
+
+    if (updatedStats) await setBasicStats(updatedStats, yearType, programmeFilter, specialGroups)
+    else throw new ApplicationError('No data', 204)
+
+    return res.json(updatedStats)
   }
+)
+
+type GetFacultyStatsResBody = ThesisWriterData
+
+router.get<GetStatsParams, CanError<GetFacultyStatsResBody, GenericApplicationError>, GetStatsReqBody, GetStatsQuery>(
+  '/:id/thesisstats',
+  auth.roles(['facultyStatistics']),
+  async (req, res) => {
+    const { id: facultyId } = req.params
+    const { year_type: yearType, programme_filter: programmeFilter, special_groups: specialGroups } = req.query
+
+    const faculty = await getFacultyCodeById(facultyId)
+    if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+    const data = await getThesisWritersStats(faculty.code, yearType, programmeFilter, specialGroups)
+    if (data) return res.json(data)
+
+    const programmes = await getDegreeProgrammesOfFaculty(faculty.code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
+    if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
+
+    const updatedStats = await combineFacultyThesisWriters(faculty.code, programmes, yearType, specialGroups)
+    if (updatedStats) await setThesisWritersStats(updatedStats, yearType, programmeFilter, specialGroups)
+    else throw new ApplicationError('No data', 204)
+
+    return res.json(updatedStats)
+  }
+)
+
+type GetCreditStatsResBody = {
+  codes: string[]
+  ids: string[]
+  programmeNames: Record<string, NameWithCode>
+}
+type GetCreditStatsQuery = {
+  year_type: YearType
 }
 
-router.get('/:id/creditstats', auth.roles(['facultyStatistics']), async (req: GetCreditStatsRequest, res: Response) => {
-  const facultyId = req.params.id
-  const code = await getFacultyCodeById(facultyId)
+router.get<
+  GetStatsParams,
+  CanError<GetCreditStatsResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetCreditStatsQuery
+>('/:id/creditstats', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
   const { year_type: yearType } = req.query
-  const stats = await getFacultyCredits(code, yearType === 'ACADEMIC_YEAR')
-  return res.json(stats)
+
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+  return res.json(await getFacultyCredits(faculty.code, yearType === 'ACADEMIC_YEAR'))
 })
 
-router.get('/:id/thesisstats', auth.roles(['facultyStatistics']), async (req: GetStatsRequest, res: Response) => {
-  const facultyId = req.params.id
-  const code = await getFacultyCodeById(facultyId)
-  if (!code) {
-    return res.status(422).end()
-  }
+type GetGraduationStatsQuery = {
+  programme_filter: ProgrammeFilter
+}
 
-  const { year_type: yearType, programme_filter: programmeFilter, special_groups: specialGroups } = req.query
-  const data = await getThesisWritersStats(code, yearType, programmeFilter, specialGroups)
-  if (data) {
-    return res.json(data)
-  }
+router.get<
+  GetStatsParams,
+  CanError<FacultyGraduationStatistics, GenericApplicationError>,
+  GetStatsReqBody,
+  GetGraduationStatsQuery
+>('/:id/graduationtimes', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
+  const { programme_filter: programmeFilter } = req.query
 
-  const programmes = await getDegreeProgrammesOfFaculty(code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
-  if (!programmes.length) {
-    return res.status(422).end()
-  }
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+  const data = await getGraduationStats(faculty.code, programmeFilter)
+  // Times were needed to compute medians/averages serverside, but are not needed in frontend.
+  // They are still kept in redis as university view needs them for the same purpose.
+  if (data) return res.json(objectKeyAnnihilator(data, 'times', []))
+
+  const programmes = await getDegreeProgrammesOfFaculty(faculty.code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
+  if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
+
+  const updatedStats = await countGraduationTimes(faculty.code, programmes)
+  if (updatedStats) await setGraduationStats(updatedStats, programmeFilter)
+
+  return res.json(objectKeyAnnihilator(updatedStats, 'times', []))
+})
+
+type GetProgressStatsResBody = {
+  bachelorsProgStats: Record<string, number[][]>
+  bcMsProgStats: Record<string, number[][]>
+  creditCounts: Record<string, Record<string, number[]>>
+  doctoralProgStats: Record<string, number[][]>
+  id: string
+  mastersProgStats: Record<string, number[][]>
+  programmeNames: Record<string, NameWithCode>
+  yearlyBachelorTitles: string[][]
+  yearlyBcMsTitles: string[][]
+  yearlyMasterTitles: string[][]
+  years: string[]
+}
+
+type GetProgressStatsQuery = {
+  graduated: Graduated
+  special_groups: SpecialGroups
+}
+
+router.get<
+  GetStatsParams,
+  CanError<GetProgressStatsResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetProgressStatsQuery
+>('/:id/progressstats', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
+  const { special_groups: specialGroups } = req.query
+
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+  const programmes = await getDegreeProgrammesOfFaculty(faculty.code, true)
+  if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
+
+  const data = await getFacultyProgressStats(faculty.code, specialGroups)
+  if (data) return res.json(data)
+
+  const updatedStats = await combineFacultyStudentProgress(faculty.code, programmes, specialGroups)
+  if (updatedStats) await setFacultyProgressStats(updatedStats, specialGroups)
 
-  let updatedStats: any = await combineFacultyThesisWriters(code, programmes, yearType, specialGroups)
-  if (updatedStats) {
-    updatedStats = await setThesisWritersStats(updatedStats, yearType, programmeFilter, specialGroups)
-  }
   return res.json(updatedStats)
 })
 
-interface GetGraduationStatsRequest extends Request {
-  query: {
-    programme_filter: ProgrammeFilter
-  }
+export type DegreeProgramme = Pick<
+  Unarray<ProgrammesOfOrganization>,
+  'code' | 'name' | 'degreeProgrammeType' | 'progId'
+>
+
+type GetStudentStatsResBody = {
+  id: string
+  years: string[]
+  facultyTableStats: Record<string, (number | string)[]>
+  facultyTableStatsExtra: Record<string, Record<string, Record<string, number>>>
+  /**
+   * NOTE: programmeStats order is random (not by the year). Make sure to index properly.
+   *
+   * (Order is based on the order in which studyrights were processed in the backend)
+   */
+  programmeStats: Record<string, Record<string, (string | number)[]>>
+  titles: string[]
+  programmeNames: Record<string, DegreeProgramme>
 }
 
-router.get(
-  '/:id/graduationtimes',
-  auth.roles(['facultyStatistics']),
-  async (req: GetGraduationStatsRequest, res: Response) => {
-    const facultyId = req.params.id
-    const code = await getFacultyCodeById(facultyId)
-    if (!code) {
-      return res.status(422).end()
-    }
+router.get<
+  GetStatsParams,
+  CanError<GetStudentStatsResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetProgressStatsQuery
+>('/:id/studentstats', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
+  const { special_groups: specialGroups } = req.query
 
-    const { programme_filter: programmeFilter } = req.query
-    const data = await getGraduationStats(code, programmeFilter)
-    if (data) {
-      return res.json(data)
-    }
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
-    const programmes = await getDegreeProgrammesOfFaculty(code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
-    if (!programmes) {
-      return res.status(422).end()
-    }
+  const data = await getFacultyStudentStats(faculty.code, specialGroups)
+  if (data) return res.json(data)
 
-    let updatedStats: any = await countGraduationTimes(code, programmes)
-    if (updatedStats) {
-      updatedStats = await setGraduationStats(updatedStats, programmeFilter)
-    }
-    return res.json(updatedStats)
-  }
-)
+  const programmes = await getDegreeProgrammesOfFaculty(faculty.code, true)
+  if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
 
-interface GetProgressStatsRequest extends Request {
-  query: {
-    graduated: Graduated
-    special_groups: SpecialGroups
-  }
+  const updatedStats = await combineFacultyStudents(faculty.code, programmes, specialGroups)
+  if (updatedStats) await setFacultyStudentStats(updatedStats, specialGroups)
+
+  return res.json(updatedStats)
+})
+
+type GetUpdateBasicViewResBody = never
+type GetUpdateBasicViewQuery = {
+  stats_type: StatsType
 }
 
-router.get(
-  '/:id/progressstats',
-  auth.roles(['facultyStatistics']),
-  async (req: GetProgressStatsRequest, res: Response) => {
-    const facultyId = req.params.id
-    const code = await getFacultyCodeById(facultyId)
-    if (!code) {
-      return res.status(422).end()
-    }
+router.get<
+  GetStatsParams,
+  CanError<GetUpdateBasicViewResBody, GenericApplicationError>,
+  GetStatsReqBody,
+  GetUpdateBasicViewQuery
+>('/:id/update_basicview', auth.roles(['facultyStatistics']), async (req, res) => {
+  const { id: facultyId } = req.params
+  const { stats_type: statsType } = req.query
 
-    const programmes = await getDegreeProgrammesOfFaculty(code, true)
-    if (!programmes) {
-      return res.status(422).end()
-    }
+  const faculty = await getFacultyCodeById(facultyId)
+  if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
-    const { graduated, special_groups: specialGroups } = req.query
-    const data = await getFacultyProgressStats(code, specialGroups, graduated)
-    if (data) {
-      return res.json(data)
-    }
-
-    let updatedStats: any = await combineFacultyStudentProgress(code, programmes, specialGroups, graduated)
-    if (updatedStats) {
-      updatedStats = await setFacultyProgressStats(updatedStats, specialGroups, graduated)
-    }
-    return res.json(updatedStats)
-  }
-)
-
-interface GetStudentStatsRequest extends GetProgressStatsRequest {}
-
-router.get(
-  '/:id/studentstats',
-  auth.roles(['facultyStatistics']),
-  async (req: GetStudentStatsRequest, res: Response) => {
-    const facultyId = req.params.id
-    const code = await getFacultyCodeById(facultyId)
-    if (!code) {
-      return res.status(422).end()
-    }
-
-    const { graduated, special_groups: specialGroups } = req.query
-
-    /* Fetch cached data if exists */
-    const data = await getFacultyStudentStats(code, specialGroups, graduated)
-    if (data) {
-      return res.json(data)
-    }
-
-    const newProgrammes = await getDegreeProgrammesOfFaculty(code, true)
-    if (!newProgrammes.length) {
-      return res.status(422).end()
-    }
-
-    const updatedStats = await combineFacultyStudents(code, newProgrammes, specialGroups, graduated)
-
-    if (updatedStats) {
-      void setFacultyStudentStats(updatedStats, specialGroups, graduated)
-    }
-    return res.json(updatedStats)
-  }
-)
-
-interface GetUpdateBasicViewRequest extends Request {
-  query: {
-    stats_type: StatsType
-  }
-}
-
-router.get(
-  '/:id/update_basicview',
-  auth.roles(['facultyStatistics']),
-  async (req: GetUpdateBasicViewRequest, res: Response) => {
-    const facultyId = req.params.id
-    const code = await getFacultyCodeById(facultyId)
-    if (!code) {
-      return res.status(422).end()
-    }
-    const { stats_type: statsType } = req.query
-    try {
-      const result = await updateFacultyOverview(code, statsType)
-      return res.json(result)
-    } catch (error) {
-      const message = `Failed to update basic stats for faculty ${code}`
-      logger.error(message, { error })
-      return res.status(500).json({ error: message })
-    }
-  }
-)
-
-router.get('/:id/update_progressview', auth.roles(['facultyStatistics']), async (req: Request, res: Response) => {
-  const facultyId = req.params.id
-  const code = await getFacultyCodeById(facultyId)
-  if (!code) {
-    return res.status(422).end()
-  }
   try {
-    const result = await updateFacultyProgressOverview(code)
-    return res.json(result)
+    await updateFacultyOverview(faculty.code, statsType)
+    return res.json()
   } catch (error) {
-    const message = `Failed to update progress tab stats for faculty ${code}`
-    logger.error(message, { error })
-    return res.status(500).json({ error: message })
+    throw new ApplicationError(`Failed to update basic stats for faculty ${faculty.code}`, 500)
   }
 })
 
+router.get<GetStatsParams, CanError<never, GenericApplicationError>, GetStatsReqBody, GetStatsQuery>(
+  '/:id/update_progressview',
+  auth.roles(['facultyStatistics']),
+  async (req, res) => {
+    const { id: facultyId } = req.params
+
+    const faculty = await getFacultyCodeById(facultyId)
+    if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
+
+    try {
+      await updateFacultyProgressOverview(faculty.code)
+      return res.json()
+    } catch (error) {
+      throw new ApplicationError(`Failed to update progress tab stats for faculty ${faculty.code}`, 500)
+    }
+  }
+)
+
 export default router

services/backend/src/routes/feedback.ts

@@ -1,6 +1,6 @@
 import { Request, Response, Router } from 'express'
 
-import { sendFeedbackToToska } from '../services/mailService'
+import { sendFeedback } from '../services/mailService'
 import { ApplicationError } from '../util/customErrors'
 import logger from '../util/logger'
 
@@ -20,7 +20,7 @@ router.post('/email', async (req: EmailRequest, res: Response) => {
     throw new ApplicationError('User not found', 404)
   }
 
-  await sendFeedbackToToska({
+  await sendFeedback({
     feedbackContent: content,
     user,
   })

services/backend/src/routes/health.ts

@@ -0,0 +1,10 @@
+import { Router } from 'express'
+
+const router = Router()
+
+/** Used for checking if backend is up eg. Docker healthchecks */
+router.get<never, { status: string }>('/', (_, res) => {
+  return res.status(200).json({ status: 'healthy' })
+})
+
+export default router

services/backend/src/routes/login.ts

@@ -1,8 +1,8 @@
 import { Router } from 'express'
 
 import { CanError } from '@oodikone/shared/routes'
+import { FormattedUser } from '@oodikone/shared/types'
 import { omitKeys } from '@oodikone/shared/util'
-import { FormattedUser } from '../types/user'
 import { ApplicationError } from '../util/customErrors'
 
 const router = Router()

services/backend/src/routes/population.ts

@@ -1,5 +1,5 @@
 import { Router } from 'express'
-import { difference, intersection, uniq } from 'lodash'
+import { difference, intersection, uniq } from 'lodash-es'
 
 import type { CanError } from '@oodikone/shared/routes'
 import {
@@ -15,18 +15,22 @@ import {
   PopulationstatisticsMaxYearsToCreatePopulationFormQuery,
   PopulationstatisticsMaxYearsToCreatePopulationFormResBody,
 } from '@oodikone/shared/routes/populations'
-import { Unification, Unarray } from '@oodikone/shared/types'
-import { mapToProviders } from '@oodikone/shared/util'
+import { Unification } from '@oodikone/shared/types'
+import { keyBy, mapToProviders } from '@oodikone/shared/util'
 import { rootOrgId } from '../config'
-import { getCourseProvidersForCourses, maxYearsToCreatePopulationFrom } from '../services/courses'
+import {
+  getCourseProvidersForCourses,
+  maxYearsToCreatePopulationFrom,
+  searchAndCombineSubstitutionGroupsToCodes,
+} from '../services/courses'
 import { encrypt } from '../services/encrypt'
-import { getDegreeProgrammesOfOrganization, ProgrammesOfOrganization } from '../services/faculty/faculty'
+import { getDegreeProgrammesOfOrganization } from '../services/faculty/faculty'
 import { getStudentTagMap } from '../services/populations/getStudentData'
 import { parseDateRangeFromParams } from '../services/populations/shared'
 import { statisticsOf } from '../services/populations/statisticsOf'
 import { getStudentNumbersWithStudyRights } from '../services/populations/studentNumbersWithStudyRights'
 import { findByCourseAndSemesters } from '../services/students'
-import { getFullStudyProgrammeRights, hasFullAccessToStudentData } from '../util'
+import { getFullStudyProgrammeRights, handleQueryArrays, hasFullAccessToStudentData } from '../util'
 
 const router = Router()
 
@@ -98,13 +102,17 @@ const obfuscateStuff = ({
 }
 
 router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReqBody, PopulationstatisticsQuery>(
-  '/v3/populationstatistics',
+  '/populationstatistics',
   async (req, res) => {
     const { id: userId, roles: userRoles, programmeRights: userProgrammeRights, studentsUserCanAccess } = req.user
-    const { years, semesters, programme, combinedProgramme, studentStatuses } = req.query
+    const { programme, combinedProgramme } = req.query
+
+    const years = handleQueryArrays(req.query.years)
+    const semesters = handleQueryArrays(req.query.semesters)
+    const studentStatuses = handleQueryArrays(req.query.studentStatuses) // NOTE: TRANSFERRED means transferred out of the program
 
     const requiredFields = [years, semesters, programme]
-    if (requiredFields.some(field => !field)) {
+    if (requiredFields.some(field => !field.length)) {
       return res.status(400).json({ error: 'The query should have years, semesters and a programme defined' })
     }
 
@@ -112,9 +120,11 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
       return res.status(400).json({ error: 'Semester should be either SPRING OR FALL' })
     }
 
-    const hasCorrectStatus = (studentStatuses: string[]) =>
-      studentStatuses.every(status => ['EXCHANGE', 'NONDEGREE', 'TRANSFERRED'].includes(status))
-    if (studentStatuses && !hasCorrectStatus(studentStatuses)) {
+    const hasCorrectStatus = (studentStatuses: (string | undefined)[]) =>
+      studentStatuses.every(status => status && ['EXCHANGE', 'NONDEGREE', 'TRANSFERRED'].includes(status))
+
+    // NOTE: This has always the length of at least 1, [undefined]
+    if (studentStatuses[0] && !hasCorrectStatus(studentStatuses)) {
       return res.status(400).json({ error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' })
     }
 
@@ -128,10 +138,8 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
       return res.status(403).json({ error: 'Trying to request unauthorized students data' })
     }
 
-    const { startDate, endDate } = parseDateRangeFromParams({
-      ...req.query,
-      years: req.query.years,
-    })
+    // NOTE: Years are academic years
+    const { startDate, endDate } = parseDateRangeFromParams({ semesters, years })
 
     const studentNumbers = await getStudentNumbersWithStudyRights({
       programmeCodes: [programme],
@@ -164,14 +172,15 @@ router.get<
   CanError<PopulationstatisticsbycourseResBody>,
   PopulationstatisticsbycourseReqBody,
   PopulationstatisticsbycourseParams
->('/v3/populationstatisticsbycourse', async (req, res) => {
+>('/populationstatisticsbycourse', async (req, res) => {
   const {
     id: userId,
     roles: userRoles,
     programmeRights: userProgrammeRights,
     studentsUserCanAccess: allStudentsUserCanAccess,
   } = req.user
-  const { coursecodes: coursecodeJSON, from, to, separate, unifyCourses } = req.query
+  const { coursecodes: coursecodeJSON, from, to, separate, unifyCourses, ...restParams } = req.query
+  const includeSubstitutions = restParams.includeSubstitutions === 'true'
 
   if (!coursecodeJSON || !from || !to) {
     return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
@@ -179,16 +188,17 @@ router.get<
 
   const isSeparate = separate === 'true'
   const coursecodes = JSON.parse(coursecodeJSON)
+  const codes = includeSubstitutions ? await searchAndCombineSubstitutionGroupsToCodes(coursecodes) : coursecodes
 
   const toFromDiff = Math.abs(Number(to) - Number(from) + 1)
   const requestedYearsToCreatePopulationFrom = Math.ceil(isSeparate ? toFromDiff / 2 : toFromDiff) // 2 semesters = 1 year
 
-  const maxYearsForPopulation = await maxYearsToCreatePopulationFrom(coursecodes, Unification.REGULAR)
+  const maxYearsForPopulation = await maxYearsToCreatePopulationFrom(codes, Unification.REGULAR)
   if (requestedYearsToCreatePopulationFrom > maxYearsForPopulation) {
     return res.status(400).json({ error: `Max years to create population from is ${maxYearsForPopulation}` })
   }
 
-  const studentNumbers = await findByCourseAndSemesters(coursecodes, Number(from), Number(to), isSeparate, unifyCourses)
+  const studentNumbers = await findByCourseAndSemesters(codes, Number(from), Number(to), isSeparate, unifyCourses)
 
   const courseProviders: string[] = await getCourseProvidersForCourses(coursecodes)
   const fullStudyProgrammeRights = getFullStudyProgrammeRights(userProgrammeRights)
@@ -209,18 +219,25 @@ router.get<
     studentsUserCanAccess,
   })
 
-  return res.json(processed)
+  return res.json({ ...processed, mainCourseCodes: coursecodes, allCourseCodes: codes })
 })
 
 // Used in custom population and single study guidance groups
-router.post<never, GetCustomPopulationResBody, CustomPopulationQuery>(
-  '/v3/populationstatisticsbystudentnumbers',
+router.post<never, CanError<GetCustomPopulationResBody>, CustomPopulationQuery>(
+  '/populationstatisticsbystudentnumbers',
   async (req, res) => {
-    const { studentNumbers, tags } = req.body
+    const { studentNumbers, tags } = req.body ?? { studentNumbers: [] }
     const { id: userId, roles, studentsUserCanAccess } = req.user
+
+    const students = handleQueryArrays(studentNumbers)
+
+    if (!students.length) {
+      return res.status(422).json({ error: 'Body should include student numbers' })
+    }
+
     const filteredStudentNumbers = hasFullAccessToStudentData(roles)
-      ? studentNumbers
-      : intersection(studentNumbers, studentsUserCanAccess)
+      ? students
+      : intersection(students, studentsUserCanAccess)
 
     const studyProgrammeCode = tags?.studyProgramme?.split('+')[0]
 
@@ -234,14 +251,14 @@ router.post<never, GetCustomPopulationResBody, CustomPopulationQuery>(
     const result = await statisticsOf(filteredStudentNumbers, studyRights, tagMap, startDate)
 
     const resultWithStudyProgramme = { ...result, studyProgramme: tags?.studyProgramme }
-    const discardedStudentNumbers = difference(studentNumbers, filteredStudentNumbers)
+    const discardedStudentNumbers = difference(students, filteredStudentNumbers)
 
     res.status(200).json({ ...resultWithStudyProgramme, discardedStudentNumbers })
   }
 )
 
 router.get<never, PopulationstatisticsStudyprogrammesResBody>(
-  '/v3/populationstatistics/studyprogrammes',
+  '/populationstatistics/studyprogrammes',
   async (req, res) => {
     const { roles, programmeRights } = req.user
     const programmes = await getDegreeProgrammesOfOrganization(rootOrgId, false)
@@ -252,14 +269,13 @@ router.get<never, PopulationstatisticsStudyprogrammesResBody>(
     const filteredProgrammes = hasFullAccessToStudentData(roles)
       ? programmes
       : programmes.filter(programme => allRights.includes(programme.code))
-    const formattedProgrammes = filteredProgrammes.reduce<Record<string, Unarray<ProgrammesOfOrganization>>>(
-      (acc, curr) => {
-        acc[curr.code] = curr
-        return acc
-      },
-      {}
-    )
-    res.json(formattedProgrammes)
+    const formattedFilteredProgrammes = keyBy(filteredProgrammes, 'code')
+    const formattedAllProgrammes = keyBy(programmes, 'code')
+
+    res.json({
+      allProgrammes: formattedAllProgrammes,
+      filteredProgrammes: formattedFilteredProgrammes,
+    })
   }
 )
 
@@ -268,7 +284,7 @@ router.get<
   PopulationstatisticsMaxYearsToCreatePopulationFormResBody,
   never,
   PopulationstatisticsMaxYearsToCreatePopulationFormQuery
->('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
+>('/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
   const courseCodes = JSON.parse(req.query.courseCodes) as string[]
   const maxYearsToCreatePopulationFromOpen = await maxYearsToCreatePopulationFrom(courseCodes, Unification.OPEN)
   const maxYearsToCreatePopulationFromUni = await maxYearsToCreatePopulationFrom(courseCodes, Unification.REGULAR)

services/backend/src/routes/programmeModules.ts

@@ -1,16 +1,21 @@
-import { Request, Response, Router } from 'express'
+import { Router } from 'express'
 
+import { CurriculumOption } from '@oodikone/shared/types'
 import { getCoursesAndModules, getCurriculumOptions } from '../services/programmeModules'
 
 const router = Router()
 
-router.get('/v3/curriculum-options/:code', async (req: Request, res: Response) => {
+type CurriculumOptionParams = { code: string }
+
+router.get<CurriculumOptionParams, CurriculumOption[] | null>('/curriculum-options/:code', async (req, res) => {
   const { code } = req.params
   const result = await getCurriculumOptions(code)
   res.json(result)
 })
 
-router.get('/v3/curriculum/:code/:periodIds', async (req: Request, res: Response) => {
+type CurriculumPeriodParams = { code: string; periodIds: string }
+
+router.get<CurriculumPeriodParams>('/curriculum/:code/:periodIds', async (req, res) => {
   const { code, periodIds } = req.params
   const { defaultProgrammeCourses, secondProgrammeCourses } = await getCoursesAndModules(code, periodIds)
 

services/backend/src/routes/studyProgramme.ts

@@ -1,6 +1,7 @@
-import { Request, Response, Router } from 'express'
+import { Router } from 'express'
 
 import { Graduated, SpecialGroups, YearType } from '@oodikone/shared/types'
+import { CreditStatsPayload } from '@oodikone/shared/types/studyProgramme'
 import {
   getBasicStats,
   setBasicStats,
@@ -11,10 +12,8 @@ import {
 } from '../services/analyticsService'
 import { getCreditsProduced } from '../services/providerCredits'
 import { getBasicStatsForStudytrack } from '../services/studyProgramme/studyProgrammeBasics'
-import {
-  getStudyProgrammeCoursesForStudyTrack,
-  getStudyProgrammeStatsForColorizedCoursesTable,
-} from '../services/studyProgramme/studyProgrammeCourses'
+import { getStudyProgrammeCoursesForStudyTrack } from '../services/studyProgramme/studyProgrammeCourses'
+import { getStudyProgrammeStatsForColorizedCoursesTable } from '../services/studyProgramme/studyProgrammeCoursesBySemester'
 import { getGraduationStatsForStudyTrack } from '../services/studyProgramme/studyProgrammeGraduations'
 import { updateBasicView, updateStudyTrackView } from '../services/studyProgramme/studyProgrammeUpdates'
 import { getStudyRightsInProgramme, getStudyTracksForProgramme } from '../services/studyProgramme/studyRightFinders'
@@ -24,32 +23,30 @@ import { logInfoForGrafana } from '../util/logInfoForGrafana'
 
 const router = Router()
 
-interface GetCreditStatsRequest extends Request {
-  query: {
-    codes: string
-    specialGroups: SpecialGroups
-    yearType: YearType
-  }
+type StudyProgrammeParams = { id: string }
+type UpdateStudyProgrammeQuery = { combined_programme: string }
+type CreditStatsQuery = {
+  codes: string
+  specialGroups: SpecialGroups
+  yearType: YearType
 }
 
-router.get('/creditstats', async (req: GetCreditStatsRequest, res: Response) => {
+router.get<StudyProgrammeParams, unknown, never, CreditStatsQuery>('/creditstats', async (req, res) => {
   const { codes, specialGroups, yearType } = req.query
-  const stats = {}
+  const stats: CreditStatsPayload = {}
   for (const code of JSON.parse(codes)) {
     stats[code] = await getCreditsProduced(code, yearType === 'ACADEMIC_YEAR', specialGroups === 'SPECIAL_INCLUDED')
   }
-  return res.json({ stats })
+  return res.json(stats)
 })
 
-interface GetStatsRequest extends Request {
-  query: {
-    year_type: YearType
-    special_groups: SpecialGroups
-    combined_programme: string
-  }
+type StatsRequestQuery = {
+  year_type: YearType
+  special_groups: SpecialGroups
+  combined_programme: string
 }
 
-router.get('/:id/basicstats', async (req: GetStatsRequest, res: Response) => {
+router.get<StudyProgrammeParams, unknown, never, StatsRequestQuery>('/:id/basicstats', async (req, res) => {
   const code = req.params.id
   const { year_type: yearType, special_groups: specialGroups, combined_programme: combinedProgramme } = req.query
   if (!code) {
@@ -76,7 +73,7 @@ router.get('/:id/basicstats', async (req: GetStatsRequest, res: Response) => {
   return res.json(updatedStats)
 })
 
-router.get('/:id/graduationstats', async (req: GetStatsRequest, res: Response) => {
+router.get<StudyProgrammeParams, unknown, never, StatsRequestQuery>('/:id/graduationstats', async (req, res) => {
   const code = req.params.id
   const { year_type: yearType, special_groups: specialGroups, combined_programme: combinedProgramme } = req.query
   if (!code) {
@@ -103,14 +100,12 @@ router.get('/:id/graduationstats', async (req: GetStatsRequest, res: Response) =
   return res.json(updatedStats)
 })
 
-interface GetCourseStatsRequest extends Request {
-  query: {
-    combinedProgramme: string
-    yearType: YearType
-  }
+type CourseStatsQuery = {
+  combinedProgramme: string
+  yearType: YearType
 }
 
-router.get('/:id/coursestats', async (req: GetCourseStatsRequest, res: Response) => {
+router.get<StudyProgrammeParams, unknown, never, CourseStatsQuery>('/:id/coursestats', async (req, res) => {
   const code = req.params.id
   const { combinedProgramme, yearType } = req.query
   const date = new Date()
@@ -124,23 +119,21 @@ router.get('/:id/coursestats', async (req: GetCourseStatsRequest, res: Response)
   }
 })
 
-interface GetStudyTrackStatsRequest extends Request {
-  query: {
-    graduated: Graduated
-    special_groups: SpecialGroups
-    combined_programme: string
-  }
+type StudyTrackStatsQuery = {
+  graduated: Graduated
+  special_groups: SpecialGroups
+  combined_programme: string
 }
 
-router.get('/:id/studytrackstats', async (req: GetStudyTrackStatsRequest, res: Response) => {
+router.get<StudyProgrammeParams, unknown, never, StudyTrackStatsQuery>('/:id/studytrackstats', async (req, res) => {
   const code = req.params.id
-  const { graduated, special_groups: specialGroups, combined_programme: combinedProgramme } = req.query
+  const { special_groups: specialGroups, combined_programme: combinedProgramme } = req.query
   if (!code) {
     return res.status(422).end()
   }
 
   void logInfoForGrafana(code, combinedProgramme)
-  const data = await getStudyTrackStats(code, combinedProgramme, graduated, specialGroups)
+  const data = await getStudyTrackStats(code, combinedProgramme, specialGroups)
   if (data) {
     return res.json(data)
   }
@@ -150,18 +143,20 @@ router.get('/:id/studytrackstats', async (req: GetStudyTrackStatsRequest, res: R
     studyProgramme: code,
     combinedProgramme,
     settings: {
-      graduated: graduated === 'GRADUATED_INCLUDED',
       specialGroups: specialGroups === 'SPECIAL_INCLUDED',
     },
     studyRightsOfProgramme,
   })
+
   if (updated) {
-    await setStudyTrackStats(updated, graduated, specialGroups)
+    await setStudyTrackStats(updated, specialGroups)
+    return res.json(updated)
   }
-  return res.json(updated)
+
+  return res.status(500).end()
 })
 
-router.get('/:id/colorizedtablecoursestats', async (req: Request, res: Response) => {
+router.get<StudyProgrammeParams>('/:id/colorizedtablecoursestats', async (req, res) => {
   const code = req.params.id
   try {
     const data = await getStudyProgrammeStatsForColorizedCoursesTable(code)
@@ -171,7 +166,7 @@ router.get('/:id/colorizedtablecoursestats', async (req: Request, res: Response)
   }
 })
 
-router.get('/:id/studytracks', async (req: Request, res: Response) => {
+router.get<StudyProgrammeParams>('/:id/studytracks', async (req, res) => {
   const code = req.params.id
   if (!code) {
     return res.status(422).end()
@@ -184,42 +179,42 @@ router.get('/:id/studytracks', async (req: Request, res: Response) => {
   }
 })
 
-interface GetUpdateViewRequest extends Request {
-  query: {
-    combined_programme: string
+router.get<StudyProgrammeParams, unknown, never, UpdateStudyProgrammeQuery>(
+  '/:id/update_basicview',
+  async (req, res) => {
+    const code = req.params.id
+    const combinedProgramme = req.query?.combined_programme
+    if (!code) {
+      return res.status(400).json({ error: 'Missing code' })
+    }
+    try {
+      const result = await updateBasicView(code, combinedProgramme)
+      return res.json(result)
+    } catch (error) {
+      const message = `Failed to update basic stats for programme ${code}${combinedProgramme ? `+${combinedProgramme}` : ''}`
+      logger.error(message, { error })
+      return res.status(500).json({ error: message })
+    }
   }
-}
+)
 
-router.get('/:id/update_basicview', async (req: GetUpdateViewRequest, res: Response) => {
-  const code = req.params.id
-  const combinedProgramme = req.query?.combined_programme
-  if (!code) {
-    return res.status(400).json({ error: 'Missing code' })
+router.get<StudyProgrammeParams, unknown, never, UpdateStudyProgrammeQuery>(
+  '/:id/update_studytrackview',
+  async (req, res) => {
+    const code = req.params.id
+    const combinedProgramme = req.query?.combined_programme
+    if (!code) {
+      return res.status(400).json({ error: 'Missing code' })
+    }
+    try {
+      const result = await updateStudyTrackView(code, combinedProgramme)
+      return res.json(result)
+    } catch (error) {
+      const message = `Failed to update study track stats for programme ${code}${combinedProgramme ? `+${combinedProgramme}` : ''}`
+      logger.error(message, { error })
+      return res.status(500).json({ error: message })
+    }
   }
-  try {
-    const result = await updateBasicView(code, combinedProgramme)
-    return res.json(result)
-  } catch (error) {
-    const message = `Failed to update basic stats for programme ${code}${combinedProgramme ? `+${combinedProgramme}` : ''}`
-    logger.error(message, { error })
-    return res.status(500).json({ error: message })
-  }
-})
-
-router.get('/:id/update_studytrackview', async (req: GetUpdateViewRequest, res: Response) => {
-  const code = req.params.id
-  const combinedProgramme = req.query?.combined_programme
-  if (!code) {
-    return res.status(400).json({ error: 'Missing code' })
-  }
-  try {
-    const result = await updateStudyTrackView(code, combinedProgramme)
-    return res.json(result)
-  } catch (error) {
-    const message = `Failed to update study track stats for programme ${code}${combinedProgramme ? `+${combinedProgramme}` : ''}`
-    logger.error(message, { error })
-    return res.status(500).json({ error: message })
-  }
-})
+)
 
 export default router

services/backend/src/routes/tags.ts

@@ -1,5 +1,5 @@
 import { Router } from 'express'
-import { difference } from 'lodash'
+import { difference } from 'lodash-es'
 
 import { CanError } from '@oodikone/shared/routes'
 import { NewTag, Role, StudentTag, Tag } from '@oodikone/shared/types'