[Docker][Dev] fix permissions with node_modules...................

We use xlsx which is distributed by their own cdn rather than npm. Npm
v12 by defaults blocks this installation.

.github/actions/build/backend/action.yaml

@@ -0,0 +1,41 @@
+name: 'Build backend'
+description: 'Build backend in staging or production version'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+
+outputs:
+  image:
+    description: 'Image name provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.image }}
+  tags:
+    description: 'Tag names provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@v6
+
+    - name: Build backend image
+      id: build-image-action
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: oodikone-backend
+        tags: ${{ inputs.environment }} ${{ github.sha }}
+        context: services
+        containerfiles: |
+          services/backend/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          BASE_PATH=${{ case(inputs.environment == 'staging', '/oodikone', '') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}

.github/actions/build/frontend/action.yaml

@@ -0,0 +1,50 @@
+name: 'Build frontend'
+description: 'Build frontend in staging or production versions'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+  sentry_dns:
+    required: false
+    description: 'Sentry url to send error logs to (only production, enforced)'
+    default: ''
+
+outputs:
+  image:
+    description: 'Image name provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.image }}
+  tags:
+    description: 'Tag names provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - name: Get current time
+      id: get_current_time
+      shell: bash
+      run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
+
+    - name: Build frontend image
+      id: build-image-action
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: oodikone-frontend
+        tags: ${{ inputs.environment }} ${{ github.sha }}
+        context: services
+        containerfiles: |
+          services/frontend/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          BASE_PATH=${{ case(inputs.environment == 'staging', '/oodikone', '') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}
+          BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+          SENTRY_DSN=${{ case(inputs.environment == 'production', inputs.sentry_dns, '') }}

.github/actions/build/scheduler/action.yaml

@@ -0,0 +1,40 @@
+name: 'Build updater scheduler'
+description: 'Build updater scheduler in staging or production version'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+
+outputs:
+  image:
+    description: 'Image name provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.image }}
+  tags:
+    description: 'Tag names provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@v6
+
+    - name: Build scheduler image
+      id: build-image-action
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: updater-scheduler
+        tags: ${{ inputs.environment }} ${{ github.sha }}
+        context: updater/sis-updater-scheduler
+        containerfiles: |
+          updater/sis-updater-scheduler/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}

.github/actions/build/worker/action.yaml

@@ -0,0 +1,40 @@
+name: 'Build updater worker'
+description: 'Build updater worker in staging or production version'
+
+inputs:
+  environment:
+    required: true
+    description: 'staging or production'
+    default: 'production'
+  release_version:
+    required: false
+    description: 'Release version added to production builds'
+    default: ${{ github.sha }}
+
+outputs:
+  image:
+    description: 'Image name provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.image }}
+  tags:
+    description: 'Tag names provided by buildah action'
+    value: ${{ steps.build-image-action.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@v6
+
+    - name: Build worker image
+      id: build-image-action
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: updater-worker
+        tags: ${{ inputs.environment }} ${{ github.sha }}
+        context: updater/sis-updater-worker
+        containerfiles: |
+          updater/sis-updater-worker/Dockerfile
+        build-args: |
+          GIT_SHA=${{ github.sha }}
+          STAGING=${{ case(inputs.environment == 'staging', 'true', 'false') }}
+          SENTRY_ENVIRONMENT=${{ inputs.environment }}
+          SENTRY_RELEASE=${{ inputs.release_version }}

.github/actions/test/api/action.yaml

@@ -0,0 +1,37 @@
+name: 'Run api tests'
+description: 'Runs api tests in backend'
+
+inputs:
+  username:
+    required: true
+    description: 'Toska registry username'
+  password:
+    required: true
+    description: 'Toska registry password'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Login to toska docker registry
+      uses: docker/login-action@v4
+      with:
+        registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+        username: ${{ inputs.username }}
+        password: ${{ inputs.password }}
+
+    - name: Start oodikone (without frontend and backend)
+      shell: bash
+      run: docker compose --file docker-compose.ci.yml up --quiet-pull --detach --wait
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/backend
+
+    - name: Run api-tests
+      shell: bash
+      run: npm run test:api --prefix ./services/backend

.github/actions/test/component/action.yaml

@@ -0,0 +1,31 @@
+name: 'Run component tests'
+description: 'Runs frontend component tests'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/shared && npm ci --prefix ./services/frontend
+
+    - name: Install Playwright Browsers
+      shell: bash
+      run: npm exec -- playwright install --with-deps
+      working-directory: ./services/frontend
+
+    - name: Run Playwright tests
+      shell: bash
+      run: npm run test:component --prefix ./services/frontend
+
+    - uses: actions/upload-artifact@v4
+      if: |
+        !cancelled()
+      with:
+        name: playwright-report
+        path: playwright-report/
+        retention-days: 7

.github/actions/test/e2e/action.yaml

@@ -32,7 +32,7 @@ runs:
     - name: 'Run cypress'
       uses: cypress-io/github-action@v7
       with:
-        browser: chrome
+        browser: firefox
         spec: ${{ inputs.spec }}
       env:
         CYPRESS_baseUrl: http://localhost:1337
@@ -43,8 +43,9 @@ runs:
       with:
         name: 'Cypress videos - ${{ inputs.test_name }}'
         path: ./cypress/videos/*
+        retention-days: 7
 
     - name: 'Print logs on failure'
       shell: bash
       if: failure()
-      run: docker compose --file docker-compose.ci.yml --profile oodikone logs && sleep 15
+      run: npm run logs:ci && sleep 15

.github/actions/test/services/action.yaml

@@ -0,0 +1,37 @@
+name: 'Run service tests'
+description: 'Runs service tests in backend'
+
+inputs:
+  username:
+    required: true
+    description: 'Toska registry username'
+  password:
+    required: true
+    description: 'Toska registry password'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Login to toska docker registry
+      uses: docker/login-action@v4
+      with:
+        registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+        username: ${{ inputs.username }}
+        password: ${{ inputs.password }}
+
+    - name: Start oodikone (without frontend and backend)
+      shell: bash
+      run: docker compose --file docker-compose.ci.yml up --quiet-pull --detach --wait
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/backend
+
+    - name: Run service-tests
+      shell: bash
+      run: npm run test:services --prefix ./services/backend

.github/actions/test/unit/action.yaml

@@ -0,0 +1,22 @@
+name: 'Run unit tests'
+description: 'Runs unit tests in backend and frontend'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./services/shared && npm ci --prefix ./services/backend && npm ci --prefix ./services/frontend
+
+    - name: Run frontend unit tests
+      shell: bash
+      run: npm run test:unit --prefix ./services/frontend
+
+    - name: Run backend unit tests
+      shell: bash
+      run: npm run test:unit --prefix ./services/backend

.github/workflows/production.yaml

@@ -5,45 +5,6 @@ on:
     types: [published]
 
 jobs:
-  release_frontend:
-    name: 'Build and release frontend'
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: 'Get current time'
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-
-      - name: Extract release version
-        id: extract_version
-        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
-
-      - name: Build frontend image
-        id: build-image
-        uses: redhat-actions/buildah-build@v2
-        with:
-          image: oodikone-frontend
-          tags: production ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/frontend/Dockerfile
-          build-args: |
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
-            SENTRY_DSN=${{ secrets.OODIKONE_FRONTEND_SENTRY_DSN }}
-
-      - name: Push to quay.io
-        uses: redhat-actions/push-to-registry@v2
-        with:
-          image: ${{ steps.build-image.outputs.image }}
-          tags: ${{ steps.build-image.outputs.tags }}
-          registry: quay.io/toska
-          username: toska+github
-          password: ${{ secrets.QUAY_IO_TOKEN }}
-
   release_backend:
     name: 'Build and release backend'
     runs-on: ubuntu-latest
@@ -55,18 +16,40 @@ jobs:
         id: extract_version
         run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
 
-      - name: Build backend image
+      - name: 'Build backend'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/backend
         with:
-          image: oodikone-backend
-          tags: production ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/backend/Dockerfile
-          build-args: |
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
+          environment: 'production'
+          release_version: ${{ steps.extract_version.outputs.VERSION }}
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_frontend:
+    name: 'Build and release frontend'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Extract release version
+        id: extract_version
+        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+
+      - name: 'Build frontend'
+        id: build-image
+        uses: ./.github/actions/build/frontend
+        with:
+          environment: 'production'
+          release_version: ${{ steps.extract_version.outputs.VERSION }}
+          sentry_dns: ${{ secrets.OODIKONE_FRONTEND_SENTRY_DSN }} # Used only in production
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2

.github/workflows/pull_request.yaml

@@ -5,56 +5,60 @@ on:
     branches:
       - master
     paths:
+      - '.github/actions/**'
+      - '.github/workflows/pull_request.yaml'
+      - '.github/workflows/test.yaml'
+      - 'services/**'
+      - 'updater/**'
+      - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
+      - 'package*.json'
       - 'cypress/**'
       - 'cypress.config.json'
-      - 'services/**'
-      - 'docker-compose.ci.yml'
-      - 'package*.json'
 
 jobs:
   lint_packages:
     name: 'Lint packages'
     runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
+
     steps:
       - uses: actions/checkout@v6
       - name: Lint backend, frontend and shared
         uses: ./.github/actions/lint
 
-  test_api:
+  test_unit:
+    name: 'Run unit tests'
+    runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
     needs:
       - lint_packages
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-24.04
+
     steps:
-      - name: Checkout
-        uses: actions/checkout@v6
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/test/unit
 
-      - name: Install node-24 for cypress
-        uses: actions/setup-node@v6
-        with:
-          node-version: 24
+  test_api:
+    name: 'Test backend api'
+    runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
+    needs:
+      - lint_packages
 
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v4
-        with:
-          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-
-      - name: Start oodikone detached
-        run: npm run oodikone:ci:setup
-
-      - name: Run api-tests
-        run: npm run oodikone:ci:api
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/test/api
 
   test_end_to_end:
+    name: ${{ matrix.name }}
     runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
     needs:
       - lint_packages
       - test_api
     strategy:
       fail-fast: false
+
       matrix:
         include:
           - name: 'Test course statistics'
@@ -83,11 +87,13 @@ jobs:
             spec: cypress/e2e/Study_guidance_groups.js
           - name: 'Test university'
             spec: cypress/e2e/University.js
-    name: ${{ matrix.name }}
+          - name: 'Test numbers match'
+            spec: cypress/e2e/Numbers_match.js
+
     steps:
       - uses: actions/checkout@v6
       - name: 'Run tests'
-        uses: ./.github/actions/run-tests
+        uses: ./.github/actions/test/e2e
         with:
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}

.github/workflows/staging.yaml

@@ -8,12 +8,13 @@ on:
       - '.github/actions/**'
       - '.github/workflows/staging.yaml'
       - '.github/workflows/test.yaml'
+      - 'services/**'
+      - 'updater/**'
+      - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
+      - 'package*.json'
       - 'cypress/**'
       - 'cypress.config.json'
-      - 'services/**'
-      - 'updater/sis-updater-worker/src/db/migrations/**'
-      - 'docker-compose.ci.yml'
-      - 'package*.json'
 
 jobs:
   test:
@@ -22,31 +23,20 @@ jobs:
       username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
       password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
 
-  release_frontend:
-    name: 'Build and release frontend'
+  release_backend:
+    name: 'Build and release backend'
     runs-on: ubuntu-latest
     needs: test
 
     steps:
       - uses: actions/checkout@v6
 
-      - name: Get current time
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-
-      - name: Build frontend image
+      - name: 'Build backend'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/backend
         with:
-          image: oodikone-frontend
-          tags: staging ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/frontend/Dockerfile
-          build-args: |
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-            BASE_PATH=/oodikone
-            STAGING=true
+          environment: 'staging'
+          release_version: ${{ github.sha }}
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2
@@ -57,27 +47,68 @@ jobs:
           username: toska+github
           password: ${{ secrets.QUAY_IO_TOKEN }}
 
-  release_backend:
-    name: 'Build and release backend'
+  release_frontend:
+    name: 'Build and release frontend'
     runs-on: ubuntu-latest
     needs: test
 
     steps:
       - uses: actions/checkout@v6
 
-      - name: Build backend image
+      - name: 'Build frontend'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/frontend
         with:
-          image: oodikone-backend
-          tags: staging ${{ github.sha }}
-          context: services
-          containerfiles: |
-            services/backend/Dockerfile
-          build-args: |
-            BASE_PATH=/oodikone
-            GIT_SHA=${{ github.sha }}
-            STAGING=true
+          environment: 'staging'
+          release_version: ${{ github.sha }}
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_updater_worker:
+    name: 'Build and release updater worker'
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: 'Build worker'
+        id: build-image
+        uses: ./.github/actions/build/worker
+        with:
+          environment: 'staging'
+          release_version: ${{ github.sha }}
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_updater_scheduler:
+    name: 'Build and release updater scheduler'
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: 'Build scheduler'
+        id: build-image
+        uses: ./.github/actions/build/scheduler
+        with:
+          environment: 'staging'
+          release_version: ${{ github.sha }}
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2
@@ -91,11 +122,12 @@ jobs:
   report_success:
     name: 'Report deployment success to the "oodikone_notifications" channel'
     runs-on: ubuntu-latest
+    if: success()
     needs:
       - release_frontend
       - release_backend
-
-    if: success()
+      - release_updater_worker
+      - release_updater_scheduler
 
     steps:
       - uses: actions/checkout@v6
@@ -110,11 +142,12 @@ jobs:
   report_failure:
     name: 'Report deployment failure to the "oodikone_notifications" channel'
     runs-on: ubuntu-latest
+    if: failure()
     needs:
       - release_frontend
       - release_backend
-
-    if: failure()
+      - release_updater_worker
+      - release_updater_scheduler
 
     steps:
       - uses: actions/checkout@v6

.github/workflows/test.yaml

@@ -26,94 +26,22 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: docker/setup-buildx-action@v4
-      - name: 'Build image and export it'
-        uses: docker/build-push-action@v7
+      - name: Build backend
+        uses: ./.github/actions/build/backend
         with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          context: services
-          file: services/backend/Dockerfile
-          push: false
-          tags: oodikone-backend:staging
-          outputs: type=docker,dest=/tmp/backend.tar
-        env:
-          DOCKER_BUILD_SUMMARY: false
-
-      - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v7
-        with:
-          if-no-files-found: error
-          name: backend-image
-          path: /tmp/backend.tar
+          environment: 'staging'
+          release_version: ${{ github.sha }}
 
   build_frontend:
     name: 'Build frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: docker/setup-buildx-action@v4
-
-      - name: Get current time
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-
-      - name: Build frontend image
-        uses: docker/build-push-action@v7
+      - name: Build frontend
+        uses: ./.github/actions/build/frontend
         with:
-          build-args: |
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-            BASE_PATH=/oodikone
-            STAGING=true
-          context: services
-          file: services/frontend/Dockerfile
-          push: false
-          tags: oodikone-frontend:staging
-          outputs: type=docker,dest=/tmp/frontend.tar
-        env:
-          DOCKER_BUILD_SUMMARY: false
-
-      # - name: 'Upload image for other jobs'
-      #   uses: actions/upload-artifact@v7
-      #   with:
-      #     if-no-files-found: error
-      #     name: frontend-image
-      #     path: /tmp/frontend.tar
-
-  pull_docker_images:
-    name: 'Pull all needed docker images and cache them'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v6
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v4
-        with:
-          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-
-      - name: 'Pull required images'
-        run: docker compose --file docker-compose.ci.yml --profile oodikone --profile updater pull --ignore-buildable --quiet
-
-      - name: 'Save Docker images for downstream jobs'
-        run: |
-          docker save \
-            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db \
-            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db \
-            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db \
-            redis:7.2.6 \
-            postgres:15.10 \
-            "quay.io/toska/jami:production" \
-            nginx:latest \
-          | zstd -T0 -3 -o /tmp/docker-images.tar.zst
-
-      - name: 'Upload Docker images artifact'
-        uses: actions/upload-artifact@v7
-        with:
-          name: docker-images
-          path: /tmp/docker-images.tar.zst
-          retention-days: 1
+          environment: 'staging'
+          release_version: ${{ github.sha }}
 
   # === Update databases ===
   check_migration_files:
@@ -123,11 +51,12 @@ jobs:
       kone_db_migrations: ${{ steps.check_migrations.outputs.kone_db_migrations }}
       user_db_migrations: ${{ steps.check_migrations.outputs.user_db_migrations }}
       sis_db_migrations: ${{ steps.check_migrations.outputs.sis_db_migrations }}
+
     steps:
       - uses: actions/checkout@v6
       - name: Check if new migrations have been added
         id: check_migrations
-        uses: dorny/paths-filter@v3
+        uses: dorny/paths-filter@v4
         with:
           filters: |
             kone_db_migrations:
@@ -142,11 +71,16 @@ jobs:
 
   build_and_push_anon_sis_db:
     name: 'Build anon sis-db and push to toska hub if new migrations have been added'
-    needs:
-      - pull_docker_images
-      - check_migration_files
     runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.sis_db_migrations == 'true'
+    if: |
+      !cancelled() && !failure() &&
+      needs.check_migration_files.outputs.sis_db_migrations == 'true'
+    needs:
+      - check_migration_files
+      - build_backend
+      - build_frontend
+      - lint_packages
+
     steps:
       - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
@@ -156,15 +90,6 @@ jobs:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-
       - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
         run: |
           docker compose --file docker-compose.ci.yml up --detach --quiet-pull sis-updater-worker
@@ -173,31 +98,19 @@ jobs:
 
   build_and_push_anon_kone_db:
     name: 'Build anon kone-db and push to toska hub if new migrations have been added'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure() &&
+      needs.check_migration_files.outputs.kone_db_migrations == 'true'
     needs:
-      - pull_docker_images
+      - check_migration_files
       - build_backend
       - build_frontend
-      - check_migration_files
-    runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.kone_db_migrations == 'true'
+      - lint_packages
+
     steps:
       - uses: actions/checkout@v6
 
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-image
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
-
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v4
         with:
@@ -207,43 +120,24 @@ jobs:
 
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         run: |
-          docker compose --file docker-compose.ci.yml up --detach backend
+          docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
           sleep 30 # wait for backend to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
 
   build_and_push_anon_user_db:
     name: 'Build anon user-db and push to toska hub if new migrations have been added'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure() &&
+      needs.check_migration_files.outputs.user_db_migrations == 'true'
     needs:
-      - pull_docker_images
+      - check_migration_files
       - build_backend
       - build_frontend
-      - check_migration_files
-    runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.user_db_migrations == 'true'
+      - lint_packages
+
     steps:
       - uses: actions/checkout@v6
-      - uses: dorny/paths-filter@v3
-        id: check_migrations
-        with:
-          filters: |
-            migrations:
-              - 'services/backend/src/database/migrations_user/**'
-              - '.github/workflows/test.yaml'
-
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-image
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
 
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v4
@@ -254,39 +148,49 @@ jobs:
 
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         run: |
-          docker compose --file docker-compose.ci.yml up --detach backend
+          docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
           sleep 30 # wait for backend to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
 
-  # === Backend api-tests ===
-  test_api:
+  # === Update sis-db with updater ===
+  check_updater_changed:
+    name: 'Check if updater has been updated'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    outputs:
+      has_updater_changed: ${{ steps.check_updater_changed.outputs.has_updater_changed }}
+
+    steps:
+      - uses: actions/checkout@v6
+      - uses: dorny/paths-filter@v4
+        id: check_updater_changed
+        with:
+          filters: |
+            has_updater_changed:
+              - 'updater/**'
+              - '.github/workflows/test.yaml'
+              - '.github/workflows/staging.yaml'
+              - 'docker-compose.ci.yml'
+
+  run_updater_updates:
+    name: 'Run updater tests and updates'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure() &&
+      (needs.check_updater_changed.outputs.has_updater_changed == 'true' ||
+      needs.check_migration_files.outputs.kone_db_migrations == 'true' ||
+      needs.check_migration_files.outputs.user_db_migrations == 'true' ||
+      needs.check_migration_files.outputs.sis_db_migrations == 'true')
     needs:
-      - pull_docker_images
+      - check_migration_files
+      - check_updater_changed
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
-      - lint_packages
-    if: always() && !cancelled() && needs.lint_packages.result == 'success'
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-24.04
+
     steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-
-      - name: Install node-24
-        uses: actions/setup-node@v6
-        with:
-          node-version: 24
-
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
+      - uses: actions/checkout@v6
 
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v4
@@ -295,21 +199,103 @@ jobs:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: Start oodikone (without frontend and backend)
-        run: npm run oodikone:ci:setup -- --quiet-pull
+      - name: Start worker containers
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-worker --quiet-pull --detach --wait
 
-      - name: Run api-tests
-        run: npm run oodikone:ci:api
+      - name: Run hourly jobs
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
+        env:
+          SCHEDULE_IMMEDIATE: hourly
+          EXIT_AFTER_IMMEDIATES: yes
+
+      - name: Run weekly jobs
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
+        env:
+          SCHEDULE_IMMEDIATE: weekly
+          EXIT_AFTER_IMMEDIATES: yes
+
+      # TODO: Push the updated sis-db to toska-hub when updater is deterministic with test-data
+      # - name: Push newly build db to toska hub
+      #   if: success()
+      #   run: ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
+
+      - name: Dump logs on failure
+        if: failure()
+        run: npm run logs:ci
+
+  # === Backend api-tests ===
+  test_api:
+    name: 'Run backend api tests'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - run_updater_updates
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Run api tests
+        uses: ./.github/actions/test/api
+        with:
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+  #
+  # === Backend service-tests ===
+  test_services:
+    name: 'Run backend service-tests'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - run_updater_updates
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Run service-tests
+        uses: ./.github/actions/test/services
+        with:
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+
+  # === Unit testing ===
+  test_unit:
+    name: 'Run unit tests'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - run_updater_updates
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Run unit tests (frontend and backend)
+        uses: ./.github/actions/test/unit
+
+  # === Frontend component testing ===
+  test_components:
+    name: 'Test frontend components'
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - run_updater_updates
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Run frontend component tests
+        uses: ./.github/actions/test/component
 
   # === End-to-end tests ===
   test_end_to_end:
+    runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
     needs:
-      - pull_docker_images
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-      - lint_packages
-    if: always() && !cancelled() && needs.lint_packages.result == 'success'
+      - run_updater_updates
     strategy:
       fail-fast: false
       matrix:
@@ -340,9 +326,11 @@ jobs:
             spec: cypress/e2e/Study_guidance_groups.js
           - name: 'Test university'
             spec: cypress/e2e/University.js
-
+          - name: 'Test numbers match'
+            spec: cypress/e2e/Numbers_match.js
+          - name: 'Test access rights'
+            spec: cypress/e2e/Routes.js
     name: ${{ matrix.name }}
-    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -352,29 +340,8 @@ jobs:
         with:
           node-version: 24
 
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-image
-      # - name: 'Download frontend image'
-      #   uses: actions/download-artifact@v8
-      #   with:
-      #     name: frontend-image
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
-      # - name: 'Load built frontend image'
-      #   run: docker load --input frontend.tar
-
       - name: 'Run tests'
-        uses: ./.github/actions/run-tests
+        uses: ./.github/actions/test/e2e
         with:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
@@ -384,14 +351,12 @@ jobs:
   # === Other tests ===
   test_service_exit_statuses:
     name: 'Check services run correctly'
-    needs:
-      - pull_docker_images
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-      - lint_packages
-    if: always() && !cancelled() && needs.lint_packages.result == 'success'
     runs-on: ubuntu-latest
+    if: |
+      !cancelled() && !failure()
+    needs:
+      - run_updater_updates
+
     steps:
       - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
@@ -401,29 +366,8 @@ jobs:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: 'Download Docker images'
-        uses: actions/download-artifact@v8
-        with:
-          name: docker-images
-          path: /tmp
-      - name: 'Download backend image'
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-image
-      # - name: 'Download frontend image'
-      #   uses: actions/download-artifact@v8
-      #   with:
-      #     name: frontend-image
-
-      - name: 'Load Docker images'
-        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
-      - name: 'Load built backend image'
-        run: docker load --input backend.tar
-      # - name: 'Load built frontend image'
-      #   run: docker load --input frontend.tar
-
       - name: 'Run services, build rest of the needed images'
-        run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull
+        run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull --wait
 
       - name: 'Check that exit statuses are not errors'
         run: |
@@ -435,4 +379,4 @@ jobs:
 
       - name: 'Print logs on failure'
         if: failure()
-        run: docker compose --file docker-compose.ci.yml --profile oodikone logs && sleep 15
+        run: npm run logs:ci && sleep 15

.github/workflows/updater.yaml

@@ -5,8 +5,7 @@ on:
     types: [published]
     paths:
       - '.github/workflows/updater.yaml'
-      - 'updater/sis-updater-worker/**'
-      - 'updater/sis-updater-scheduler/**'
+      - 'updater/**'
 
 jobs:
   lint_scheduler_and_worker:
@@ -18,9 +17,6 @@ jobs:
         with:
           node-version: 24
 
-      - name: Install safe-chain
-        run: set -o pipefail && curl -fsSL https://github.com/AikidoSec/safe-chain/releases/latest/download/install-safe-chain.sh | sh -s -- --ci
-
       - name: 'Install dependencies'
         run: npm ci && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker
       - name: 'Run linter'
@@ -29,6 +25,7 @@ jobs:
   build_worker:
     name: 'Build Updater worker'
     runs-on: ubuntu-latest
+
     steps:
       - uses: actions/checkout@v6
       - uses: docker/setup-buildx-action@v4
@@ -41,15 +38,18 @@ jobs:
           tags: toska/sis-updater-worker:latest
         env:
           DOCKER_BUILD_SUMMARY: false
+
       - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v7
         with:
           name: updater-worker-image
           path: /tmp/worker.tar
+          retention-days: 7
 
   build_scheduler:
     name: 'Build Updater scheduler'
     runs-on: ubuntu-latest
+
     steps:
       - uses: actions/checkout@v6
       - uses: docker/setup-buildx-action@v4
@@ -62,11 +62,13 @@ jobs:
           tags: toska/sis-updater-scheduler:latest
         env:
           DOCKER_BUILD_SUMMARY: false
+
       - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v7
         with:
           name: updater-scheduler-image
           path: /tmp/scheduler.tar
+          retention-days: 7
 
   test_updater:
     name: 'Test Updater'
@@ -75,6 +77,7 @@ jobs:
       - lint_scheduler_and_worker
       - build_worker
       - build_scheduler
+
     steps:
       - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
@@ -83,34 +86,41 @@ jobs:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+
       - name: 'Download built images'
         uses: actions/download-artifact@v8
         with:
           pattern: 'updater-{worker,scheduler}-image'
           merge-multiple: true
+
       - name: 'Load worker image'
         run: docker load --input worker.tar
       - name: 'Load scheduler image'
         run: docker load --input scheduler.tar
+
       - name: 'Start worker'
-        run: docker compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
+        run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull --no-recreate --wait sis-updater-worker
+
       - name: 'Run hourly jobs'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: hourly
           EXIT_AFTER_IMMEDIATES: yes
+
       - name: 'Run weekly jobs'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: weekly
           EXIT_AFTER_IMMEDIATES: yes
+
       - name: 'Run prepurge'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: prepurge
           EXIT_AFTER_IMMEDIATES: yes
+
       - name: 'Run purge'
-        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
         env:
           SCHEDULE_IMMEDIATE: purge
           EXIT_AFTER_IMMEDIATES: yes
@@ -122,18 +132,12 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - name: Build worker image
+      - name: 'Build updater worker'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/worker
         with:
-          image: updater-worker
-          tags: staging production ${{ github.sha }}
-          context: updater/sis-updater-worker
-          containerfiles: |
-            updater/sis-updater-worker/Dockerfile
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=production
+          environment: 'production'
+          release_version: ${{ github.sha }}
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2
@@ -151,18 +155,12 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - name: Build scheduler image
+      - name: 'Build udpater scheduler'
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: ./.github/actions/build/scheduler
         with:
-          image: updater-scheduler
-          tags: staging production ${{ github.sha }}
-          context: updater/sis-updater-scheduler
-          containerfiles: |
-            updater/sis-updater-scheduler/Dockerfile
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=production
+          environment: 'production'
+          release_version: ${{ github.sha }}
 
       - name: Push to quay.io
         uses: redhat-actions/push-to-registry@v2

.gitignore

@@ -56,7 +56,7 @@ cypress/videos/
 coverage/
 instrumented/
 
-# Pycharm 
+# Pycharm
 .idea
 
 # VS Code
@@ -68,3 +68,5 @@ instrumented/
 # Target of shared symlink
 services/**/src/shared
 
+
+!services/backend/.env.test

README.md

@@ -123,6 +123,8 @@ Some useful commands are defined in `package.json` and can be run with `npm run
 
 - `npm run oodikone`: starts Oodikone with anonymized data
 - `npm run oodikone:real`: starts Oodikone with real data
+- `npm run oodikone:test`: starts Oodikone with non-persistent database (wipes after every reset)
+- `npm run oodikone:ci`: starts Oodikone in CI mode
 - `npm run updater`: starts updater with anonymized data
 - `npm run updater:real`: starts updater with real data
 - `npm run both`: starts Oodikone and updater with anonymized data
@@ -149,8 +151,8 @@ It is recommended to spend some time to become familiar with `docker` and `docke
 - `docker compose ps`: view the containers in the running environment
 - `docker compose logs frontend`: print logs for just frontend
 - `docker compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
-- `docker exec -it backend sh`: open bash terminal inside backend container
-- `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database
+- `docker exec -it oodikone-backend sh`: open bash terminal inside backend container
+- `docker exec -it oodikone-sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database
 
 ### User types for development
 
@@ -191,7 +193,7 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 
 ### Modules are missing after updating package.json
 
-You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container or `npm run install:docker` to install modules inside all packages (backend, frontend, sis-updater-scheduler and sis-updater-worker).
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Prefer `npm run ci:docker` or `npm run install:docker`, which also fix `node_modules` ownership for the non-root runtime user. If you run installs manually, use root inside the container and restore ownership afterwards (`chown -R 1001:0 node_modules`) to avoid permission issues.
 
 ### Study guidance groups don't work on my machine
 

anonyymioodi/commit_and_push_to_toska_hub.sh

@@ -43,6 +43,6 @@ fi
 
 infomsg "Committing and pushing to registry"
 database=${args[0]}
-docker commit "$database" "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
+docker commit "oodikone-$database" "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
 docker push "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
 successmsg "Succesfully pushed"

anonyymioodi/create_container_from_dump.sh

@@ -52,13 +52,13 @@ fi
 
 
 infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
-database=${args[0]}
-docker-compose down --remove-orphans
-docker-compose up -d "$database"
+database="${args[0]}"
+docker compose down --remove-orphans
+docker compose up -d "$database"
 
 database_dump="$database.sqz"
 infomsg "Attempting to restore database $database from dump $database_dump"
-retry docker-compose exec "$database" pg_isready --dbname="$database"
-docker exec -i "$database" /bin/bash -c "pg_restore --username=postgres \
+retry docker compose exec "$database" pg_isready --dbname="$database"
+docker exec -i "oodikone-$database" /bin/bash -c "pg_restore --username=postgres \
 --format=custom --dbname=$database --no-owner" < "$database_dump"
 successmsg "Database $database succesfully created"

anonyymioodi/docker-compose.yml

@@ -1,40 +1,57 @@
 x-defaults: &defaults
   restart: unless-stopped
 
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
 services:
   kone-db:
     <<: *defaults
-    container_name: kone-db
+    container_name: oodikone-kone-db
     environment:
       PGDATA: /data
       POSTGRES_DB: kone-db
       POSTGRES_HOST_AUTH_METHOD: trust
     image: postgres:15.10
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
+
+  user-db:
+    <<: *defaults
+    container_name: oodikone-user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: postgres:15.10
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-db:
     <<: *defaults
-    container_name: sis-db
+    container_name: oodikone-sis-db
     environment:
       PGDATA: /data
       POSTGRES_DB: sis-db
       POSTGRES_HOST_AUTH_METHOD: trust
     image: postgres:15.10
     shm_size: 2gb
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-importer-db:
     <<: *defaults
-    container_name: sis-importer-db
+    container_name: oodikone-sis-importer-db
     environment:
       PGDATA: /data
       POSTGRES_DB: sis-importer-db
       POSTGRES_HOST_AUTH_METHOD: trust
     image: postgres:15.10
-
-  user-db:
-    <<: *defaults
-    container_name: user-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: user-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults

anonyymioodi/update_staging_db.sh

@@ -56,7 +56,7 @@ function create_dumps() {
 function create_dump_from_db() {
   echo "Dumping db $1..."
 
-  docker exec "$1" pg_dump -Fc -U postgres "$1" > "$dump_directory/$1.sqz" || die "Failed to dump db $1"
+  docker exec "oodikone-$1" pg_dump -Fc -U postgres "$1" > "$dump_directory/$1.sqz" || die "Failed to dump db $1"
 }
 
 

cli.sh

@@ -18,11 +18,11 @@ source "$PROJECT_ROOT"/scripts/functions_for_setup.sh
 # Try to run cleanup function if things fail.
 trap cleanup SIGINT SIGTERM ERR EXIT
 
-# Run docker-compose down on cleanup
+# Run docker compose down on cleanup
 cleanup() {
   trap - SIGINT SIGTERM ERR EXIT
-  warningmsg "Trying to run docker-compose down and remove orphans"
-  docker-compose --profile oodikone --profile updater down --remove-orphans
+  warningmsg "Trying to run docker compose down and remove orphans"
+  docker compose --profile oodikone --profile updater down --remove-orphans
 }
 
 # === CLI ===
@@ -49,7 +49,7 @@ set_custom_select_prompt() {
 
   options=(
     "Set up Oodikone from scratch"
-    "Reset all real data"
+    "Reset all data"
     "Reset single database"
     "Restore data from dumps"
     "Docker system prune"
@@ -68,8 +68,8 @@ while true; do
     case $opt in
       "Set up Oodikone from scratch")
         set_up_oodikone_from_scratch;;
-      "Reset all real data")
-        reset_all_real_data;;
+      "Reset all data")
+        reset_all_data;;
       "Reset single database")
         reset_single_database
         set_custom_select_prompt # Set prompt back to initial values

cypress/e2e/Course_statistics.js

@@ -468,10 +468,10 @@ describe('Basic user', () => {
       cy.cs('course-population-for-2019-2020').click()
 
       cy.contains('Tietorakenteet ja algoritmit')
-      cy.contains('Class of 2019-2020, 33 students')
+      cy.contains('Class of 2019-2020, 32 students')
       cy.contains('Include substitutions, Open and normal')
 
-      cy.contains('Students (33)').click()
+      cy.contains('Students (32)').click()
       cy.contains('td', '394776').siblings().eq(2).contains('3')
       cy.contains('td', '497388').siblings().eq(2).contains('2')
     })
@@ -486,10 +486,10 @@ describe('Basic user', () => {
       cy.cs('course-population-for-2019-2020').click()
 
       cy.contains('Tietorakenteet ja algoritmit')
-      cy.contains('Class of 2019-2020, 33 students')
+      cy.contains('Class of 2019-2020, 32 students')
       cy.contains('Include substitutions, Open and normal')
 
-      cy.contains('Students (33)').click()
+      cy.contains('Students (32)').click()
       cy.get('table tbody td').filter(':contains("Hidden")').should('have.length', 9)
     })
   })
@@ -537,7 +537,8 @@ describe('Basic user', () => {
         cy.contains('Show population').should('be.enabled')
       })
 
-      describe('Students tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Students tab', () => {
         describe('Info boxes', () => {
           it('Student statistics table', () => {
             cy.cs('StudentStatistics-info-box-button').click()
@@ -639,7 +640,8 @@ describe('Basic user', () => {
         })
       })
 
-      describe('Attempts tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Attempts tab', () => {
         beforeEach(() => {
           openAttemptsTab()
         })
@@ -774,7 +776,8 @@ describe('Basic user', () => {
         cy.contains('Show population').should('be.enabled')
       })
 
-      describe('Students tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Students tab', () => {
         it('Show grades off, Separate by semesters off', () => {
           const tableContents = [
             // [Time, Total students, Passed, Failed, Enrolled no grade, Pass rate, Fail rate]
@@ -960,7 +963,8 @@ describe('Basic user', () => {
         })
       })
 
-      describe('Attempts tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Attempts tab', () => {
         beforeEach(() => {
           openAttemptsTab()
         })
@@ -1206,7 +1210,8 @@ describe('Basic user', () => {
   })
 })
 
-describe('Only course statistics', () => {
+// TODO: Writing these as backend api tests
+describe.skip('Only course statistics', () => {
   beforeEach(() => {
     cy.init('/coursestatistics', 'onlycoursestatistics')
     cy.url().should('include', '/coursestatistics')

cypress/e2e/Degree_programme_overview.js

@@ -32,36 +32,21 @@ const deleteTag = tagName => {
 
 describe('Degree programme overview', () => {
   describe('Degree programme selector', () => {
-    beforeEach(() => {
-      cy.init('/study-programme', 'admin')
-    })
-
-    it('Correct categories', () => {
-      cy.contains('Bachelor programmes')
-      cy.contains('Master programmes')
-      cy.contains('Combined programmes')
-      cy.contains('Doctoral programmes')
-    })
-
     it('Degree programme search filter', () => {
+      cy.init('/study-programme', 'admin')
       cy.contains('Tietojenkäsittelytieteen kandiohjelma').should('exist')
       cy.contains('Matemaattisten tieteiden kandiohjelma').should('exist')
       cy.cs('study-programme-filter').type('Tietojenkäsittelytieteen')
       cy.contains('Tietojenkäsittelytieteen kandiohjelma').should('exist')
       cy.contains('Matemaattisten tieteiden kandiohjelma').should('not.exist')
     })
-
-    it('Old and specialized programmes filter', () => {
-      cy.contains('Biotekniikka (ylempi)').should('not.exist')
-      cy.cs('filter-old-programmes-toggle').click()
-      cy.contains('Biotekniikka (ylempi)').should('exist')
-    })
   })
 
   describe('Basic information tab works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
       cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click({ force: true })
+      cy.cs('year-toggle').click() // NOTE: Tests are written for calendar years
     })
 
     // If the backend breaks for one of the sections, the section header is not rendered and this will fail
@@ -79,11 +64,11 @@ describe('Degree programme overview', () => {
         // [Year, Started studying, Accepted, Graduated, Transferred Away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
         [2023, 8, 8, 26, 0, 0],
-        [2022, 25, 26, 48, 1, 4],
-        [2021, 29, 32, 47, 0, 2],
+        [2022, 25, 26, 47, 1, 3],
+        [2021, 29, 32, 48, 0, 2],
         [2020, 26, 27, 12, 1, 3],
         [2019, 28, 34, 1, 0, 1],
-        [2018, 40, 45, 0, 0, 0],
+        [2018, 40, 45, 0, 0, 1],
         [2017, 41, 47, 0, 0, 0],
       ]
 
@@ -112,8 +97,8 @@ describe('Degree programme overview', () => {
         // [Year, Graduated, Wrote thesis]
         ...years.map(year => [year, 0, 0]),
         [2023, 26, 5],
-        [2022, 48, 13],
-        [2021, 47, 76],
+        [2022, 47, 13],
+        [2021, 48, 76],
         [2020, 12, 23],
         [2019, 1, 19],
         [2018, 0, 0],
@@ -130,8 +115,8 @@ describe('Degree programme overview', () => {
         // [Year, Started studying, Accepted, Graduated]
         ...years.map(year => [year, 0, 0, 0]),
         [2023, 8, 8, 24],
-        [2022, 25, 26, 44],
-        [2021, 29, 32, 46],
+        [2022, 25, 26, 43],
+        [2021, 29, 32, 47],
         [2020, 26, 27, 11],
         [2019, 28, 34, 1],
         [2018, 40, 45, 0],
@@ -144,8 +129,8 @@ describe('Degree programme overview', () => {
         // [Year, Graduated, Wrote thesis]
         ...years.map(year => [year, 0, 0]),
         [2023, 24, 5],
-        [2022, 44, 12],
-        [2021, 46, 69],
+        [2022, 43, 12],
+        [2021, 47, 69],
         [2020, 11, 22],
         [2019, 1, 18],
         [2018, 0, 0],
@@ -163,11 +148,11 @@ describe('Degree programme overview', () => {
         // [Year, Started studying, Accepted, Graduated, Transferred away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
         ['2023 - 2024', 8, 8, 4, 0, 0],
-        ['2022 - 2023', 25, 26, 36, 0, 0],
-        ['2021 - 2022', 29, 32, 57, 1, 6],
+        ['2022 - 2023', 25, 26, 35, 0, 0],
+        ['2021 - 2022', 29, 32, 58, 1, 5],
         ['2020 - 2021', 26, 27, 30, 0, 3],
         ['2019 - 2020', 28, 34, 7, 1, 1],
-        ['2018 - 2019', 40, 45, 0, 0, 0],
+        ['2018 - 2019', 40, 45, 0, 0, 1],
         ['2017 - 2018', 41, 47, 0, 0, 0],
       ]
       cy.checkTableStats(studentTableContents, 'students-of-the-study-programme')
@@ -207,19 +192,20 @@ describe('Degree programme overview', () => {
         .should('contain', 47)
         .should('contain', 76)
 
-      cy.cs('unset-breakdown-bar-chart')
-      cy.cs('graduation-time-toggle').click()
+      cy.cs('graduation-mode-selector').within(() => {
+        cy.cs('select-median').click()
+      })
       cy.cs('unset-median-bar-chart').within(() => {
-        cy.contains('48 graduated').should('be.visible')
-        cy.contains('48 graduated').hover()
+        cy.contains('47 graduated').should('be.visible')
+        cy.contains('47 graduated').hover()
       })
 
-      cy.get('.grad-vals').contains('48 students graduated in year 2022').should('be.visible')
+      cy.get('.grad-vals').contains('47 students graduated in year 2022').should('be.visible')
 
-      cy.contains('median study time: 6.5 semesters')
+      cy.contains('median study time: 6 semesters')
       cy.contains('24 graduated on time')
       cy.contains('12 graduated max year overtime')
-      cy.contains('12 graduated over year late')
+      cy.contains('11 graduated over year late')
 
       cy.cs('programmes-before-or-after-stacked-bar-chart-section')
         .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
@@ -232,14 +218,16 @@ describe('Degree programme overview', () => {
   })
 
   describe('Graduation times of master programmes', () => {
-    it('are split into two graphs', () => {
+    it('are split into three graphs', () => {
       cy.init('/study-programme')
       cy.contains('a', 'Matematiikan ja tilastotieteen maisteriohjelma').click({ force: true })
+      cy.cs('year-toggle').click() // Tests are written for calendar year
 
-      cy.cs('unset-breakdown-bar-chart')
       cy.cs('unset-breakdown-bar-chart')
 
-      cy.cs('graduation-time-toggle').click()
+      cy.cs('graduation-mode-selector').within(() => {
+        cy.cs('select-average').click()
+      })
       cy.cs('unset-graduation-times-section')
         .eq(1)
         .within(() => {
@@ -249,7 +237,7 @@ describe('Degree programme overview', () => {
         })
 
       cy.get('.grad-vals').contains('2 students graduated in year 2021').should('be.visible')
-      cy.contains('median study time: 4.5 semesters')
+      cy.contains('average study time: 4.5 semesters')
       cy.contains('1 graduated on time')
       cy.contains('1 graduated max year overtime')
       cy.contains('0 graduated over year late')
@@ -263,7 +251,7 @@ describe('Degree programme overview', () => {
         })
 
       cy.get('.grad-vals').contains('11 students graduated in year 2023').should('be.visible')
-      cy.contains('median study time: 12 semesters')
+      cy.contains('average study time: 11.18 semesters')
       cy.contains('4 graduated on time')
       cy.contains('6 graduated max year overtime')
       cy.contains('1 graduated over year late')
@@ -307,15 +295,15 @@ describe('Degree programme overview', () => {
 
     it('Students of the degree programme are shown correctly', () => {
       const tableContents = [
-        // [Year, All, Started studying, Present, Absent, Passive, Graduated, Men, Women, Other/Unknown, Finland, Other]
-        ['2023 - 2024', 8, 8, 0, 0, 8, 0, 5, 3, 0, 8, 1],
-        ['2022 - 2023', 26, 25, 0, 0, 24, 2, 19, 7, 0, 25, 1],
-        ['2021 - 2022', 38, 29, 0, 0, 33, 5, 29, 9, 0, 36, 4],
-        ['2020 - 2021', 30, 26, 0, 0, 11, 19, 15, 15, 0, 29, 3],
-        ['2019 - 2020', 35, 28, 0, 0, 8, 27, 22, 13, 0, 35, 2],
-        ['2018 - 2019', 45, 40, 0, 0, 6, 39, 26, 19, 0, 44, 1],
-        ['2017 - 2018', 47, 41, 0, 0, 5, 42, 31, 16, 0, 47, 0],
-        ['Total', 229, 197, 0, 0, 95, 134, 147, 82, 0, 224, 12],
+        // [Year, All, Started studying, Present, Absent, Passive, Graduated, Has recent attainments, Men, Women, Other/Unknown, Finland, Other]
+        ['2023 - 2024', 8, 8, 0, 0, 8, 0, 0, 5, 3, 0, 8, 1],
+        ['2022 - 2023', 26, 25, 0, 0, 24, 2, 0, 19, 7, 0, 25, 1],
+        ['2021 - 2022', 37, 29, 0, 0, 33, 4, 0, 29, 8, 0, 35, 4],
+        ['2020 - 2021', 30, 26, 0, 0, 11, 19, 0, 15, 15, 0, 29, 3],
+        ['2019 - 2020', 35, 28, 0, 0, 8, 27, 0, 22, 13, 0, 35, 2],
+        ['2018 - 2019', 46, 40, 0, 0, 6, 40, 0, 26, 20, 0, 45, 1],
+        ['2017 - 2018', 47, 41, 0, 0, 5, 42, 0, 31, 16, 0, 47, 0],
+        ['Total', 229, 197, 0, 0, 95, 134, 0, 147, 82, 0, 224, 12],
       ]
 
       cy.checkTableStats(tableContents, 'study-tracks-and-class-statistics')
@@ -390,10 +378,10 @@ describe('Degree programme overview', () => {
         ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
         ['2023 - 2024', 8, 8, 0, 0, 0, 0, 0, 0, 0],
         ['2022 - 2023', 26, 9, 8, 4, 3, 0, 0, 0, 2],
-        ['2021 - 2022', 38, 8, 4, 9, 8, 4, 0, 0, 5],
+        ['2021 - 2022', 37, 8, 4, 9, 8, 4, 0, 0, 4],
         ['2020 - 2021', 30, 2, 1, 2, 5, 0, 0, 1, 19],
         ['2019 - 2020', 35, 1, 0, 2, 1, 0, 0, 4, 27],
-        ['2018 - 2019', 45, 0, 1, 1, 2, 0, 0, 2, 39],
+        ['2018 - 2019', 46, 0, 1, 1, 2, 0, 0, 2, 40],
         ['2017 - 2018', 47, 0, 1, 3, 0, 0, 1, 0, 42],
         ['Total', 229, 28, 15, 21, 19, 4, 1, 7, 134],
       ]
@@ -466,14 +454,14 @@ describe('Degree programme overview', () => {
       it('Students of the study track are shown correctly', () => {
         cy.cs('study-track-overview-section').contains('Students of the study track MAT-MAT by starting year')
         const tableContents = [
-          // [Year, All, Started studying, Present, Absent, Passive, Graduated, Men, Women, Other/Unknown, Finland, Other]
-          ['2022 - 2023', 3, 3, 0, 0, 1, 2, 2, 1, 0, 3, 0],
-          ['2021 - 2022', 5, 1, 0, 0, 1, 4, 3, 2, 0, 5, 0],
-          ['2020 - 2021', 13, 10, 0, 0, 1, 12, 7, 6, 0, 12, 3],
-          ['2019 - 2020', 17, 14, 0, 0, 0, 17, 10, 7, 0, 17, 1],
-          ['2018 - 2019', 23, 21, 0, 0, 2, 21, 11, 12, 0, 23, 0],
-          ['2017 - 2018', 28, 24, 0, 0, 1, 27, 15, 13, 0, 28, 0],
-          ['Total', 89, 73, 0, 0, 6, 83, 48, 41, 0, 88, 4],
+          // [Year, All, Started studying, Present, Absent, Passive, Graduated, Has recent attainment, Men, Women, Other/Unknown, Finland, Other]
+          ['2022 - 2023', 3, 3, 0, 0, 1, 2, 0, 2, 1, 0, 3, 0],
+          ['2021 - 2022', 4, 1, 0, 0, 1, 3, 0, 3, 1, 0, 4, 0],
+          ['2020 - 2021', 13, 10, 0, 0, 1, 12, 0, 7, 6, 0, 12, 3],
+          ['2019 - 2020', 17, 14, 0, 0, 0, 17, 0, 10, 7, 0, 17, 1],
+          ['2018 - 2019', 24, 21, 0, 0, 2, 22, 0, 11, 13, 0, 24, 0],
+          ['2017 - 2018', 28, 24, 0, 0, 1, 27, 0, 15, 13, 0, 28, 0],
+          ['Total', 89, 73, 0, 0, 6, 83, 0, 48, 41, 0, 88, 4],
         ]
         cy.checkTableStats(tableContents, 'study-tracks-and-class-statistics')
       })
@@ -530,6 +518,8 @@ describe('Degree programme overview', () => {
 
     describe('By credit type tab', () => {
       beforeEach(() => {
+        cy.cs('show-credits-students-toggle').click()
+        cy.cs('year-toggle').click()
         cy.get('thead').within(() => {
           cy.get('tr')
             .eq(0)
@@ -544,7 +534,7 @@ describe('Degree programme overview', () => {
               cy.get('td').eq(0).contains('Course')
               cy.get('td').eq(1).contains('MAT11004')
               cy.get('td').eq(2).contains('Differentiaalilaskenta')
-              cy.get('td').eq(3).contains('1170')
+              cy.get('td').eq(3).contains('1240')
             })
         })
       })
@@ -568,7 +558,7 @@ describe('Degree programme overview', () => {
       })
 
       it('year toggle works', () => {
-        cy.cs('year-toggle').click()
+        cy.cs('year-toggle').click() // Default view now Academic years
 
         cy.cs('from-year-select').click()
         cy.cs('from-year-select-option-2017').click()
@@ -582,7 +572,7 @@ describe('Degree programme overview', () => {
               cy.get('td').eq(0).contains('Course')
               cy.get('td').eq(1).contains('MAT11001')
               cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
-              cy.get('td').eq(3).contains('1220')
+              cy.get('td').eq(3).contains('1265')
             })
         })
       })
@@ -641,6 +631,8 @@ describe('Degree programme overview', () => {
       })
 
       it("'Show credits/Show students' toggle works", () => {
+        cy.cs('year-toggle').click()
+        cy.wait(500) // So that the toggle has time to load
         cy.get('thead tr')
           .eq(0)
           .within(() => {
@@ -715,7 +707,7 @@ describe('Degree programme overview', () => {
               cy.get('td').eq(0).contains('Course')
               cy.get('td').eq(1).contains('MAT11001')
               cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
-              cy.get('td').eq(3).contains('375')
+              cy.get('td').eq(3).contains('272')
             })
         })
       })

cypress/e2e/Faculty_statistics.js

@@ -110,15 +110,15 @@ describe('Faculty statistics', () => {
       })
 
       it('Toggle years works', () => {
-        cy.cs('credits-produced-by-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('thesis-writers-of-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('students-of-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('graduated-of-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('year-toggle').click()
         cy.cs('credits-produced-by-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
         cy.cs('thesis-writers-of-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
         cy.cs('students-of-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
         cy.cs('graduated-of-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
+        cy.cs('year-toggle').click()
+        cy.cs('credits-produced-by-the-faculty-interactive-data-table').should('contain', '2022')
+        cy.cs('thesis-writers-of-the-faculty-interactive-data-table').should('contain', '2022')
+        cy.cs('students-of-the-faculty-interactive-data-table').should('contain', '2022')
+        cy.cs('graduated-of-the-faculty-interactive-data-table').should('contain', '2022')
       })
 
       it('Toggle programmes works', () => {

cypress/e2e/Filters.js

@@ -19,7 +19,7 @@ describe("Population statistics with a master's programme", { testIsolation: fal
 
   it('Study track filter works', () => {
     cy.cs('studyTrackFilter-header').click()
-    cy.selectFromDropdown('studyTrackFilter', 0, true)
+    cy.selectFromDropdown('studyTrackFilter', 'Matematiikka ja soveltava matematiikka (MAST-MSM)', true)
     checkFilteredStudentCount(15)
 
     cy.cs('studyTrackFilter-clear').click()

cypress/e2e/Language_center.js

@@ -44,7 +44,7 @@ describe('When language center is opened', () => {
         cy.get('table > tbody > tr:first').within(() => {
           cy.get('td').should('have.length', 15)
           cy.get('td').eq(0).contains('All courses total')
-          const numbers = [2076, 36, 9, 33, 757, 15, 40, 6, 48, 1, 24, 1059, 48, 0]
+          const numbers = [2076, 36, 9, 34, 759, 15, 40, 6, 51, 1, 24, 1059, 42, 0]
           numbers.forEach((number, index) => {
             cy.get('td')
               .eq(index + 1)
@@ -55,12 +55,12 @@ describe('When language center is opened', () => {
 
       it('Faculties tab "exceeding" button works', () => {
         cy.cs('difference-button').click()
-        checkNumbers([63, 2, 0, 3, 13, 0, 1, 1, 2, 0, 0, 30, 10, 1], 15, 'faculties')
+        checkNumbers([66, 3, 0, 4, 18, 0, 1, 1, 4, 0, 0, 31, 3, 1], 15, 'faculties')
       })
 
       it('Faculties tab semester selector changes numbers', () => {
         chooseSemester('Syksy 2020', 'from')
-        checkNumbers([1184, 28, 6, 27, 497, 4, 10, 2, 33, 0, 9, 535, 33, 0], 15, 'faculties')
+        checkNumbers([1184, 28, 6, 28, 499, 4, 10, 2, 36, 0, 9, 535, 27, 0], 15, 'faculties')
       })
     })
 

cypress/e2e/Numbers_match.js

@@ -0,0 +1,122 @@
+/// <reference types="cypress" />
+
+const selectCourseStatus = (courseCode, status) => {
+  cy.cs(`courseFilter-${courseCode}-selector`).click()
+  cy.get(`[role="listbox"][aria-labelledby=courseFilter-${courseCode}]`).within(() => {
+    cy.get('li').contains(status).click()
+  })
+}
+
+describe('Numbers should match between', () => {
+  describe('Programme courses, Course statistics and Course population (no substitutions)', () => {
+    const test = [
+      // Code, name, total students, passed, not completed, failed*, enrolled no grade* (not used by Programme courses)
+      ['Course', 'MAT11001', 'Johdatus yliopistomatematiikkaan', '272', '253', (1 + 18).toString(), '1', '18'],
+      ['Course', 'MAT11003', 'Raja-arvot', '270', '249', (1 + 20).toString(), '1', '20'],
+      ['Course', 'MAT11004', 'Differentiaalilaskenta', '262', '248', (1 + 13).toString(), '1', '13'],
+      ['Course', 'MAT21002', 'Sarjat', '256', '250', (0 + 6).toString(), '0', '6'],
+      ['Course', 'MAT11005', 'Integraalilaskenta', '251', '234', (0 + 17).toString(), '0', '17'],
+      ['Course', 'MAT11002', 'Lineaarialgebra ja matriisilaskenta I', '247', '242', (1 + 4).toString(), '1', '4'],
+      ['Course', 'MAT12003', 'Todennäköisyyslaskenta I', '234', '227', (3 + 4).toString(), '3', '4'],
+      ['Course', 'MAT21001', 'Lineaarialgebra ja matriisilaskenta II', '230', '216', (4 + 10).toString(), '4', '10'],
+      ['Course', 'MAT21003', 'Vektorianalyysi I', '228', '202', (3 + 23).toString(), '3', '23'],
+      ['Course', 'MAT20005', 'Akateemiset taidot', '201', '178', (0 + 23).toString(), '0', '23'],
+      ['Module', 'MAT110', 'Matematiikka, perusopinnot', '195', '195', (0 + 0).toString(), '0', '0'],
+      ['Course', 'MAT21014', 'Johdatus logiikkaan I', '188', '170', (0 + 18).toString(), '0', '18'],
+    ]
+
+    it('in Programme courses', () => {
+      // Move to Math bachelor programme's programme courses
+      cy.init('/study-programme')
+      cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click()
+      cy.cs('ProgrammeCoursesTab').click()
+
+      // Sort by Total students
+      cy.get('thead').within(() => {
+        cy.get('tr')
+          .eq(0)
+          .within(() => {
+            cy.get('th').eq(3).click()
+          })
+      })
+
+      cy.get('tbody').within(() => {
+        test.forEach((course, index) => {
+          cy.get('tr')
+            .eq(index)
+            .within(() => {
+              // Check a course's fields
+              course.forEach((field, fieldIndex) => {
+                if (fieldIndex <= 5) {
+                  // Failed, Enrolled no grade values not used
+                  cy.get('td').eq(fieldIndex).contains(field)
+                }
+              })
+
+              // Course link button, (name is second entry)
+              // cy.cs(`${course.at(1).toLocaleLowerCase()}-course-statistics-link-button`).click()
+            })
+        })
+      })
+    })
+
+    it('in Course statistics', () => {
+      test.forEach(course => {
+        // Move to Course statistics
+        cy.init(`/coursestatistics?courseCodes=%5B%22${course.at(1)}%22%5D&combineSubstitutions=false`, 'basic')
+        cy.url().should('include', `/coursestatistics?courseCodes=%5B%22${course.at(1)}%22%5D`)
+        cy.contains('Course statistics')
+        cy.contains(course.at(2)) // course name
+
+        cy.cs('FromYearSelector').click()
+        cy.cs('FromYearSelectorOption2017-2018').click()
+
+        cy.get('tbody').within(() => {
+          cy.get('tr')
+            .eq(0)
+            .within(() => {
+              cy.get('td').eq(1).contains(course.at(3))
+              cy.get('td').eq(2).contains(course.at(4))
+              cy.get('td').eq(3).contains(course.at(6)) // jump over combined Not completed
+              cy.get('td').eq(4).contains(course.at(7))
+            })
+        })
+      })
+    })
+
+    it('in Course population', () => {
+      test.forEach(course => {
+        // Academic years between 2017-2023
+        cy.init(
+          `/coursepopulation?from=68&to=74&coursecodes=%5B%22${course.at(1)}%22%5D&includeSubstitutions=false`,
+          'basic'
+        )
+        cy.contains(course.at(2)) // Course name
+
+        // Select course from Course-filter
+        cy.cs('courseFilter-filter-card').click()
+        cy.cs('courseFilter-selector').click()
+        cy.cs('courseFilter-popper').within(() => {
+          cy.contains('li', `${course.at(1)} - ${course.at(2)}`).click() // "code - name"
+        })
+
+        selectCourseStatus(course.at(1), 'All')
+        cy.contains(`Students (${course.at(3)})`)
+
+        selectCourseStatus(course.at(1), 'Passed')
+        cy.contains(`Students (${course.at(4)})`)
+
+        selectCourseStatus(course.at(1), 'Failed')
+        cy.contains(`Students (${course.at(6)})`)
+
+        selectCourseStatus(course.at(1), 'Enrolled, No Grade')
+        cy.contains(`Students (${course.at(7)})`)
+      })
+    })
+  })
+
+  describe('Course statistics and Course population', () => {
+    it.skip('when selecting one year')
+    it.skip('when selecting the all years')
+  })
+})

cypress/e2e/Routes.js

@@ -0,0 +1,85 @@
+import { userHeaders } from '../support/commands.js'
+
+const getBaseRoutes = [
+  ['/', {}],
+  ['/changelog', {}],
+  ['/completedcoursessearch', {}],
+  ['/university', {}],
+  ['/feedback', {}],
+  ['/populations', { roles: ['fullSisuAccess'], programmeRights: [] }],
+  ['/study-programme', { roles: ['fullSisuAccess'], programmeRights: [] }],
+  ['/study-programme/KH50_001', { roles: ['fullSisuAccess'], programmeRights: ['KH50_001'] }],
+  ['/coursepopulation', { roles: ['fullSisuAccess'], programmeRights: [] }],
+  ['/users', { roles: ['admin'] }],
+  ['/users/334', { roles: ['admin'] }],
+  ['/updater', { roles: ['admin'] }],
+  ['/coursestatistics', { roles: ['fullSisuAccess', 'courseStatistics'], programmeRights: [] }],
+  ['/faculties', { roles: ['fullSisuAccess', 'facultyStatistics'] }],
+  ['/faculties/hy-org-1000000911', { roles: ['fullSisuAccess', 'facultyStatistics'] }],
+  ['/students', { roles: ['fullSisuAccess', 'studyGuidanceGroups'], programmeRights: [] }],
+  ['/students/545400', { roles: ['fullSisuAccess', 'studyGuidanceGroups'], programmeRights: [] }],
+  ['/custompopulation', { roles: ['fullSisuAccess', 'studyGuidanceGroups'], programmeRights: [] }],
+  ['/teachers', { roles: ['teachers'] }],
+  ['/teachers/hy-hlo-1463447', { roles: ['teachers'] }],
+  ['/studyguidancegroups', { roles: ['studyGuidanceGroups'] }],
+  ['/studyguidancegroups/otm-b3b52ff5-88f6-4d52-bd18-0588023b6322', { userId: '', roles: ['studyGuidanceGroups'] }],
+  ['/close-to-graduation', { roles: ['fullSisuAccess', 'studyGuidanceGroups'] }],
+  ['/languagecenterview', { iamGroups: ['grp-kielikeskus-esihenkilot'] }],
+]
+
+for (const user of userHeaders) {
+  if (user.uid === 'admin') continue
+  describe(`${user.displayname} has access only to views that were intended`, () => {
+    let userInfo
+    before(() => {
+      cy.request({ method: 'GET', url: '/api/login', headers: user }).then(({ body }) => (userInfo = body?.user))
+    })
+
+    beforeEach(() => cy.wrap(userInfo).should('not.be.undefined'))
+    beforeEach(() => {
+      cy.intercept('**/api/**', req => {
+        req.on('response', res => {
+          expect(res.statusCode, `${req.method} ${req.url}`).to.be.lessThan(500)
+        })
+      })
+    })
+
+    for (const [route, requirements] of getBaseRoutes) {
+      it(`Checking access ${route}`, () => {
+        cy.init(route, user.uid)
+
+        const requiredUserId = !requirements.userId || requirements.userId === userInfo.userId
+
+        const requiredRoles = !requirements.roles || requirements.roles?.some(role => userInfo.roles?.includes(role))
+
+        const requiredRights =
+          !requirements.requireProgrammeRights ||
+          userInfo.roles?.includes('fullSisuAccess') ||
+          requirements.programmeRights.every(role => userInfo.programmeRights?.includes(role))
+
+        // This is defined in frontend ProtectedRoute component.
+        const specialNeedsRoutes = [
+          'populations',
+          'study-programme',
+          'coursepopulation',
+          'coursestatistics',
+          'students',
+          'custompopulation',
+        ]
+
+        const mankeli = specialNeedsRoutes.some(path => route.includes(path))
+          ? requiredRoles || requiredRights
+          : requiredRoles && requiredRights
+
+        const shouldHaveAccess = mankeli && requiredUserId
+
+        if (shouldHaveAccess) {
+          cy.contains('Something broke').should('not.exist')
+          cy.contains('Access denied').should('not.exist')
+        } else {
+          cy.contains('Access denied')
+        }
+      })
+    }
+  })
+}

cypress/e2e/Students.js

@@ -259,8 +259,10 @@ describe('Students tests', () => {
 
     it('When a study plan is selected, courses included in the study plan are highlighted with a blue background', () => {
       cy.visit('/students/550789')
-      cy.contains('table tbody tr', 'Kulttuurien tutkimuksen kandiohjelma (01.08.2020–30.06.2023)').within(() => {
-        cy.get('td').eq(0).click()
+      cy.contains('Study rights')
+      cy.get('[data-cy=KH40_005-0]').contains('Kulttuurien tutkimuksen kandiohjelma')
+      cy.get('[data-cy=KH40_005-0]').within(() => {
+        cy.get('td').eq(1).click()
       })
       cy.contains('table tbody tr', 'Kandidaatintutkielma (KUKA-LIS222)').should(
         'have.css',
@@ -269,27 +271,23 @@ describe('Students tests', () => {
       )
     })
 
-    it('When a study plan is selected, the time frame of the credit graph is updated', () => {
-      cy.clock(new Date('2024-08-30').getTime(), ['Date'])
-      cy.visit('/students/550789')
-      cy.contains('table tbody tr', 'Kulttuurien tutkimuksen kandiohjelma (01.08.2020–30.06.2023)').within(() => {
-        cy.get('td').eq(0).click()
-      })
-    })
-
     it("If there's a study plan corresponding to the degree programme, completed credits are displayed", () => {
       cy.visit('/students/550789')
-      cy.contains('table tbody tr', 'Kulttuurien tutkimuksen kandiohjelma (01.08.2020–30.06.2023)').within(() => {
+      cy.contains('Study rights')
+      cy.get('[data-cy=KH40_005-0]').contains('Kulttuurien tutkimuksen kandiohjelma')
+      cy.get('[data-cy=KH40_005-0]').within(() => {
         cy.get('td')
-          .eq(-1)
+          .eq(4)
           .contains(/^185 cr$/)
       })
     })
 
     it("If there's a study plan corresponding to the degree programme and the student hasn't graduated, percentage of completion is also displayed", () => {
       cy.visit('/students/458723')
-      cy.contains('table tbody tr', 'Oikeusnotaarin koulutusohjelma (01.08.2020–31.07.2027)').within(() => {
-        cy.get('td').eq(-1).contains('69% (125 cr)')
+      cy.contains('Study rights')
+      cy.get('[data-cy=KH20_001-0]').contains('Oikeusnotaarin koulutusohjelma')
+      cy.get('[data-cy=KH20_001-0]').within(() => {
+        cy.get('td').eq(4).contains('69% (125 cr)')
       })
     })
   })

cypress/e2e/Teachers.js

@@ -35,7 +35,7 @@ describe('Teachers page tests', () => {
     cy.cs('Search').click()
     cy.cs('teacher-search').type(teacher1.split(' ')[0])
     cy.get('table tbody tr').should('have.length', 4)
-    cy.get('table tbody tr').eq(1).contains('td', teacher1)
+    cy.get('table tbody tr').eq(2).contains('td', teacher1)
   })
 
   it('Can check teacher page', () => {

cypress/e2e/University.js

@@ -10,21 +10,18 @@ const checkProgressTables = () => {
   progressLevels.forEach(level => cy.cs(`${level}-faculty-progress-table`))
 }
 
-const graduationTimesLevels = ['bachelor', 'master', 'doctor'] // ? Bachelor + master seems to be missing in test data
+const graduationTimesLevels = ['bachelor', 'bcMsCombo', 'master', 'doctor']
 
-const checkAverageGraduationTimesBreakdownBarCharts = () => {
+const checkGraduationCharts = mode => {
   graduationTimesLevels.forEach(level => {
-    cy.cs(`${level}-breakdown-bar-chart`)
+    cy.cs(`${level}-${mode}-bar-chart`).within(() => {
+      // Check that bar has loaded
+      cy.contains('Loading content').should('not.exist')
+    })
   })
 }
 
-const checkAverageGraduationTimesMedianBarCharts = () => {
-  graduationTimesLevels.forEach(level => {
-    cy.cs(`${level}-median-bar-chart`)
-  })
-}
-
-describe('University view', () => {
+describe('University view', { defaultCommandTimeout: 60 * 1000 }, () => {
   beforeEach(() => {
     cy.init('/university')
     cy.contains('University')
@@ -73,17 +70,31 @@ describe('University view', () => {
       cy.cs('faculty-graduations-tab').click()
     })
 
-    it('has all the correct median time bar charts', () => {
-      checkAverageGraduationTimesBreakdownBarCharts()
-    })
+    describe('Different modes work', () => {
+      it('Breakdown', () => {
+        cy.cs('graduation-mode-selector').within(() => {
+          cy.cs('select-breakdown').click()
+        })
+        checkGraduationCharts('breakdown')
+      })
 
-    it("'Breakdown/Median times' toggle works", () => {
-      cy.cs('graduation-time-toggle').click()
-      checkAverageGraduationTimesMedianBarCharts()
+      it('Median', () => {
+        cy.cs('graduation-mode-selector').within(() => {
+          cy.cs('select-median').click()
+        })
+        checkGraduationCharts('median')
+      })
+
+      it('Average', () => {
+        cy.cs('graduation-mode-selector').within(() => {
+          cy.cs('select-average').click()
+        })
+        checkGraduationCharts('average')
+      })
     })
 
     it('info boxes contain correct information', () => {
-      cy.cs('average-graduation-times-info-box-button').click()
+      cy.cs('average-graduation-times-info-box-button').hover()
       cy.cs('average-graduation-times-info-box-content').contains('Opiskelijoiden keskimääräiset valmistumisajat')
     })
   })

cypress/support/commands.js

@@ -58,7 +58,7 @@ const onlyStudyGuidanceGroupsUser = {
   hypersonsisuid: 'hy-hlo-2222222',
 }
 
-const userHeaders = [
+export const userHeaders = [
   adminUserHeaders,
   basicUserHeaders,
   onlycoursestatisticsUserHeaders,

docker-compose.ci.yml

@@ -1,15 +1,30 @@
+name: oodikone
+
 x-defaults: &defaults
   restart: unless-stopped
 
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
 services:
   backend:
     <<: *defaults
-    container_name: backend
+    container_name: oodikone-backend
+    image: oodikone-backend:staging
+    build:
+      context: services
+      dockerfile: backend/Dockerfile
     depends_on:
-      - kone-db
-      - redis
-      - sis-db
-      - user-db
+      kone-db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      user-db:
+        condition: service_healthy
     environment:
       CI: 'true'
       DB_URL_USER: postgres://postgres@user-db:5432/user-db
@@ -22,132 +37,152 @@ services:
       IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
       JAMI_URL: 'http://jami:3003/'
       CRYPT_KEY: 'e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'
-    build:
-      context: services
-      dockerfile: backend/Dockerfile
-    image: oodikone-backend:staging
+    ports:
+      - 8080:8080
     profiles:
       - oodikone
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8080/health'] # In CI/prod the routing happens at a different level, hence no /api/ needed
+      <<: *healthcheck-defaults
 
   frontend:
     <<: *defaults
-    container_name: frontend
+    container_name: oodikone-frontend
+    image: oodikone-frontend:staging
     build:
       context: services
       dockerfile: frontend/Dockerfile
-    image: oodikone-frontend:staging
+    depends_on:
+      backend:
+        condition: service_healthy
+    ports:
+      - 5000:5000
     profiles:
       - oodikone
 
   kone-db:
     <<: *defaults
-    container_name: kone-db
+    container_name: oodikone-kone-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
     environment:
       PGDATA: /data
       POSTGRES_DB: kone-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
+    ports:
+      - 4321:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
+
+  user-db:
+    <<: *defaults
+    container_name: oodikone-user-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    ports:
+      - 4320:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   redis:
     <<: *defaults
+    container_name: oodikone-redis
+    image: redis:7.2.6
     command: >
       sh -c 'redis-server --appendonly yes && redis-cli flushall'
-    image: redis:7.2.6
-    container_name: redis
     ports:
       - 6379:6379
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping', '|', 'grep', 'PONG']
+      <<: *healthcheck-defaults
 
   sis-db:
     <<: *defaults
-    container_name: sis-db
+    container_name: oodikone-sis-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
     environment:
       PGDATA: /data
       POSTGRES_DB: sis-db
       POSTGRES_HOST_AUTH_METHOD: trust
     shm_size: 2gb
+    ports:
+      - 12345:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
-  user-db:
+  sis-importer-db:
     <<: *defaults
-    container_name: user-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: user-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    container_name: oodikone-sis-importer-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
+    profiles:
+      - updater
+    ports:
+      - 12346:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   jami:
     <<: *defaults
+    container_name: oodikone-jami
     image: quay.io/toska/jami:production
+    depends_on:
+      jami-db:
+        condition: service_healthy
     environment:
       - npm_config_cache=/tmp
       - PORT=3003
-      - NODE_ENV=dev
+      - NODE_ENV=test # previously was development which borks CI
       - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
     ports:
       - 3003:3003
-    container_name: jami
 
   jami-db:
     <<: *defaults
+    container_name: oodikone-jami-db
     image: postgres:15.10
     environment:
       - PGDATA=/data
       - POSTGRES_PASSWORD=postgres
-    container_name: jami-db
     ports:
       - 5433:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
+  # This is still needed to make backend calls correctly route from localhost:5000/api to localhost:8080
   nginx:
+    container_name: oodikone-nginx
     image: nginx:latest
-    container_name: nginx
     depends_on:
-      - frontend
-      - backend
+      frontend:
+        condition: service_started
+      backend:
+        condition: service_healthy
     ports:
       - 1337:80
-    volumes:
-      - ./nginx.conf:/etc/nginx/nginx.conf
     profiles:
       - oodikone
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
 
-  # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
-  sis-importer-db:
-    <<: *defaults
-    container_name: sis-importer-db
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
-    profiles:
-      - updater
-
-  sis-updater-worker:
-    <<: *defaults
-    build:
-      context: updater/sis-updater-worker
-    container_name: sis-updater-worker
-    depends_on:
-      - redis
-      - sis-db
-      - sis-importer-db
-    environment:
-      CI: 'true'
-      DB_URL: postgres://postgres@sis-db:5432/sis-db
-      REDIS_HOST: redis
-      SIS_IMPORTER_DATABASE: sis-importer-db
-      SIS_IMPORTER_HOST: sis-importer-db
-      SIS_IMPORTER_PASSWORD: postgres
-      SIS_IMPORTER_USER: postgres
-      SIS_PASSWORD: postgres
-    profiles:
-      - updater
-
+  # Following services + sis-importer-db are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
   sis-updater-scheduler:
-    <<: *defaults
+    restart: no
+    container_name: oodikone-updater-scheduler
     build:
       context: updater/sis-updater-scheduler
-    container_name: sis-updater-scheduler
+      dockerfile: ./Dockerfile
     depends_on:
-      - redis
-      - sis-importer-db
+      redis:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
     environment:
       CI: 'true'
       REDIS_HOST: redis
@@ -162,3 +197,31 @@ services:
       - 8082:8082
     profiles:
       - updater
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8082/health']
+      <<: *healthcheck-defaults
+
+  sis-updater-worker:
+    <<: *defaults
+    container_name: oodikone-updater-worker
+    build:
+      context: updater/sis-updater-worker
+      dockerfile: ./Dockerfile
+    depends_on:
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
+    environment:
+      CI: 'true'
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_HOST: redis
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_PASSWORD: postgres
+    profiles:
+      - updater

docker-compose.real.yml

@@ -1,3 +1,5 @@
+name: oodikone
+
 services:
   backend:
     environment:
@@ -20,3 +22,4 @@ services:
 
 volumes:
   redis-data-real:
+    name: oodikone-redis-data-real

docker-compose.test.yml

@@ -1,26 +1,27 @@
-services:
-  api-tests:
-    volumes:
-      - backend-tests-nodemod:/opt/app-root/backend/node_modules
-      - ./services/backend/:/opt/app-root/backend
-      - ./services/shared/:/opt/app-root/shared
-    environment:
-      CI: 'true'
-      NODE_ENV: 'test'
-      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
-      DB_URL_USER: postgres://postgres@user-db:5432/user-db
-      FRONT_URL: http://localhost:3000
-      REDIS: redis
-      SECRET_TOKEN: test
-      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
-      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      JAMI_URL: 'http://jami:3003/'
-      IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
-      CRYPT_KEY: 'e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'
-    build:
-      context: services/backend
-      dockerfile: dev.Dockerfile
-    command: npm run test
+name: oodikone
 
-volumes:
-  backend-tests-nodemod:
+x-defaults: &defaults
+  restart: unless-stopped
+
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
+services:
+  sis-db:
+    <<: *defaults
+    container_name: oodikone-sis-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
+    pull_policy: always
+    volumes: !reset [] # Remove previous volume definitions
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    shm_size: 2gb
+    ports:
+      - 12345:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults

docker-compose.yml

@@ -1,13 +1,20 @@
+name: oodikone
+
 x-defaults: &defaults
   restart: unless-stopped
 
 x-build_defaults: &build_defaults
   dockerfile: dev.Dockerfile
 
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 1s
+  timeout: 3s
+  retries: 30
+
 services:
   adminer:
     <<: *defaults
-    container_name: adminer
+    container_name: oodikone-adminer
     environment:
       ADMINER_DESIGN: pepa-linha
     image: adminer:4.8.1
@@ -18,15 +25,19 @@ services:
 
   backend:
     <<: *defaults
+    container_name: oodikone-backend
     build:
-      <<: *build_defaults
-      context: services/backend
-    container_name: backend
+      dockerfile: backend/dev.Dockerfile
+      context: services
     depends_on:
-      - kone-db
-      - redis
-      - sis-db
-      - user-db
+      kone-db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      user-db:
+        condition: service_healthy
     env_file:
       - .env
     tty: true
@@ -47,14 +58,21 @@ services:
     volumes:
       - ./services/backend/:/opt/app-root/backend
       - ./services/shared/:/opt/app-root/shared
+      - shared-nodemod:/opt/app-root/shared/node_modules
       - backend-nodemod:/opt/app-root/backend/node_modules
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8080/api/health']
+      <<: *healthcheck-defaults
 
   frontend:
     <<: *defaults
+    container_name: oodikone-frontend
     build:
-      <<: *build_defaults
-      context: services/frontend
-    container_name: frontend
+      dockerfile: frontend/dev.Dockerfile
+      context: services
+    depends_on:
+      backend:
+        condition: service_healthy
     ports:
       - 3000:3000
     profiles:
@@ -62,40 +80,66 @@ services:
     volumes:
       - ./services/frontend/:/opt/app-root/frontend
       - ./services/shared/:/opt/app-root/shared
+      - shared-nodemod:/opt/app-root/shared/node_modules
       - frontend-nodemod:/opt/app-root/frontend/node_modules
 
   kone-db:
     <<: *defaults
-    container_name: kone-db
+    container_name: oodikone-kone-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
     profiles:
       - oodikone
+    ports:
+      - 4321:5432
     volumes:
       - kone-db-data:/data
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
+
+  user-db:
+    <<: *defaults
+    container_name: oodikone-user-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    profiles:
+      - oodikone
+    ports:
+      - 4320:5432
+    volumes:
+      - user-db-data:/data
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   redis:
     <<: *defaults
+    container_name: oodikone-redis
     command: 'redis-server --appendonly yes'
-    container_name: redis
     image: redis:7.2.6
     ports:
-      - '6379:6379'
+      - 6379:6379
     volumes:
       - redis-data:/data
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping', '|', 'grep', 'PONG']
+      <<: *healthcheck-defaults
 
   sis-db:
     <<: *defaults
-    container_name: sis-db
+    container_name: oodikone-sis-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
     volumes:
       - sis-db-data:/data
     shm_size: 2gb
     ports:
       - 12345:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-importer-db:
     <<: *defaults
-    container_name: sis-importer-db
+    container_name: oodikone-sis-importer-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
     profiles:
       - updater
@@ -103,41 +147,55 @@ services:
       - sis-importer-db-data:/data
     ports:
       - 12346:5432
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
-  sis-updater-scheduler:
+  jami:
     <<: *defaults
-    build:
-      <<: *build_defaults
-      context: updater/sis-updater-scheduler
-    container_name: sis-updater-scheduler
+    container_name: oodikone-jami
+    image: quay.io/toska/jami:production
     depends_on:
-      - redis
-      - sis-importer-db
+      jami-db:
+        condition: service_healthy
+    env_file:
+      - .env
     environment:
-      REDIS_HOST: redis
-      SECRET_TOKEN: dev
-      SIS_IMPORTER_DATABASE: sis-importer-db
-      SIS_IMPORTER_HOST: sis-importer-db
-      SIS_IMPORTER_PASSWORD: postgres
-      SIS_IMPORTER_USER: postgres
+      - npm_config_cache=/tmp
+      - PORT=3003
+      - NODE_ENV=development
+      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/jami-db-real
     ports:
-      - 8082:8082
-    profiles:
-      - updater
+      - 3003:3003
+
+  jami-db:
+    <<: *defaults
+    container_name: oodikone-jami-db
+    image: postgres:15.10
+    environment:
+      - PGDATA=/data
+      - POSTGRES_PASSWORD=postgres
+    ports:
+      - 5433:5432
     volumes:
-      - sis-updater-scheduler-nodemod:/opt/app-root/src/node_modules
-      - ./updater/sis-updater-scheduler/:/opt/app-root/src
+      - jami-data:/data
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres']
+      <<: *healthcheck-defaults
 
   sis-updater-worker:
     <<: *defaults
+    container_name: oodikone-updater-worker
     build:
       <<: *build_defaults
       context: updater/sis-updater-worker
-    container_name: sis-updater-worker
     depends_on:
-      - redis
-      - sis-db
-      - sis-importer-db
+      redis:
+        condition: service_healthy
+      sis-db:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
     environment:
       DB_URL: postgres://postgres@sis-db:5432/sis-db
       REDIS_HOST: redis
@@ -149,52 +207,60 @@ services:
     profiles:
       - updater
     volumes:
-      - sis-updater-worker-nodemod:/opt/app-root/src/node_modules
       - ./updater/sis-updater-worker/:/opt/app-root/src
+      - sis-updater-worker-nodemod:/opt/app-root/src/node_modules
 
-  user-db:
+  sis-updater-scheduler:
     <<: *defaults
-    container_name: user-db
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
+    container_name: oodikone-updater-scheduler
+    build:
+      <<: *build_defaults
+      context: updater/sis-updater-scheduler
+    depends_on:
+      redis:
+        condition: service_healthy
+      sis-importer-db:
+        condition: service_healthy
+    environment:
+      REDIS_HOST: redis
+      SECRET_TOKEN: dev
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
+    ports:
+      - 8082:8082
     profiles:
-      - oodikone
+      - updater
     volumes:
-      - user-db-data:/data
-
-  jami:
-    <<: *defaults
-    image: quay.io/toska/jami:production
-    env_file:
-      - .env
-    environment:
-      - npm_config_cache=/tmp
-      - PORT=3003
-      - NODE_ENV=dev
-      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/jami-db-real
-    ports:
-      - 3003:3003
-    container_name: jami
-
-  jami-db:
-    <<: *defaults
-    image: postgres:15.10
-    environment:
-      - PGDATA=/data
-      - POSTGRES_PASSWORD=postgres
-    container_name: jami-db
-    ports:
-      - 5433:5432
-    volumes:
-      - jami-data:/data
+      - ./updater/sis-updater-scheduler/:/opt/app-root/src
+      - sis-updater-scheduler-nodemod:/opt/app-root/src/node_modules
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8082/health']
+      <<: *healthcheck-defaults
 
 volumes:
   backend-nodemod:
+    name: oodikone-backend-nodemod
   frontend-nodemod:
+    name: oodikone-frontend-nodemod
+  shared-nodemod:
+    name: oodikone-shared-nodemod
   kone-db-data:
+    name: oodikone-kone-db-data
   redis-data:
+    name: oodikone-redis-data
   sis-db-data:
+    name: oodikone-sis-db-data
   sis-importer-db-data:
+    name: oodikone-sis-importer-db-data
   sis-updater-scheduler-nodemod:
+    name: oodikone-updater-scheduler-nodemod
   sis-updater-worker-nodemod:
+    name: oodikone-updater-worker-nodemod
   user-db-data:
+    name: oodikone-user-db-data
   jami-data:
+    name: oodikone-jami-data

documentation/migrations.md

@@ -8,7 +8,7 @@ These instructions explain how to update the schemas for the following databases
 
 The steps to **add, remove, or modify columns** are mostly the same for each database, but the file paths and container names differ. Always **double-check** that you have the correct function name, column name, data type, and other relevant details in your migration file.
 
-> [!NOTE]  
+> [!NOTE]
 > If you need to change the `sis-importer-db` schema, see the instructions in the [sis-importer repo](https://github.com/UniversityOfHelsinkiCS/sis-importer/blob/master/README.md#tricks--tips).
 
 ## General guidelines
@@ -25,7 +25,7 @@ The steps to **add, remove, or modify columns** are mostly the same for each dat
 - Use **snake_case** in the actual database columns.
 - If a model has `underscored: true`, Sequelize automatically converts camelCase to snake_case in all queries. Otherwise, specify the column name explicitly.
 
-> [!IMPORTANT]  
+> [!IMPORTANT]
 > The column name **in the database** (snake_case) must be used in the migration file, not the camelCase name from your model.
 
 ### Verifying the migration locally
@@ -73,7 +73,7 @@ If you need to **revert a migration locally**, follow these steps:
 
 ## `sis-db`
 
-> [!NOTE]  
+> [!NOTE]
 > These instructions assume that needed data is already in the `sis-importer-db`. If not, you must add the data there first. See the [sis-importer repo](https://github.com/UniversityOfHelsinkiCS/sis-importer/blob/master/README.md#tricks--tips) for more details.
 
 1. **Update the updater model:**

eslint.config.mjs

@@ -263,5 +263,26 @@ export default tseslint.config(
       'no-promise-executor-return': 'off',
     },
   },
+
+
+  // Backend tests
+  {
+    files: ["services/backend/tests/**/*.ts", "services/backend/vitest.config.ts"],
+    rules: {
+      'import-x/no-extraneous-dependencies': 'off',
+    }
+  },
+
+  // Frontend tests
+  {
+    files: ['services/frontend/tests/**/*.{ts,tsx}', "services/frontend/playwright/**/*.{ts,tsx}", 'services/frontend/playwright-ct.config.ts'],
+    rules: {
+      'import-x/no-extraneous-dependencies': 'off',
+      'import-x/no-unused-modules': 'off',
+      'import-x/no-default-export': 'off',
+      'react/jsx-filename-extension': 'off',
+    },
+  },
+
   eslintPluginPrettierRecommended // This must be the last plugin so it can override other configs
 )

package.json

@@ -12,23 +12,26 @@
     "stylelint": "stylelint",
     "prettier": "prettier",
     "cli": "./cli.sh",
-    "oodikone": "./run.sh oodikone anon up --detach",
+    "logs": "docker compose --file docker-compose.yml logs backend frontend sis-updater-worker sis-updater-scheduler jami -f",
+    "logs:all": "docker compose --file docker-compose.yml --profile oodikone --profile updater logs -f",
+    "logs:ci": "docker compose --file docker-compose.ci.yml logs backend frontend sis-updater-worker sis-updater-scheduler jami",
+    "oodikone": "npm run docker:down && ./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
-    "oodikone:ci:setup": "docker compose --file docker-compose.ci.yml --profile tests up --detach",
-    "oodikone:ci:api": "docker compose --file docker-compose.test.yml up --exit-code-from api-tests",
+    "oodikone:test": "npm run docker:down && docker compose --profile oodikone up --detach && docker compose --file docker-compose.test.yml up --detach",
+    "oodikone:ci": "npm run docker:down && docker compose --file docker-compose.ci.yml --profile oodikone up",
     "updater": "./run.sh updater anon up --detach",
     "updater:real": "./run.sh updater real up --detach",
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
     "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
-    "testupdater": "docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start",
-    "flushredis": "docker exec -it redis redis-cli flushall",
+    "testupdater": "docker compose --file docker-compose.ci.yml up --detach --no-recreate --wait sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker compose --file docker-compose.ci.yml up sis-updater-scheduler --exit-code-from sis-updater-scheduler",
+    "flushredis": "docker exec -it oodikone-redis redis-cli flushall",
     "prepare": "husky",
     "install:local": "npm i && npm i --prefix services/shared && npm i --prefix services/frontend && npm i --prefix services/backend && npm i --prefix updater/sis-updater-scheduler && npm i --prefix updater/sis-updater-worker",
-    "install:docker": "docker exec frontend npm i && docker exec backend npm i && docker exec sis-updater-scheduler npm i && docker exec sis-updater-worker npm i",
+    "install:docker": "docker exec --user 0 oodikone-frontend sh -lc \"npm i --prefix /opt/app-root/shared && npm i --prefix /opt/app-root/frontend && chown -R 1001:0 /opt/app-root/shared/node_modules /opt/app-root/frontend/node_modules\" && docker exec --user 0 oodikone-backend sh -lc \"npm i --prefix /opt/app-root/backend && chown -R 1001:0 /opt/app-root/backend/node_modules\" && docker exec --user 0 oodikone-updater-scheduler sh -lc \"npm i --prefix /opt/app-root/src && chown -R 1001:0 /opt/app-root/src/node_modules\" && docker exec --user 0 oodikone-updater-worker sh -lc \"npm i --prefix /opt/app-root/src && chown -R 1001:0 /opt/app-root/src/node_modules\"",
     "install:both": "npm run install:local && npm run install:docker",
     "ci:local": "npm ci && npm ci --prefix services/shared && npm ci --prefix services/frontend && npm ci --prefix services/backend && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker",
-    "ci:docker": "docker exec frontend npm ci && docker exec backend npm ci && docker exec sis-updater-scheduler npm ci && docker exec sis-updater-worker npm ci",
+    "ci:docker": "docker exec --user 0 oodikone-frontend sh -lc \"npm ci --prefix /opt/app-root/shared && npm ci --prefix /opt/app-root/frontend && chown -R 1001:0 /opt/app-root/shared/node_modules /opt/app-root/frontend/node_modules\" && docker exec --user 0 oodikone-backend sh -lc \"npm ci --prefix /opt/app-root/backend && chown -R 1001:0 /opt/app-root/backend/node_modules\" && docker exec --user 0 oodikone-updater-scheduler sh -lc \"npm ci --prefix /opt/app-root/src && chown -R 1001:0 /opt/app-root/src/node_modules\" && docker exec --user 0 oodikone-updater-worker sh -lc \"npm ci --prefix /opt/app-root/src && chown -R 1001:0 /opt/app-root/src/node_modules\"",
     "ci:both": "npm run ci:local && npm run ci:docker"
   },
   "repository": {

run.sh

@@ -23,7 +23,7 @@ Usage: $(basename "${BASH_SOURCE[0]}") option [version] command --flag
 Parameters:
 * Option: oodikone/updater/both
 * Version: anon/real. Not necessary when running down/logs/pull.
-* Command: will be passed to docker-compose.
+* Command: will be passed to docker compose.
 EOF
 }
 

scripts/functions_for_setup.sh

@@ -12,7 +12,7 @@ DUMP_DIR="$PROJECT_ROOT/.databasedumps"
 S3_CONFIG_FILE=~/.s3cfg
 S3_BUCKET="s3://psyduck"
 
-## Following the naming convention in docker-compose, these are names for database
+## Following the naming convention in docker compose, these are names for database
 ## services. The real data databases inside service will have names suffixed by "-real".
 KONE_DB_NAME="kone-db"
 SIS_DB_NAME="sis-db"
@@ -112,11 +112,11 @@ reset_databases() {
   infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
 
   npm run docker:down
-  docker-compose up -d "${databases[@]}"
+  docker compose up -d "${databases[@]}"
 
   for database in "${databases[@]}"; do
     local database_dump="$database_dump_dir/$database.sql.gz"
-    local database_container="$database"
+    local database_container="oodikone-$database"
     local database_name="$database-real"
 
     infomsg "Attempting to create database $database_name from dump $database_dump inside container $database_container"
@@ -138,7 +138,7 @@ reset_databases() {
   successmsg "Database setup finished"
 }
 
-reset_all_real_data() {
+reset_all_data() {
   infomsg "Downloading real data dumps"
   for i in ${!DATABASES[*]}; do
     local database="${DATABASES[$i]}"
@@ -215,6 +215,7 @@ set_up_oodikone_from_scratch() {
     "$PROJECT_ROOT"
     "$PROJECT_ROOT/services/frontend"
     "$PROJECT_ROOT/services/backend"
+    "$PROJECT_ROOT/services/shared"
     "$PROJECT_ROOT/updater/sis-updater-scheduler"
     "$PROJECT_ROOT/updater/sis-updater-worker"
   )
@@ -223,7 +224,7 @@ set_up_oodikone_from_scratch() {
   "$PROJECT_ROOT"/run.sh both down --remove-orphans --volumes --rmi all || infomsg "Cleaning errored, but will continue"
   for folder in "${folders_to_set_up[@]}"; do
     cd "$folder" || die "Couldn't change directory to folder $folder"
-    rm -rf node_modules|| die "Couldn't remove node_modules in folder $folder"
+    rm -rf node_modules || die "Couldn't remove node_modules in folder $folder"
   done
 
   infomsg "Installing npm packages locally to root and each subfolder to enable linting and formatting"

services/.dockerignore

@@ -1,7 +1,116 @@
-*/.dockerignore
-*/.eslintrc.json
-*/.gitignore
-*/*Dockerfile
-*/build
-*/dist
-*/node_modules
+# disallow directory named "private", it can be used to store things that should not end up in version control or images
+# NOTE: needs to be at the root of the project
+private/
+gitignored/
+
+# disallow version control, package managment, IDEs, tools etc
+**/node_modules
+**/.venv
+**/.git
+**/.github
+**/.vscode
+**/.idea
+**/.zed
+**/coverage/
+**/.eslintcache
+**/.DS_Store
+
+# do not include any data by default (except json)
+**/*.csv
+**/*.tsv
+
+**/*.db
+**/*.sqlite
+**/*.sql
+**/*.psql
+**/*.sqlz
+**/*.sqz
+**/*.sql.gz
+**/*.bson
+**/*.jsonl
+
+**/*.bin
+**/*.dat
+**/*.data
+
+**/*.swp
+**/*.tmp
+**/.~lock*
+**/*.pid
+**/*.seed
+**/*.lock
+
+**/*.gz
+**/*.tgz
+**/*.xz
+**/*.zip
+**/*.gzip
+**/*.tar
+**/*.xz
+**/*.7z
+**/*.rar
+
+**/*.docx
+**/*.doc
+**/*.xlsx
+**/*.xls
+**/*.pptx
+**/*.ppt
+**/*.odt
+**/*.odp
+**/*.ods
+**/*.pdf
+
+
+# disallow logs
+**/*.log
+**/*.log*
+**/logs/
+**/log/
+**/*.cpuprofile
+**/*.heapprofile
+**/*.heapsnapshot
+
+# disallow configuration
+**/*.yml
+**/*.yaml
+**/*.toml
+**/*.conf
+**/*.ini
+**/toska_login/
+**/toska_login/*
+**/cypress.env.json
+cypress/
+
+
+# disallow secrects
+**/.env
+**/.env*
+**/*.env.*
+**/.secrects
+**/*.s3cfg
+
+# disallow backups
+**/backups/*
+**/backups
+**/backup/*
+**/backup
+**/dump/
+**/dump
+**/dumps/
+**/dumps
+**/databasedumps/
+**/privateData
+**/temp
+**/minio-data/*
+
+
+# additional inclusions here
+
+# exclusions here
+**/.dockerignore
+**/.gitignore
+**/.eslintrc.json
+**/*Dockerfile
+**/build
+**/dist

services/backend/.env.test

@@ -0,0 +1,12 @@
+CI='true'
+NODE_ENV='test'
+DB_URL_KONE=postgres://postgres@localhost:4321/kone-db
+DB_URL_USER=postgres://postgres@localhost:4320/user-db
+FRONT_URL=http://localhost:3000
+REDIS=localhost
+SECRET_TOKEN=test
+SIS_DB_URL=postgres://postgres@localhost:12345/sis-db
+SIS_UPDATER_URL=http://localhost:8082
+JAMI_URL='http://localhost:3003/'
+IMPORTER_DB_API_URL='https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
+CRYPT_KEY='e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'

services/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi9/nodejs-24-minimal AS build
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal AS build
 
 ENV TZ="Europe/Helsinki"
 
@@ -29,7 +29,7 @@ RUN npm ci --include=dev --prefix shared && npm ci --include=dev --prefix backen
 RUN npm run build --prefix shared && npm run build --prefix backend
 
 
-FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 
@@ -50,6 +50,9 @@ ENV SENTRY_RELEASE=$SENTRY_RELEASE
 
 WORKDIR /opt/app-root/shared
 COPY --from=build /opt/app-root/shared/dist .
+COPY --from=build /opt/app-root/shared/.npmrc .
+COPY --from=build /opt/app-root/shared/package*.json ./
+RUN npm ci --omit=dev
 
 WORKDIR /opt/app-root/backend
 COPY --from=build /opt/app-root/backend/.npmrc .

services/backend/dev.Dockerfile

@@ -1,15 +1,21 @@
-FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
+FROM registry.access.redhat.com/ubi10/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 
 ENV NODE_ENV=development
 ENV NODE_OPTIONS=--max-old-space-size=4096
 
-WORKDIR /opt/app-root/backend
-COPY --chown=1001:1001 . .
+USER 0
 
-USER 1001
-RUN npm ci
+WORKDIR /opt/app-root/shared
+COPY --chown=1001:0 ./shared .
+RUN npm ci && chown -R 1001:0 /opt/app-root/shared
+
+WORKDIR /opt/app-root/backend
+COPY --chown=1001:0 ./backend .
+RUN npm ci && chown -R 1001:0 /opt/app-root/backend
 
 EXPOSE 8080
+
+USER 1001
 CMD ["node_modules/.bin/nodemon", "--exec", "node --import tsx index.ts", "--ext", "ts,js,json", "--watch", "src", "--watch", "index.ts", "--legacy-watch"]

services/backend/package.json

@@ -3,7 +3,11 @@
   "private": true,
   "scripts": {
     "build": "tsc",
-    "test": "node --import tsx --test --test-reporter spec --test-force-exit"
+    "test:all": "vitest",
+    "test:api": "vitest ./tests/api",
+    "test:services": "vitest ./tests/services",
+    "test:unit": "vitest ./tests/unit",
+    "test:silent": "SILENT_TESTING=true vitest"
   },
   "dependencies": {
     "@oodikone/shared": "file:../shared",
@@ -27,7 +31,6 @@
     "umzug": "^3.8.2",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "winston": "^3.17.0",
-    "winston-gelf-transporter": "^1.0.2",
     "winston-loki": "^6.1.3"
   },
   "devDependencies": {
@@ -42,11 +45,7 @@
     "nodemon": "^3.1.11",
     "supertest": "^7.2.2",
     "tsx": "^4.19.4",
-    "typescript": "^5.9.2"
-  },
-  "overrides": {
-    "winston-gelf-transporter": {
-      "gelf-pro": "^1.4.0"
-    }
+    "typescript": "^5.9.2",
+    "vitest": "4.1.9"
   }
 }

services/backend/src/config/index.ts

@@ -6,6 +6,8 @@ export const isProduction = !isStaging && process.env.NODE_ENV === 'production'
 
 export const runningInCI = process.env.CI === 'true'
 
+export const silentTesting = process.env.SILENT_TESTING === 'true'
+
 // IAM group
 export const requiredGroup = ['grp-oodikone-users', 'grp-oodikone-basic-users']
 
@@ -36,9 +38,6 @@ export const { DB_URL_KONE, DB_URL_USER, SECRET_TOKEN, SIS_DB_URL, SIS_UPDATER_U
 
 export const rootOrgId = process.env.ROOT_ORG_ID ?? 'hy-university-root-id'
 
-export const languageCenterViewEnabled =
-  process.env.LANGUAGE_CENTER_VIEW_ENABLED == null ? true : process.env.LANGUAGE_CENTER_VIEW_ENABLED === 'true'
-
 export const concurrentWorkers = parseInt(process.env.CONCURRENT_WORKERS!, 10) || 2
 
 export const DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS!, 10) || 5 // sequelize's default

services/backend/src/database/connection.ts

@@ -24,6 +24,7 @@ import {
   TeacherModel,
 } from '../models'
 import {
+  BannerModel,
   CustomPopulationSearchModel,
   ExcludedCourseModel,
   OpenUniPopulationSearchModel,
@@ -107,6 +108,7 @@ const sequelizeKone = new Sequelize(conf.DB_URL_KONE!, {
   logging: false,
   password: conf.KONE_PASSWORD,
   models: [
+    BannerModel,
     CustomPopulationSearchModel,
     ExcludedCourseModel,
     OpenUniPopulationSearchModel,

services/backend/src/database/migrations_kone/20260709_01_create_table_banners.js

@@ -0,0 +1,46 @@
+const { INTEGER, STRING, DATE } = require('sequelize')
+
+module.exports = {
+  up: async ({ context: queryInterface }) => {
+    await queryInterface.createTable('banners', {
+      id: {
+        type: INTEGER,
+        primaryKey: true,
+        autoIncrement: true,
+      },
+      text: {
+        type: STRING,
+        allowNull: false,
+      },
+      color: {
+        type: STRING,
+        allowNull: false,
+      },
+      lightness: {
+        type: STRING,
+        allowNull: false,
+      },
+      start_date: {
+        type: DATE,
+        allowNull: false,
+      },
+      end_date: {
+        type: DATE,
+        allowNull: false,
+      },
+      last_modified_by: {
+        type: STRING,
+        allowNull: false,
+      },
+      created_at: {
+        type: DATE,
+      },
+      updated_at: {
+        type: DATE,
+      },
+    })
+  },
+  down: async ({ context: queryInterface }) => {
+    await queryInterface.dropTable('banners')
+  },
+}

services/backend/src/events.ts

@@ -1,6 +1,6 @@
 import { CronJob } from 'cron'
 
-import { isProduction, runningInCI, languageCenterViewEnabled, CRON_SCHEDULE } from './config'
+import { isProduction, runningInCI, CRON_SCHEDULE } from './config'
 import { getDegreeProgrammesOfFaculty } from './services/faculty/faculty'
 import { getFaculties } from './services/faculty/facultyHelpers'
 import { updateFacultyOverview, updateFacultyProgressOverview } from './services/faculty/facultyUpdates'
@@ -98,10 +98,7 @@ const dailyJobs = async () => {
     await refreshTeacherLeaderboard()
     await refreshProgrammesAndFaculties()
 
-    if (languageCenterViewEnabled) {
-      jobQueue.refreshLanguagecenter()
-    }
-
+    jobQueue.refreshLanguagecenter()
     jobQueue.refreshCloseToGraduation()
   } catch (error) {
     logger.error('Daily jobs failed', error)

services/backend/src/middleware/accessLogger.ts

@@ -38,6 +38,11 @@ const accessLogger = morgan((tokens, req: Request, res: Response): undefined =>
     meta['req-route'] = req.route.path
   }
 
+  // Ignore admin route that is frequently pinged for updates.
+  if (/.*\/updater\/jobs$/.test(meta.url ?? '')) {
+    return
+  }
+
   const message = [
     `${user.mockedBy ? '(mocking) ' : ''}${user.name}`,
     `${(tokens['response-time'](req, res) ?? '0').split('.')[0]} ms`.padEnd(8, ' '),

services/backend/src/middleware/currentUser.ts

@@ -2,7 +2,7 @@ import * as Sentry from '@sentry/node'
 import { NextFunction, Request, Response } from 'express'
 import { intersection } from 'lodash-es'
 
-import { requiredGroup } from '../config'
+import { requiredGroup, baseUrl } from '../config'
 import { getMockedUser, getOrganizationAccess, getUser } from '../services/userService'
 import { ApplicationError } from '../util/customErrors'
 import logger from '../util/logger'
@@ -36,6 +36,11 @@ export default async (req: Request, _: Response, next: NextFunction) => {
     uid: username,
   } = req.headers as Headers
 
+  if (req.path === `${baseUrl}/health`) {
+    next()
+    return
+  }
+
   if (!username || !sessionId)
     throw new ApplicationError(
       `${username ?? 'Anonymous user'} requested ${req.url} without authorization headers.`,

services/backend/src/models/kone/banner.ts

@@ -0,0 +1,33 @@
+import { Column, DataType, Model, PrimaryKey, Table, AutoIncrement } from 'sequelize-typescript'
+
+import type { Banner } from '@oodikone/shared/models/kone'
+
+@Table({
+  underscored: true,
+  modelName: 'banner',
+  tableName: 'banners',
+})
+export class BannerModel extends Model implements Banner {
+  @PrimaryKey
+  @AutoIncrement
+  @Column(DataType.INTEGER)
+  declare id: number
+
+  @Column(DataType.DATE)
+  declare startDate: Date
+
+  @Column(DataType.DATE)
+  declare endDate: Date
+
+  @Column(DataType.STRING)
+  declare text: string
+
+  @Column(DataType.STRING)
+  declare color: string
+
+  @Column(DataType.STRING)
+  declare lightness: 'light' | 'main' | 'dark'
+
+  @Column(DataType.STRING)
+  declare lastModifiedBy: string
+}

services/backend/src/models/kone/index.ts

@@ -1,3 +1,4 @@
+export { BannerModel } from './banner'
 export { CustomPopulationSearchModel } from './customPopulationSearch'
 export { ExcludedCourseModel } from './excludedCourse'
 export { OpenUniPopulationSearchModel } from './openUniPopulationSearch'

services/backend/src/routes.ts

@@ -3,12 +3,13 @@ import compression from 'compression'
 import cors from 'cors'
 import { Express, json as jsonExpress } from 'express'
 
-import { frontUrl, languageCenterViewEnabled } from './config'
+import { frontUrl } from './config'
 import accessLogger from './middleware/accessLogger'
 import * as auth from './middleware/auth'
 import currentUserMiddleware from './middleware/currentUser'
 import errorHandler from './middleware/errorHandler'
 import headersMiddleware from './middleware/headers'
+import banners from './routes/banners'
 import changelog from './routes/changelog'
 import closeToGraduation from './routes/closeToGraduation'
 import completedCoursesSearch from './routes/completedCoursesSearch'
@@ -18,6 +19,7 @@ import curriculumPeriods from './routes/curriculumPeriods'
 import customPopulationSearch from './routes/customPopulationSearch'
 import faculties from './routes/faculties'
 import feedback from './routes/feedback'
+import health from './routes/health'
 import languageCenterData from './routes/languageCenterData'
 import login from './routes/login'
 import population from './routes/population'
@@ -44,6 +46,7 @@ const routes = (app: Express, url: string) => {
   app.use(currentUserMiddleware)
   app.use(accessLogger)
   app.use(`${url}/login`, login)
+  app.use(`${url}/banners`, banners)
   app.use(url, courses)
   app.use(`${url}/students`, students)
   app.use(url, population)
@@ -56,9 +59,7 @@ const routes = (app: Express, url: string) => {
   app.use(`${url}/programme-criteria`, studyProgrammeCriteria)
   app.use(`${url}/changelog`, changelog)
   app.use(`${url}/completedcoursessearch`, completedCoursesSearch)
-  if (languageCenterViewEnabled) {
-    app.use(`${url}/languagecenterdata`, languageCenterData)
-  }
+  app.use(`${url}/languagecenterdata`, languageCenterData)
   app.use(`${url}/faculties`, faculties)
   app.use(`${url}/university`, university)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
@@ -70,6 +71,7 @@ const routes = (app: Express, url: string) => {
   app.use(`${url}/close-to-graduation`, auth.roles(['fullSisuAccess', 'studyGuidanceGroups']), closeToGraduation)
   app.use(`${url}/study-programme-pins`, studyProgrammePins)
   app.use(`${url}/curriculum-periods`, curriculumPeriods)
+  app.use(`${url}/health`, health)
   app.get('', (_, res) => {
     res.status(404).json({ error: 'unknown endpoint' })
   })

services/backend/src/routes/banners.ts

@@ -0,0 +1,53 @@
+import { Request, Response, Router } from 'express'
+
+import { Banner } from '@oodikone/shared/models/kone'
+import { saveBanner, getActiveBanners, getAllBanners } from '../services/banners'
+
+const router = Router()
+
+router.get<never, Banner[]>('/', async (_req: Request, res: Response) => {
+  const banners = await getActiveBanners()
+  res.json(banners).end()
+})
+
+router.get<never, Banner[]>('/all', async (req: Request, res: Response) => {
+  const { user } = req
+  if (!user.isAdmin || !user.iamGroups.includes('grp-toska')) {
+    return res.status(403).end()
+  }
+
+  const banners = await getAllBanners()
+  res.json(banners)
+})
+
+router.post('/new', async (req: Request, res: Response) => {
+  const { user } = req
+  if (!user.isAdmin || !user.iamGroups.includes('grp-toska')) {
+    return res.status(403).end()
+  }
+
+  const payload = {
+    ...req.body,
+    lastModifiedBy: user.username,
+  }
+
+  await saveBanner(payload)
+  res.status(201).end()
+})
+
+router.put('/update', async (req: Request, res: Response) => {
+  const { user } = req
+  if (!user.isAdmin || !user.iamGroups.includes('grp-toska')) {
+    return res.status(403).end()
+  }
+
+  const payload = {
+    ...req.body,
+    lastModifiedBy: user.username,
+  }
+
+  await saveBanner(payload)
+  res.status(201).end()
+})
+
+export default router

services/backend/src/routes/courses.ts

@@ -51,7 +51,7 @@ router.get<never, CanError<CoursesMultiResBody>, CoursesMultiReqBody, CoursesMul
   }
 )
 
-type CourseYearlyStatsResBody = Awaited<ReturnType<typeof getCourseYearlyStats>>
+export type CourseYearlyStatsResBody = Awaited<ReturnType<typeof getCourseYearlyStats>>
 
 router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody, CourseYearlyStatsQuery>(
   '/courseyearlystats',
@@ -83,7 +83,7 @@ router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody,
     const useSeparate = separate === 'true'
 
     const results = await getCourseYearlyStats(courseCodes, useSeparate, anonymizationSalt, useCombineSubstitutions)
-    res.json(results)
+    return res.json(results)
   }
 )
 

services/backend/src/routes/faculties.ts

@@ -2,6 +2,7 @@ import { Router } from 'express'
 
 import type { CanError, GenericApplicationError } from '@oodikone/shared/routes'
 import type {
+  FacultyGraduationStatistics,
   Graduated,
   Name,
   NameWithCode,
@@ -22,7 +23,6 @@ import {
   setBasicStats,
   getThesisWritersStats,
   setThesisWritersStats,
-  getGraduationStats,
   setGraduationStats,
   getFacultyStudentStats,
   setFacultyStudentStats,
@@ -30,12 +30,13 @@ import {
   setFacultyProgressStats,
   type BasicData,
   type ThesisWriterData,
-  GraduationData,
+  getGraduationStats,
 } from '../services/faculty/facultyService'
 import { combineFacultyStudentProgress } from '../services/faculty/facultyStudentProgress'
 import { combineFacultyStudents } from '../services/faculty/facultyStudents'
 import { combineFacultyThesisWriters } from '../services/faculty/facultyThesisWriters'
 import { updateFacultyOverview, updateFacultyProgressOverview } from '../services/faculty/facultyUpdates'
+import { objectKeyAnnihilator } from '../util'
 import { ApplicationError } from '../util/customErrors'
 
 // Faculty uses a lot of tools designed for Degree programme.
@@ -135,14 +136,13 @@ router.get<
   return res.json(await getFacultyCredits(faculty.code, yearType === 'ACADEMIC_YEAR'))
 })
 
-type GetGraduationStatsResBody = GraduationData
 type GetGraduationStatsQuery = {
   programme_filter: ProgrammeFilter
 }
 
 router.get<
   GetStatsParams,
-  CanError<GetGraduationStatsResBody, GenericApplicationError>,
+  CanError<FacultyGraduationStatistics, GenericApplicationError>,
   GetStatsReqBody,
   GetGraduationStatsQuery
 >('/:id/graduationtimes', auth.roles(['facultyStatistics']), async (req, res) => {
@@ -153,7 +153,9 @@ router.get<
   if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
   const data = await getGraduationStats(faculty.code, programmeFilter)
-  if (data) return res.json(data)
+  // Times were needed to compute medians/averages serverside, but are not needed in frontend.
+  // They are still kept in redis as university view needs them for the same purpose.
+  if (data) return res.json(objectKeyAnnihilator(data, 'times', []))
 
   const programmes = await getDegreeProgrammesOfFaculty(faculty.code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
   if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
@@ -161,7 +163,7 @@ router.get<
   const updatedStats = await countGraduationTimes(faculty.code, programmes)
   if (updatedStats) await setGraduationStats(updatedStats, programmeFilter)
 
-  return res.json(updatedStats)
+  return res.json(objectKeyAnnihilator(updatedStats, 'times', []))
 })
 
 type GetProgressStatsResBody = {
@@ -212,7 +214,7 @@ export type DegreeProgramme = Pick<
   'code' | 'name' | 'degreeProgrammeType' | 'progId'
 >
 
-type GetStudetStatsResBody = {
+type GetStudentStatsResBody = {
   id: string
   years: string[]
   facultyTableStats: Record<string, (number | string)[]>
@@ -229,7 +231,7 @@ type GetStudetStatsResBody = {
 
 router.get<
   GetStatsParams,
-  CanError<GetStudetStatsResBody, GenericApplicationError>,
+  CanError<GetStudentStatsResBody, GenericApplicationError>,
   GetStatsReqBody,
   GetProgressStatsQuery
 >('/:id/studentstats', auth.roles(['facultyStatistics']), async (req, res) => {

services/backend/src/routes/health.ts

@@ -0,0 +1,10 @@
+import { Router } from 'express'
+
+const router = Router()
+
+/** Used for checking if backend is up eg. Docker healthchecks */
+router.get<never, { status: string }>('/', (_, res) => {
+  return res.status(200).json({ status: 'healthy' })
+})
+
+export default router

services/backend/src/routes/population.ts

@@ -112,7 +112,7 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
     const studentStatuses = handleQueryArrays(req.query.studentStatuses) // NOTE: TRANSFERRED means transferred out of the program
 
     const requiredFields = [years, semesters, programme]
-    if (requiredFields.some(field => !field)) {
+    if (requiredFields.some(field => !field.length)) {
       return res.status(400).json({ error: 'The query should have years, semesters and a programme defined' })
     }
 
@@ -223,14 +223,18 @@ router.get<
 })
 
 // Used in custom population and single study guidance groups
-router.post<never, GetCustomPopulationResBody, CustomPopulationQuery>(
+router.post<never, CanError<GetCustomPopulationResBody>, CustomPopulationQuery>(
   '/populationstatisticsbystudentnumbers',
   async (req, res) => {
-    const { studentNumbers, tags } = req.body
+    const { studentNumbers, tags } = req.body ?? { studentNumbers: [] }
     const { id: userId, roles, studentsUserCanAccess } = req.user
 
     const students = handleQueryArrays(studentNumbers)
 
+    if (!students.length) {
+      return res.status(422).json({ error: 'Body should include student numbers' })
+    }
+
     const filteredStudentNumbers = hasFullAccessToStudentData(roles)
       ? students
       : intersection(students, studentsUserCanAccess)

services/backend/src/routes/university.ts

@@ -1,11 +1,12 @@
 import { Request, Response, Router } from 'express'
 import { cloneDeep } from 'lodash-es'
 
-import { Graduated, NameWithCode } from '@oodikone/shared/types'
+import { FacultyGraduationStatistics, Graduated, NameWithCode } from '@oodikone/shared/types'
+import { MediansByCategory, UniversityGraduationStatistics } from '@oodikone/shared/types/graduations'
 import { magicFacultyCode } from '../config/organizationConstants'
 import { OrganizationModel } from '../models'
 import { getDegreeProgrammesOfFaculty } from '../services/faculty/faculty'
-import { countGraduationTimes, LevelGraduationStats } from '../services/faculty/facultyGraduationTimes'
+import { countGraduationTimes } from '../services/faculty/facultyGraduationTimes'
 import { getSortedFaculties } from '../services/faculty/facultyHelpers'
 import {
   getFacultyProgressStats,
@@ -16,19 +17,19 @@ import {
 import { combineFacultyStudentProgress, FacultyProgressData } from '../services/faculty/facultyStudentProgress'
 import { GraduationTarget } from '../services/graduationHelpers'
 import { getMedian } from '../services/studyProgramme/studyProgrammeHelpers'
+import { objectKeyAnnihilator } from '../util'
 import logger from '../util/logger'
 
 const router = Router()
 
 const degreeNames = ['bachelor', 'bachelorMaster', 'master', 'doctor'] as const
 
-const getProgrammeNames = (faculties: OrganizationModel[]) => {
-  return faculties.reduce<Record<string, NameWithCode>>((obj, faculty) => {
+const getProgrammeNames = (faculties: OrganizationModel[]) =>
+  faculties.reduce<Record<string, NameWithCode>>((obj, faculty) => {
     const { name, code } = faculty.toJSON()
     obj[faculty.code] = { code, ...name }
     return obj
   }, {})
-}
 
 interface GetProgressStatsRequest extends Request {
   query: {
@@ -120,16 +121,20 @@ router.get('/allprogressstats', async (req: GetProgressStatsRequest, res: Respon
   res.status(200).json(universityData)
 })
 
-router.get('/allgraduationstats', async (_req: Request, res: Response) => {
-  const degreeNames = ['bachelor', 'bcMsCombo', 'master', 'doctor'] as const
+router.get('/allgraduationstats', async (_req: Request, res: Response<UniversityGraduationStatistics>) => {
+  const degreeTypes = ['bachelor', 'bcMsCombo', 'master', 'doctor'] as const
+  /** University view does not support old degree programmes */
+  const programmeFilter = 'NEW_DEGREE_PROGRAMMES' as const
+
   const allFaculties = await getSortedFaculties()
   const facultyCodes = allFaculties.map(faculty => faculty.code)
-  const facultyData: Record<string, Awaited<ReturnType<typeof countGraduationTimes>>> = {}
-  const programmeFilter = 'NEW_DEGREE_PROGRAMMES'
+  const facultyData: Record<string, FacultyGraduationStatistics> = {}
+
   for (const facultyCode of facultyCodes) {
-    let data: any = await getGraduationStats(facultyCode, programmeFilter, true)
+    let data = await getGraduationStats(facultyCode, programmeFilter, true)
+
     if (!data) {
-      logger.info(`Data missing from server: Refreshing graduation faculty data for faculty ${facultyCode}`)
+      logger.info(`Data missing from redis: Refreshing graduation faculty data for faculty ${facultyCode}`)
       const programmes = await getDegreeProgrammesOfFaculty(facultyCode, true)
       data = await countGraduationTimes(facultyCode, programmes)
       await setGraduationStats(data, programmeFilter)
@@ -137,48 +142,50 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
     facultyData[facultyCode] = data
   }
 
-  const universityData = {
-    goals: {
-      bachelor: GraduationTarget.THREE_YEARS,
-      bcMsCombo: GraduationTarget.FIVE_YEARS,
-      master: GraduationTarget.TWO_YEARS,
-      doctor: GraduationTarget.FOUR_YEARS,
+  const goals: UniversityGraduationStatistics['goals'] = {
+    bachelor: GraduationTarget.THREE_YEARS,
+    bcMsCombo: GraduationTarget.FIVE_YEARS,
+    master: GraduationTarget.TWO_YEARS,
+    doctor: GraduationTarget.FOUR_YEARS,
+  }
+  const programmeNames = getProgrammeNames(allFaculties)
+
+  const byGradYear: UniversityGraduationStatistics['byGradYear'] = {
+    medians: {
+      bachelor: [],
+      bcMsCombo: [],
+      master: [],
+      doctor: [],
     },
-    programmeNames: getProgrammeNames(allFaculties),
-    byGradYear: {
-      medians: {} as Record<string, LevelGraduationStats[]>,
-      programmes: {
-        medians: {} as Record<
-          string,
-          Record<string, { programmes: string[]; data: Array<LevelGraduationStats & { name: string; code: string }> }>
-        >,
+    programmes: {
+      medians: {
+        bachelor: {},
+        bcMsCombo: {},
+        master: {},
+        doctor: {},
       },
     },
-    classSizes: {
-      bachelor: null as Record<string, number> | null,
-      bcMsCombo: null as Record<string, number> | null,
-      master: null as Record<string, number> | null,
-      doctor: null as Record<string, number> | null,
-      programmes: {} as Record<string, Record<string, Record<string, number>>>,
-    },
+  }
+  const classSizes: UniversityGraduationStatistics['classSizes'] = {
+    programmes: {},
   }
 
-  const unifyTotals = (facultyData: Record<string, LevelGraduationStats[]>, isLast: boolean) => {
-    const mediansForUniversity = universityData.byGradYear.medians
-    for (const degree of degreeNames) {
-      if (!mediansForUniversity[degree]) mediansForUniversity[degree] = []
-      if (!facultyData[degree]) continue
-      for (const yearStats of facultyData[degree]) {
-        const universityStats = mediansForUniversity[degree]
-        const universityYearStats = mediansForUniversity[degree].find(stats => stats.name === yearStats.name)
+  /** Computes university level stats */
+  const unifyTotals = (facultyData: MediansByCategory, isLast: boolean) => {
+    const mediansForUniversity = byGradYear.medians
+    for (const degreeType of degreeTypes) {
+      if (!facultyData[degreeType]) continue
+
+      for (const facultyYearStats of facultyData[degreeType]) {
+        const universityYearStats = mediansForUniversity[degreeType].find(stats => stats.name === facultyYearStats.name)
         if (!universityYearStats) {
-          universityStats.push(yearStats)
+          mediansForUniversity[degreeType].push(facultyYearStats)
         } else {
-          universityYearStats.times.push(...yearStats.times)
-          universityYearStats.amount += yearStats.amount
-          universityYearStats.statistics.onTime += yearStats.statistics.onTime
-          universityYearStats.statistics.yearOver += yearStats.statistics.yearOver
-          universityYearStats.statistics.wayOver += yearStats.statistics.wayOver
+          universityYearStats.times.push(...facultyYearStats.times)
+          universityYearStats.amount += facultyYearStats.amount
+          universityYearStats.statistics.onTime += facultyYearStats.statistics.onTime
+          universityYearStats.statistics.yearOver += facultyYearStats.statistics.yearOver
+          universityYearStats.statistics.wayOver += facultyYearStats.statistics.wayOver
           if (isLast) {
             universityYearStats.median = getMedian(universityYearStats.times)
           }
@@ -187,29 +194,28 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
     }
   }
 
-  const unifyProgrammeStats = (facultyData: Record<string, LevelGraduationStats[]>, facultyCode: string) => {
-    const mediansForUniversity = universityData.byGradYear.programmes.medians
-    for (const degree of degreeNames) {
-      if (!facultyData[degree]) continue
-      if (!mediansForUniversity[degree]) mediansForUniversity[degree] = {}
-      for (const yearData of facultyData[degree]) {
-        if (!mediansForUniversity[degree][yearData.name]) {
-          mediansForUniversity[degree][yearData.name] = { programmes: [], data: [] }
-        }
-        const uniYearStats = mediansForUniversity[degree][yearData.name]
+  /** Computes faculty breakdown */
+  const unifyFacultyStats = (facultyData: MediansByCategory, facultyCode: string) => {
+    const mediansForUniversity = byGradYear.programmes.medians
+    for (const degreeType of degreeTypes) {
+      if (!facultyData[degreeType]) continue
+
+      for (const facultyYearStats of facultyData[degreeType]) {
+        mediansForUniversity[degreeType][facultyYearStats.name] ??= { programmes: [], data: [] }
+        const uniYearStats = mediansForUniversity[degreeType][facultyYearStats.name]
         if (!uniYearStats.programmes.find(prog => prog === facultyCode)) {
           uniYearStats.programmes.push(facultyCode)
         }
         const uniYearFacultyStats = uniYearStats.data.find(item => item.code === facultyCode)
         if (!uniYearFacultyStats) {
-          uniYearStats.data.push({ ...cloneDeep(yearData), name: facultyCode, code: facultyCode })
+          uniYearStats.data.push({ ...cloneDeep(facultyYearStats), name: facultyCode, code: facultyCode })
         } else {
           uniYearFacultyStats.times.push(...uniYearFacultyStats.times)
           uniYearFacultyStats.median = getMedian(uniYearFacultyStats.times)
-          uniYearFacultyStats.amount += yearData.amount
-          uniYearFacultyStats.statistics.onTime += yearData.statistics.onTime
-          uniYearFacultyStats.statistics.yearOver += yearData.statistics.yearOver
-          uniYearFacultyStats.statistics.wayOver += yearData.statistics.wayOver
+          uniYearFacultyStats.amount += facultyYearStats.amount
+          uniYearFacultyStats.statistics.onTime += facultyYearStats.statistics.onTime
+          uniYearFacultyStats.statistics.yearOver += facultyYearStats.statistics.yearOver
+          uniYearFacultyStats.statistics.wayOver += facultyYearStats.statistics.wayOver
         }
       }
     }
@@ -218,27 +224,23 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
   for (let i = 0; i < facultyCodes.length; i++) {
     const facultyCode = facultyCodes[i]
     const data = facultyData[facultyCode]
-    unifyTotals(data.byGradYear.medians, i === facultyCodes.length - 1)
-    unifyProgrammeStats(data.byGradYear.medians, facultyCode)
-    for (const degree of degreeNames) {
-      if (!data.classSizes[degree]) {
-        continue
-      }
-      const facultyClassSizes = data.classSizes[degree] as Record<string, number>
-      if (!universityData.classSizes[degree]) {
-        universityData.classSizes[degree] = facultyClassSizes
-      } else {
-        Object.entries(facultyClassSizes).forEach(([key, value]) => {
-          universityData.classSizes[degree]![key] += value
-        })
-      }
-    }
 
-    const { programmes, ...rest } = data.classSizes
-    universityData.classSizes.programmes[facultyCode] = rest as Record<string, Record<string, number>>
+    unifyTotals(data.byGradYear.medians, i === facultyCodes.length - 1)
+    unifyFacultyStats(data.byGradYear.medians, facultyCode)
+
+    const { programmes: _, ...rest } = data.classSizes
+    classSizes.programmes[facultyCode] = rest
   }
 
-  res.status(200).json(universityData)
+  // Individual times are no longer needed as medians/averages were already computed.
+  objectKeyAnnihilator(byGradYear, 'times', [])
+
+  res.status(200).json({
+    goals,
+    programmeNames,
+    byGradYear,
+    classSizes,
+  })
 })
 
 export default router

services/backend/src/routes/updater.ts

@@ -1,6 +1,5 @@
 import { Request, Response, Router } from 'express'
 
-import { languageCenterViewEnabled } from '../config'
 import { refreshFaculties, refreshProgrammes } from '../events'
 import {
   abortUpdate,
@@ -109,8 +108,6 @@ router.post('/refresh_faculties_v2', async (req: Request, res: Response) => {
 
 router.post('/refresh_language_center_data', (req: Request, res: Response) => {
   logger.info(`${req.user.username} requested refresh of language center data`)
-  if (!languageCenterViewEnabled)
-    res.status(418).json({ error: 'The language center functionality is not activated in your environment.' })
   jobQueue.refreshLanguagecenter()
   res.status(200).json('Added job for refreshing language center data')
 })

services/backend/src/services/banners.ts

@@ -0,0 +1,28 @@
+import { Op } from 'sequelize'
+import { Banner } from '@oodikone/shared/models/kone'
+import { BannerModel } from '../models/kone/banner'
+import { SetOptional } from '../types'
+
+export const getActiveBanners = async () =>
+  await BannerModel.findAll({
+    attributes: ['id', 'text', 'lightness', 'color', 'startDate', 'endDate'],
+    where: {
+      endDate: {
+        [Op.gt]: new Date(),
+      },
+      startDate: {
+        [Op.lt]: new Date(),
+      },
+    },
+    raw: true,
+  })
+
+export const getAllBanners = async () => await BannerModel.findAll({ raw: true })
+
+export const saveBanner = async (payload: SetOptional<Banner, 'id'>) => {
+  if (!payload.id) {
+    await BannerModel.create(payload)
+  } else {
+    await BannerModel.upsert(payload)
+  }
+}

services/backend/src/services/completedCoursesSearch.ts

@@ -7,7 +7,7 @@ import {
   EnrollmentState,
   StudentCredits,
 } from '@oodikone/shared/types'
-import { omitKeys } from '@oodikone/shared/util'
+import { enrollmentTimeDateThreshold, omitKeys } from '@oodikone/shared/util'
 import { CourseModel, CreditModel, EnrollmentModel, StudentModel, StudyplanModel } from '../models'
 
 type StudentWithStudyplanNested = Pick<
@@ -113,6 +113,7 @@ const getEnrollments = async (
         state: {
           [Op.eq]: EnrollmentState.ENROLLED,
         },
+        enrollment_date_time: { [Op.gte]: enrollmentTimeDateThreshold },
       },
     })
 

services/backend/src/services/courses/courseYearlyStatsCounter.ts

@@ -124,6 +124,7 @@ export class CourseYearlyStatsCounter {
     this.groups[groupCode].students.grades[studentNumber] = { grade, passed }
   }
 
+  /** NOTE: Duplicates are counted by year eg. you can be both in 2022 and 2023 enrollments */
   public markEnrollmentToGroup(
     studentNumber: string,
     enrollmentDateTime: Date,

services/backend/src/services/courses/creditsAndEnrollmentsOfCourse.ts

@@ -1,7 +1,8 @@
 import { Op } from 'sequelize'
 
 import { Credit, Enrollment } from '@oodikone/shared/models'
-import { EnrollmentState, Unification } from '@oodikone/shared/types'
+import { CreditTypeCode, EnrollmentState, Unification } from '@oodikone/shared/types'
+import { enrollmentTimeDateThreshold } from '@oodikone/shared/util'
 import {
   StudentModel,
   CreditModel,
@@ -39,6 +40,7 @@ export const getCreditsForCourses = async (codeGroups: string[][], unification:
             [Op.in]: allCourseCodes,
           },
           is_open: getIsOpen(unification),
+          credittypecode: { [Op.not]: CreditTypeCode.IMPROVED }, // We do not care about improved grades
         },
         order: [['attainment_date', 'ASC']],
         include: [
@@ -142,7 +144,7 @@ export const getEnrollmentsForCourses = async (codeGroups: string[][], unificati
     include: [
       {
         model: EnrollmentModel,
-        attributes: ['studentnumber', 'enrollment_date_time', 'course_code'],
+        attributes: ['studentnumber', 'enrollment_date_time', 'course_code', 'studyright_id'],
         include: [
           {
             model: SemesterModel,
@@ -158,7 +160,8 @@ export const getEnrollmentsForCourses = async (codeGroups: string[][], unificati
           course_code: {
             [Op.in]: allCourseCodes,
           },
-          enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
+          // Date when OK changed from Oodi to Sisu data, studyright_id is null before that date
+          enrollment_date_time: { [Op.gte]: enrollmentTimeDateThreshold },
           state: EnrollmentState.ENROLLED,
           is_open: getIsOpen(unification),
         },

services/backend/src/services/courses/index.ts

@@ -3,10 +3,12 @@ import { Op, fn as dbFn, col as dbCol } from 'sequelize'
 
 import { Credit, Enrollment } from '@oodikone/shared/models'
 import { Name, EnrollmentState, Unification } from '@oodikone/shared/types'
+import { enrollmentTimeDateThreshold, getSemesterCodeAt } from '@oodikone/shared/util'
 import { dateIsBetween } from '@oodikone/shared/util/datetime'
 import logger from '../../../src/util/logger'
 import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, SISStudyRightElementModel } from '../../models'
 import { isOpenUniCourseCode } from '../../util'
+import { getSemestersAndYears, SemestersAndYears } from '../semesters'
 import { CourseYearlyStatsCounter } from './courseYearlyStatsCounter'
 import {
   getCreditsForCourses,
@@ -52,26 +54,25 @@ const parseCredit = (
   studyRightElements: Array<SISStudyRightElementModel>
 ): FormattedCredit => {
   const singleCourse = isSingleCourse(creditGroup)
-  // Latest attainment for a course
-  const credit =
+
+  // TODO: Calculate substitutions correctly once course_unit_realisation hits
+  const attainment =
     creditGroup
-      .sort((a, b) => b.attainment_date.getTime() - a.attainment_date.getTime())
-      .find(credit => !CreditModel.failed(credit)) ?? creditGroup.at(0)!
+      .sort((a, b) => b.attainment_date.getTime() - a.attainment_date.getTime()) // desc
+      .find(credit => CreditModel.passed(credit)) ??
+    creditGroup.find(credit => CreditModel.failed(credit)) ??
+    creditGroup.at(0)!
 
-  const courseCode = singleCourse ? credit.course_code : mainCourseCode
-  const grade = singleCourse ? credit.grade : 'substituted'
-  const credits = singleCourse ? credit.credits : creditGroup.reduce((acc, credit) => acc + credit.credits, 0)
+  const courseCode = singleCourse ? attainment.course_code : mainCourseCode
+  const grade = singleCourse ? attainment.grade : 'substituted'
+  const credits = singleCourse ? attainment.credits : creditGroup.reduce((acc, credit) => acc + credit.credits, 0)
 
-  // Take the first attainment for the course (see Student statistics info-box)
-  // TODO: Substitution groups should be marked for the semester when
-  // the group was first finished eg. when was the first time the last missing
-  // course of the group was passed
   const {
     semester,
     attainment_date: attainmentDate,
     student_studentnumber: studentNumber,
     studyright_id: studyRightId,
-  } = creditGroup.at(-1)!
+  } = attainment
 
   const { yearcode: yearCode, yearname: yearName, semestercode: semesterCode, name: semesterName } = semester
 
@@ -99,7 +100,7 @@ const parseCredit = (
     attainmentDate,
     courseCode,
     grade,
-    passed: singleCourse ? !CreditModel.failed(credit) : true,
+    passed: singleCourse ? !CreditModel.failed(attainment) : true,
     studentNumber: anonymizationSalt ? anonymizeStudentNumber(studentNumber, anonymizationSalt) : studentNumber,
     programme,
     credits,
@@ -107,31 +108,44 @@ const parseCredit = (
 }
 
 type FormattedEnrollment = {
-  yearCode: number
-  yearName: string
-  semesterCode: number
-  semesterName: Name
   courseCode: string
   enrollmentDateTime: Date
+  programme: FormattedProgramme
   studentNumber: string
 }
 
-const parseEnrollment = (enrollment: Enrollment, anonymizationSalt: string | null): FormattedEnrollment => {
+const parseEnrollment = (
+  enrollment: Enrollment,
+  anonymizationSalt: string | null,
+  studyRightElements: Array<SISStudyRightElementModel>
+): FormattedEnrollment => {
   const {
     studentnumber: studentNumber,
-    semester,
     enrollment_date_time: enrollmentDateTime,
     course_code: courseCode,
+    studyright_id: studyRightId,
   } = enrollment
-  const { yearcode: yearCode, yearname: yearName, semestercode: semesterCode, name: semesterName } = semester
+
+  const programmeOfEnrollment: SISStudyRightElementModel | undefined =
+    studyRightElements.find(studyRightElement => studyRightElement.studyRightId === studyRightId) ??
+    studyRightElements
+      .filter(({ startDate, endDate }) => dateIsBetween(enrollmentDateTime, startDate, endDate))
+      .sort((a, b) => b.startDate.getTime() - a.startDate.getTime())
+      .at(0) // The newest studyRightElement
+
+  const programme = programmeOfEnrollment
+    ? formatStudyRightElement(programmeOfEnrollment)
+    : {
+        code: 'OTHER',
+        name: { en: 'Other', fi: 'Muu', sv: 'Andra' },
+        facultyCode: 'OTHER',
+        organization: { name: { en: 'Other', fi: 'Muu', sv: 'Andra' } },
+      }
 
   return {
-    yearCode,
-    yearName,
-    semesterCode,
-    semesterName,
     courseCode,
     enrollmentDateTime,
+    programme,
     studentNumber: anonymizationSalt ? anonymizeStudentNumber(studentNumber, anonymizationSalt) : studentNumber,
   }
 }
@@ -150,6 +164,19 @@ const getSubstitutionGroupDetails = async (codeGroups: string[][]) => {
   )
 }
 
+const getSemesterAndYearByDate = (
+  date: Date,
+  semesters: SemestersAndYears['semesters'],
+  years: SemestersAndYears['years']
+) => {
+  const semesterCode = getSemesterCodeAt(semesters, date) ?? getSemesterCodeAt(semesters, new Date())!
+
+  const semester = semesters[semesterCode]
+  const year = years[semester.yearcode]
+
+  return { year, semester }
+}
+
 const getYearlyStatsOfNew = async (
   course: Pick<CourseModel, 'code' | 'name' | 'substitution_groups'> | null,
   courseCode: string,
@@ -179,6 +206,8 @@ const getYearlyStatsOfNew = async (
       break
   }
 
+  const { semesters, years } = await getSemestersAndYears()
+
   const [creditGroups, enrollmentGroups] = await Promise.all([
     getCreditsForCourses(filteredCreditGroupCodes, unification),
     getEnrollmentsForCourses(filteredCreditGroupCodes, unification),
@@ -219,6 +248,7 @@ const getYearlyStatsOfNew = async (
       programme.organization
     )
 
+    // Credits/attainments have quaranteed matching attainment_date and semesters/years
     const groupCode = separate ? semesterCode : yearCode
     const groupName = separate ? semesterName : yearName
     counter.markCreditToGroup(studentNumber, passed, grade, groupCode, groupName, creditCourseCode, yearCode)
@@ -231,16 +261,32 @@ const getYearlyStatsOfNew = async (
     for (const enrollment of enrollments) {
       const {
         studentNumber,
-        semesterCode,
-        semesterName,
-        yearCode,
-        yearName,
         courseCode: enrollmentCourseCode,
         enrollmentDateTime,
-      } = parseEnrollment(enrollment, anonymizationSalt)
+        programme,
+      } = parseEnrollment(
+        enrollment,
+        anonymizationSalt,
+        studentNumberToSrElementsMap[enrollment.studentnumber ?? 0] ?? []
+      )
 
-      const groupCode = separate ? semesterCode : yearCode
-      const groupName = separate ? semesterName : yearName
+      // Enrollments can have conflicting enrollment_date_time and semestercode, so we need to manually select
+      // semestercode matching enrollment_date_time
+      const { semester, year } = getSemesterAndYearByDate(enrollment.enrollment_date_time, semesters, years)
+
+      counter.markStudyProgramme(
+        studentNumber,
+        year.yearcode,
+        false, // passed
+        0, // credits
+        programme.code,
+        programme.name,
+        programme.facultyCode,
+        programme.organization
+      )
+
+      const groupCode = separate ? semester.semestercode : year.yearcode
+      const groupName = separate ? semester.name : year.yearname
 
       counter.markEnrollmentToGroup(
         studentNumber,
@@ -248,7 +294,7 @@ const getYearlyStatsOfNew = async (
         groupCode,
         groupName,
         enrollmentCourseCode,
-        yearCode
+        year.yearcode
       )
     }
   }
@@ -268,6 +314,11 @@ const getYearlyStatsOfNew = async (
   }
 }
 
+/**
+  Mostly-legacy function that prevents user from opening too big of a populations.
+  At the moment the threshold to prevent a population is so high it will never trigger.
+
+ */
 export const maxYearsToCreatePopulationFrom = async (courseCodes: string[], unification: Unification) => {
   const lastAttainmentDate = (await CourseModel.findOne({
     attributes: [[dbFn('MAX', dbCol('max_attainment_date')), 'date']],
@@ -291,7 +342,7 @@ export const maxYearsToCreatePopulationFrom = async (courseCodes: string[], unif
   })
 
   // MAGIC NUMBER
-  const maxAllowedAttainments = 1000000 // * Lower this value to get a smaller result if necessary
+  const maxAllowedAttainments = 1_000_000 // * Lower this value to get a smaller result if necessary
   return Math.max(1, maxAllowedAttainments / attainmentsWithinThreshold)
 }
 
@@ -313,6 +364,7 @@ export const getCourseYearlyStats = async (
           [Op.in]: courseCodes,
         },
         state: EnrollmentState.ENROLLED,
+        enrollment_date_time: { [Op.gte]: enrollmentTimeDateThreshold },
       },
     }),
   ])
@@ -334,11 +386,12 @@ export const getCourseYearlyStats = async (
       const course: Pick<CourseModel, 'code' | 'name' | 'substitution_groups'> | null = await CourseModel.findOne({
         attributes: ['code', 'name', 'substitution_groups'],
         where: { code: courseCode },
+        raw: true,
       })
 
       if (!course) {
         logger.error('Course for course stats not found with code' + courseCode)
-        return []
+        return {}
       }
 
       const [openStats, regularStats, unifyStats] = await Promise.all([
@@ -378,8 +431,8 @@ export const getCourseYearlyStats = async (
   return statsRegular
 }
 
-export const getCourseProvidersForCourses = async (codes: string[]) => {
-  return (
+export const getCourseProvidersForCourses = async (codes: string[]) =>
+  (
     await OrganizationModel.findAll({
       attributes: ['code'],
       include: {
@@ -393,7 +446,6 @@ export const getCourseProvidersForCourses = async (codes: string[]) => {
       raw: true,
     })
   ).map(({ code }) => code)
-}
 
 export const getCourseDetails = async (codes: string[]) =>
   CourseModel.findAll({

services/backend/src/services/faculty/facultyGraduationTimes.ts

@@ -1,36 +1,77 @@
-import { cloneDeep } from 'lodash-es'
-
-import { Name, DegreeProgrammeType, Unarray } from '@oodikone/shared/types'
-import { ProgrammeGraduationStats } from '@oodikone/shared/types/studyProgramme'
+import { Name, DegreeProgrammeType, FacultyGraduationStatistics, ClassSizes } from '@oodikone/shared/types'
+import type {
+  GraduationStatistics,
+  GraduationStats,
+  MediansByCategory,
+  MediansByProgrammes,
+  ProgrammeGraduationStats,
+  ProgrammeMedians,
+} from '@oodikone/shared/types'
 import { omitKeys } from '@oodikone/shared/util'
 import { getGraduationStats, getStudyTrackStats, setGraduationStats, setStudyTrackStats } from '../analyticsService'
 import { GraduationTarget } from '../graduationHelpers'
-import { getGraduationStatsForStudyTrack, GraduationTimes } from '../studyProgramme/studyProgrammeGraduations'
-import { getMedian } from '../studyProgramme/studyProgrammeHelpers'
+import { getGraduationStatsForStudyTrack } from '../studyProgramme/studyProgrammeGraduations'
+import { getAverage, getMedian } from '../studyProgramme/studyProgrammeHelpers'
 import { getStudyRightsInProgramme } from '../studyProgramme/studyRightFinders'
 import { getStudyTrackStatsForStudyProgramme } from '../studyProgramme/studyTrackStats'
 import type { ProgrammesOfOrganization } from './faculty'
 
-export type LevelGraduationStats = Omit<Unarray<GraduationTimes['medians']>, 'y'> & { median: number }
-
-type ProgrammeStats = {
-  data: Array<Omit<LevelGraduationStats, 'times'> & { code: string }>
-  programmes: string[]
-}
-
 export const programmeTypes = {
   [DegreeProgrammeType.BACHELOR]: 'bachelor',
   [DegreeProgrammeType.MASTER]: 'master',
   [DegreeProgrammeType.DOCTOR]: 'doctor',
 } as const
 
-const getStatsByGraduationYear = async (facultyProgrammes: ProgrammesOfOrganization) => {
-  const newStats: ProgrammeGraduationStats[] = []
-  const medians: Record<string, LevelGraduationStats[]> = {}
-  const programmes: { medians: Record<string, Record<string, ProgrammeStats>> } = { medians: {} }
+const setYearlyBreakdown = (code: string, name: string, level: ProgrammeMedians, year: GraduationStats) => {
+  if (!year.times.length) return
+
+  level[year.name] ??= { data: [], programmes: [] }
+  level[year.name].programmes.push(code)
+  level[year.name].data.push(omitKeys({ ...structuredClone(year), name, code }, ['classSize']))
+}
+
+const mergeStats = (
+  medians: MediansByCategory,
+  level: keyof MediansByCategory,
+  statsForYear: GraduationStats,
+  classSizes?: ClassSizes
+) => {
+  if (classSizes) {
+    classSizes[level][statsForYear.name] ??= 0
+    classSizes[level][statsForYear.name] += statsForYear.classSize ?? 0
+  }
+
+  const gradStatsForYear = medians[level].find(entry => entry.name === statsForYear.name)
+
+  if (!gradStatsForYear) {
+    medians[level].push(structuredClone(statsForYear))
+  } else {
+    gradStatsForYear.times.push(...statsForYear.times)
+    gradStatsForYear.median = getMedian(gradStatsForYear.times)
+    gradStatsForYear.average = getAverage(gradStatsForYear.times)
+    gradStatsForYear.amount += statsForYear.amount
+    gradStatsForYear.statistics.onTime += statsForYear.statistics.onTime
+    gradStatsForYear.statistics.yearOver += statsForYear.statistics.yearOver
+    gradStatsForYear.statistics.wayOver += statsForYear.statistics.wayOver
+  }
+}
+
+const getStatsByGraduationYear = async (
+  facultyProgrammes: ProgrammesOfOrganization
+): Promise<GraduationStatistics['byGradYear']> => {
+  const newStats: Pick<ProgrammeGraduationStats, 'id' | 'graduationTimes' | 'comboTimes'>[] = []
+  const medians: MediansByCategory = { bachelor: [], master: [], bcMsCombo: [], doctor: [] }
+  const programmes: { medians: MediansByProgrammes } = {
+    medians: {
+      bachelor: {},
+      master: {},
+      bcMsCombo: {},
+      doctor: {},
+    },
+  }
 
   for (const programme of facultyProgrammes) {
-    if (programme.degreeProgrammeType == null || !(programme.degreeProgrammeType in programmeTypes)) {
+    if (!programme.degreeProgrammeType || !(programme.degreeProgrammeType in programmeTypes)) {
       continue
     }
     const statsFromRedis = await getGraduationStats(programme.code, '', 'CALENDAR_YEAR', 'SPECIAL_INCLUDED')
@@ -51,92 +92,44 @@ const getStatsByGraduationYear = async (facultyProgrammes: ProgrammesOfOrganizat
   }
 
   for (const { id: programmeCode, graduationTimes, comboTimes } of newStats) {
-    const { degreeProgrammeType, progId, code } = facultyProgrammes.find(({ code }) => code === programmeCode)!
+    const { degreeProgrammeType, progId } = facultyProgrammes.find(({ code }) => code === programmeCode)!
     const level = programmeTypes[degreeProgrammeType as keyof typeof programmeTypes]
 
-    if (!medians[level]) {
-      medians[level] = cloneDeep(graduationTimes.medians).map(year => ({ ...omitKeys(year, ['y']), median: year.y }))
-    } else {
-      for (const statsForYear of graduationTimes.medians) {
-        const correctYear = medians[level].find(entry => entry.name === statsForYear.name)
-        if (!correctYear) {
-          medians[level].push({ ...omitKeys(cloneDeep(statsForYear), ['y']), median: statsForYear.y })
-          continue
-        }
-        correctYear.times.push(...statsForYear.times)
-        correctYear.median = getMedian(correctYear.times)
-        correctYear.amount += statsForYear.amount
-        correctYear.statistics.onTime += statsForYear.statistics.onTime
-        correctYear.statistics.yearOver += statsForYear.statistics.yearOver
-        correctYear.statistics.wayOver += statsForYear.statistics.wayOver
-      }
-    }
-
-    if (level === 'master' && comboTimes.medians.length) {
-      if (!medians.bcMsCombo) {
-        medians.bcMsCombo = cloneDeep(comboTimes.medians).map(year => ({ ...omitKeys(year, ['y']), median: year.y }))
-      } else {
-        for (const statsForYear of comboTimes.medians) {
-          const correctYear = medians.bcMsCombo.find(entry => entry.name === statsForYear.name)
-          if (!correctYear) {
-            medians.bcMsCombo.push({ ...omitKeys(cloneDeep(statsForYear), ['y']), median: statsForYear.y })
-            continue
-          }
-          correctYear.times.push(...statsForYear.times)
-          correctYear.median = getMedian(correctYear.times)
-          correctYear.amount += statsForYear.amount
-          correctYear.statistics.onTime += statsForYear.statistics.onTime
-          correctYear.statistics.yearOver += statsForYear.statistics.yearOver
-          correctYear.statistics.wayOver += statsForYear.statistics.wayOver
-        }
-      }
-    }
-
-    if (!programmes.medians[level]) {
-      programmes.medians[level] = {}
-    }
-
     for (const year of graduationTimes.medians) {
-      if (year.y === 0) continue
-      if (!programmes.medians[level][year.name]) {
-        programmes.medians[level][year.name] = { data: [], programmes: [] }
-      }
-      programmes.medians[level][year.name].data.push(
-        omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y'])
-      )
-      programmes.medians[level][year.name].programmes.push(code)
+      mergeStats(medians, level, year)
+      setYearlyBreakdown(programmeCode, progId, programmes.medians[level], year)
     }
 
     if (level === 'master' && comboTimes.medians.length) {
-      if (!programmes.medians.bcMsCombo) {
-        programmes.medians.bcMsCombo = {}
-      }
       for (const year of comboTimes.medians) {
-        if (year.y === 0) continue
-        if (!programmes.medians.bcMsCombo[year.name]) {
-          programmes.medians.bcMsCombo[year.name] = { data: [], programmes: [] }
-        }
-        programmes.medians.bcMsCombo[year.name].data.push(
-          omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y'])
-        )
-        programmes.medians.bcMsCombo[year.name].programmes.push(code)
+        mergeStats(medians, 'bcMsCombo', year)
+        setYearlyBreakdown(programmeCode, progId, programmes.medians.bcMsCombo, year)
       }
     }
   }
   return { medians, programmes }
 }
 
-const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization) => {
-  const newStats: Array<Awaited<ReturnType<typeof getStudyTrackStatsForStudyProgramme>>> = []
-  const medians: Record<string, LevelGraduationStats[]> = {}
-  const programmes: { medians: Record<string, Record<string, ProgrammeStats>> } = { medians: {} }
-  const classSizes: Record<string, Record<string, number | Record<string, number>>> = { programmes: {} }
+const getStatsByStartYear = async (
+  facultyProgrammes: ProgrammesOfOrganization
+): Promise<GraduationStatistics['byStartYear'] & Pick<GraduationStatistics, 'classSizes'>> => {
+  const classSizes: ClassSizes = { bachelor: {}, master: {}, bcMsCombo: {}, doctor: {}, programmes: {} }
+  const newStats: Awaited<ReturnType<typeof getStudyTrackStatsForStudyProgramme>>[] = []
+  const medians: MediansByCategory = { bachelor: [], master: [], bcMsCombo: [], doctor: [] }
+  const programmes: { medians: MediansByProgrammes } = {
+    medians: {
+      bachelor: {},
+      master: {},
+      bcMsCombo: {},
+      doctor: {},
+    },
+  }
 
   for (const programme of facultyProgrammes) {
-    if (programme.degreeProgrammeType == null || !(programme.degreeProgrammeType in programmeTypes)) {
+    if (!programme.degreeProgrammeType || !(programme.degreeProgrammeType in programmeTypes)) {
       continue
     }
-    const statsFromRedis = await getStudyTrackStats(programme.code, '', 'SPECIAL_EXCLUDED')
+    const statsFromRedis = await getStudyTrackStats(programme.code, null, 'SPECIAL_EXCLUDED')
     if (statsFromRedis) {
       newStats.push(statsFromRedis)
       continue
@@ -156,9 +149,13 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
   }
 
   for (const { id: programmeCode, graduationTimes } of newStats) {
-    const { degreeProgrammeType, progId, code } = facultyProgrammes.find(({ code }) => code === programmeCode)!
+    const { degreeProgrammeType, progId } = facultyProgrammes.find(({ code }) => code === programmeCode)!
     const level = programmeTypes[degreeProgrammeType as keyof typeof programmeTypes]
-    if (!(programmeCode in graduationTimes)) continue
+
+    if (!(programmeCode in graduationTimes)) {
+      continue
+    }
+
     const statsForProgramme = graduationTimes[programmeCode]
     const basicStats = statsForProgramme.medians.basic
     const comboStats = statsForProgramme.medians.combo
@@ -166,91 +163,31 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
       (acc, year) => ({ ...acc, [year.name]: year.classSize ?? 0 }),
       {}
     )
-    classSizes.programmes[code] = classSizesByYear
+    classSizes.programmes[programmeCode] = classSizesByYear
 
-    if (!medians[level]) {
-      medians[level] = cloneDeep(basicStats).map(year => ({ ...omitKeys(year, ['y', 'classSize']), median: year.y }))
-    }
-    classSizes[level] ??= {}
     for (const statsForYear of basicStats) {
-      classSizes[level][statsForYear.name] ??= 0
-      ;(classSizes[level][statsForYear.name] as number) += statsForYear.classSize ?? 0
-
-      const correctYear = medians[level].find(entry => entry.name === statsForYear.name)
-      if (!correctYear) {
-        medians[level].push({ ...omitKeys(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
-        continue
-      }
-      correctYear.times.push(...statsForYear.times)
-      correctYear.median = getMedian(correctYear.times)
-      correctYear.amount += statsForYear.amount
-      correctYear.statistics.onTime += statsForYear.statistics.onTime
-      correctYear.statistics.yearOver += statsForYear.statistics.yearOver
-      correctYear.statistics.wayOver += statsForYear.statistics.wayOver
+      mergeStats(medians, level, statsForYear, classSizes)
+      setYearlyBreakdown(programmeCode, progId, programmes.medians[level], statsForYear)
     }
 
     if (level === 'master' && comboStats.length) {
-      if (!medians.bcMsCombo) {
-        medians.bcMsCombo = cloneDeep(comboStats).map(year => ({
-          ...omitKeys(year, ['y', 'classSize']),
-          median: year.y,
-        }))
-      }
-      classSizes.bcMsCombo ??= {}
       for (const statsForYear of comboStats) {
-        classSizes.bcMsCombo[statsForYear.name] ??= 0
-        ;(classSizes.bcMsCombo[statsForYear.name] as number) += statsForYear.classSize ?? 0
-        const correctYear = medians.bcMsCombo.find(entry => entry.name === statsForYear.name)
-        if (!correctYear) {
-          medians.bcMsCombo.push({ ...omitKeys(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
-          continue
-        }
-        correctYear.times.push(...statsForYear.times)
-        correctYear.median = getMedian(correctYear.times)
-        correctYear.amount += statsForYear.amount
-        correctYear.statistics.onTime += statsForYear.statistics.onTime
-        correctYear.statistics.yearOver += statsForYear.statistics.yearOver
-        correctYear.statistics.wayOver += statsForYear.statistics.wayOver
-      }
-    }
-
-    if (!programmes.medians[level]) {
-      programmes.medians[level] = {}
-    }
-
-    for (const year of basicStats) {
-      if (year.y === 0) continue
-      if (!programmes.medians[level][year.name]) {
-        programmes.medians[level][year.name] = { data: [], programmes: [] }
-      }
-      programmes.medians[level][year.name].data.push(
-        omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y', 'classSize'])
-      )
-      programmes.medians[level][year.name].programmes.push(code)
-    }
-
-    if (level === 'master' && comboStats.length) {
-      if (!programmes.medians.bcMsCombo) {
-        programmes.medians.bcMsCombo = {}
-      }
-      for (const year of comboStats) {
-        if (year.y === 0) continue
-        if (!programmes.medians.bcMsCombo[year.name]) {
-          programmes.medians.bcMsCombo[year.name] = { data: [], programmes: [] }
-        }
-        programmes.medians.bcMsCombo[year.name].data.push(
-          omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y', 'classSize'])
-        )
-        programmes.medians.bcMsCombo[year.name].programmes.push(code)
+        mergeStats(medians, 'bcMsCombo', statsForYear, classSizes)
+        setYearlyBreakdown(programmeCode, progId, programmes.medians.bcMsCombo, statsForYear)
       }
     }
   }
+
   return { medians, programmes, classSizes }
 }
 
-export const countGraduationTimes = async (faculty: string, programmesOfFaculty: ProgrammesOfOrganization) => {
+export const countGraduationTimes = async (
+  facultyCode: string,
+  programmesOfFaculty: ProgrammesOfOrganization
+): Promise<FacultyGraduationStatistics> => {
   const { medians: mediansByGraduationYear, programmes: programmesByGraduationYear } =
     await getStatsByGraduationYear(programmesOfFaculty)
+
   const {
     medians: mediansByStartYear,
     programmes: programmesByStartYear,
@@ -259,7 +196,7 @@ export const countGraduationTimes = async (faculty: string, programmesOfFaculty:
 
   const goals = {
     bachelor: GraduationTarget.THREE_YEARS,
-    bcMsCombo: faculty === 'H90' ? GraduationTarget.SIX_YEARS : GraduationTarget.FIVE_YEARS,
+    bcMsCombo: facultyCode === 'H90' ? GraduationTarget.SIX_YEARS : GraduationTarget.FIVE_YEARS,
     master: GraduationTarget.TWO_YEARS,
     doctor: GraduationTarget.FOUR_YEARS,
     exceptions: {
@@ -283,7 +220,7 @@ export const countGraduationTimes = async (faculty: string, programmesOfFaculty:
   }, {})
 
   return {
-    id: faculty,
+    id: facultyCode,
     goals,
     byGradYear: {
       medians: mediansByGraduationYear,

services/backend/src/services/faculty/facultyService.ts

@@ -1,4 +1,4 @@
-import { Name, ProgrammeFilter, SpecialGroups, YearType } from '@oodikone/shared/types'
+import { FacultyGraduationStatistics, Name, ProgrammeFilter, SpecialGroups, YearType } from '@oodikone/shared/types'
 import { DegreeProgramme } from '../../routes/faculties'
 import { redisClient } from '../redis'
 import { FacultyProgressData } from './facultyStudentProgress'
@@ -29,16 +29,16 @@ const createRedisKeyForFacultyStudents = (id: string, specialGroups: SpecialGrou
   return `FACULTY_STUDENTS_STATS_${id}_${specialGroups}`
 }
 
-/*
+/**
   Faculty data objects have graduation times left in as arrays, so that
-  university-level evaluation overview can count median-times by itself.
+  university-level evaluation overview can count median-times by itself (in backend).
   Faculties, however, don't need this, and it isn't needed in frontend
-  for universityview either. This removes them from the object.
+  for university view either. This clears the array.
 */
-const removeGraduationTimes = (data: GraduationData) => {
+const removeGraduationTimes = (data: FacultyGraduationStatistics) => {
   Object.values(data.byGradYear.medians).forEach(array =>
     array.forEach(yearStat => {
-      yearStat.times = null
+      yearStat.times = []
     })
   )
 }
@@ -158,7 +158,7 @@ export type GraduationData = {
   classSizes: Record<string, Record<string, number | Record<string, number>>>
 }
 
-export const setGraduationStats = async (data: GraduationData, programmeFilter: ProgrammeFilter) => {
+export const setGraduationStats = async (data: FacultyGraduationStatistics, programmeFilter: ProgrammeFilter) => {
   const { id } = data
   const redisKey = createRedisKeyForGraduationTimeStats(id, programmeFilter)
   await redisClient.set(redisKey, JSON.stringify(data))
@@ -170,7 +170,7 @@ export const getGraduationStats = async (id: string, programmeFilter: ProgrammeF
   if (!dataFromRedis) {
     return null
   }
-  const data = JSON.parse(dataFromRedis) as GraduationData
+  const data = JSON.parse(dataFromRedis) as FacultyGraduationStatistics
   if (!keepGraduationTimes) {
     removeGraduationTimes(data)
   }

services/backend/src/services/faculty/facultyStudents.ts

@@ -48,9 +48,7 @@ const calculateCombinedStats = (programmeCodes: string[], stats: StudyTrackStats
 
     if (!programmeOtherCountriesCount) continue
     for (const [year, countryStats] of Object.entries(programmeOtherCountriesCount)) {
-      if (!(year in facultyTableStatsExtra)) {
-        facultyTableStatsExtra[year] = {}
-      }
+      facultyTableStatsExtra[year] ??= {}
       facultyTableStatsExtra[year][programmeCode] = countryStats
     }
   }

services/backend/src/services/graduationHelpers.ts

@@ -8,7 +8,7 @@ export const GraduationTarget = {
   FOUR_YEARS: 8,
   FIVE_YEARS: 10,
   SIX_YEARS: 12,
-} as const
+}
 
 /** Uses semesters, not months */
 export const countTimeCategories = (times: number[], target: number) => {

services/backend/src/services/languageCenterData.ts

@@ -1,16 +1,18 @@
 import { orderBy } from 'lodash-es'
 import { Op } from 'sequelize'
 
+import { EnrollmentState } from '@oodikone/shared/types'
 import { CourseModel, CreditModel, EnrollmentModel, SISStudyRightModel } from '../models'
 import { redisClient } from './redis'
 
 export const LANGUAGE_CENTER_REDIS_KEY = 'LANGUAGE_CENTER_DATA'
 
-const isBetween = (start, date, end) => {
+const isBetween = (start: Date, date: Date, end: Date) => {
   return new Date(start).getTime() <= new Date(date).getTime() && new Date(date).getTime() <= new Date(end).getTime()
 }
 
-const findStudyRight = (studyRights, date) => {
+/** Assumes studyrights are in correct order */
+const findStudyRight = (studyRights: SISStudyRightModel[], date: Date) => {
   return studyRights?.find(studyRight => isBetween(studyRight.startDate, date, studyRight.endDate))
 }
 
@@ -23,7 +25,7 @@ const getDifference = stats => {
 }
 
 const getLanguageCenterCourses = async () => {
-  const courses = await CourseModel.findAll({
+  const courses: Pick<CourseModel, 'code' | 'name'>[] = await CourseModel.findAll({
     attributes: ['code', 'name'],
     where: {
       [Op.or]: [{ code: { [Op.like]: 'KK%' } }, { code: { [Op.like]: 'AYKK%' } }],
@@ -33,29 +35,37 @@ const getLanguageCenterCourses = async () => {
   return courses
 }
 
-export const createArrayOfCourses = async (attempts, courses) => {
+export const createArrayOfCourses = (attempts: Attempt[], courses: Pick<CourseModel, 'code' | 'name'>[]) => {
   const fields = { completions: 0, enrollments: 0, difference: 0, rejected: 0 }
-  const semesters = {}
-  const faculties = {}
-  const semesterStatsMap = attempts.reduce((obj, cur) => {
+  const semesters: Record<Attempt['semestercode'], boolean> = {}
+  // Remove undefined as a possibility for type
+  const faculties: Record<Exclude<Attempt['faculty'], undefined>, boolean> = {}
+
+  // TODO: Type this properly. Good luck :)
+  const semesterStatsMap = attempts.reduce((acc, cur) => {
     const semester = cur.semestercode
-    faculties[cur.faculty] = true
-    semesters[semester] = true
-    if (!obj[cur.courseCode]) {
-      obj[cur.courseCode] = {}
-    }
-    if (!obj[cur.courseCode][semester]) {
-      obj[cur.courseCode][semester] = { ...fields }
-    }
-    if (cur.faculty && !obj[cur.courseCode][semester][cur.faculty]) {
-      obj[cur.courseCode][semester][cur.faculty] = { ...fields }
-    }
-    const allFacultiesTotal = obj[cur.courseCode][semester]
-    const allStats = [allFacultiesTotal]
     if (cur.faculty) {
-      const semesterFacultyStats = obj[cur.courseCode][semester][cur.faculty]
+      faculties[cur.faculty] = true
+    }
+    semesters[semester] = true
+
+    acc[cur.courseCode] ??= {}
+    // NOTE: This has to be spread because otherwise a ref to fields is saved to
+    // acc[cur.courseCode][semester] which would in the end result in a massive circular object
+    acc[cur.courseCode][semester] ??= { ...fields }
+
+    if (cur.faculty) {
+      acc[cur.courseCode][semester][cur.faculty] ??= { ...fields }
+    }
+
+    const allFacultiesTotal = acc[cur.courseCode][semester]
+    const allStats = [allFacultiesTotal]
+
+    if (cur.faculty) {
+      const semesterFacultyStats = acc[cur.courseCode][semester][cur.faculty]
       allStats.push(semesterFacultyStats)
     }
+
     for (const stats of allStats) {
       if (cur.completed) {
         stats.completions += 1
@@ -65,7 +75,7 @@ export const createArrayOfCourses = async (attempts, courses) => {
         stats.rejected += 1
       }
     }
-    return obj
+    return acc
   }, {})
 
   const courseList = courses
@@ -88,55 +98,60 @@ export const createArrayOfCourses = async (attempts, courses) => {
   return courseList
 }
 
+type Attempt = {
+  studentNumber: string
+  courseCode: string
+  completed: boolean
+  date: Date
+  semestercode: number
+  faculty?: string
+  enrolled?: boolean
+}
+
 export const computeLanguageCenterData = async () => {
   const courses = await getLanguageCenterCourses()
   const autumnSemester2017 = 135
 
   const credits = await CreditModel.findAll({
+    raw: true,
     attributes: ['course_code', 'student_studentnumber', 'semestercode', 'attainment_date', 'studyright_id'],
     where: {
       [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
       semestercode: { [Op.gte]: autumnSemester2017 },
       credittypecode: 4,
     },
-    raw: true,
   })
 
   const enrollments = await EnrollmentModel.findAll({
+    raw: true,
     attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'studyright_id', 'state'],
     where: {
       [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
-      semestercode: { [Op.gte]: autumnSemester2017 },
       state: { [Op.in]: ['ENROLLED', 'REJECTED'] },
+      // EnrollmentDateTimeThreshold need not be used here as we are not counting failed courses
     },
-    raw: true,
   })
 
-  const studyRights = await SISStudyRightModel.findAll({ raw: true })
+  const studyRights = await SISStudyRightModel.findAll({
+    raw: true,
+    order: [['end_date', 'DESC NULLS LAST']],
+  })
 
-  const attemptStudyRightToFacultyMap = studyRights.reduce((obj, cur) => {
-    if (!cur.id) return obj
-    obj[cur.id] = cur.facultyCode
-    return obj
+  const attemptStudyRightToFacultyMap = studyRights.reduce<
+    Record<SISStudyRightModel['id'], SISStudyRightModel['facultyCode']>
+  >((acc, cur) => {
+    if (!cur.id) return acc
+    acc[cur.id] = cur.facultyCode
+    return acc
   }, {})
 
-  credits.forEach(credit => {
-    credit.faculty = attemptStudyRightToFacultyMap[credit.studyright_id]
-  })
-
-  enrollments.forEach(enrollment => {
-    enrollment.faculty = attemptStudyRightToFacultyMap[enrollment.studyright_id]
-  })
-
-  const studentNumbers = new Set()
-  const attemptsByStudents = {}
+  const studentNumbers = new Set<string>()
+  const attemptsByStudents: Record<string, Attempt[]> = {}
 
   credits.forEach(credit => {
     const studentNumber = credit.student_studentnumber
     studentNumbers.add(studentNumber)
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
+    attemptsByStudents[studentNumber] ??= []
     attemptsByStudents[studentNumber].push({
       studentNumber,
       courseCode: credit.course_code,
@@ -149,10 +164,8 @@ export const computeLanguageCenterData = async () => {
 
   enrollments.forEach(enrollment => {
     const studentNumber = enrollment.studentnumber
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
     studentNumbers.add(studentNumber)
+    attemptsByStudents[studentNumber] ??= []
     if (
       attemptsByStudents[studentNumber].find(
         attempt =>
@@ -163,6 +176,7 @@ export const computeLanguageCenterData = async () => {
     ) {
       return
     }
+
     attemptsByStudents[studentNumber].push({
       studentNumber,
       courseCode: enrollment.course_code,
@@ -170,23 +184,25 @@ export const computeLanguageCenterData = async () => {
       date: enrollment.enrollment_date_time,
       faculty: attemptStudyRightToFacultyMap[enrollment.studyright_id],
       semestercode: enrollment.semestercode,
-      enrolled: enrollment.state === 'ENROLLED',
+      enrolled: enrollment.state === EnrollmentState.ENROLLED,
     })
   })
 
-  const studentNumberToStudyRightsMap = studyRights.reduce((obj, cur) => {
-    if (!obj[cur.studentNumber]) obj[cur.studentNumber] = []
-    obj[cur.studentNumber].push(cur)
-    return obj
+  const studentNumberToStudyRightsMap = studyRights.reduce<
+    Record<SISStudyRightModel['studentNumber'], SISStudyRightModel[]>
+  >((acc, cur) => {
+    acc[cur.studentNumber] ??= []
+    acc[cur.studentNumber].push(cur)
+    return acc
   }, {})
 
-  const attempts = []
+  const attempts: Attempt[] = []
   studentNumbers.forEach(studentNumber => attempts.push(...attemptsByStudents[studentNumber]))
   // 93033 Avoin yliopisto, yhteistyöoppilaitokset
   // 93034 Kesäyliopistot
-  const isOpenUni = facultyCode => ['9301', 'H930', '93033', '93034'].includes(facultyCode)
-  const isMisc = facultyCode => ['H906', 'H401'].includes(facultyCode) // Language center, Alexander institute (very few numbers)
-  const isHyFaculty = facultyCode => facultyCode.substring(0, 3).match('^H\\d')
+  const isOpenUni = (facultyCode: string) => ['9301', 'H930', '93033', '93034'].includes(facultyCode)
+  const isMisc = (facultyCode: string) => ['H906', 'H401'].includes(facultyCode) // Language center, Alexander institute (very few numbers)
+  const isHyFaculty = (facultyCode: string) => /^H\d/.exec(facultyCode.substring(0, 3))
 
   attempts.forEach(attempt => {
     if (attempt.faculty) {
@@ -210,7 +226,7 @@ export const computeLanguageCenterData = async () => {
     }
   })
 
-  const unorderedTableData = await createArrayOfCourses(attempts, courses)
+  const unorderedTableData = createArrayOfCourses(attempts, courses)
   const tableData = orderBy(unorderedTableData, 'code')
 
   const faculties = [...new Set(attempts.map(({ faculty }) => faculty))]

services/backend/src/services/openUni/format.ts

@@ -1,33 +0,0 @@
-import { CourseModel, CreditModel, EnrollmentModel, StudentModel } from '../../models'
-
-export const formatCourseInfo = (course: CourseModel) => {
-  return {
-    label: course.code,
-    name: course.name,
-  }
-}
-
-export const formatOpenCredits = (credit: CreditModel) => {
-  return {
-    attainmentCourseCode: credit.course_code,
-    attainmentDate: credit.attainment_date,
-    attainmentGrade: credit.grade,
-    attainmentStudentNumber: credit.student_studentnumber,
-  }
-}
-
-export const formatOpenEnrollments = (enrollment: EnrollmentModel) => {
-  return {
-    enrollmentCourseCode: enrollment.course_code,
-    enrollmentDateTime: enrollment.enrollment_date_time,
-    enrollmentStudentNumber: enrollment.studentnumber,
-  }
-}
-
-export const formatStudentInfo = (student: StudentModel) => {
-  return {
-    studentNumber: student.studentnumber,
-    email: student.email,
-    secondaryEmail: student.secondary_email,
-  }
-}

services/backend/src/services/openUni/openUniSearches.ts

@@ -1,94 +1,4 @@
-import { Op, WhereOptions } from 'sequelize'
-
-import { SISStudyRight } from '@oodikone/shared/models'
-import { EnrollmentState, ExtentCode } from '@oodikone/shared/types'
-import { CourseModel, CreditModel, EnrollmentModel, StudentModel, SISStudyRightModel } from '../../models'
 import { OpenUniPopulationSearchModel } from '../../models/kone'
-import { formatCourseInfo, formatOpenCredits, formatOpenEnrollments, formatStudentInfo } from './format'
-
-export const getCredits = async (courseCodes: string[], startdate: Date) =>
-  (
-    await CreditModel.findAll({
-      attributes: ['attainment_date', 'course_code', 'grade', 'student_studentnumber'],
-      where: {
-        course_code: {
-          [Op.in]: courseCodes,
-        },
-        attainment_date: {
-          [Op.gte]: startdate,
-        },
-      },
-    })
-  ).map(formatOpenCredits)
-
-export const getStudentInfo = async (studentNumbers: string[]) =>
-  (
-    await StudentModel.findAll({
-      attributes: ['studentnumber', 'email', 'secondary_email'],
-      where: {
-        studentnumber: {
-          [Op.in]: studentNumbers,
-        },
-      },
-    })
-  ).map(formatStudentInfo)
-
-export const getEnrollments = async (courseCodes: string[], startDate: Date, endDate: Date) =>
-  (
-    await EnrollmentModel.findAll({
-      attributes: ['course_code', 'enrollment_date_time', 'studentnumber'],
-      where: {
-        course_code: {
-          [Op.in]: courseCodes,
-        },
-        is_open: true,
-        enrollment_date_time: {
-          [Op.and]: {
-            [Op.lte]: endDate,
-            [Op.gte]: startDate,
-          },
-        },
-        state: EnrollmentState.ENROLLED,
-      },
-    })
-  ).map(formatOpenEnrollments)
-
-export const getCourseNames = async (courseCodes: string[]) =>
-  (
-    await CourseModel.findAll({
-      attributes: ['code', 'name'],
-      where: {
-        code: {
-          [Op.in]: courseCodes,
-        },
-      },
-    })
-  ).map(formatCourseInfo)
-
-export const getStudyRights = async (studentNumbers: string[]) => {
-  const where: WhereOptions<SISStudyRight> = {
-    extentCode: {
-      [Op.in]: [
-        ExtentCode.BACHELOR,
-        ExtentCode.BACHELOR_AND_MASTER,
-        ExtentCode.MASTER,
-        ExtentCode.LICENTIATE,
-        ExtentCode.DOCTOR,
-      ],
-    },
-  }
-
-  if (studentNumbers.length > 0) {
-    where.studentNumber = {
-      [Op.in]: studentNumbers,
-    }
-  }
-
-  return await SISStudyRightModel.findAll({
-    attributes: ['startDate', 'endDate', 'studentNumber'],
-    where,
-  })
-}
 
 export const getOpenUniSearchesByUser = async (userId: string) => {
   return await OpenUniPopulationSearchModel.findAll({

services/backend/src/services/populations/getStudentData.ts

@@ -3,6 +3,8 @@ import { Op } from 'sequelize'
 import { Credit, Enrollment, SISStudyRight, SISStudyRightElement, Student, Studyplan } from '@oodikone/shared/models'
 import { Tag, TagStudent } from '@oodikone/shared/models/kone'
 import { EnrollmentState } from '@oodikone/shared/types'
+import { enrollmentTimeDateThreshold } from '@oodikone/shared/util'
+import { dateMaxFromList } from '@oodikone/shared/util/datetime'
 import {
   CreditModel,
   EnrollmentModel,
@@ -48,8 +50,8 @@ export type StudentEnrollment = Pick<
   Enrollment,
   'course_code' | 'state' | 'enrollment_date_time' | 'semestercode' | 'studentnumber' | 'studyright_id'
 >
-export const getEnrollments = (studentNumbers: string[], startDate: string): Promise<Array<StudentEnrollment>> =>
-  EnrollmentModel.findAll({
+export const getEnrollments = (studentNumbers: string[], startDate: string): Promise<Array<StudentEnrollment>> => {
+  return EnrollmentModel.findAll({
     attributes: ['course_code', 'state', 'enrollment_date_time', 'semestercode', 'studentnumber', 'studyright_id'],
     where: {
       studentnumber: {
@@ -57,11 +59,12 @@ export const getEnrollments = (studentNumbers: string[], startDate: string): Pro
       },
       state: EnrollmentState.ENROLLED,
       enrollment_date_time: {
-        [Op.gte]: startDate,
+        [Op.gte]: dateMaxFromList(new Date(startDate), enrollmentTimeDateThreshold),
       },
     },
     raw: true,
   })
+}
 
 type StudentPersonalData = Pick<
   Student,

services/backend/src/services/semesters.ts

@@ -4,7 +4,7 @@ import { Name } from '@oodikone/shared/types'
 import { dateMinFromList, dateMaxFromList } from '@oodikone/shared/util/datetime'
 import { SemesterModel } from '../models'
 
-type SemestersAndYears = {
+export type SemestersAndYears = {
   years: Record<
     string,
     {

services/backend/src/services/students.ts

@@ -1,14 +1,15 @@
-import { Op, QueryTypes } from 'sequelize'
+import { Op } from 'sequelize'
 
 import type { Credit, Student } from '@oodikone/shared/models'
-import { EnrollmentState, UnifyStatus } from '@oodikone/shared/types'
+import { CreditTypeCode, EnrollmentState, UnifyStatus } from '@oodikone/shared/types'
 import { FormattedStudentForSearch, StudentPageStudent } from '@oodikone/shared/types/studentData'
-import { splitByEmptySpace } from '@oodikone/shared/util'
-import { dbConnections } from '../database/connection'
+import { enrollmentTimeDateThresholdAcademicYear, splitByEmptySpace } from '@oodikone/shared/util'
+import { dateIsBetween, dateMaxFromList } from '@oodikone/shared/util/datetime'
 import {
   StudentModel,
   CreditModel,
   CourseModel,
+  EnrollmentModel,
   ProgrammeModuleModel,
   SemesterModel,
   StudyplanModel,
@@ -17,8 +18,6 @@ import {
 } from '../models'
 import { TagModel, TagStudentModel } from '../models/kone'
 
-const { sequelize } = dbConnections
-
 const byStudentNumber = async (studentNumber: string) => {
   const [student, tags] = await Promise.all([
     StudentModel.findByPk(studentNumber, {
@@ -61,7 +60,14 @@ const byStudentNumber = async (studentNumber: string) => {
         },
         {
           model: StudyplanModel,
-          attributes: ['id', 'included_courses', 'programme_code', 'completed_credits', 'sis_study_right_id'],
+          attributes: [
+            'id',
+            'included_courses',
+            'includedModules',
+            'programme_code',
+            'completed_credits',
+            'sis_study_right_id',
+          ],
         },
       ],
       order: [
@@ -116,7 +122,13 @@ const getUnifyStatus = (unifyCourses: UnifyStatus): [boolean] | [true, false] =>
   }
 }
 
-/* from & to are semestercodes if separate = false, or yearcodes in case separate is true. */
+/**
+  Returns students whose latest "active" attainment/enrollment is between to and from.
+
+  NOTE: If you have failed a course in 2021 and passed the same course in 2023
+  you would not be included in from=2020 to=2022 query
+
+ */
 export const findByCourseAndSemesters = async (
   codes: string[],
   from: number,
@@ -124,6 +136,7 @@ export const findByCourseAndSemesters = async (
   separate: boolean,
   unifyCourses: UnifyStatus = 'unifyStats'
 ) => {
+  /* from & to are semestercodes if separate = false, or yearcodes in case separate is true. */
   const startSemester = await SemesterModel.findOne({
     where: {
       [separate ? 'semestercode' : 'yearcode']: from,
@@ -146,61 +159,108 @@ export const findByCourseAndSemesters = async (
     return []
   }
 
-  const { startdate, semestercode: fromSemester } = startSemester
-  const { enddate, semestercode: toSemester } = endSemester
+  const { startdate } = startSemester
+  const { enddate } = endSemester
 
   const unifyStatus = getUnifyStatus(unifyCourses)
 
-  type QueryResult = {
-    studentnumber: string
-  }[]
+  const credits = (await CreditModel.findAll({
+    raw: true,
+    attributes: [
+      ['student_studentnumber', 'studentnumber'],
+      'course_code',
+      ['attainment_date', 'date'],
+      'credittypecode',
+    ],
+    where: {
+      course_code: { [Op.in]: codes },
+      is_open: unifyStatus,
+      credittypecode: { [Op.not]: CreditTypeCode.IMPROVED }, // We do not care about improved grades
+    },
+    order: [
+      ['attainment_date', 'DESC'],
+      ['credittypecode', 'ASC'],
+    ],
+  })) as unknown as Array<{
+    studentnumber: CreditModel['student_studentnumber']
+    course_code: CreditModel['course_code']
+    date: CreditModel['attainment_date']
+    credittypecode: CreditModel['credittypecode']
+  }>
+  // HACK: Sequelize doesn't like renaming attributes so we have to force-cast
 
-  const queryResult: QueryResult = await sequelize.query(
-    `
-    SELECT
-      studentnumber
-    FROM
-      student s
-    WHERE
-      EXISTS (
-        SELECT
-          1
-        FROM
-          credit c
-        WHERE
-          c.student_studentnumber = s.studentnumber
-          AND c.course_code IN (:codes)
-          AND c.is_open IN (:isOpen)
-          AND c.attainment_date BETWEEN '${startdate.toISOString()}' AND '${enddate.toISOString()}'
+  const enrollments = (await EnrollmentModel.findAll({
+    raw: true,
+    attributes: ['studentnumber', 'course_code', ['enrollment_date_time', 'date']],
+    where: {
+      course_code: { [Op.in]: codes },
+      is_open: unifyStatus,
+      state: EnrollmentState.ENROLLED,
+    },
+    order: [['enrollment_date_time', 'DESC']],
+  })) as unknown as Array<{
+    studentnumber: EnrollmentModel['studentnumber']
+    course_code: EnrollmentModel['course_code']
+    date: EnrollmentModel['enrollment_date_time']
+  }>
+  // HACK: Sequelize doesn't like renaming attributes so we have to force-cast
+
+  const studentCreditsAndEnrollments: Record<
+    string,
+    Array<{
+      studentnumber: string
+      course_code: string
+      date: Date
+      type: 'credit' | 'enrollment'
+      credittypecode?: CreditTypeCode
+    }>
+  > = {}
+
+  for (const credit of credits) {
+    studentCreditsAndEnrollments[credit.studentnumber] ??= []
+    studentCreditsAndEnrollments[credit.studentnumber].push({ ...credit, type: 'credit' })
+  }
+  for (const enrollment of enrollments) {
+    studentCreditsAndEnrollments[enrollment.studentnumber] ??= []
+    studentCreditsAndEnrollments[enrollment.studentnumber].push({ ...enrollment, type: 'enrollment' })
+  }
+
+  const passedCreditTypes = [CreditTypeCode.PASSED, CreditTypeCode.APPROVED]
+
+  const filteredStudentNumbers = new Set<string>()
+  Object.entries(studentCreditsAndEnrollments).map(([studentNumber, creditsOrEnrollments]) => {
+    const latestCredit = creditsOrEnrollments.find(
+      entry => entry.type === 'credit' && codes.includes(entry.course_code)
+    )
+    const latestEnrollment = creditsOrEnrollments.find(
+      entry => entry.type === 'enrollment' && codes.includes(entry.course_code)
+    )
+
+    if (
+      latestCredit &&
+      passedCreditTypes.includes(latestCredit.credittypecode!) &&
+      dateIsBetween(latestCredit.date, startdate, enddate)
+    ) {
+      filteredStudentNumbers.add(studentNumber)
+    } else if (
+      latestCredit &&
+      latestCredit.credittypecode === CreditTypeCode.FAILED &&
+      dateIsBetween(latestCredit.date, startdate, enddate)
+    ) {
+      filteredStudentNumbers.add(studentNumber)
+    } else if (
+      latestEnrollment &&
+      dateIsBetween(
+        latestEnrollment.date,
+        dateMaxFromList(startdate, enrollmentTimeDateThresholdAcademicYear)!,
+        enddate
       )
-      OR EXISTS (
-        SELECT
-          1
-        FROM
-          enrollment e
-        WHERE
-          e.studentnumber = s.studentnumber
-          AND e.course_code IN (:codes)
-          AND e.semestercode BETWEEN ${fromSemester} AND ${toSemester}
-          AND e.enrollment_date_time >= '2021-05-31'
-          AND e.state = :enrollmentState
-      );
-    `,
-    {
-      replacements: {
-        codes,
-        minYearCode: from,
-        maxYearCode: to,
-        isOpen: unifyStatus,
-        enrollmentState: EnrollmentState.ENROLLED,
-      },
-      type: QueryTypes.SELECT,
-      raw: true,
+    ) {
+      filteredStudentNumbers.add(studentNumber)
     }
-  )
+  })
 
-  const studentNumbers = queryResult.map(result => result.studentnumber)
-  return studentNumbers
+  return [...filteredStudentNumbers]
 }
 
 const formatSharedStudentData = ({

services/backend/src/services/studyProgramme/creditGetters.ts

@@ -39,7 +39,7 @@ export const getCreditsForProvider = async (provider: string, codes: string[], s
   }, {})
 
   const courseIdToShareMap = providers.reduce<Record<string, CourseProviderModel[]>>((obj, provider) => {
-    if (!obj[provider.coursecode]) obj[provider.coursecode] = []
+    obj[provider.coursecode] ??= []
     obj[provider.coursecode].push(provider)
     return obj
   }, {})

services/backend/src/services/studyProgramme/index.ts

@@ -1,13 +1,16 @@
 import { memoize } from 'lodash-es'
 import { Op, QueryTypes } from 'sequelize'
 
-import { Name, CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
+import { Name, EnrollmentState } from '@oodikone/shared/types'
+import { enrollmentTimeDateThresholdAcademicYear } from '@oodikone/shared/util'
+import { dateIsBetween } from '@oodikone/shared/util/datetime'
 import { dbConnections } from '../../database/connection'
 import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, ProgrammeModuleModel } from '../../models'
 import logger from '../../util/logger'
 
 export const getAllProgrammeCourses = async (providerCode: string) => {
   const courses: Pick<CourseModel, 'id' | 'code' | 'name' | 'substitution_groups'>[] = await CourseModel.findAll({
+    raw: true,
     attributes: ['id', 'code', 'name', 'substitution_groups'],
     include: [
       {
@@ -22,59 +25,76 @@ export const getAllProgrammeCourses = async (providerCode: string) => {
         },
       },
     ],
-    raw: true,
   })
   return courses
 }
 
-const getCourseCode = (code: string) => {
-  if (code.startsWith('AY')) {
-    return code.replace('AY', '')
-  }
-  if (code.startsWith('A')) {
-    return code.replace('A', '')
-  }
-  return code
-}
-
 export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, programmeCourses: string[]) => {
   try {
-    const enrollmentsCourses = await EnrollmentModel.findAll({
-      attributes: ['studentnumber', 'course_code'],
+    const enrollments: Array<
+      Pick<EnrollmentModel, 'studentnumber' | 'course_code' | 'enrollment_date_time' | 'course'>
+    > = await EnrollmentModel.findAll({
+      raw: true,
+      nest: true,
+      attributes: ['studentnumber', 'course_code', 'enrollment_date_time'],
+      include: {
+        model: CourseModel,
+        attributes: ['is_study_module'],
+      },
       where: {
         course_code: {
           [Op.in]: programmeCourses,
         },
         enrollment_date_time: {
-          [Op.between]: [from, to],
+          [Op.gte]: enrollmentTimeDateThresholdAcademicYear, // This has to be Academic year to match with Course statistics which doesn't display enrollments for ...-2021
         },
         state: EnrollmentState.ENROLLED,
       },
+      order: [['enrollment_date_time', 'DESC']],
     })
 
-    const allEnrollments = new Map<string, Set<string>>()
-    for (const { studentnumber, course_code: courseCode } of enrollmentsCourses) {
-      const code = getCourseCode(courseCode)
-      if (!allEnrollments.has(code)) {
-        allEnrollments.set(code, new Set())
-      }
-      allEnrollments.get(code)!.add(studentnumber)
-    }
-
-    const credits = await CreditModel.findAll({
-      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'isStudyModule'],
+    // TODO: This is being called for each year, could be optimized to call only once
+    const credits: Array<
+      Pick<
+        CreditModel,
+        'course_code' | 'student_studentnumber' | 'credittypecode' | 'isStudyModule' | 'attainment_date'
+      >
+    > = await CreditModel.findAll({
+      raw: true,
+      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'isStudyModule', 'attainment_date'],
       where: {
         course_code: {
           [Op.in]: programmeCourses,
         },
-        attainment_date: {
-          [Op.between]: [from, to],
-        },
       },
+      order: [
+        ['attainment_date', 'DESC'],
+        ['credittypecode', 'ASC'],
+      ], // Passed credits for a student have to be handled first (only PASSED, and FAILED matter)
     })
 
+    const enrolledStudentsByCourseCode = new Map<string, Set<string>>()
+
+    // Pick only this (between to, from) year's credits
+    const filteredCredits = credits.filter(credit => dateIsBetween(credit.attainment_date, from, to))
+
+    const filteredEnrollments = enrollments
+      // Only include the first enrollment for a course for a student.
+      // This assumes that enrollments are sorted in DESC order by enrollment_date_time
+      .filter(({ course_code: courseCode, studentnumber: studentNumber }) => {
+        const courseEnrollments = enrolledStudentsByCourseCode.get(courseCode)
+        const hasEnrolled = courseEnrollments?.has(studentNumber) ?? false
+
+        if (!courseEnrollments) enrolledStudentsByCourseCode.set(courseCode, new Set<string>())
+        if (!hasEnrolled) enrolledStudentsByCourseCode.get(courseCode)!.add(studentNumber)
+
+        return !hasEnrolled
+      })
+      .filter(enrollment => dateIsBetween(enrollment.enrollment_date_time, from, to))
+
     const courseCodeToName = (
       await CourseModel.findAll({
+        raw: true,
         attributes: ['code', 'name'],
         where: {
           code: programmeCourses,
@@ -85,50 +105,102 @@ export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, p
       return acc
     }, new Map())
 
-    const creditCourses = credits.map(credit => ({
-      code: getCourseCode(credit.course_code),
+    const creditCourses = filteredCredits.map(credit => ({
+      code: credit.course_code,
       studentNumber: credit.student_studentnumber,
       creditTypeCode: credit.credittypecode,
       courseName: courseCodeToName.get(credit.course_code)!,
       isStudyModule: credit.isStudyModule,
+      attainmentDate: credit.attainment_date,
     }))
 
     const passedByCourseCodes = new Map<string, Set<string>>()
+    const failedByCourseCodes = new Map<string, Set<string>>()
     const notCompletedByCourseCodes = new Map<string, Set<string>>()
     const courses = new Map<string, { name: Name; isStudyModule: boolean }>()
 
-    for (const { code, courseName, creditTypeCode, isStudyModule, studentNumber } of creditCourses) {
-      if (!courses.has(code)) {
-        courses.set(code, {
+    const studentHasPassedCourse = (studentNumber: string, courseCode: string): boolean =>
+      (passedByCourseCodes.get(courseCode)?.has(studentNumber) ?? false) ||
+      !!credits.find(
+        credit =>
+          credit.student_studentnumber === studentNumber &&
+          credit.course_code === courseCode &&
+          CreditModel.passed(credit)
+      )
+
+    /** Student has already been counted as failed or there is a newer failed credit */
+    const studentHasFailedCourse = (
+      studentNumber: string,
+      courseCode: string,
+      attainmentDate: Date = new Date(0)
+    ): boolean =>
+      (failedByCourseCodes.get(courseCode)?.has(studentNumber) ?? false) ||
+      !!credits.find(
+        credit =>
+          credit.student_studentnumber === studentNumber &&
+          credit.course_code === courseCode &&
+          CreditModel.failed(credit) &&
+          credit.attainment_date > attainmentDate
+      )
+
+    const studentHasCredit = (studentNumber: string, courseCode: string): boolean =>
+      studentHasPassedCourse(studentNumber, courseCode) || studentHasFailedCourse(studentNumber, courseCode)
+
+    for (const {
+      code: courseCode,
+      courseName,
+      creditTypeCode,
+      isStudyModule,
+      studentNumber,
+      attainmentDate,
+    } of creditCourses) {
+      if (!courses.has(courseCode)) {
+        courses.set(courseCode, {
           isStudyModule,
           name: courseName,
         })
-        passedByCourseCodes.set(code, new Set())
-        notCompletedByCourseCodes.set(code, new Set())
+        passedByCourseCodes.set(courseCode, new Set())
+        failedByCourseCodes.set(courseCode, new Set())
+        notCompletedByCourseCodes.set(courseCode, new Set())
       }
-      if ([CreditTypeCode.PASSED, CreditTypeCode.IMPROVED, CreditTypeCode.APPROVED].includes(creditTypeCode)) {
-        passedByCourseCodes.get(code)!.add(studentNumber)
+
+      if (CreditModel.passed({ credittypecode: creditTypeCode })) {
+        passedByCourseCodes.get(courseCode)!.add(studentNumber)
       }
-      if (creditTypeCode === CreditTypeCode.FAILED && !passedByCourseCodes.get(code)!.has(studentNumber)) {
-        notCompletedByCourseCodes.get(code)!.add(studentNumber)
+      if (
+        CreditModel.failed({ credittypecode: creditTypeCode }) &&
+        !studentHasPassedCourse(studentNumber, courseCode) &&
+        !studentHasFailedCourse(studentNumber, courseCode, attainmentDate)
+      ) {
+        failedByCourseCodes.get(courseCode)!.add(studentNumber)
       }
     }
 
-    // If student has enrollments, but no attainment for a particular course, they have no credit info.
-    programmeCourses.forEach(courseCode => {
-      allEnrollments.get(courseCode)?.forEach(studentnumber => {
-        if (passedByCourseCodes.get(courseCode)?.has(studentnumber)) {
-          notCompletedByCourseCodes.get(courseCode)?.add(studentnumber)
-        }
-      })
-    })
+    // Add course details to courseMap if there are only enrollments
+    for (const { studentnumber: studentNumber, course_code: courseCode, course } of filteredEnrollments) {
+      if (!courses.has(courseCode)) {
+        courses.set(courseCode, {
+          isStudyModule: course.is_study_module,
+          name: courseCodeToName.get(courseCode)!,
+        })
+        passedByCourseCodes.set(courseCode, new Set())
+        failedByCourseCodes.set(courseCode, new Set())
+        notCompletedByCourseCodes.set(courseCode, new Set())
+      }
 
-    return [...courses.entries()].map(([code, { isStudyModule, name }]) => ({
-      code,
-      name,
-      isStudyModule,
-      allNotPassed: notCompletedByCourseCodes.get(code)?.size ?? 0,
-    }))
+      if (!studentHasCredit(studentNumber, courseCode)) {
+        notCompletedByCourseCodes.get(courseCode)!.add(studentNumber)
+      }
+    }
+
+    return [...courses.entries()].map(([code, { isStudyModule, name }]) => {
+      return {
+        code,
+        name,
+        isStudyModule,
+        allNotPassed: (notCompletedByCourseCodes.get(code)?.size ?? 0) + (failedByCourseCodes.get(code)?.size ?? 0),
+      }
+    })
   } catch (error) {
     logger.error(`getNotCompletedForProgrammeCourses failed ${error}`)
     return []

services/backend/src/services/studyProgramme/studentGetters.ts

@@ -95,7 +95,7 @@ export const getProgrammeCourseAggregates = async (params: {
           WHERE
             cr.course_code IN (:courseCodes)
             AND cr.attainment_date BETWEEN :from AND :to
-            AND cr.credittypecode = :creditType
+            AND cr.credittypecode IN (:creditTypes)
         )
         SELECT
           fc."courseCode",
@@ -176,7 +176,7 @@ export const getProgrammeCourseAggregates = async (params: {
           courseCodes,
           from,
           to,
-          creditType: CreditTypeCode.PASSED,
+          creditTypes: [CreditTypeCode.PASSED, CreditTypeCode.APPROVED],
           degreeExtentCodes,
           otherUniversityExtentCodes,
           separateStudyExtentCodes,

services/backend/src/services/studyProgramme/studyProgrammeCoursesBySemester.ts

@@ -35,7 +35,7 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
     attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'state'],
     where: {
       course_code: { [Op.in]: courseCodes },
-      semestercode: { [Op.gte]: autumnSemester2017 },
+      semestercode: { [Op.gte]: autumnSemester2017 }, // It is ok to use enrollments before OK sisu-migration, ColorizedTableData doesn't need it
       state: { [Op.in]: [EnrollmentState.ENROLLED, EnrollmentState.REJECTED] },
     },
     raw: true,
@@ -47,9 +47,7 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
   credits.forEach(credit => {
     const studentNumber = credit.student_studentnumber
     studentList.add(studentNumber)
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
+    attemptsByStudents[studentNumber] ??= []
     attemptsByStudents[studentNumber].push({
       studentNumber,
       courseCode: credit.course_code,
@@ -61,9 +59,7 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
 
   enrollments.forEach(enrollment => {
     const studentNumber = enrollment.studentnumber
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
+    attemptsByStudents[studentNumber] ??= []
     studentList.add(studentNumber)
     if (
       attemptsByStudents[studentNumber].find(
@@ -88,7 +84,7 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
   const attemptsArray = [] as Attempt[]
   studentList.forEach(studentNumber => attemptsArray.push(...attemptsByStudents[studentNumber]))
 
-  const unorderedTableData = await createArrayOfCourses(attemptsArray, courses)
+  const unorderedTableData = createArrayOfCourses(attemptsArray, courses)
   const tableData = orderBy(unorderedTableData, 'code')
 
   return { tableData }

services/backend/src/services/studyProgramme/studyProgrammeGraduations.ts

@@ -3,7 +3,7 @@ import isSameOrAfter from 'dayjs/plugin/isSameOrAfter'
 import { indexOf, orderBy } from 'lodash-es'
 
 import {
-  GraduationStatistics,
+  GraduationTimeCategories,
   Name,
   SemesterEnrollment,
   StudyTrack,
@@ -34,6 +34,7 @@ import {
   getYearsObject,
   getStudyRightElementsWithPhase,
   hasTransferredFromOrToProgramme,
+  getAverage,
 } from './studyProgrammeHelpers'
 import { getStudyRightsInProgramme } from './studyRightFinders'
 
@@ -118,7 +119,7 @@ export type GraduationTimes = {
     y: number
     amount: number
     name: string | number
-    statistics: GraduationStatistics
+    statistics: GraduationTimeCategories
     times: number[]
   }>
   goal: number
@@ -234,24 +235,22 @@ const getGraduationTimeAndThesisWriterStats = async ({
   const comboTimes: ChartGraduationTimes = { medians: [], goal: goal + GraduationTarget.THREE_YEARS }
 
   for (const year of years.toReversed()) {
-    const median = getMedian(graduationTimes[year])
-    const statistics = countTimeCategories(graduationTimes[year], goal)
     times.medians.push({
-      y: median,
+      median: getMedian(graduationTimes[year]),
+      average: getAverage(graduationTimes[year]),
       amount: graduationTimes[year].length,
       name: year.toString(),
-      statistics,
+      statistics: countTimeCategories(graduationTimes[year], goal),
       times: [...graduationTimes[year]],
     })
 
     if (doCombo) {
-      const median = getMedian(graduationTimesCombo[year])
-      const statistics = countTimeCategories(graduationTimesCombo[year], goal + GraduationTarget.THREE_YEARS)
       comboTimes.medians.push({
-        y: median,
+        median: getMedian(graduationTimesCombo[year]),
+        average: getAverage(graduationTimesCombo[year]),
         amount: graduationTimesCombo[year].length,
         name: year.toString(),
-        statistics,
+        statistics: countTimeCategories(graduationTimesCombo[year], goal + GraduationTarget.THREE_YEARS),
         times: [...graduationTimesCombo[year]],
       })
     }
@@ -302,13 +301,10 @@ const getProgrammesBeforeStarting = async ({
     if (studyRight.extentCode !== ExtentCode.BACHELOR_AND_MASTER) return acc
     const phase1Programmes = studyRight.studyRightElements.filter(elem => elem.phase === Phase.ANY)
     const [latestPhase1Programme] = orderBy(phase1Programmes, ['endDate'], ['desc'])
-    if (!acc[latestPhase1Programme.code]) {
-      const programmeWithYears: ProgrammeWithYears = {
-        code: latestPhase1Programme.code,
-        name: latestPhase1Programme.name,
-        ...getYearsObject({ years }),
-      }
-      acc[latestPhase1Programme.code] = programmeWithYears
+    acc[latestPhase1Programme.code] ??= {
+      code: latestPhase1Programme.code,
+      name: latestPhase1Programme.name,
+      ...getYearsObject({ years }),
     }
     const phase2Programmes = studyRight.studyRightElements.filter(elem => elem.phase === Phase.MASTER)
 
@@ -347,15 +343,11 @@ const getProgrammesAfterGraduation = async ({
     const [firstPhase2Programme] = orderBy(phase2Programmes, ['startDate'], ['asc'])
     if (!firstPhase2Programme) return acc
 
-    if (!acc[firstPhase2Programme.code]) {
-      const programmeWithYears: ProgrammeWithYears = {
-        code: firstPhase2Programme.code,
-        name: firstPhase2Programme.name,
-        ...getYearsObject({ years }),
-      }
-      acc[firstPhase2Programme.code] = programmeWithYears
+    acc[firstPhase2Programme.code] ??= {
+      code: firstPhase2Programme.code,
+      name: firstPhase2Programme.name,
+      ...getYearsObject({ years }),
     }
-
     ;(acc[firstPhase2Programme.code][defineYear(firstPhase2Programme.startDate, isAcademicYear)] as number) += 1
     return acc
   }, {})

services/backend/src/services/studyProgramme/studyProgrammeHelpers.ts

@@ -56,6 +56,11 @@ export const getMedian = (values: number[]) => {
   return (sorted[half - 1] + sorted[half]) / 2.0
 }
 
+export const getAverage = (values: number[]) => {
+  if (!values.length) return 0
+  return Number((values.reduce((acc, cur) => acc + cur, 0) / values.length).toFixed(2))
+}
+
 /** @returns academic starting year e.g. "2019-2020" */
 export function defineYear(date: Date, isAcademicYear: true): string
 export function defineYear(date: Date, isAcademicYear: false): number
@@ -259,7 +264,7 @@ export const tableTitles = {
     licentiate: ['Graduated bachelor', 'Graduated licentiate'],
     master: ['Graduated bachelor', 'Graduated master'],
   },
-  studytracksEnd: ['Men', 'Women', 'Other / Unknown', 'Finland', 'Other'],
+  studytracksEnd: ['Has recent attainment', 'Men', 'Women', 'Other / Unknown', 'Finland', 'Other'],
 } as const
 
 export const getId = (code: string) => {

services/backend/src/services/studyProgramme/studyRightFinders.ts

@@ -2,7 +2,7 @@ import { col, fn, Includeable, Op } from 'sequelize'
 
 import { SISStudyRight } from '@oodikone/shared/models'
 import { CreditTypeCode } from '@oodikone/shared/types'
-import { StudentModel, SISStudyRightModel, SISStudyRightElementModel, CreditModel } from '../../models'
+import { StudentModel, SISStudyRightModel, SISStudyRightElementModel, CreditModel, StudyplanModel } from '../../models'
 
 export const getStudyRightsInProgramme = async (
   programmeCode: string,
@@ -16,6 +16,7 @@ export const getStudyRightsInProgramme = async (
 
   const studyRightIds = (
     await SISStudyRightModel.findAll({
+      raw: true,
       attributes: ['id'],
       include: {
         model: SISStudyRightElementModel,
@@ -23,7 +24,6 @@ export const getStudyRightsInProgramme = async (
         attributes: [],
         where,
       },
-      raw: true,
     })
   ).map(sr => sr.id)
 
@@ -36,23 +36,34 @@ export const getStudyRightsInProgramme = async (
   ]
 
   if (includeStudentsAndCredits) {
-    include.push({
-      model: StudentModel,
-      attributes: ['gender_code', 'citizenships'],
-      include: [
-        {
-          model: CreditModel,
-          attributes: ['attainment_date', 'credits'],
-          where: {
-            isStudyModule: false,
-            credittypecode: {
-              [Op.in]: [CreditTypeCode.PASSED, CreditTypeCode.APPROVED],
+    include.push(
+      {
+        model: StudentModel,
+        attributes: ['gender_code', 'citizenships'],
+        include: [
+          {
+            model: CreditModel,
+            attributes: ['attainment_date', 'credits', 'course_code'],
+            required: false,
+            where: {
+              isStudyModule: false,
+              credittypecode: {
+                [Op.in]: [CreditTypeCode.PASSED, CreditTypeCode.APPROVED],
+              },
             },
           },
-          required: false,
+        ],
+      },
+      {
+        model: StudyplanModel,
+        as: 'studyPlans',
+        attributes: ['included_courses'],
+        required: false,
+        where: {
+          programme_code: programmeCode,
         },
-      ],
-    })
+      }
+    )
   }
 
   return (
@@ -65,7 +76,7 @@ export const getStudyRightsInProgramme = async (
         },
       },
     })
-  ).map(studyRight => studyRight.toJSON())
+  ).map(studyRight => studyRight.toJSON<SISStudyRight>())
 }
 
 export const getStudyTracksForProgramme = async (studyProgramme: string) => {

services/backend/src/services/studyProgramme/studyTrackStats.ts

@@ -1,8 +1,7 @@
+import dayjs from 'dayjs'
 import { SISStudyRight, SISStudyRightElement } from '@oodikone/shared/models'
 import {
   GraduationTimes,
-  MedianEntry,
-  ProgrammeOrStudyTrackGraduationStats,
   StudyTrackStats,
   GenderCode,
   EnrollmentType,
@@ -17,6 +16,7 @@ import { calculateDurationOfStudies, shouldIncludeComboStats } from './studyProg
 import {
   computePercentiles,
   defineYear,
+  getAverage,
   getCreditCount,
   getGoal,
   getMedian,
@@ -40,7 +40,7 @@ const getGraduationTimeStats = async (
   combinedProgramme?: string
 ) => {
   const goal = await getGoal(combinedProgramme ?? studyProgramme)
-  const finalGraduationTimes = { goals: { basic: goal, combo: goal + GraduationTarget.THREE_YEARS } }
+  const finalGraduationTimes = { goals: { basic: goal, combo: goal + GraduationTarget.THREE_YEARS } } as GraduationTimes
 
   const calculateGraduationTimes = (
     graduationTimes: Record<string, Record<string, number[]>>,
@@ -48,7 +48,7 @@ const getGraduationTimeStats = async (
   ) => {
     for (const programmeOrTrack of Object.keys(graduationTimes)) {
       const stats = graduationTimes[programmeOrTrack]
-      finalGraduationTimes[programmeOrTrack] ??= { medians: { basic: [] as MedianEntry[], combo: [] as MedianEntry[] } }
+      finalGraduationTimes[programmeOrTrack] ??= { medians: { basic: [], combo: [] } }
       for (const [year, oneYearStats] of Object.entries(stats)) {
         const classSize = mainStatsByTrack[combinedProgramme ? studyProgramme : programmeOrTrack].find(
           stats => stats[0] === year
@@ -61,21 +61,20 @@ const getGraduationTimeStats = async (
           classSize,
           name: year,
           statistics: countTimeCategories(oneYearStats, finalGraduationTimes.goals[type]),
-          y: getMedian(oneYearStats),
+          median: getMedian(oneYearStats),
+          average: getAverage(oneYearStats),
           times: [...oneYearStats],
         }
-        ;(finalGraduationTimes[programmeOrTrack] as ProgrammeOrStudyTrackGraduationStats).medians[type].push(final)
+        finalGraduationTimes[programmeOrTrack].medians[type].push(final)
       }
-      ;(finalGraduationTimes[programmeOrTrack] as ProgrammeOrStudyTrackGraduationStats).medians[type].sort(
-        createLocaleComparator('name', true)
-      )
+      finalGraduationTimes[programmeOrTrack].medians[type].sort(createLocaleComparator('name', true))
     }
   }
 
   calculateGraduationTimes(graduationTimes, 'basic')
   calculateGraduationTimes(graduationTimesCombo, 'combo')
 
-  return finalGraduationTimes as GraduationTimes
+  return finalGraduationTimes
 }
 
 const getEmptyYear = () => ({
@@ -86,6 +85,7 @@ const getEmptyYear = () => ({
   passive: 0,
   graduated: 0,
   graduatedCombinedProgramme: 0,
+  attainmentWithinYear: 0,
   male: 0,
   female: 0,
   otherUnknown: 0,
@@ -133,6 +133,8 @@ const combineStats = (
         getPercentage(yearStats.passive, yearStats.all),
         yearStats.graduated,
         getPercentage(yearStats.graduated, yearStats.all),
+        yearStats.attainmentWithinYear,
+        getPercentage(yearStats.attainmentWithinYear, yearStats.all),
         yearStats.male,
         getPercentage(yearStats.male, yearStats.all),
         yearStats.female,
@@ -158,9 +160,7 @@ const combineStats = (
       ])
       mainStatsByTrack[programmeOrStudyTrack].push([year, ...yearArray])
       if (Object.keys(yearStats.otherCountriesCounts).length > 0) {
-        if (!otherCountriesCount[programmeOrStudyTrack]) {
-          otherCountriesCount[programmeOrStudyTrack] = {}
-        }
+        otherCountriesCount[programmeOrStudyTrack] ??= {}
         otherCountriesCount[programmeOrStudyTrack][year] = yearStats.otherCountriesCounts
       }
     }
@@ -182,6 +182,8 @@ const getMainStatsByTrackAndYear = async (
   const { semesters } = await getSemestersAndYears()
   const { semestercode: currentSemester } = Object.values(semesters).find(semester => semester.enddate >= new Date())!
 
+  const yearAgo = dayjs().subtract(1, 'year')
+
   const graduationTimes: Record<string, Record<string, number[]>> = {}
   const graduationTimesCombo: Record<string, Record<string, number[]>> = {}
   const graduationTimesCombinedProgrammeCombo: Record<string, Record<string, number[]>> = {}
@@ -213,6 +215,18 @@ const getMainStatsByTrackAndYear = async (
       }
     }
 
+    const latestAttainmentWithinYearAndInHops = dayjs(
+      studyRight.student.credits
+        .filter(credit => studyRight.studyPlans.some(sp => sp.included_courses.includes(credit.course_code)))
+        .reduce((latest, { attainment_date }) => {
+          return attainment_date > latest ? attainment_date : latest
+        }, new Date('1900-01-01'))
+    ).isSameOrAfter(yearAgo)
+
+    if (latestAttainmentWithinYearAndInHops) {
+      yearlyStats[year][programmeOrStudyTrack].attainmentWithinYear++
+    }
+
     /* Gender cols */
     if (studyRight.student.gender_code === GenderCode.MALE) {
       yearlyStats[year][programmeOrStudyTrack].male += 1
@@ -285,35 +299,27 @@ const getMainStatsByTrackAndYear = async (
     )
 
     if (countAsBachelorMaster) {
-      if (!graduationTimesCombo[programmeOrStudyTrack]) {
-        graduationTimesCombo[programmeOrStudyTrack] = getYearsObject({
-          years: years.filter(year => year !== 'Total'),
-          emptyArrays: true,
-        })
-      }
+      graduationTimesCombo[programmeOrStudyTrack] ??= getYearsObject({
+        years: years.filter(year => year !== 'Total'),
+        emptyArrays: true,
+      })
       const startYearInBachelor = defineYear(startDate, true)
-      if (graduationTimesCombo[programmeOrStudyTrack][startYearInBachelor]) {
-        graduationTimesCombo[programmeOrStudyTrack][startYearInBachelor].push(duration)
-      }
+      graduationTimesCombo[programmeOrStudyTrack][startYearInBachelor]?.push(duration)
     } else {
-      if (!graduationTimes[programmeOrStudyTrack]) {
-        graduationTimes[programmeOrStudyTrack] = getYearsObject({
-          years: years.filter(year => year !== 'Total'),
-          emptyArrays: true,
-        })
-      }
+      graduationTimes[programmeOrStudyTrack] ??= getYearsObject({
+        years: years.filter(year => year !== 'Total'),
+        emptyArrays: true,
+      })
       graduationTimes[programmeOrStudyTrack][year].push(duration)
     }
 
     if (!hasGraduatedFromCombinedProgramme) return
     const bachelorStartDate = getStudyRightElementsWithPhase(studyRight, 1)[0]?.startDate
     if (!bachelorStartDate) return
-    if (!graduationTimesCombinedProgrammeCombo[combinedProgramme!]) {
-      graduationTimesCombinedProgrammeCombo[combinedProgramme!] = getYearsObject({
-        years: years.filter(year => year !== 'Total'),
-        emptyArrays: true,
-      })
-    }
+    graduationTimesCombinedProgrammeCombo[combinedProgramme!] ??= getYearsObject({
+      years: years.filter(year => year !== 'Total'),
+      emptyArrays: true,
+    })
     const startYearInNewBachelorProgramme = defineYear(studyRightElement.startDate, true)
     const combinedDuration = calculateDurationOfStudies(
       bachelorStartDate,
@@ -322,9 +328,7 @@ const getMainStatsByTrackAndYear = async (
       semesters,
       studyRight.transferInfo
     )
-    if (graduationTimesCombinedProgrammeCombo[combinedProgramme!][startYearInNewBachelorProgramme]) {
-      graduationTimesCombinedProgrammeCombo[combinedProgramme!][startYearInNewBachelorProgramme].push(combinedDuration)
-    }
+    graduationTimesCombinedProgrammeCombo[combinedProgramme!][startYearInNewBachelorProgramme]?.push(combinedDuration)
   }
 
   const creditCounts: Record<string, number[]> = getYearsObject({

services/backend/src/services/teachers/index.ts

@@ -21,6 +21,7 @@ export const getTeachersBySearchTerm = async (searchTerm: string) => {
         [Op.and]: searchTerms.map(term => ({ [Op.iLike]: `%${term}%` })),
       },
     },
+    order: [['name', 'ASC']],
   })
 }
 

services/backend/src/services/teachers/top.ts

@@ -83,9 +83,7 @@ const updatedStats = (
   transferred: boolean
 ) => {
   const { id, name } = teacher
-  if (!teacherStats[id]) {
-    teacherStats[id] = { id, name, passed: 0, failed: 0, credits: 0, transferred: 0 }
-  }
+  teacherStats[id] ??= { id, name, passed: 0, failed: 0, credits: 0, transferred: 0 }
   const stats = teacherStats[id]
   if (passed) {
     stats.passed += 1

services/backend/src/types/index.ts

@@ -1 +1,4 @@
 export { ParsedCourse } from './course'
+
+/** Sets one or more field(s) of an object to optional */
+export type SetOptional<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>

services/backend/src/util/index.ts

@@ -106,5 +106,25 @@ export const safeJSONParse = async <T>(json: string): Promise<T | null> => {
 
 // NOTE: Make sure we have only arrays, "/...&years=2021" is parsed by express as years: 2021, not [2021]
 export const handleQueryArrays = <T>(requestField: T | T[]): T[] => {
-  return Array.isArray(requestField) ? requestField : [requestField]
+  return requestField ? (Array.isArray(requestField) ? requestField : [requestField]) : []
+}
+
+/**
+ * Recursively deletes all values from object that match the given key.
+ * Alternatively instead of deleting, replaces the values with something else.
+ */
+export const objectKeyAnnihilator = (data: any, keyToNuke: string, replaceWith?: any) => {
+  if (Array.isArray(data)) {
+    data.forEach(item => objectKeyAnnihilator(item, keyToNuke, replaceWith))
+  } else if (!!data && typeof data === 'object') {
+    if (data.times) {
+      if (replaceWith) {
+        data.times = replaceWith
+      } else {
+        delete data.times
+      }
+    }
+    Object.values(data).forEach(item => objectKeyAnnihilator(item, keyToNuke, replaceWith))
+  }
+  return data
 }

services/backend/src/util/logger.ts

@@ -1,9 +1,7 @@
-import os from 'os'
 import * as winston from 'winston'
-import { WinstonGelfTransporter } from 'winston-gelf-transporter'
 import LokiTransport from 'winston-loki'
 
-import { isProduction } from '../config'
+import { isProduction, silentTesting } from '../config'
 
 const transports: winston.transport[] = [new winston.transports.Console()]
 
@@ -11,18 +9,7 @@ transports.push(
   new LokiTransport({
     host: 'http://loki-svc.toska-lokki.svc.cluster.local:3100',
     labels: { app: 'oodikone', environment: process.env.NODE_ENV ?? 'production' },
-  }),
-  new WinstonGelfTransporter({
-    handleExceptions: true,
-    host: 'svm-116.cs.helsinki.fi',
-    port: 9503,
-    protocol: 'udp',
-    hostName: os.hostname(),
-    additional: {
-      app: 'oodikone',
-      environment: 'production',
-    },
-  }) as winston.transport
+  })
 )
 
 const { colorize, combine, timestamp, printf } = winston.format
@@ -43,6 +30,7 @@ const logger = winston.createLogger({
     })
   ),
   transports,
+  silent: silentTesting,
 })
 
 logger.on('error', error => console.error('Logging failed! Reason: ', error)) // eslint-disable-line no-console

services/backend/src/worker/processor.js

@@ -6,7 +6,6 @@ const {
   refreshCloseToGraduating,
 } = require('../events')
 const { updateSISStudyPlans } = require('../services/sisUpdaterService')
-const logger = require('../util/logger')
 
 // This bullmq worker processor has to be in a different file for it to be run in a sandboxed process.
 // Beware of creating a circular dependency, e.g. don't import worker file into the files that you import to this file.
@@ -23,14 +22,9 @@ const refreshers = {
 }
 
 module.exports = async job => {
-  try {
-    if (job.name === 'studyplansUpdate') {
-      await updateSISStudyPlans(job.data.days)
-      return
-    }
+  if (job.name === 'studyplansUpdate') {
+    await updateSISStudyPlans(job.data.days)
+  } else {
     await refreshers[job.id.split('-')[0]](job.data?.code)
-  } catch (error) {
-    logger.error(error.stack)
-    throw error
   }
 }

services/backend/src/worker/worker.ts

@@ -31,12 +31,14 @@ worker.on('completed', job => {
 })
 
 // If there is no error event listener, the worker stops taking jobs after any error.
+// This is only for some internal errors. If a job fails/throws, the below "failed" event is emitted instead.
 worker.on('error', error => {
   logger.error('Job returned error:')
   logger.error(error.toString())
   Sentry.captureException(error)
 })
 
+// Errors from the processor are propagated back here. Thus no logging needs to be done there.
 worker.on('failed', (job, error) => {
   logger.error(`Job ${job?.id ?? 'unknown id'} failed. ${error.stack ?? ''}`)
   Sentry.captureException(error)

services/backend/tests/api/changelog.test.ts

@@ -1,17 +1,16 @@
 import { Express } from 'express'
-import assert from 'node:assert/strict'
-import { describe, it, before } from 'node:test'
 import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
 
-import { initTests } from './utils'
+import { initTests } from '../utils'
 
-let app: Express
-before(async () => {
-  app = await initTests()
-})
+void describe('Changelog', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
 
-void describe('Changelog', async () => {
-  await it('should return dev data in dev env', async () => {
+  it('should return dev data in dev env', async () => {
     const res = await request(app)
       .get('/changelog')
       .set('shib-session-id', 'test')

services/backend/tests/api/courses/courseyearlystats.test.ts

@@ -0,0 +1,465 @@
+import { Express } from 'express'
+import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { Unarray } from '@oodikone/shared/types'
+import { CourseYearlyStatsResBody } from '../../../src/routes/courses'
+import { initTests, ResponseWithBody } from '../../utils'
+
+void describe('Course yearly statistics', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
+
+  it('should return nothing with missing parameters', async () => {
+    const res = await request(app)
+      .get('/courseyearlystats')
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')
+
+    assert.strictEqual(res.status, 422)
+    assert.strictEqual(res.body?.error, 'Missing required query parameters')
+  })
+
+  describe('should return correct amount of values for a course without substitutions (TKT10002)', () => {
+    let body: Unarray<CourseYearlyStatsResBody>
+    beforeAll(async () => {
+      const res = (await request(app)
+        .get('/courseyearlystats?codes=TKT10002&combineSubstitutions=false')
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
+
+      assert.strictEqual(res.status, 200)
+      assert.strictEqual(res.body.length, 1, 'Query to return anything')
+      body = res.body.at(0)!
+      assert(
+        'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
+        'All keys of courseyearlystats not defined'
+      )
+    })
+
+    it('(statistic years)', () => {
+      assert.strictEqual(body.unifyStats!.statistics.length, 8, 'unifyStats years')
+      assert.strictEqual(body.regularStats!.statistics.length, 8, 'regularStats years')
+      assert.strictEqual(body.openStats!.statistics.length, 4, 'openStats years')
+
+      // Faculties should have the same amount of years as normal statistics (above)
+      assert.strictEqual(Object.keys(body.unifyStats!.facultyStats).length, 8, 'unifyStats years')
+      assert.strictEqual(Object.keys(body.regularStats!.facultyStats).length, 8, 'regularStats years')
+      assert.strictEqual(Object.keys(body.openStats!.facultyStats).length, 4, 'openStats years')
+    })
+
+    describe('(statistic contents)', () => {
+      it('- 2016-2017', () => {
+        const stats2016 = body.unifyStats!.statistics.find(year => year.name === '2016-2017')!
+        assert.deepStrictEqual(
+          stats2016.students.studentNumbers,
+          ['484144'],
+          'Incorrect students marked to 2016 stats (students)'
+        )
+
+        assert.deepStrictEqual(stats2016.attempts.grades['4'], ['484144'], 'Incorrect grades marked to 2016 stats (4)')
+        assert.strictEqual(
+          stats2016.attempts.categories.passed.length,
+          1,
+          'Incorrect students marked as passed to 2016 stats (passed)'
+        )
+        assert.strictEqual(
+          stats2016.attempts.categories.failed.length,
+          0,
+          'Incorrect students marked as failed to 2016 stats (failed)'
+        )
+
+        assert('enrollments' in stats2016, 'Missing field enrollment in statsitics')
+        assert.strictEqual(stats2016.enrollments.length, 0, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2016.allEnrollments.length, 0, 'Enrollments found when none should exist')
+      })
+
+      it('- 2017-2018', () => {
+        const stats2017 = body.unifyStats!.statistics.find(year => year.name === '2017-2018')!
+
+        assert.strictEqual(stats2017.attempts.grades['5'].length, 11, 'Incorrect grades marked to 2017 stats (5)')
+        assert.strictEqual(stats2017.attempts.grades['4'].length, 5, 'Incorrect grades marked to 2017 stats, (4)')
+        assert.strictEqual(stats2017.attempts.grades['3'].length, 2, 'Incorrect grades marked to 2017 stats (3)')
+        assert.strictEqual(stats2017.attempts.grades['2'].length, 1, 'Incorrect grades marked to 2017 stats (2)')
+        assert(!('1' in stats2017.attempts.grades), 'Incorrect grades marked to 2017 stats (1)')
+        assert.strictEqual(stats2017.attempts.grades['Hyl.'].length, 4, 'Incorrect grades marked to 2017 stats (Hyl.)')
+
+        assert.strictEqual(stats2017.students.studentNumbers.length, 23, 'Incorrect amount of total students 2017')
+
+        assert.strictEqual(
+          stats2017.attempts.categories.passed.length,
+          11 + 5 + 2 + 1 + 0,
+          'Incorrect students marked as passed to 2017 stats'
+        )
+        assert.strictEqual(
+          stats2017.attempts.categories.failed.length,
+          4,
+          'Incorrect students marked as failed to 2017 stats'
+        )
+
+        // These should not exist before 21-22
+        assert('enrollments' in stats2017, 'Missing field enrollment in statsitics')
+        assert.strictEqual(stats2017.enrollments.length, 0, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2017.allEnrollments.length, 0, 'Enrollments found when none should exist')
+      })
+
+      it('- 2018-2019', () => {
+        const stats2018 = body.unifyStats!.statistics.find(year => year.name === '2018-2019')!
+
+        // There are more students that have passed this course later even though they have failed the course during 2018-2019
+        assert.deepStrictEqual(
+          stats2018.attempts.categories.failed,
+          ['454128', '546124'],
+          'Incorrect students marked as failed 2018'
+        )
+
+        assert.strictEqual(stats2018.attempts.grades['0'].length, 2)
+        assert.strictEqual((stats2018.attempts.grades['Hyl.'] ?? []).length, 0)
+        assert.strictEqual(stats2018.attempts.categories.failed.length, 2, 'Incorrect amount of failed students 2018')
+      })
+
+      it('- 2020-2021', () => {
+        const stats2020 = body.unifyStats!.statistics.find(year => year.name === '2020-2021')!
+
+        // These should not exist before 21-22
+        assert('enrollments' in stats2020, 'Missing field enrollment in statsitics')
+        assert.strictEqual(stats2020.enrollments.length, 0, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2020.allEnrollments.length, 0, 'Enrollments found when none should exist')
+      })
+
+      it('- 2022-2023', () => {
+        const stats2022 = body.unifyStats!.statistics.find(year => year.name === '2022-2023')!
+        assert('enrollments' in stats2022, 'Missing field enrollment in statsitics')
+
+        assert.strictEqual(stats2022.students.studentNumbers.length, 27, 'Incorrect amount of total students 2022')
+
+        assert.strictEqual(stats2022.enrollments.length, 34, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2022.allEnrollments.length, 43, 'Enrollments found when none should exist')
+      })
+
+      it('- 2023-2024', () => {
+        const stats2023 = body.unifyStats!.statistics.find(year => year.name === '2023-2024')!
+        assert.strictEqual(stats2023.students.studentNumbers.length, 1, 'Incorrect amount of total students 2023')
+
+        assert.deepStrictEqual(
+          stats2023.attempts.categories.passed,
+          ['521757'],
+          'Incorrect students marked as passed in 2023 stats'
+        )
+        assert.deepStrictEqual(
+          stats2023.attempts.grades['5'],
+          ['521757'],
+          "Correct student doesn't exist under correct grade"
+        )
+
+        assert('enrollments' in stats2023, 'Missing field enrollment in statsitics')
+        assert.strictEqual(stats2023.enrollments.length, 7, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2023.allEnrollments.length, 7, 'Enrollments found when none should exist')
+      })
+    })
+
+    describe('should work correctly in specific cases (TKT10002)', () => {
+      let body: Unarray<CourseYearlyStatsResBody>
+      beforeAll(async () => {
+        const res = (await request(app)
+          .get('/courseyearlystats?codes=TKT10002&combineSubstitutions=false')
+          .set('shib-session-id', 'test')
+          .set('uid', 'basic')
+          .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
+
+        assert.strictEqual(res.status, 200)
+        assert.strictEqual(res.body.length, 1, 'Query to return anything')
+        body = res.body.at(0)!
+        assert(
+          'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
+          'All keys of courseyearlystats not defined'
+        )
+      })
+
+      it('should include students who have failed a course and then completed it only once', () => {
+        const stats2017 = body.unifyStats!.statistics.find(year => year.name === '2017-2018')!
+        const { studentNumbers } = stats2017.students
+
+        assert(
+          !(
+            '457686' in studentNumbers || // SPRING 2019
+            '455478' in studentNumbers || // FALL 2020
+            '547994' in studentNumbers // FALL 2022
+          ),
+          "Students that have completed course later should not be included in the previous year's stats"
+        )
+
+        assert(
+          !body
+            .unifyStats!.statistics.find(year => year.name === '2017-2018')
+            ?.attempts.categories.passed.includes('457686') &&
+            !body
+              .unifyStats!.statistics.find(year => year.name === '2017-2018')
+              ?.attempts.categories.failed.includes('457686'),
+          "Student was incorrectly included to the failed course code's year stats (457686)"
+        )
+        assert(
+          body
+            .unifyStats!.statistics.find(year => year.name === '2018-2019')
+            ?.attempts.categories.passed.includes('457686') &&
+            body
+              .unifyStats!.statistics.find(year => year.name === '2018-2019')
+              ?.attempts.grades['5'].includes('457686'),
+          "Student was incorrectly excluded from the passed course's completion year stats (457686)"
+        )
+
+        assert(
+          !body
+            .unifyStats!.statistics.find(year => year.name === '2017-2018')
+            ?.attempts.categories.passed.includes('455478') &&
+            !body
+              .unifyStats!.statistics.find(year => year.name === '2017-2018')
+              ?.attempts.categories.failed.includes('455478'),
+          "Student was incorrectly included to the failed course code's year stats (455478)"
+        )
+        assert(
+          body
+            .unifyStats!.statistics.find(year => year.name === '2020-2021')
+            ?.attempts.categories.passed.includes('455478') &&
+            body
+              .unifyStats!.statistics.find(year => year.name === '2020-2021')
+              ?.attempts.grades['2'].includes('455478'),
+          "Student was incorrectly excluded from the passed course's completion year stats (455478)"
+        )
+
+        assert(
+          !body
+            .unifyStats!.statistics.find(year => year.name === '2017-2018')
+            ?.attempts.categories.passed.includes('547994') &&
+            !body
+              .unifyStats!.statistics.find(year => year.name === '2017-2018')
+              ?.attempts.categories.failed.includes('547994'),
+          "Student was incorrectly included to the failed course code's year stats (547994)"
+        )
+        assert(
+          body
+            .unifyStats!.statistics.find(year => year.name === '2022-2023')
+            ?.attempts.categories.passed.includes('547994') &&
+            body
+              .unifyStats!.statistics.find(year => year.name === '2022-2023')
+              ?.attempts.grades['5'].includes('547994'),
+          "Student was incorrectly excluded from the passed course's completion year stats (547994)"
+        )
+      })
+
+      it("should mark student's enrollment time correctly with mismatched semestercode and enrollment_date_time", () => {
+        const stats2021 = body.unifyStats!.statistics.find(year => year.name === '2021-2022')!
+        const stats2022 = body.unifyStats!.statistics.find(year => year.name === '2022-2023')!
+        assert('enrollments' in stats2021, 'Missing field enrollment in statsitics (2021)')
+        assert('enrollments' in stats2022, 'Missing field enrollment in statsitics (2022)')
+        // Student with enrollment_date_time outside of semestert start and end dates
+        assert(
+          !stats2021.enrollments.map(enrollment => enrollment.studentNumber).includes('455129') &&
+            stats2022.enrollments.map(enrollment => enrollment.studentNumber).includes('455129'),
+          'Student with incorrect semester (by updater) but a correct enrollment_date was not found in the correct year (by enrollment_date)'
+        )
+      })
+    })
+
+    describe('should work correctly in specific cases (MAT11001)', () => {
+      let body: Unarray<CourseYearlyStatsResBody>
+      beforeAll(async () => {
+        const res = (await request(app)
+          .get('/courseyearlystats?codes=MAT11001&combineSubstitutions=false')
+          .set('shib-session-id', 'test')
+          .set('uid', 'basic')
+          .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
+
+        assert.strictEqual(res.status, 200)
+        assert.strictEqual(res.body.length, 1)
+        body = res.body.at(0)!
+        assert(
+          'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
+          'All keys of courseyearlystats not defined'
+        )
+      })
+
+      it('where a student has failed a course', () => {
+        const stats = body.unifyStats?.statistics!.find(year => year.name === '2018-2019')
+        assert(stats && 'enrollments' in stats, 'Missing field enrollment in statsitics')
+
+        assert(
+          stats.attempts.categories.failed.includes('542874'),
+          "Stats didn't include student with only failed course attainment (failed)"
+        )
+        assert(
+          !stats.attempts.categories.passed.includes('542874'),
+          'Stats did incorrectly include student with only failed course attainment (passed)'
+        )
+        assert(
+          !stats.enrollments.map(({ studentNumber }) => studentNumber).includes('542874'),
+          "Stats didn't include student with only failed course attainment (enrollments)"
+        )
+      })
+    })
+
+    describe('should work correctly in specific cases (MAT11002)', () => {
+      let body: Unarray<CourseYearlyStatsResBody>
+      beforeAll(async () => {
+        const res = (await request(app)
+          .get('/courseyearlystats?codes=MAT11002&combineSubstitutions=false')
+          .set('shib-session-id', 'test')
+          .set('uid', 'basic')
+          .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
+
+        assert.strictEqual(res.status, 200)
+        assert.strictEqual(res.body.length, 1)
+        body = res.body.at(0)!
+        assert(
+          'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
+          'All keys of courseyearlystats not defined'
+        )
+      })
+
+      it('should not include students for AY code', () => {
+        const stats = body.unifyStats?.statistics!.find(year => year.name === '2017-2018')
+        assert(stats && 'enrollments' in stats, 'Missing field enrollment in statsitics')
+        assert(!('534980' in stats.attempts.categories.failed))
+      })
+
+      it('should not include student with failed grade after passed grade', () => {
+        const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
+        assert(year, 'Stats missing completely')
+        assert(
+          !year.attempts.categories.failed.includes('501716'),
+          'Failed students included incorrectly the student in question'
+        )
+        assert(
+          year.attempts.categories.passed.includes('501716'),
+          "Passed students didn't include student with a passed grade"
+        )
+        assert(year.attempts.grades['1'].includes('501716'), 'Grades should include the student in the correct grade')
+      })
+
+      it('should include student with failed grade and passed AY grade', () => {
+        const year = body.unifyStats?.statistics.find(year => year.name === '2020-2021')
+        assert(year, 'Stats missing completely')
+        assert(
+          year.attempts.categories.failed.includes('0011812135'),
+          "Failed students didn't incorrectly include student in question"
+        )
+        assert(
+          !year.attempts.categories.passed.includes('0011812135'),
+          'Passed students incorrectly included student without a passing grade'
+        )
+        assert(
+          year.attempts.grades['Hyl.'].includes('0011812135'),
+          'Grades should include the student in the correct grade'
+        )
+      })
+
+      it('should include student with only approved grade', () => {
+        const year = body.unifyStats?.statistics.find(year => year.name === '2022-2023')
+        assert(year, 'Stats missing completely')
+        assert(
+          !year.attempts.categories.failed.includes('543385'),
+          'Failed students included incorrectly the student in question'
+        )
+        assert(
+          year.attempts.categories.passed.includes('543385'),
+          "Passed students didn't include student with a passed grade"
+        )
+        assert(year.attempts.grades['5'].includes('543385'), 'Grades should include the student in the correct grade')
+      })
+
+      describe('should calculate enrollments correctly', () => {
+        it('- 2021-2022', () => {
+          const year = body.unifyStats?.statistics.find(year => year.name === '2021-2022')
+          assert(year && 'enrollments' in year, 'Missing field enrollment in statsitics')
+          assert.strictEqual(year.enrollments.length, 27, 'Incorrect amount of distinct enrolled students 2021')
+          assert.strictEqual(year.allEnrollments.length, 29, 'Incorrect amount of total enrolled students 2021')
+        })
+        it('- 2022-2023', () => {
+          const year = body.unifyStats?.statistics.find(year => year.name === '2022-2023')
+          assert(year && 'enrollments' in year, 'Missing field enrollment in statsitics')
+          assert.strictEqual(year.enrollments.length, 23, 'Incorrect amount of distinct enrolled students 2022')
+          assert.strictEqual(year.allEnrollments.length, 26, 'Incorrect amount of total enrolled students 2022')
+        })
+        it('- 2023-2024', () => {
+          const year = body.unifyStats?.statistics.find(year => year.name === '2023-2024')
+          assert(year && 'enrollments' in year, 'Missing field enrollment in statsitics')
+          assert.strictEqual(year.enrollments.length, 1, 'Incorrect amount of distinct enrolled students 2023')
+          assert.strictEqual(year.allEnrollments.length, 1, 'Incorrect amount of total enrolled students 2023')
+        })
+
+        it('- total', () => {
+          assert(
+            body.unifyStats?.statistics.every(year => 'enrollments' in year),
+            'Missing field enrollment in statsitics'
+          )
+          assert.strictEqual(
+            body.unifyStats?.statistics.reduce((acc, yearStats) => acc + yearStats.enrollments.length, 0),
+            27 + 23 + 1
+          )
+          assert.strictEqual(
+            body.unifyStats?.statistics.reduce((acc, yearStats) => acc + yearStats.allEnrollments.length, 0),
+            29 + 26 + 1
+          )
+          assert.strictEqual(
+            body.unifyStats?.statistics.reduce(
+              (acc, yearStats) => acc + yearStats.attempts.categories.failed.length,
+              0
+            ),
+            1
+          )
+          assert.strictEqual(
+            body.unifyStats?.statistics.reduce(
+              (acc, yearStats) => acc + yearStats.attempts.categories.passed.length,
+              0
+            ),
+            249
+          )
+        })
+      })
+    })
+
+    describe('should work correctly in specific cases (MAT21003)', () => {
+      let body: Unarray<CourseYearlyStatsResBody>
+      beforeAll(async () => {
+        const res = (await request(app)
+          .get('/courseyearlystats?codes=MAT21003&combineSubstitutions=false')
+          .set('shib-session-id', 'test')
+          .set('uid', 'basic')
+          .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
+
+        assert.strictEqual(res.status, 200)
+        assert.strictEqual(res.body.length, 1)
+        body = res.body.at(0)!
+        assert(
+          'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
+          'All keys of courseyearlystats not defined'
+        )
+      })
+
+      describe('should not count duplicate failed grades to different years', () => {
+        it('- 2017-2018 should not include failed grade', () => {
+          const year = body.unifyStats?.statistics.find(year => year.name === '2017-2018')
+          assert.strictEqual(year?.attempts.categories.failed.length, 0, 'Failed stats should not include any students')
+          assert.deepStrictEqual(year?.attempts.categories.failed, [], 'Failed stats should not include any students')
+        })
+        it('- 2018-2019 should include a failed grade', () => {
+          const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
+          assert.strictEqual(year?.attempts.categories.failed.length, 1, 'Failed stats should include only one student')
+          assert.deepStrictEqual(
+            year?.attempts.categories.failed,
+            ['539036'],
+            'Failed stats included the incorrect student'
+          )
+        })
+      })
+    })
+  })
+
+  it.todo('should return correct amount of students for a course with substitutions')
+  it.todo('should return correct amount of students for multiple coursecodes without substitutions')
+  it.todo('should return correct amount of students for multiple coursecodes with substitutions')
+})

services/backend/tests/api/population/populationStatistics.test.ts

@@ -0,0 +1,226 @@
+import { Express } from 'express'
+import { Sequelize } from 'sequelize'
+import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { CanError } from '@oodikone/shared/routes'
+import { PopulationstatisticsResBody } from '@oodikone/shared/routes/populations'
+import { dbConnections } from '../../../src/database/connection'
+import { initTests, ResponseWithBody } from '../../utils'
+
+const populationUrl = (
+  programme = '',
+  years: string[] = [],
+  semesters: ('SPRING' | 'FALL')[] = [],
+  students: string[] = []
+) => {
+  return `/populationstatistics?${programme && 'programme=' + programme}${years.map(year => '&years=' + year).join('')}${semesters.map(sem => '&semesters=' + sem).join('')}${students.map(s => '&studentStatuses=' + s).join('')}`
+}
+
+void describe('Population statistics', () => {
+  let app: Express
+  let connection: Sequelize
+
+  beforeAll(async () => {
+    app = await initTests()
+    connection = dbConnections.sequelize
+  })
+
+  describe('should fail when all fields not defined', () => {
+    it('(all)', async () => {
+      const resAllMissing = await request(app)
+        .get(populationUrl())
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')
+      assert.strictEqual(resAllMissing.status, 400)
+    })
+
+    it('(all but code)', async () => {
+      const resProgramme = await request(app)
+        .get(populationUrl('KH50_001'))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')
+
+      assert.strictEqual(resProgramme.status, 400)
+    })
+
+    it('(all but code and years)', async () => {
+      const resProgrammeAndYears = await request(app)
+        .get(populationUrl('KH50_001', ['2021', '2022']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')
+
+      assert.strictEqual(resProgrammeAndYears.status, 400)
+    })
+  })
+
+  it('should not return any data to unauthorized user', async () => {
+    const res = (await request(app)
+      .get(populationUrl('KH50_001', ['2021', '2022'], ['SPRING', 'FALL']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'norights')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CanError<PopulationstatisticsResBody>>
+
+    assert.deepStrictEqual(res.body, { error: 'Trying to request unauthorized students data' })
+    assert.strictEqual(res.status, 403)
+  })
+
+  it('should fail when trying to access a programme that user has no permissions to', async () => {
+    const res = (await request(app)
+      .get(populationUrl('MH50_001', ['2021', '2022'], ['SPRING', 'FALL']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'onlyiamrights')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CanError<PopulationstatisticsResBody>>
+
+    assert.strictEqual(res.status, 403)
+    assert.deepStrictEqual(res.body, { error: 'Trying to request unauthorized students data' })
+  })
+
+  it('should return students in the programme for authorized user', async () => {
+    const res = (await request(app)
+      .get(populationUrl('KH50_001', ['2021', '2022'], ['SPRING', 'FALL']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    assert.strictEqual(res.status, 200)
+    assert.notStrictEqual(res.body.students.length, 0)
+  })
+
+  describe('should work with the programme flag', () => {
+    // TODO: Fix this asap
+    it.skip('(bachelor)', async () => {
+      const resBachelor = (await request(app)
+        .get(populationUrl('KH50_001', ['2021'], ['SPRING', 'FALL']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+      assert.strictEqual(resBachelor.status, 200)
+      assert.strictEqual(resBachelor.body.students.length, 37)
+    })
+
+    it('(master)', async () => {
+      const resMaster = (await request(app)
+        .get(populationUrl('MH50_001', ['2021'], ['SPRING', 'FALL']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+      assert.strictEqual(resMaster.status, 200)
+      assert.strictEqual(resMaster.body.students.length, 37)
+    })
+  })
+
+  describe('should work with semester flag', () => {
+    it('(FALL)', async () => {
+      const resFall = (await request(app)
+        .get(populationUrl('KH50_001', ['2021'], ['FALL']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+      assert.strictEqual(resFall.status, 200)
+      assert.strictEqual(resFall.body.students.length, 34)
+    })
+
+    // TODO: Fix this asap
+    it.skip('(SPRING)', async () => {
+      const resSpring = (await request(app)
+        .get(populationUrl('KH50_001', ['2021'], ['SPRING']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+      assert.strictEqual(resSpring.status, 200)
+      assert.strictEqual(resSpring.body.students.length, 3)
+      // 524071 had previously start date in math of 2022-05-27
+      assert(
+        !resSpring.body.students.map(({ studentNumber }) => studentNumber).includes('524071'),
+        'Most likely student with incorrectly calculated study right start date included in to students'
+      )
+    })
+  })
+
+  describe('should work with year flag correctly', () => {
+    it('(2021)', async () => {
+      const res2021 = (await request(app)
+        .get(populationUrl('KH50_001', ['2021'], ['FALL', 'SPRING']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+      const dbCount2021 = await connection.query(
+        `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2021-07-31' and "sis_study_right_elements".start_date < '2022-08-01'`,
+        { raw: true, plain: true }
+      )
+      assert.strictEqual(res2021.status, 200)
+      assert.strictEqual(res2021.body.students.length, dbCount2021?.count)
+    })
+
+    it('(2022)', async () => {
+      const res2022 = (await request(app)
+        .get(populationUrl('KH50_001', ['2022'], ['FALL', 'SPRING']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+      const dbCount2022 = await connection.query(
+        `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2022-07-31' and "sis_study_right_elements".start_date < '2023-08-01'`,
+        { raw: true, plain: true }
+      )
+
+      assert.strictEqual(res2022.status, 200)
+      assert.strictEqual(res2022.body.students.length, dbCount2022?.count)
+    })
+
+    it('(2021-2022)', async () => {
+      const res2021_2022 = (await request(app)
+        .get(populationUrl('KH50_001', ['2021', '2022'], ['FALL', 'SPRING']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+      const dbCount2021_2022 = await connection.query(
+        `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2021-07-31' and "sis_study_right_elements".start_date < '2023-08-01'`,
+        { raw: true, plain: true }
+      )
+      assert.strictEqual(res2021_2022.status, 200)
+      assert.strictEqual(res2021_2022.body.students.length, dbCount2021_2022?.count)
+    })
+  })
+
+  it.skip('should return only students who have transferred out of the programme when TRANSFERRED flag is set', async () => {
+    // TODO: This doesn't work correctly...
+    const resWithTransOut = (await request(app)
+      .get(populationUrl('KH50_001', ['2019'], ['FALL', 'SPRING'], ['']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'mluukkai')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    const resWithoutTransOut = (await request(app)
+      .get(populationUrl('KH50_001', ['2019'], ['FALL', 'SPRING'], ['TRANSFERRED']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'mluukkai')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    const dbCountWithoutTransferredOut = await connection.query(
+      `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2019-07-31' and "sis_study_right_elements".start_date < '2020-08-01'`,
+      { raw: true, plain: true }
+    )
+    const dbCountWithTransferredOut = await connection.query(
+      `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2019-07-31' and "sis_study_right_elements".start_date < '2020-08-01'`,
+      { raw: true, plain: true }
+    )
+
+    assert.strictEqual(resWithTransOut.status, 200)
+    assert.strictEqual(resWithoutTransOut.status, 200)
+
+    assert.strictEqual(resWithTransOut.body.students.length, dbCountWithTransferredOut?.count)
+    assert.strictEqual(resWithoutTransOut.body.students.length, dbCountWithoutTransferredOut?.count)
+  })
+  it.todo('should return only degree-students and non-degree students when NONDEGREE flag is set')
+  it.todo('should return only degree-students and exchange students when EXCHANGE flag is set')
+})

services/backend/tests/api/population/populationStatisticsByCourse.test.ts

@@ -1,17 +1,16 @@
 import { Express } from 'express'
-import assert from 'node:assert/strict'
-import { describe, it, before } from 'node:test'
 import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
 
-import { initTests } from '../utils'
+import { initTests } from '../../utils'
 
-let app: Express
-before(async () => {
-  app = await initTests()
-})
+void describe('Population statistics by course', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
 
-void describe('Population statistics by course', async () => {
-  await it('should not return anything with missing parameters', async () => {
+  it('should not return anything with missing parameters', async () => {
     const res = await request(app)
       .get('/populationstatisticsbycourse')
       .set('shib-session-id', 'test')
@@ -22,7 +21,7 @@ void describe('Population statistics by course', async () => {
     assert.deepEqual(res.body.error, 'The body should have a yearcode and coursecode defined')
   })
 
-  await it('should return correct amount of students for a course', async () => {
+  it('should return correct amount of students for a course', async () => {
     const res = await request(app)
       .get('/populationstatisticsbycourse?coursecodes=["MAT21005"]&from=70&to=70')
       .set('shib-session-id', 'test')
@@ -126,6 +125,16 @@ void describe('Population statistics by course', async () => {
 
     assert.strictEqual(res.status, 200)
     assert.strictEqual(res.body.students.length, 40)
-    assert.deepStrictEqual(res.body.students.sort((a, b) => b.studentNumber - a.studentNumber).at(-1), testStudent)
+    const responseStudent: typeof testStudent = res.body.students
+      .sort((a, b) => b.studentNumber - a.studentNumber)
+      .at(-1)
+
+    assert.strictEqual(responseStudent.name, testStudent.name)
+    assert.strictEqual(responseStudent.studentNumber, testStudent.studentNumber)
+    assert.strictEqual(responseStudent.credits, testStudent.credits)
+    assert.strictEqual(responseStudent.credits, testStudent.credits)
+    assert.strictEqual(responseStudent.started, testStudent.started)
+    assert.strictEqual(responseStudent.sis_person_id, testStudent.sis_person_id)
+    assert.deepStrictEqual(responseStudent.studyRights, testStudent.studyRights)
   })
 })

services/backend/tests/api/population/populationStatisticsByStudentNumbers.ts

@@ -0,0 +1,23 @@
+import { Express } from 'express'
+import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { initTests } from '../../utils'
+
+void describe('Population statistics by student number', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
+
+  it('should not return anything with missing parameters', async () => {
+    const res = await request(app)
+      .post('/populationstatisticsbystudentnumbers')
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')
+
+    assert.deepEqual(res.status, 422)
+    assert.deepEqual(res.body.error, 'Body should include student numbers')
+  })
+})

services/backend/tests/api/population/populationStudyProgrammes.test.ts

@@ -1,17 +1,16 @@
 import { Express } from 'express'
-import assert from 'node:assert/strict'
-import { describe, it, before } from 'node:test'
 import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
 
-import { initTests } from '../utils'
+import { initTests } from '../../utils'
 
-let app: Express
-before(async () => {
-  app = await initTests()
-})
+void describe('Population statistics (study programme)', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
 
-void describe('Population statistics (study programme)', async () => {
-  await it('should return correctly formatted responses', async () => {
+  it('should return correctly formatted responses', async () => {
     const res = await request(app)
       .get('/populationstatistics/studyprogrammes')
       .set('shib-session-id', 'test')
@@ -27,7 +26,7 @@ void describe('Population statistics (study programme)', async () => {
     assert.notStrictEqual(Object.keys(res.body.filteredProgrammes).length, 0)
   })
 
-  await it('should return correct programmes to allProgrammes', async () => {
+  it('should return correct programmes to allProgrammes', async () => {
     const res = await request(app)
       .get('/populationstatistics/studyprogrammes')
       .set('shib-session-id', 'test')
@@ -46,7 +45,7 @@ void describe('Population statistics (study programme)', async () => {
         'hy-lv-69',
         'hy-lv-68',
         'hy-lv-70',
-      ],
+      ].sort(),
       degreeProgrammeType: 'urn:code:degree-program-type:bachelors-degree',
       name: {
         en: "Bachelor's Programme in Mathematical Sciences",
@@ -56,11 +55,14 @@ void describe('Population statistics (study programme)', async () => {
       progId: 'MAT',
     }
 
+    // Sort curriculum periods so that order doesn't mess up tests
+    res.body.allProgrammes.KH50_001.curriculumPeriodIds.sort()
+
     assert.strictEqual(Object.keys(res.body.allProgrammes).length, 972)
     assert.deepStrictEqual(res.body.allProgrammes.KH50_001, testMathProgramme)
   })
 
-  await it('should return the programmes, that user has access to, to filteredProgrammes', async () => {
+  it('should return the programmes, that user has access to, to filteredProgrammes', async () => {
     const res = await request(app)
       .get('/populationstatistics/studyprogrammes')
       .set('shib-session-id', 'test')
@@ -79,7 +81,7 @@ void describe('Population statistics (study programme)', async () => {
         'hy-lv-69',
         'hy-lv-68',
         'hy-lv-70',
-      ],
+      ].sort(),
       degreeProgrammeType: 'urn:code:degree-program-type:bachelors-degree',
       name: {
         en: "Bachelor's Programme in Mathematical Sciences",
@@ -91,11 +93,15 @@ void describe('Population statistics (study programme)', async () => {
 
     assert.strictEqual(Object.keys(res.body.filteredProgrammes).length, 2) // Basic has access to math bachelor + master
 
+    // Sort curriculum periods so that order doesn't mess up tests
+    res.body.filteredProgrammes.KH50_001.curriculumPeriodIds.sort()
+    res.body.allProgrammes.KH50_001.curriculumPeriodIds.sort()
+
     assert.deepStrictEqual(res.body.filteredProgrammes.KH50_001, testMathProgramme)
     assert.deepStrictEqual(res.body.allProgrammes.KH50_001, testMathProgramme) // All programmes should still include filteredProgramme programmes
   })
 
-  await it.todo(
+  it.todo(
     'should return both degree levels of study programme when searching KH90_001 or MH90_001 (ELK, ELL) in filteredProgrammes'
   )
 })