[META] Fix Fetchios lint error

include substitutions button

.github/actions/run-tests/action.yaml

@@ -19,7 +19,7 @@ runs:
   using: composite
   steps:
     - name: 'Login to toska Docker registry'
-      uses: docker/login-action@v3
+      uses: docker/login-action@v4
       with:
         registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
         username: ${{ inputs.username }}
@@ -27,10 +27,10 @@ runs:
 
     - name: 'Run services, pull rest of the needed images'
       shell: bash
-      run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull
+      run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull
 
     - name: 'Run cypress'
-      uses: cypress-io/github-action@v6
+      uses: cypress-io/github-action@v7
       with:
         browser: chrome
         spec: ${{ inputs.spec }}
@@ -38,7 +38,7 @@ runs:
         CYPRESS_baseUrl: http://localhost:1337
 
     - name: 'Upload video if tests fail'
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v7
       if: failure()
       with:
         name: 'Cypress videos - ${{ inputs.test_name }}'
@@ -47,4 +47,4 @@ runs:
     - name: 'Print logs on failure'
       shell: bash
       if: failure()
-      run: docker compose --file docker-compose.ci.yml logs
+      run: docker compose --file docker-compose.ci.yml --profile oodikone logs

.github/workflows/production.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: 'Get current time'
         id: get_current_time
@@ -49,7 +49,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Extract release version
         id: extract_version
@@ -85,7 +85,7 @@ jobs:
       - release_backend
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Extract release version
         id: extract_version
@@ -119,7 +119,7 @@ jobs:
       - release_backend
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Extract release version
         id: extract_version
@@ -157,7 +157,7 @@ jobs:
     if: success()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
@@ -178,7 +178,7 @@ jobs:
     if: failure()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:

.github/workflows/pull_request.yaml

@@ -17,14 +17,43 @@ jobs:
     name: 'Lint packages'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Lint backend, frontend and shared
         uses: ./.github/actions/lint
 
+  test_api:
+    needs:
+      - lint_packages
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install node-24 for cypress
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v4
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+
+      - name: Start oodikone detached
+        run: npm run oodikone:ci:setup
+
+      - name: Run api-tests
+        run: npm run oodikone:ci:api
+
   test_end_to_end:
     runs-on: ubuntu-latest
     needs:
       - lint_packages
+      - test_api
     strategy:
       fail-fast: false
       matrix:
@@ -59,7 +88,7 @@ jobs:
             spec: cypress/e2e/University.js
     name: ${{ matrix.name }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:

.github/workflows/staging.yaml

@@ -29,7 +29,7 @@ jobs:
     needs: test
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Get current time
         id: get_current_time
@@ -64,7 +64,7 @@ jobs:
     needs: test
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Build backend image
         id: build-image
@@ -99,7 +99,7 @@ jobs:
     if: success()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
@@ -118,7 +118,7 @@ jobs:
     if: failure()
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:

.github/workflows/test.yaml

@@ -16,7 +16,7 @@ jobs:
     name: 'Lint packages'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Lint backend, frontend and shared
         uses: ./.github/actions/lint
 
@@ -25,10 +25,10 @@ jobs:
     name: 'Build backend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: docker/setup-buildx-action@v3
+      - uses: actions/checkout@v6
+      - uses: docker/setup-buildx-action@v4
       - name: 'Build image and export it'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           build-args: |
             SENTRY_RELEASE=${{ github.sha }}
@@ -41,7 +41,7 @@ jobs:
         env:
           DOCKER_BUILD_SUMMARY: false
       - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v7
         with:
           name: backend-image
           path: /tmp/backend.tar
@@ -50,7 +50,7 @@ jobs:
     name: 'Build frontend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Get current time
         id: get_current_time
         run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
@@ -76,7 +76,7 @@ jobs:
       user_db_migrations: ${{ steps.check_migrations.outputs.user_db_migrations }}
       sis_db_migrations: ${{ steps.check_migrations.outputs.sis_db_migrations }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Check if new migrations have been added
         id: check_migrations
         uses: dorny/paths-filter@v3
@@ -98,9 +98,9 @@ jobs:
     runs-on: ubuntu-latest
     if: needs.check_migration_files.outputs.sis_db_migrations == 'true'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
@@ -120,15 +120,15 @@ jobs:
     runs-on: ubuntu-latest
     if: needs.check_migration_files.outputs.kone_db_migrations == 'true'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Download backend image'
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v8
         with:
           name: backend-image
       - name: 'Load built backend image'
         run: docker load --input backend.tar
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
@@ -148,7 +148,7 @@ jobs:
     runs-on: ubuntu-latest
     if: needs.check_migration_files.outputs.user_db_migrations == 'true'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: dorny/paths-filter@v3
         id: check_migrations
         with:
@@ -157,13 +157,13 @@ jobs:
               - 'services/backend/src/database/migrations_user/**'
               - '.github/workflows/test.yaml'
       - name: 'Download backend image'
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v8
         with:
           name: backend-image
       - name: 'Load built backend image'
         run: docker load --input backend.tar
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
@@ -174,14 +174,104 @@ jobs:
           sleep 30 # wait for backend to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
 
-  # === End-to-end tests ===
-  test_end_to_end:
+  # === Other tests ===
+  test_service_exit_statuses:
+    name: 'Check services run correctly'
     needs:
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
-    # Always run the tests, even if some of the needed jobs failed or were skipped
-    if: always()
+      - lint_packages
+    if: always() && !cancelled() && needs.lint_packages.result == 'success'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v4
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+
+      - name: 'Run services, pull rest of the needed images'
+        run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull
+
+      - name: 'Check that exit statuses are not errors'
+        run: |
+          docker ps -a
+          if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
+            echo 'Some process had nonzero exit code';
+            exit 1;
+          fi
+
+      - name: 'Save Docker images for downstream jobs'
+        run: |
+          docker save \
+            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db \
+            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db \
+            registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db \
+            redis:7.2.6 \
+            postgres:15.10 \
+            "quay.io/toska/jami:production" \
+            nginx:latest \
+          | zstd -T0 -3 -o /tmp/docker-images.tar.zst
+
+      - name: 'Upload Docker images artifact'
+        uses: actions/upload-artifact@v7
+        with:
+          name: docker-images
+          path: /tmp/docker-images.tar.zst
+          retention-days: 1
+
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker compose --file docker-compose.ci.yml --profile oodikone logs
+
+  # === Backend api-tests ===
+  test_api:
+    needs:
+      - test_service_exit_statuses
+    if: always() && !cancelled() && needs.test_service_exit_statuses.result == 'success'
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install node-24 for cypress
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: 'Download Docker images'
+        uses: actions/download-artifact@v8
+        with:
+          name: docker-images
+          path: /tmp
+
+      - name: 'Load Docker images'
+        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
+
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v4
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+
+      - name: Start oodikone (without frontend and backend)
+        run: npm run oodikone:ci:setup
+
+      - name: Run api-tests
+        run: npm run oodikone:ci:api
+
+  # === End-to-end tests ===
+  test_end_to_end:
+    needs:
+      - test_service_exit_statuses
+      - test_api
+    if: always() && !cancelled() && needs.test_service_exit_statuses.result == 'success'
     strategy:
       fail-fast: false
       matrix:
@@ -219,7 +309,21 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
+
+      - name: Install node-24 for cypress
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: 'Download Docker images'
+        uses: actions/download-artifact@v8
+        with:
+          name: docker-images
+          path: /tmp
+
+      - name: 'Load Docker images'
+        run: zstd -d /tmp/docker-images.tar.zst --stdout | docker load
 
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
@@ -228,36 +332,3 @@ jobs:
           password: ${{ secrets.password }}
           spec: ${{ matrix.spec }}
           test_name: ${{ matrix.name }}
-
-  # === Other tests ===
-  test_service_exit_statuses:
-    name: 'Check services run correctly'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    if: always()
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v5
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
-        with:
-          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-
-      - name: 'Run services, pull rest of the needed images'
-        run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull
-
-      - name: 'Check that exit statuses are not errors'
-        run: |
-          docker ps -a
-          if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
-            echo 'Some process had nonzero exit code';
-            exit 1;
-          fi
-
-      - name: 'Print logs on failure'
-        if: failure()
-        run: docker compose --file docker-compose.ci.yml logs

.github/workflows/updater.yaml

@@ -1,26 +1,19 @@
 name: 'Test (and release) Updater'
 
 on:
-  push:
-    branches:
-      - master
+  release:
+    types: [published]
     paths:
       - '.github/workflows/updater.yaml'
       - 'updater/sis-updater-worker/**'
       - 'updater/sis-updater-scheduler/**'
-  pull_request:
-    branches:
-      - master
-    paths:
-      - 'updater/sis-updater-worker/**'
-      - 'updater/sis-updater-scheduler/**'
 
 jobs:
   lint_scheduler_and_worker:
     name: 'Lint scheduler and worker'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: actions/setup-node@v6
         with:
           node-version: 24
@@ -33,10 +26,10 @@ jobs:
     name: 'Build Updater worker'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: docker/setup-buildx-action@v3
+      - uses: actions/checkout@v6
+      - uses: docker/setup-buildx-action@v4
       - name: 'Build the image'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: updater/sis-updater-worker/
           push: false
@@ -45,7 +38,7 @@ jobs:
         env:
           DOCKER_BUILD_SUMMARY: false
       - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v7
         with:
           name: updater-worker-image
           path: /tmp/worker.tar
@@ -54,10 +47,10 @@ jobs:
     name: 'Build Updater scheduler'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: docker/setup-buildx-action@v3
+      - uses: actions/checkout@v6
+      - uses: docker/setup-buildx-action@v4
       - name: 'Build the image'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: updater/sis-updater-scheduler/
           push: false
@@ -66,7 +59,7 @@ jobs:
         env:
           DOCKER_BUILD_SUMMARY: false
       - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v7
         with:
           name: updater-scheduler-image
           path: /tmp/scheduler.tar
@@ -79,15 +72,15 @@ jobs:
       - build_worker
       - build_scheduler
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
       - name: 'Download built images'
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v8
         with:
           pattern: 'updater-{worker,scheduler}-image'
           merge-multiple: true
@@ -122,9 +115,8 @@ jobs:
     name: 'Build and release Updater worker'
     runs-on: ubuntu-latest
     needs: test_updater
-    if: github.event_name == 'push'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Build worker image
         id: build-image
@@ -152,9 +144,8 @@ jobs:
     name: 'Build and release Updater scheduler'
     runs-on: ubuntu-latest
     needs: test_updater
-    if: github.event_name == 'push'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Build scheduler image
         id: build-image

.gitignore

@@ -65,9 +65,6 @@ instrumented/
 .vscode/launch.json
 .vscode/terminals.json
 
-# Funidata specific
-environments/
-
 # Target of shared symlink
 services/**/src/shared
 

.nycrc

@@ -1,5 +0,0 @@
-{
-  "all": true,
-  "include": ["services/frontend/src/**/*.js", "services/frontend/src/**/*.jsx"],
-  "extends": "./nyc-config.js"
-}

anonyymioodi/README.md

@@ -6,14 +6,14 @@ The quick guide should be enough for most purposes: updating schema, data, or th
 
 ## Quick guide to updating a test database
 
-1. Ensure you have the latest test-db. Just pulling images might not replace your existing volume, so you may need to remove it manually. This command might work: `docker volume rm oodikone_sis-db-data` (replace `oodikone_sis-db-data` with the correct volume name). The new image, including data, should be pulled the next time you launch Oodikone normally.
+1. Ensure you have the latest test-db. Just pulling images might not replace your existing volume, so you may need to remove it manually. This command might work: `docker volume rm oodikone-sis-db-data` (replace `oodikone-sis-db-data` with the correct volume name). The new image, including data, should be pulled the next time you launch Oodikone normally.
 
 2. Do your dev work as usual with the test database, for example, using `npm run both`. Edit data as needed, (for example, run the updater via the frontend), and then run tests to see if they pass (follow the instructions in the [README](../README.md) to run the tests more quickly).
 
 3. If you're updating `sis-db`:
 
    - Modify the scheduler limit in [config.js](../updater/sis-updater-scheduler/src/config.js) by setting `module.exports.DEV_SCHEDULE_COUNT = null`.
-   - Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone_sis-updater-worker_1`).
+   - Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone-sis-updater-worker`).
 
 4. When you're satisfied with the database, create a dump:
 
@@ -67,6 +67,6 @@ The quick guide should be enough for most purposes: updating schema, data, or th
 
 5. Navigate to <http://localhost:3000/updater> and click the three buttons (Update meta, Update students, Update curriculums) one by one. This will populate `sis-db` with data from the new `sis-importer-db`. You might want to do each update a couple of times, as the database tables affect each other, and not all necessary data might be generated on the first try.
 
-6. Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone_sis-updater-worker_1`). If you see any errors, try clicking the buttons again.
+6. Follow the `sis-updater-worker` logs to ensure everything is running smoothly (`docker logs --follow oodikone-sis-updater-worker`). If you see any errors, try clicking the buttons again.
 
 7. Run all the tests to ensure everything is working as expected (follow the instructions in the [README](../README.md) to run the tests more quickly). If any tests fail, update the test files accordingly.

cypress.config.js

@@ -3,7 +3,6 @@
 const { defineConfig } = require('cypress')
 
 module.exports = defineConfig({
-  experimentalStudio: process.env.NODE_ENV === 'development',
   projectId: 'c3jsph',
   defaultCommandTimeout: 30000,
   requestTimeout: 30000,
@@ -14,7 +13,6 @@ module.exports = defineConfig({
     // We've imported your old cypress plugins here.
     // You may want to clean this up later by importing these.
     setupNodeEvents(on, config) {
-      require('@cypress/code-coverage/task')(on, config)
       require('./cypress/plugins/index.js')(on, config)
       return config
     },

cypress/e2e/Course_statistics.js

@@ -33,6 +33,7 @@ const checkGradeTable = gradesTableContents => {
 }
 
 const checkTableContents = contents => {
+  cy.get('table tbody').scrollIntoView()
   cy.get('table tbody').within(() => {
     contents.forEach((values, trIndex) => {
       cy.get('tr')
@@ -396,7 +397,6 @@ describe('Basic user', () => {
     cy.contains('Search for courses')
     searchByCourseCode('TKT')
     cy.cs('select-multiple-courses-toggle').should('not.have.class', 'Mui-checked')
-    cy.contains('TKT10004, BSCS2001, 581328, A581328, AYTKT10004')
     cy.contains('Select all search results').should('not.exist')
     cy.cs('select-multiple-courses-toggle').click()
     cy.cs('select-multiple-courses-toggle').should('have.class', 'Mui-checked')

cypress/e2e/Custom_population.js

@@ -12,7 +12,7 @@ const save = () => {
 }
 
 const selectSavedPopulation = name => {
-  cy.cs('history-search').click().type(`${name}{enter}`)
+  cy.cs('history-search').type(`${name}{enter}`)
 }
 
 const deleteAllSearches = () => {

cypress/e2e/Degree_programme_overview.js

@@ -93,31 +93,14 @@ describe('Degree programme overview', () => {
     it('Basic information contains correct credits', () => {
       const years = getEmptyYears()
       const tableContents = [
-        ...years.map(year => [year, 0, 0, 0, 0, 0]),
-        [2023, 1519, 1519, 0, 0, 222],
-        [2022, 3235, 3205, 0, 30, 209],
-        [2021, 5133, 5108, 0, 25, 428],
-        [2020, 5801, 5796, 0, 5, 94],
-        [2019, 5305, 5305, 0, 0, 162],
-        [2018, 3442, 3432, 0, 10, 21],
-        [2017, 1211, 1211, 0, 0, 189],
-      ]
-
-      cy.checkTableStats(tableContents, 'credits-produced-by-the-study-programme')
-    })
-
-    it("Toggling 'Show special categories' on displays additional information", () => {
-      cy.cs('special-categories-toggle').click()
-      const years = getEmptyYears()
-      const tableContents = [
-        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0]),
-        [2023, 1519, 1519, 0, 0, 0, 0, 222],
-        [2022, 3235, 3205, 0, 30, 0, 0, 209],
-        [2021, 5133, 5108, 0, 25, 0, 0, 428],
-        [2020, 5801, 5796, 0, 5, 0, 0, 94],
-        [2019, 5305, 5305, 0, 0, 0, 0, 162],
-        [2018, 3442, 3432, 0, 10, 0, 0, 21],
-        [2017, 1211, 1211, 0, 0, 0, 0, 189],
+        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+        [2023, 1519, 1519, 0, 0, 0, 0, 0, 222, 0],
+        [2022, 3235, 3205, 0, 0, 30, 0, 0, 209, 0],
+        [2021, 5133, 5108, 0, 0, 25, 0, 0, 428, 25],
+        [2020, 5801, 5796, 0, 0, 5, 0, 0, 94, 0],
+        [2019, 5305, 5305, 0, 0, 0, 0, 0, 162, 0],
+        [2018, 3442, 3432, 0, 0, 10, 0, 0, 21, 0],
+        [2017, 1211, 1211, 0, 0, 0, 0, 0, 189, 0],
       ]
 
       cy.checkTableStats(tableContents, 'credits-produced-by-the-study-programme')
@@ -190,14 +173,14 @@ describe('Degree programme overview', () => {
       cy.checkTableStats(studentTableContents, 'students-of-the-study-programme')
 
       const creditTableContents = [
-        ...years.map(year => [year, 0, 0, 0, 0, 0]),
-        ['2023 - 2024', 160, 160, 0, 0, 67],
-        ['2022 - 2023', 2725, 2720, 0, 5, 337],
-        ['2021 - 2022', 4092, 4042, 0, 50, 198],
-        ['2020 - 2021', 5420, 5415, 0, 5, 321],
-        ['2019 - 2020', 6043, 6043, 0, 0, 101],
-        ['2018 - 2019', 4856, 4851, 0, 5, 107],
-        ['2017 - 2018', 2350, 2345, 0, 5, 26],
+        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+        ['2023 - 2024', 160, 160, 0, 0, 0, 0, 0, 67, 0],
+        ['2022 - 2023', 2725, 2720, 0, 0, 5, 0, 0, 337, 0],
+        ['2021 - 2022', 4092, 4042, 0, 0, 50, 0, 0, 198, 0],
+        ['2020 - 2021', 5420, 5415, 0, 0, 5, 0, 0, 321, 25],
+        ['2019 - 2020', 6043, 6043, 0, 0, 0, 0, 0, 101, 0],
+        ['2018 - 2019', 4856, 4851, 0, 0, 5, 0, 0, 107, 0],
+        ['2017 - 2018', 2350, 2345, 0, 0, 5, 0, 0, 26, 0],
       ]
 
       cy.checkTableStats(creditTableContents, 'credits-produced-by-the-study-programme')
@@ -231,10 +214,10 @@ describe('Degree programme overview', () => {
         cy.contains('2022')
         cy.contains('48 graduated').trigger('mouseover')
         cy.contains('48 students graduated in year 2022')
-        cy.contains('median study time: 44.5 months')
-        cy.contains('22 graduated on time')
-        cy.contains('12 graduated max year overtime')
-        cy.contains('14 graduated over year late')
+        cy.contains('median study time: 6 semesters')
+        cy.contains('26 graduated on time')
+        cy.contains('10 graduated max year overtime')
+        cy.contains('12 graduated over year late')
       })
 
       cy.cs('programmes-before-or-after-stacked-bar-chart-section')
@@ -262,7 +245,7 @@ describe('Degree programme overview', () => {
         cy.contains('2021')
         cy.contains('2 graduated').trigger('mouseover')
         cy.contains('2 students graduated in year 2021')
-        cy.contains('median study time: 25 months')
+        cy.contains('median study time: 4.5 semesters')
         cy.contains('1 graduated on time')
         cy.contains('1 graduated max year overtime')
         cy.contains('0 graduated over year late')
@@ -274,9 +257,9 @@ describe('Degree programme overview', () => {
         cy.contains('2023')
         cy.contains('11 graduated').trigger('mouseover')
         cy.contains('11 students graduated in year 2023')
-        cy.contains('median study time: 69 months')
-        cy.contains('3 graduated on time')
-        cy.contains('7 graduated max year overtime')
+        cy.contains('median study time: 12 semesters')
+        cy.contains('4 graduated on time')
+        cy.contains('6 graduated max year overtime')
         cy.contains('1 graduated over year late')
       })
     })
@@ -286,7 +269,7 @@ describe('Degree programme overview', () => {
     beforeEach(() => {
       cy.intercept(
         'GET',
-        'api/v2/studyprogrammes/KH50_001/studytrackstats?graduated=GRADUATED_INCLUDED&special_groups=SPECIAL_INCLUDED&combined_programme='
+        'api/v2/studyprogrammes/KH50_001/studytrackstats?special_groups=SPECIAL_INCLUDED&combined_programme='
       ).as('stQuery')
       cy.init('/study-programme')
       cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click()
@@ -398,44 +381,46 @@ describe('Degree programme overview', () => {
     it('Student progress data is shown correctly', () => {
       const years = getEmptyYears(true)
       const tableContents = [
-        // [Year, All, < 30 credits, 30–60 credits, 60–90 credits, 90–120 credits, 120–150 credits, 150–180 credits, ≥ 180 credits]
-        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0]),
-        ['2023 - 2024', 8, 8, 0, 0, 0, 0, 0, 0],
-        ['2022 - 2023', 26, 9, 9, 4, 3, 0, 1, 0],
-        ['2021 - 2022', 38, 8, 6, 11, 8, 5, 0, 0],
-        ['2020 - 2021', 30, 2, 1, 3, 7, 4, 4, 9],
-        ['2019 - 2020', 35, 1, 0, 2, 1, 0, 4, 27],
-        ['2018 - 2019', 45, 0, 1, 1, 2, 0, 3, 38],
-        ['2017 - 2018', 47, 0, 1, 3, 0, 1, 2, 40],
-        ['Total', 229, 28, 18, 24, 21, 10, 14, 114],
+        // [Year, All, < 30 credits, 30–60 credits, 60–90 credits, 90–120 credits, 120–150 credits, 150–180 credits, ≥ 180 credits, Graduated]
+        ...years.map(year => [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+        ['2023 - 2024', 8, 8, 0, 0, 0, 0, 0, 0, 0],
+        ['2022 - 2023', 26, 9, 8, 4, 3, 0, 0, 0, 2],
+        ['2021 - 2022', 38, 8, 4, 9, 8, 4, 0, 0, 5],
+        ['2020 - 2021', 30, 2, 1, 2, 5, 0, 0, 1, 19],
+        ['2019 - 2020', 35, 1, 0, 2, 1, 0, 0, 4, 27],
+        ['2018 - 2019', 45, 0, 1, 1, 2, 0, 0, 2, 39],
+        ['2017 - 2018', 47, 0, 1, 3, 0, 0, 1, 0, 42],
+        ['Total', 229, 28, 15, 21, 19, 4, 1, 7, 134],
       ]
 
       cy.checkTableStats(tableContents, 'study-programme-progress')
     })
 
     it('Progress section', () => {
+      cy.cs('programme-progress-bar-chart-section').scrollIntoView()
+
       cy.cs('programme-progress-bar-chart-section')
         .should('contain', 'Less than 30 credits')
         .should('contain', '30–60 credits')
         .should('contain', 'At least 180 credits')
-        .should('contain', '49.8%') // The percentage for total, at least 180 credits, to check that the graph renders
+        .should('contain', '58.5%') // The percentage for total graduated, to check that the graph renders
 
-      cy.cs('programme-progress-bar-chart-section').contains('49.8%').trigger('mouseover', { force: true })
-      cy.contains('At least 180 credits: 114')
+      cy.cs('programme-progress-bar-chart-section').contains('58.5%').trigger('mouseover', { force: true })
+      cy.contains('Graduated: 134')
     })
 
     it('Average graduation times section', () => {
       cy.cs('average-graduation-times-section').within(() => {
         cy.contains('2020 - 2021')
-        cy.get('[aria-label="2020 - 2021, 15. On time."]').trigger('mouseover')
-        cy.contains('On time: 15')
+        cy.get('[aria-label="2020 - 2021, 16. On time."]').trigger('mouseover')
+        cy.contains('On time: 16')
         cy.contains("Click a bar to view that year's study track level breakdown")
         cy.get('[aria-label="2019 - 2020, 22. On time."]').click()
         cy.contains('Year 2019 - 2020 by start year')
-        cy.get('[aria-label="MAT-MAT, 14. On time."]').trigger('mouseover')
+        cy.get('[aria-label="MAT-MAT, 13. On time."]').trigger('mouseover')
         cy.contains('Matematiikka')
         cy.contains('MAT-MAT')
-        cy.contains('On time: 14')
+        cy.contains('On time: 13')
       })
 
       cy.cs('graduation-time-toggle').click()
@@ -444,9 +429,9 @@ describe('Degree programme overview', () => {
         cy.contains('2020 - 2021')
         cy.contains('19 graduated').trigger('mouseover')
         cy.contains('From class of 2020 - 2021, 19/30 students have graduated')
-        cy.contains('median study time: 33 months')
-        cy.contains('15 graduated on time')
-        cy.contains('3 graduated max year overtime')
+        cy.contains('median study time: 6 semesters')
+        cy.contains('16 graduated on time')
+        cy.contains('2 graduated max year overtime')
         cy.contains('1 graduated over year late')
       })
     })
@@ -528,14 +513,21 @@ describe('Degree programme overview', () => {
 
     describe('By credit type tab', () => {
       beforeEach(() => {
-        cy.get('tbody').within(() => {
+        cy.get('thead').within(() => {
           cy.get('tr')
             .eq(0)
+            .within(() => {
+              cy.get('th').eq(3).click()
+            })
+        })
+        cy.get('tbody').within(() => {
+          cy.get('tr')
+            .eq(8)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('341')
+              cy.get('td').eq(1).contains('MAT11004')
+              cy.get('td').eq(2).contains('Differentiaalilaskenta')
+              cy.get('td').eq(3).contains('1170')
             })
         })
       })
@@ -548,12 +540,12 @@ describe('Degree programme overview', () => {
 
         cy.get('tbody').within(() => {
           cy.get('tr')
-            .eq(0)
+            .eq(9)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('121')
+              cy.get('td').eq(1).contains('MAT21009')
+              cy.get('td').eq(2).contains('Kandidaatintutkielma')
+              cy.get('td').eq(3).contains('414')
             })
         })
       })
@@ -568,12 +560,12 @@ describe('Degree programme overview', () => {
 
         cy.get('tbody').within(() => {
           cy.get('tr')
-            .eq(0)
+            .eq(5)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('311')
+              cy.get('td').eq(1).contains('MAT11001')
+              cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
+              cy.get('td').eq(3).contains('1220')
             })
         })
       })
@@ -632,31 +624,67 @@ describe('Degree programme overview', () => {
       })
 
       it("'Show credits/Show students' toggle works", () => {
-        cy.get('thead tr').within(() => {
-          cy.get('th').should('have.length', 8)
-          const headers = ['Type', 'Code', 'Name', 'Total', 'Major', 'Non-major', 'Non-degree', 'Transferred']
-          headers.forEach((header, index) => {
-            cy.get('th').eq(index).contains(header)
+        cy.get('thead tr')
+          .eq(0)
+          .within(() => {
+            cy.get('th').should('have.length', 6)
+            const headers = ['Type', 'Code', 'Name', 'Total credits', 'Credits produced by', 'Transferred credits']
+            headers.forEach((header, index) => {
+              cy.get('th').eq(index).contains(header)
+            })
+          })
+        cy.get('thead tr')
+          .eq(1)
+          .within(() => {
+            cy.get('th').should('have.length', 7)
+            const headers = [
+              'Degree students',
+              'Open university (hetu)',
+              'Open university (no hetu)',
+              'Exchange students',
+              'Other universities',
+              'Separate studies',
+              'Other',
+            ]
+            headers.forEach((header, index) => {
+              cy.get('th').eq(index).contains(header)
+            })
           })
-        })
 
         cy.cs('show-credits-students-toggle').click()
 
+        cy.get('thead tr')
+          .eq(0)
+          .within(() => {
+            cy.get('th').should('have.length', 7)
+            const headers = [
+              'Type',
+              'Code',
+              'Name',
+              'Total students',
+              'Breakdown of total',
+              'Breakdown of passed',
+              'Not included in total or passed',
+            ]
+            headers.forEach((header, index) => {
+              cy.get('th').eq(index).contains(header)
+            })
+          })
         cy.get('thead tr')
           .eq(1)
           .within(() => {
             cy.get('th').should('have.length', 10)
             const headers = [
-              'Type',
-              'Code',
-              'Name',
-              'Total',
               'Passed',
-              'Not',
-              'Major',
-              'Non-major',
-              'Non-degree',
-              'Transferred',
+              'Not completed',
+              'Degree students',
+              'Open university students (with hetu)',
+              'Open university students (without hetu)',
+              'Exchange students',
+              'Other university students',
+              'Separate studies',
+              'Other students',
+              'Students with transferred credits',
             ]
             headers.forEach((header, index) => {
               cy.get('th').eq(index).contains(header)
@@ -665,12 +693,12 @@ describe('Degree programme overview', () => {
 
         cy.get('tbody').within(() => {
           cy.get('tr')
-            .eq(0)
+            .eq(5)
             .within(() => {
               cy.get('td').eq(0).contains('Course')
-              cy.get('td').eq(1).contains('MAT21001')
-              cy.get('td').eq(2).contains('Lineaarialgebra ja matriisilaskenta II')
-              cy.get('td').eq(3).contains('341')
+              cy.get('td').eq(1).contains('MAT11001')
+              cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
+              cy.get('td').eq(3).contains('375')
             })
         })
       })
@@ -682,7 +710,7 @@ describe('Degree programme overview', () => {
       })
 
       // FIXME: Flaky
-      it('test', () => {
+      it.skip('test', () => {
         // TODO: Implement some real tests for this tab
         cy.contains('From')
       })

cypress/e2e/Population_statistics.js

@@ -383,7 +383,7 @@ describe('Population statistics tests', () => {
       cy.contains(nonExisting).should('not.exist')
       cy.contains('button', 'Check student numbers').click()
       cy.contains('Check for student numbers')
-      cy.cs('check-student-numbers').click().type(`${existing}{enter}${nonExisting}`)
+      cy.cs('check-student-numbers').type(`${existing}{enter}${nonExisting}`)
       cy.contains('button', 'Check students').click()
       cy.get('#checkstudentsresults').within(() => {
         cy.cs('found-title').click()

cypress/e2e/University.js

@@ -39,12 +39,6 @@ describe('University view', () => {
       checkProgressTables()
     })
 
-    it("'Graduated included / Graduated excluded' toggle works", () => {
-      cy.cs('graduated-toggle').click()
-      checkProgressBarCharts()
-      checkProgressTables()
-    })
-
     it("'All study rights / Special study rights excluded' toggle works", () => {
       cy.cs('study-right-toggle').click()
       checkProgressBarCharts()

cypress/support/e2e.js

@@ -13,7 +13,6 @@
 // https://on.cypress.io/configuration
 // ***********************************************************
 
-require('@cypress/code-coverage/support')
 require('./commands')
 
 Cypress.on('uncaught:exception', error => {

docker-compose.ci.yml

@@ -25,6 +25,8 @@ services:
     build:
       context: services
       dockerfile: backend/Dockerfile
+    profiles:
+      - oodikone
 
   frontend:
     <<: *defaults
@@ -32,6 +34,8 @@ services:
     build:
       context: services
       dockerfile: frontend/Dockerfile
+    profiles:
+      - oodikone
 
   kone-db:
     <<: *defaults
@@ -47,6 +51,7 @@ services:
     command: >
       sh -c 'redis-server --appendonly yes && redis-cli flushall'
     image: redis:7.2.6
+    container_name: redis
     ports:
       - 6379:6379
 
@@ -101,6 +106,8 @@ services:
       - 1337:80
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf
+    profiles:
+      - oodikone
 
   # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
   sis-importer-db:
@@ -114,6 +121,7 @@ services:
     <<: *defaults
     build:
       context: updater/sis-updater-worker
+    container_name: sis-updater-worker
     depends_on:
       - redis
       - sis-db

docker-compose.fd.ci.yml

@@ -1,47 +0,0 @@
-services:
-  oodikone-backend:
-    build:
-      context: ${OODIKONE_REPO:-.}/services
-      dockerfile: backend/Dockerfile
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        SERVICE_PROVIDER: fd
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/oodikone-backend:${DOCKER_IMAGE_TAG:-dev}
-
-  oodikone-frontend:
-    build:
-      context: ${OODIKONE_REPO:-.}/services
-      dockerfile: frontend/Dockerfile
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        SERVICE_PROVIDER: fd
-        LANGUAGE_CENTER_VIEW_ENABLED: 'false' # currently no intention to be enabled
-        SIS_URL: 'https://sis-qa.funidata.fi'
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/oodikone-frontend:${DOCKER_IMAGE_TAG:-dev}
-
-  updater-scheduler:
-    build:
-      context: ${OODIKONE_REPO:-.}/updater/sis-updater-scheduler
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/updater-scheduler:${DOCKER_IMAGE_TAG:-dev}
-
-  updater-worker:
-    build:
-      context: ${OODIKONE_REPO:-.}/updater/sis-updater-worker
-      args:
-        SENTRY_ENVIRONMENT: ''
-        SENTRY_RELEASE: ''
-        SENTRY_DSN: ''
-        SERVICE_PROVIDER: fd
-        NODE_ENV: production
-    image: ${DOCKER_OODIKONE_NAMESPACE:-637423527834.dkr.ecr.eu-north-1.amazonaws.com/oodikone}/updater-worker:${DOCKER_IMAGE_TAG:-dev}

docker-compose.fd.yml

@@ -1,190 +0,0 @@
-x-defaults: &defaults
-  restart: unless-stopped
-
-x-build_defaults: &build_defaults
-  dockerfile: dev.Dockerfile
-
-services:
-  adminer:
-    <<: *defaults
-    container_name: adminer
-    environment:
-      ADMINER_DESIGN: pepa-linha
-    image: adminer:4.8.1
-    ports:
-      - 5050:8080
-    volumes:
-      - ./adminer-plugins:/var/www/html/plugins-enabled
-
-  backend:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/backend
-    depends_on:
-      - kone-db
-      - redis
-      - sis-db
-      - user-db
-    env_file:
-      - .env
-    environment:
-      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
-      DB_URL_USER: postgres://postgres@user-db:5432/user-db
-      FRONT_URL: http://localhost:3000
-      REDIS: redis
-      SECRET_TOKEN: dev
-      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
-      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      IMPORTER_DB_API_URL: 'https://localhost:3005'
-      JAMI_URL: 'http://jami:3003/'
-      SERVICE_PROVIDER: fd
-      SIS_HOST: ${SIS_HOST}
-      SIS_GRAPHQL_APPLICATION_ACCOUNT: ${SIS_GRAPHQL_APPLICATION_ACCOUNT}
-      SIS_GRAPHQL_APPLICATION_KEY: ${SIS_GRAPHQL_APPLICATION_KEY}
-      IMPORTER_DB_API_USER: dev_user
-      IMPORTER_DB_API_PASSWORD: dev_password
-      ROOT_ORG_ID: ${ROOT_ORG_ID}
-      LANGUAGE_CENTER_VIEW_ENABLED: 'false'
-    ports:
-      - 8080:8080
-    volumes:
-      - backend-nodemod:/opt/app-root/backend/node_modules
-      - ./environments/testu:/opt/app-root/backend/src/environment
-      - ./services/backend/:/opt/app-root/backend
-      - ./services/shared/:/opt/app-root/shared
-
-  frontend:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/frontend
-    container_name: frontend
-    ports:
-      - 3000:3000
-    environment:
-      REACT_APP_SERVICE_PROVIDER: fd
-      REACT_APP_LANGUAGE_CENTER_VIEW_ENABLED: 'false'
-      REACT_APP_SIS_URL: 'https://sis-qa.funidata.fi'
-    volumes:
-      - frontend-nodemod:/opt/app-root/frontend/node_modules
-      - ./services/frontend/:/opt/app-root/frontend
-      - ./services/shared/:/opt/app-root/shared
-
-  kone-db:
-    <<: *defaults
-    container_name: kone-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: kone-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
-    ports:
-      - 12349:5432
-    volumes:
-      - kone-db-data:/data
-
-  redis:
-    <<: *defaults
-    command: 'redis-server --appendonly yes'
-    container_name: redis
-    image: redis:6.2.16
-    ports:
-      - '6379:6379'
-    volumes:
-      - redis-data:/data
-
-  sis-db:
-    <<: *defaults
-    container_name: sis-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: sis-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
-    volumes:
-      - sis-db-data:/data
-    shm_size: 2gb
-    ports:
-      - 12345:5432
-
-  sis-updater-scheduler:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: updater/sis-updater-scheduler
-    container_name: sis-updater-scheduler
-    extra_hosts:
-      - 'host.docker.internal:host-gateway'
-    depends_on:
-      - redis
-    environment:
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      SECRET_TOKEN: dev
-      SIS_IMPORTER_HOST: ${SIS_IMPORTER_DB_HOST:-host.docker.internal}
-      SIS_IMPORTER_PORT: ${SIS_IMPORTER_DB_PORT:-15234}
-      SIS_IMPORTER_DATABASE: importer-db
-      SIS_IMPORTER_PASSWORD: dev
-      SIS_IMPORTER_USER: dev
-      SERVICE_PROVIDER: fd
-    ports:
-      - 8082:8082
-    volumes:
-      - ./updater/sis-updater-scheduler/:/opt/app-root/src
-
-  sis-updater-worker:
-    <<: *defaults
-    deploy:
-      replicas: 2
-      mode: replicated
-    build:
-      <<: *build_defaults
-      context: updater/sis-updater-worker
-    depends_on:
-      - redis
-      - sis-db
-    extra_hosts:
-      - 'host.docker.internal:host-gateway'
-    environment:
-      DB_URL: postgres://postgres@sis-db:5432/sis-db
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      SIS_IMPORTER_HOST: ${SIS_IMPORTER_DB_HOST:-host.docker.internal}
-      SIS_IMPORTER_PORT: ${SIS_IMPORTER_DB_PORT:-15234}
-      SIS_IMPORTER_DATABASE: importer-db
-      SIS_IMPORTER_PASSWORD: dev
-      SIS_IMPORTER_USER: dev
-      SIS_PASSWORD: postgres
-      ROOT_ORG_ID: ${ROOT_ORG_ID}
-      SERVICE_PROVIDER: fd
-    volumes:
-      - ./updater/sis-updater-worker/:/opt/app-root/src
-
-  user-db:
-    <<: *defaults
-    container_name: user-db
-    environment:
-      PGDATA: /data
-      POSTGRES_DB: user-db
-      POSTGRES_HOST_AUTH_METHOD: trust
-    image: postgres:15.10
-    volumes:
-      - user-db-data:/data
-    ports:
-      - 12347:5432
-
-volumes:
-  backend-nodemod:
-  frontend-nodemod:
-  kone-db-data:
-  redis-data:
-  sis-db-data:
-  user-db-data:
-
-networks:
-  default:
-    name: oodikone
-  importer:
-    name: importer_network
-    external: true

docker-compose.test.yml

@@ -1,5 +1,26 @@
 services:
-  frontend:
+  api-tests:
     volumes:
-      - frontend-nodemod:/opt/app-root/frontend/node_modules
-      - ./services/frontend/instrumented:/opt/app-root/frontend/src
+      - backend-tests-nodemod:/opt/app-root/backend/node_modules
+      - ./services/backend/:/opt/app-root/backend
+      - ./services/shared/:/opt/app-root/shared
+    environment:
+      CI: 'true'
+      NODE_ENV: 'test'
+      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
+      FRONT_URL: http://localhost:3000
+      REDIS: redis
+      SECRET_TOKEN: test
+      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
+      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
+      JAMI_URL: 'http://jami:3003/'
+      IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
+      CRYPT_KEY: 'e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'
+    build:
+      context: services/backend
+      dockerfile: dev.Dockerfile
+    command: npm run test
+
+volumes:
+  backend-tests-nodemod:

docker-compose.yml

@@ -21,6 +21,7 @@ services:
     build:
       <<: *build_defaults
       context: services/backend
+    container_name: backend
     depends_on:
       - kone-db
       - redis
@@ -132,6 +133,7 @@ services:
     build:
       <<: *build_defaults
       context: updater/sis-updater-worker
+    container_name: sis-updater-worker
     depends_on:
       - redis
       - sis-db
@@ -168,7 +170,7 @@ services:
       - npm_config_cache=/tmp
       - PORT=3003
       - NODE_ENV=dev
-      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
+      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/jami-db-real
     ports:
       - 3003:3003
     container_name: jami

documentation/resources.md

@@ -2,26 +2,19 @@
 
 A collection of useful tools, documentation and other resources.
 
-## AI tools
-
-- [ChatGPT](https://chatgpt.com)
-- [CurreChat](https://curre.helsinki.fi/chat)
-- [GitHub Copilot](https://github.com/features/copilot)
-  - Free for students!
-- [Microsoft Copilot](https://copilot.microsoft.com)
-
 ## Code style and maintainability
 
-- [CodeClimate documentation](https://docs.codeclimate.com/docs)
-  - [Advanced configuration](https://docs.codeclimate.com/docs/advanced-configuration)
 - [ESLint documentation](https://eslint.org/docs/v8.x)
 - [Prettier documentation](https://prettier.io/docs/en)
+- [qlty.sh](https://qlty.sh/gh/UniversityOfHelsinkiCS/projects/oodikone)
+  - Qlty.sh replaced Code climate which was introduced by Riku
+  - The usefulness of qlty.sh is sometimes questionable
 
 ## Database
 
 - [pgcli](https://www.pgcli.com)
   - A useful commandline tool to interact with the database
-  - Install: `$ pip install pgcli`
+  - Can be installed from some package managers or optionally via `$ pip install pgcli`
 - [Postgres documentation](https://www.postgresql.org/docs/15/index.html)
 - [Sequelize documentation](https://sequelize.org/docs/v6/)
 
@@ -29,8 +22,6 @@ A collection of useful tools, documentation and other resources.
 
 - [GitHub documentation](https://docs.github.com/en)
   - [GitHub Actions](https://docs.github.com/en/actions)
-- [GitHub Education](https://github.com/education/students)
-  - Free benefits for students!
 - [Mermaid.js documentation](https://mermaid.js.org/intro/syntax-reference.html)
 
 ## TypeScript

eslint.config.mjs

@@ -62,7 +62,7 @@ export default tseslint.config(
       'camelcase': 'off', // TODO: Enable. old db/backend logic is still using snake_case every now and then and needs a big refactor
       'class-methods-use-this': 'off', // Disabled in favor of @typescript-eslint
       'consistent-return': 'off', // Disabled in favor of @typescript-eslint
-      'id-denylist': ['error', 'c', 'd', 'e', 'err', 't'],
+      'id-denylist': ['error', 'err'],
       'no-alert': 'error',
       'no-await-in-loop': 'error',
       'no-console': 'error',
@@ -75,7 +75,14 @@ export default tseslint.config(
       'prefer-const': 'error',
       'prefer-destructuring': 'off', // Disabled in favor of @typescript-eslint
       'dot-notation': 'off', // Disabled in favor of @typescript-eslint
-      'no-restricted-imports': ['error', { 'patterns': [{ 'regex': '^@mui/[^/]+$' }] }],
+      'no-restricted-imports': ['error',
+        {
+          'patterns':
+            [
+              { 'regex': '^@mui/[^/]+$', 'message': 'Import only the needed module. This has no effect in prod due to tree-shake, but speeds up development builds.' },
+              { 'regex': '^src/.*', 'message': 'Root imports will not resolve in production build (instead silently crash).' }
+            ]
+        }],
       'quotes': ['error', 'single',
         { avoidEscape: true, allowTemplateLiterals: false }
       ],
@@ -205,6 +212,17 @@ export default tseslint.config(
     ]
   },
 
+  // Override for backend api tests
+  {
+    files: [
+      "services/backend/tests**/*.ts"
+    ],
+    rules: {
+      "import-x/no-extraneous-dependencies": "off",
+      'no-console': 'off',
+    }
+  },
+
   // Overrides for CommonJS
   {
     files: [

nyc-config.cjs

@@ -1,6 +0,0 @@
-const { parserPlugins } = require('@istanbuljs/schema').defaults.nyc
-
-module.exports = {
-  cache: false,
-  parserPlugins: parserPlugins.concat('typescript', 'jsx'),
-}

package.json

@@ -16,7 +16,8 @@
     "cli": "./cli.sh",
     "oodikone": "./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
-    "oodikone:test": "./run.sh oodikone test up --detach",
+    "oodikone:ci:setup": "docker compose --file docker-compose.ci.yml --profile tests up --detach",
+    "oodikone:ci:api": "docker compose --file docker-compose.test.yml up --exit-code-from api-tests",
     "updater": "./run.sh updater anon up --detach",
     "updater:real": "./run.sh updater real up --detach",
     "both": "./run.sh both anon up --detach",
@@ -25,11 +26,12 @@
     "testupdater": "docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start",
     "flushredis": "docker exec -it redis redis-cli flushall",
     "prepare": "husky",
-    "rapodiff": "docker exec -t oodikone_backend_1 /opt/app-root/backend/node_modules/.bin/tsx /opt/app-root/backend/src/rapodiff/rapodiff.js",
+    "rapodiff": "docker exec -t oodikone-backend /opt/app-root/backend/node_modules/.bin/tsx /opt/app-root/backend/src/rapodiff/rapodiff.js",
     "install:local": "npm i && npm i --prefix services/shared && npm i --prefix services/frontend && npm i --prefix services/backend && npm i --prefix updater/sis-updater-scheduler && npm i --prefix updater/sis-updater-worker",
+    "i:both": "npm run install:both",
     "ci:local": "npm ci && npm ci --prefix services/shared && npm ci --prefix services/frontend && npm ci --prefix services/backend && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker",
-    "install:docker": "docker exec frontend npm i && docker exec oodikone_backend_1 npm i && docker exec sis-updater-scheduler npm i && docker exec oodikone_sis-updater-worker_1 npm i",
-    "ci:docker": "docker exec frontend npm ci && docker exec oodikone_backend_1 npm ci && docker exec sis-updater-scheduler npm ci && docker exec oodikone_sis-updater-worker_1 npm ci",
+    "install:docker": "docker exec frontend npm i && docker exec backend npm i && docker exec sis-updater-scheduler npm i && docker exec sis-updater-worker npm i",
+    "ci:docker": "docker exec frontend npm ci && docker exec backend npm ci && docker exec sis-updater-scheduler npm ci && docker exec sis-updater-worker npm ci",
     "install:both": "npm run install:local && npm run install:docker",
     "ci:both": "npm run ci:local && npm run ci:docker"
   },
@@ -43,10 +45,8 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@cypress/code-coverage": "^3.14.4",
     "@eslint/js": "^9.28.0",
-    "@istanbuljs/schema": "^0.1.3",
-    "cypress": "^15.0.0",
+    "cypress": "^15.9.0",
     "eslint": "^9.28.0",
     "eslint-config-prettier": "^10.1.5",
     "eslint-import-resolver-typescript": "^4.4.3",
@@ -58,13 +58,9 @@
     "globals": "^16.2.0",
     "husky": "^9.1.7",
     "lint-staged": "^16.1.0",
-    "nyc": "^17.1.0",
     "prettier": "3.5.3",
     "stylelint": "^16.20.0",
     "stylelint-config-standard": "^39.0.0",
     "typescript-eslint": "^8.34.0"
-  },
-  "dependencies": {
-    "lodash": "^4.17.21"
   }
 }

run.sh

@@ -35,7 +35,7 @@ parse_params() {
   option=${args[0]}
 
   #  parse arguments
-  [[ ("$option" != "oodikone" && "$option" != "updater" && "$option" != "both") ]] && \
+  [[ ("$option" != "oodikone" && "$option" != "updater" && "$option" != "both" && "$option" != "tests") ]] && \
     usage && die "Wrong option: $option"
 
   [[ ${#args[@]} -eq 1 ]] && usage && die "Wrong number of arguments"
@@ -47,7 +47,7 @@ parse_params() {
     compose_command=${args[*]:1}
   else
     version=${args[1]}
-    [[ "$version" != "anon" && "$version" != "real" && "$version" != "test" ]] && usage && \
+    [[ "$version" != "anon" && "$version" != "real" && "$version" != "api" ]] && usage && \
       die "Wrong version $version"
     compose_command=${args[*]:2}
   fi
@@ -70,8 +70,8 @@ parse_env() {
   if [[ "$version" == "real" ]]; then
     env="--file docker-compose.yml --file docker-compose.real.yml"
   fi
-  if [[ "$version" == "test" ]]; then
-    env="--file docker-compose.yml --file docker-compose.test.yml"
+  if [[ "$version" == "api" ]]; then
+    env="--file docker-compose.ci.yml"
   fi
   return 0
 }
@@ -85,9 +85,9 @@ parse_params "$@"
 parse_profiles
 parse_env
 if [[ "$compose_command" == "" ]]; then
-  final_command="docker-compose ${env}"
+  final_command="docker compose ${env}"
 else
-  final_command="docker-compose ${env} ${profiles} ${compose_command}"
+  final_command="docker compose ${env} ${profiles} ${compose_command}"
 fi
 
 msg "${BLUE}Running: ${final_command}${NOFORMAT}"

scripts/functions_for_setup.sh

@@ -11,7 +11,6 @@ PROJECT_ROOT="$(git rev-parse --show-toplevel)"
 DUMP_DIR="$PROJECT_ROOT/.databasedumps"
 S3_CONFIG_FILE=~/.s3cfg
 S3_BUCKET="s3://psyduck"
-DOCKER_COMPOSE=$PROJECT_ROOT/docker-compose.yml
 
 ## Following the naming convention in docker-compose, these are names for database
 ## services. The real data databases inside service will have names suffixed by "-real".
@@ -139,29 +138,6 @@ reset_databases() {
   successmsg "Database setup finished"
 }
 
-reset_jami_data() {
-  local database="jami-db"
-  local dump_destination="$DUMP_DIR/$database.sql.gz"
-  local s3_path="jami"
-
-  infomsg "Removing old data"
-
-
-
-  infomsg "Removing database and related volume"
-  docker volume rm oodikone_jami-data || warningmsg "This is okay, continuing"
-  docker-compose -f "$DOCKER_COMPOSE" down "$database" || warningmsg "This is okay, continuing"
-
-  infomsg "Starting postgres in the background"
-  docker-compose -f "$DOCKER_COMPOSE" up -d "$database"
-  retry docker-compose -f "$DOCKER_COMPOSE" exec "$database" pg_isready --dbname="postgres"
-
-  infomsg "Populating Jami"
-  docker exec -i "$database" /bin/bash -c "gunzip | psql -U postgres" < "$dump_destination" 2> /dev/null
-  docker-compose stop "$database"
-  msg ""
-}
-
 reset_all_real_data() {
   infomsg "Downloading real data dumps"
   for i in ${!DATABASES[*]}; do

scripts/instrument.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-
-# This script is used for instrumenting the frontend code
-# Counters are added for calculating the code coverage
-# Instrumented code is used for generating code coverage reports
-
-# Ensure the script exits if any command fails
-set -euo pipefail
-
-# Define the directories
-SOURCE_DIR="services/frontend/src"
-INSTRU_DIR="services/frontend/instrumented"
-
-# Print status
-echo -n "Instrumenting frontend code... "
-
-# Create the instrumented directory if it does not exist
-mkdir -p $INSTRU_DIR
-
-# Instrument the code
-nyc instrument --all --compact=false $SOURCE_DIR $INSTRU_DIR
-
-# Copy remaining files without overwriting instrumented files
-# 
-# Images, CSS and other static files are not instrumended
-# but they are still needed to run the application
-rsync -av --quiet --ignore-existing $SOURCE_DIR/ $INSTRU_DIR/
-
-# Print status
-echo "Done!"

services/.dockerignore

@@ -0,0 +1,7 @@
+*/.dockerignore
+*/.eslintrc.json
+*/.gitignore
+*/*Dockerfile
+*/build
+*/dist
+*/node_modules

services/backend/.dockerignore

@@ -1,3 +0,0 @@
-.dockerignore
-*Dockerfile
-node_modules/

services/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi9/nodejs-22-minimal AS build
+FROM registry.access.redhat.com/ubi9/nodejs-24-minimal AS build
 
 ENV TZ="Europe/Helsinki"
 
@@ -29,7 +29,7 @@ RUN npm ci --include=dev --prefix shared && npm ci --include=dev --prefix backen
 RUN npm run build --prefix shared && npm run build --prefix backend
 
 
-FROM registry.access.redhat.com/ubi9/nodejs-22-minimal
+FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 

services/backend/dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM node:22-alpine
+FROM node:24-alpine
 
 ENV TZ="Europe/Helsinki"
 
@@ -10,4 +10,4 @@ COPY . .
 RUN npm ci
 
 EXPOSE 8080
-CMD ["node_modules/.bin/tsx", "watch", "--clear-screen=false", "index.js"]
+CMD ["node_modules/.bin/nodemon", "--exec", "node --import tsx index.ts", "--ext", "ts,js,json", "--watch", "src", "--watch", "index.ts", "--legacy-watch"]

services/backend/package.json

@@ -2,13 +2,15 @@
   "name": "backend",
   "private": true,
   "scripts": {
-    "build": "tsc"
+    "build": "tsc -p .",
+    "test": "node --import tsx --test --test-reporter spec --test-force-exit"
   },
   "dependencies": {
     "@oodikone/shared": "file:../shared",
-    "@sentry/node": "^10.5.0",
-    "@sentry/profiling-node": "^10.5.0",
-    "axios": "^1.9.0",
+    "@sentry/node": "^10.43.0",
+    "@sentry/profiling-node": "^10.43.0",
+    "@types/lodash-es": "^4.17.12",
+    "@types/pg": "^8.18.0",
     "bullmq": "^5.53.2",
     "compression": "^1.8.0",
     "cors": "^2.8.5",
@@ -16,18 +18,17 @@
     "dayjs": "^1.11.13",
     "express": "^4.21.2",
     "express-async-errors": "^3.1.1",
-    "lodash": "^4.17.21",
+    "lodash-es": "^4.17.23",
     "lru-cache": "^11.1.0",
     "morgan": "^1.10.0",
     "papaparse": "^5.5.3",
-    "pg": "^8.16.0",
+    "pg": "^8.20.0",
     "redis": "^4.7.1",
     "reflect-metadata": "^0.2.2",
-    "sequelize": "^6.37.7",
+    "sequelize": "^6.37.8",
     "sequelize-typescript": "^2.1.6",
     "umzug": "^3.8.2",
     "unfuck-utf8-headers-middleware": "^1.0.1",
-    "vite": "^7.1.12",
     "winston": "^3.17.0",
     "winston-gelf-transporter": "^1.0.2",
     "winston-loki": "^6.1.3"
@@ -36,9 +37,11 @@
     "@types/compression": "^1.8.1",
     "@types/cors": "^2.8.19",
     "@types/express": "^4.17.23",
-    "@types/lodash": "^4.17.17",
     "@types/morgan": "^1.9.10",
     "@types/node": "^24.10.1",
+    "@types/supertest": "^6.0.3",
+    "nodemon": "^3.1.11",
+    "supertest": "^7.2.2",
     "tsx": "^4.19.4",
     "typescript": "^5.9.2"
   },

services/backend/src/config/index.ts

@@ -1,5 +1,6 @@
 // Node env to use
 export const isDev = process.env.NODE_ENV === 'development'
+export const isTest = process.env.NODE_ENV === 'test'
 export const isStaging = process.env.REACT_APP_STAGING === 'true'
 export const isProduction = !isStaging && process.env.NODE_ENV === 'production'
 
@@ -32,12 +33,6 @@ export const baseUrl = isDev ? '/api' : ''
 export const frontUrl = process.env.FRONT_URL
 export const backendPort = 8080
 
-// System run in whose environment
-export const serviceProvider = process.env.SERVICE_PROVIDER?.toLowerCase() ?? 'toska'
-
-// Optional logout-url configuration
-export const configLogoutUrl = serviceProvider === 'toska' ? undefined : process.env.LOGOUT_URL
-
 // Other stuff
 export const { DB_URL_KONE, DB_URL_USER, SECRET_TOKEN, SIS_DB_URL, SIS_UPDATER_URL, CRYPT_KEY } = process.env
 

services/backend/src/middleware/accessLogger.ts

@@ -1,5 +1,5 @@
 import { Request, Response } from 'express'
-import { omit } from 'lodash'
+import { omit } from 'lodash-es'
 import morgan from 'morgan'
 
 import { FormattedUser } from '../types'

services/backend/src/middleware/currentUser.ts

@@ -1,10 +1,9 @@
 import * as Sentry from '@sentry/node'
 import { NextFunction, Request, Response } from 'express'
-import { intersection } from 'lodash'
+import { intersection } from 'lodash-es'
 
-import { configLogoutUrl, isDev, requiredGroup, serviceProvider } from '../config'
-import { getMockedUser, getMockedUserFd, getOrganizationAccess, getUserFd, getUserToska } from '../services/userService'
-import { FormattedUser } from '../types'
+import { requiredGroup } from '../config'
+import { getMockedUser, getOrganizationAccess, getUser } from '../services/userService'
 import { ApplicationError } from '../util/customErrors'
 import logger from '../util/logger'
 
@@ -25,7 +24,7 @@ type Headers = {
   'x-show-as-user'?: string
 }
 
-const toskaUserMiddleware = async (req: Request, _: Response, next: NextFunction) => {
+export default async (req: Request, _: Response, next: NextFunction) => {
   const {
     'shib-session-id': sessionId,
     'x-show-as-user': showAsUser,
@@ -61,7 +60,7 @@ const toskaUserMiddleware = async (req: Request, _: Response, next: NextFunction
   const user =
     showAsUser && specialGroup?.superAdmin
       ? await getMockedUser({ userToMock: showAsUser, mockedBy: username })
-      : await getUserToska({
+      : await getUser({
           username,
           name: name!,
           email: email!,
@@ -80,34 +79,3 @@ const toskaUserMiddleware = async (req: Request, _: Response, next: NextFunction
 
   next()
 }
-
-const fdUserMiddleware = async (req: Request, _res: Response, next: NextFunction) => {
-  const { remote_user: remoteUser, 'x-show-as-user': showAsUser } = req.headers
-
-  if (!remoteUser) {
-    throw new ApplicationError('Not enough data in request headers, remote_user was missing', 403, { configLogoutUrl })
-  }
-
-  let user: FormattedUser | null
-
-  // getMockedUser in production requires the superAdmin role, which is only available
-  // via iamGroups, so it's now only implemented for the dev environment
-  if (showAsUser && isDev) {
-    user = await getMockedUserFd({ userToMock: showAsUser as string, mockedBy: remoteUser as string })
-  } else {
-    user = await getUserFd({ username: remoteUser as string })
-  }
-
-  if (!user) {
-    throw new ApplicationError(`Could not grant access with the eppn ${remoteUser}.`, 403, { configLogoutUrl })
-  }
-
-  req.user = user
-  req.logoutUrl = configLogoutUrl
-
-  next()
-}
-
-const currentUserMiddleware = serviceProvider === 'toska' ? toskaUserMiddleware : fdUserMiddleware
-
-export default currentUserMiddleware

services/backend/src/middleware/headers.ts

@@ -1,10 +1,4 @@
 import headersMiddleware from 'unfuck-utf8-headers-middleware'
 
-import { serviceProvider } from '../config'
-
-const headersToska = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
-const headersOther = ['remote_user']
-
-const headers = serviceProvider === 'toska' ? headersToska : headersOther
-
+const headers = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
 export default headersMiddleware(headers)

services/backend/src/models/SISStudyRight.ts

@@ -71,6 +71,9 @@ export class SISStudyRightModel extends Model implements SISStudyRight {
   @Column(DataType.JSONB)
   declare semesterEnrollments: SISStudyRight['semesterEnrollments']
 
+  @Column(DataType.JSONB)
+  declare transferInfo: SISStudyRight['transferInfo']
+
   @HasMany(() => CreditModel, { foreignKey: 'studyright_id', sourceKey: 'id' })
   declare credits: SISStudyRight['credits']
 

services/backend/src/models/course.ts

@@ -73,6 +73,9 @@ export class CourseModel extends Model implements Course {
   @Column(DataType.JSONB)
   declare substitutions: Course['substitutions']
 
+  @Column(DataType.JSONB)
+  declare substitution_groups: Course['substitution_groups']
+
   @Column(DataType.STRING)
   declare course_unit_type: Course['course_unit_type']
 

services/backend/src/rapodiff/creditsDiff.js

@@ -1,4 +1,4 @@
-const { orderBy } = require('lodash')
+const { orderBy } = require('lodash-es')
 
 // dbConnections must be imported to avoid Sequelize errors even if it's not used
 // eslint-disable-next-line @typescript-eslint/no-unused-vars

services/backend/src/routes.ts

@@ -3,7 +3,7 @@ import compression from 'compression'
 import cors from 'cors'
 import { Express, json as jsonExpress } from 'express'
 
-import { frontUrl, languageCenterViewEnabled, serviceProvider } from './config'
+import { frontUrl, languageCenterViewEnabled } from './config'
 import accessLogger from './middleware/accessLogger'
 import * as auth from './middleware/auth'
 import currentUserMiddleware from './middleware/currentUser'
@@ -34,8 +34,7 @@ import tags from './routes/tags'
 import teachers from './routes/teachers'
 import university from './routes/university'
 import updater from './routes/updater'
-import usersToska from './routes/users'
-import usersFd from './routes/usersFd'
+import users from './routes/users'
 
 const routes = (app: Express, url: string) => {
   app.use(cors({ credentials: true, origin: frontUrl }))
@@ -66,12 +65,7 @@ const routes = (app: Express, url: string) => {
   app.use(`${url}/university`, university)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, teachers)
-  if (serviceProvider === 'toska') {
-    app.use(`${url}/users`, usersToska)
-  } else {
-    app.use(`${url}/users`, usersToska)
-    app.use(`${url}/users`, usersFd)
-  }
+  app.use(`${url}/users`, users)
   app.use(`${url}/feedback`, feedback)
   app.use(`${url}/custom-population-search`, customPopulationSearch)
   app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)

services/backend/src/routes/changelog.ts

@@ -1,8 +1,8 @@
-import axios from 'axios'
 import { Router } from 'express'
 
 import { Release } from '@oodikone/shared/types'
-import { isDev } from '../config'
+import { Fetchios } from '@oodikone/shared/util/fetchios'
+import { isDev, isTest } from '../config'
 
 const router = Router()
 
@@ -13,7 +13,7 @@ router.get<never, Release[]>('/', async (_, res) => {
     return res.status(200).send(changelog.data)
   }
 
-  if (isDev) {
+  if (isDev || isTest) {
     const fakeRelease: Release[] = [
       {
         description: '**Feature 1**\n- Added a fancy new feature \n\n**Feature 2**\n- Fixed a bug\n- Fixed another bug',
@@ -39,16 +39,24 @@ router.get<never, Release[]>('/', async (_, res) => {
     return res.status(200).json(fakeRelease)
   }
 
-  const response = await axios.get('https://api.github.com/repos/UniversityOfHelsinkiCS/oodikone/releases')
-  const releasesFromAPI: Release[] = response.data.map((release: Record<string, any>) => ({
-    description: release.body,
-    time: release.published_at,
-    title: release.name,
-    version: release.tag_name,
-  }))
+  const { data } = await Fetchios.get<Record<string, any>[]>(
+    'https://api.github.com/repos/UniversityOfHelsinkiCS/oodikone/releases',
+    {}
+  )
 
-  changelog.data = releasesFromAPI
-  res.status(200).json(releasesFromAPI)
+  if (data) {
+    const releasesFromAPI: Release[] = data.map(release => ({
+      description: release.body,
+      time: release.published_at,
+      title: release.name,
+      version: release.tag_name,
+    }))
+
+    changelog.data = releasesFromAPI
+    res.status(200).json(releasesFromAPI)
+  }
+
+  res.status(500).send()
 })
 
 export default router

services/backend/src/routes/completedCoursesSearch.ts

@@ -1,5 +1,5 @@
 import { Router } from 'express'
-import { difference, intersection } from 'lodash'
+import { difference, intersection } from 'lodash-es'
 
 import { CanError } from '@oodikone/shared/routes'
 import { tryCatch } from '@oodikone/shared/util'
@@ -46,7 +46,7 @@ router.get<never, SearchResBody, SearchReqBody, SearchQuery>('/', async (req, re
   // the students have enrolled to their courses in last 8 months
   // (acual logic a bit different, see Importer)
   const { data: teacherRightsToStudents, error } = importerClient
-    ? await tryCatch<{ data: string[] }>(importerClient.post('/teacher-rights/', { teacherId, studentNumbers }))
+    ? await tryCatch(importerClient.post<string[]>('/teacher-rights/', { teacherId, studentNumbers }, {}))
     : { data: null, error: null }
 
   if (error) {

services/backend/src/routes/courses.ts

@@ -10,9 +10,8 @@ import {
   CourseYearlyStatsQuery,
 } from '@oodikone/shared/routes/courses'
 import type { CourseWithSubsId } from '@oodikone/shared/types/course'
-import { uniq } from '@oodikone/shared/util'
 import { getCourseYearlyStats } from '../services/courses'
-import { getCoursesByNameAndOrCode, getCoursesByCodes } from '../services/courses/courseFinders'
+import { getCoursesByNameAndOrCode } from '../services/courses/courseFinders'
 import { getFullStudyProgrammeRights, hasFullAccessToStudentData, validateParamLength } from '../util'
 
 const router = Router()
@@ -20,49 +19,30 @@ const router = Router()
 router.get<never, CanError<CoursesMultiResBody>, CoursesMultiReqBody, CoursesMultiQuery>(
   '/v2/coursesmulti',
   async (req, res) => {
-    const { name, code, combineSubstitutions, includeSpecial } = req.query
+    const { name, code, includeSpecial } = req.query
     if (!(validateParamLength(name, 5) || validateParamLength(code, 2))) {
       return res.status(400).json({ error: 'Query parameter name or code is invalid' })
     }
 
     const courses = await getCoursesByNameAndOrCode(name, code, includeSpecial === 'true')
 
-    if (combineSubstitutions === 'false') {
-      const courseCodes = courses.map(course => course.code)
-      const substitutions = uniq(
-        courses.flatMap(course => course.substitutions).filter(code => !courseCodes.includes(code))
-      )
-      const substitutionDetails = await getCoursesByCodes(substitutions)
-      for (const substitution of substitutionDetails) {
-        courses.push(substitution.toJSON())
-      }
-    }
-
-    const mergedCourses: Record<string, CourseWithSubsId> = {}
+    const mergedCourses = new Map<string, CourseWithSubsId>()
 
     for (const course of courses) {
       if (!(course.max_attainment_date && course.min_attainment_date)) continue
 
-      const groupId = combineSubstitutions === 'true' ? course.subsId?.toString() : course.code
-      if (!groupId) continue
+      if (!mergedCourses.has(course.code)) mergedCourses.set(course.code, structuredClone(course))
+      const mergedCourse = mergedCourses.get(course.code)!
 
-      if (!mergedCourses[groupId]) {
-        mergedCourses[groupId] = {
-          ...course,
-          substitutions: combineSubstitutions === 'true' ? course.substitutions : [],
-        }
-      } else {
-        const mergedCourse = mergedCourses[groupId]
-        if (mergedCourse.max_attainment_date < course.max_attainment_date) {
-          mergedCourse.max_attainment_date = course.max_attainment_date
-        }
-        if (mergedCourse.min_attainment_date > course.min_attainment_date) {
-          mergedCourse.min_attainment_date = course.min_attainment_date
-        }
+      if (mergedCourse.max_attainment_date < course.max_attainment_date) {
+        mergedCourse.max_attainment_date = course.max_attainment_date
+      }
+      if (mergedCourse.min_attainment_date > course.min_attainment_date) {
+        mergedCourse.min_attainment_date = course.min_attainment_date
       }
     }
 
-    res.json({ courses: Object.values(mergedCourses) })
+    res.json({ courses: Array.from(mergedCourses.values()) })
   }
 )
 

services/backend/src/routes/faculties.ts

@@ -190,7 +190,7 @@ router.get<
   GetProgressStatsQuery
 >('/:id/progressstats', auth.roles(['facultyStatistics']), async (req, res) => {
   const { id: facultyId } = req.params
-  const { graduated, special_groups: specialGroups } = req.query
+  const { special_groups: specialGroups } = req.query
 
   const faculty = await getFacultyCodeById(facultyId)
   if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
@@ -198,11 +198,11 @@ router.get<
   const programmes = await getDegreeProgrammesOfFaculty(faculty.code, true)
   if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
 
-  const data = await getFacultyProgressStats(faculty.code, specialGroups, graduated)
+  const data = await getFacultyProgressStats(faculty.code, specialGroups)
   if (data) return res.json(data)
 
-  const updatedStats = await combineFacultyStudentProgress(faculty.code, programmes, specialGroups, graduated)
-  if (updatedStats) await setFacultyProgressStats(updatedStats, specialGroups, graduated)
+  const updatedStats = await combineFacultyStudentProgress(faculty.code, programmes, specialGroups)
+  if (updatedStats) await setFacultyProgressStats(updatedStats, specialGroups)
 
   return res.json(updatedStats)
 })
@@ -234,19 +234,19 @@ router.get<
   GetProgressStatsQuery
 >('/:id/studentstats', auth.roles(['facultyStatistics']), async (req, res) => {
   const { id: facultyId } = req.params
-  const { graduated, special_groups: specialGroups } = req.query
+  const { special_groups: specialGroups } = req.query
 
   const faculty = await getFacultyCodeById(facultyId)
   if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
-  const data = await getFacultyStudentStats(faculty.code, specialGroups, graduated)
+  const data = await getFacultyStudentStats(faculty.code, specialGroups)
   if (data) return res.json(data)
 
   const programmes = await getDegreeProgrammesOfFaculty(faculty.code, true)
   if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
 
-  const updatedStats = await combineFacultyStudents(faculty.code, programmes, specialGroups, graduated)
-  if (updatedStats) await setFacultyStudentStats(updatedStats, specialGroups, graduated)
+  const updatedStats = await combineFacultyStudents(faculty.code, programmes, specialGroups)
+  if (updatedStats) await setFacultyStudentStats(updatedStats, specialGroups)
 
   return res.json(updatedStats)
 })

services/backend/src/routes/feedback.ts

@@ -1,6 +1,6 @@
 import { Request, Response, Router } from 'express'
 
-import { sendFeedbackToToska } from '../services/mailService'
+import { sendFeedback } from '../services/mailService'
 import { ApplicationError } from '../util/customErrors'
 import logger from '../util/logger'
 
@@ -20,7 +20,7 @@ router.post('/email', async (req: EmailRequest, res: Response) => {
     throw new ApplicationError('User not found', 404)
   }
 
-  await sendFeedbackToToska({
+  await sendFeedback({
     feedbackContent: content,
     user,
   })

services/backend/src/routes/population.ts

@@ -1,5 +1,5 @@
 import { Router } from 'express'
-import { difference, intersection, uniq } from 'lodash'
+import { difference, intersection, uniq } from 'lodash-es'
 
 import type { CanError } from '@oodikone/shared/routes'
 import {
@@ -15,12 +15,12 @@ import {
   PopulationstatisticsMaxYearsToCreatePopulationFormQuery,
   PopulationstatisticsMaxYearsToCreatePopulationFormResBody,
 } from '@oodikone/shared/routes/populations'
-import { Unification, Unarray } from '@oodikone/shared/types'
-import { mapToProviders } from '@oodikone/shared/util'
+import { Unification } from '@oodikone/shared/types'
+import { keyBy, mapToProviders } from '@oodikone/shared/util'
 import { rootOrgId } from '../config'
 import { getCourseProvidersForCourses, maxYearsToCreatePopulationFrom } from '../services/courses'
 import { encrypt } from '../services/encrypt'
-import { getDegreeProgrammesOfOrganization, ProgrammesOfOrganization } from '../services/faculty/faculty'
+import { getDegreeProgrammesOfOrganization } from '../services/faculty/faculty'
 import { getStudentTagMap } from '../services/populations/getStudentData'
 import { parseDateRangeFromParams } from '../services/populations/shared'
 import { statisticsOf } from '../services/populations/statisticsOf'
@@ -101,7 +101,15 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
   '/v3/populationstatistics',
   async (req, res) => {
     const { id: userId, roles: userRoles, programmeRights: userProgrammeRights, studentsUserCanAccess } = req.user
-    const { years, semesters, programme, combinedProgramme, studentStatuses } = req.query
+    const { programme, combinedProgramme } = req.query
+
+    // NOTE: Make sure we have only arrays, "/...&years=2021" is parsed by express as years: 2021, not [2021]
+    const handleQueryArrays = <T>(requestField: T | T[]): T[] => {
+      return Array.isArray(requestField) ? requestField : [requestField]
+    }
+    const years = handleQueryArrays(req.query.years)
+    const semesters = handleQueryArrays(req.query.semesters)
+    const studentStatuses = handleQueryArrays(req.query.studentStatuses) // NOTE: TRANSFERRED means transferred out of the program
 
     const requiredFields = [years, semesters, programme]
     if (requiredFields.some(field => !field)) {
@@ -112,9 +120,11 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
       return res.status(400).json({ error: 'Semester should be either SPRING OR FALL' })
     }
 
-    const hasCorrectStatus = (studentStatuses: string[]) =>
-      studentStatuses.every(status => ['EXCHANGE', 'NONDEGREE', 'TRANSFERRED'].includes(status))
-    if (studentStatuses && !hasCorrectStatus(studentStatuses)) {
+    const hasCorrectStatus = (studentStatuses: (string | undefined)[]) =>
+      studentStatuses.every(status => status && ['EXCHANGE', 'NONDEGREE', 'TRANSFERRED'].includes(status))
+
+    // NOTE: This has always the length of at least 1, [undefined]
+    if (studentStatuses[0] && !hasCorrectStatus(studentStatuses)) {
       return res.status(400).json({ error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' })
     }
 
@@ -128,10 +138,8 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
       return res.status(403).json({ error: 'Trying to request unauthorized students data' })
     }
 
-    const { startDate, endDate } = parseDateRangeFromParams({
-      ...req.query,
-      years: req.query.years,
-    })
+    // NOTE: Years are academic years
+    const { startDate, endDate } = parseDateRangeFromParams({ semesters, years })
 
     const studentNumbers = await getStudentNumbersWithStudyRights({
       programmeCodes: [programme],
@@ -252,14 +260,13 @@ router.get<never, PopulationstatisticsStudyprogrammesResBody>(
     const filteredProgrammes = hasFullAccessToStudentData(roles)
       ? programmes
       : programmes.filter(programme => allRights.includes(programme.code))
-    const formattedProgrammes = filteredProgrammes.reduce<Record<string, Unarray<ProgrammesOfOrganization>>>(
-      (acc, curr) => {
-        acc[curr.code] = curr
-        return acc
-      },
-      {}
-    )
-    res.json(formattedProgrammes)
+    const formattedFilteredProgrammes = keyBy(filteredProgrammes, 'code')
+    const formattedAllProgrammes = keyBy(programmes, 'code')
+
+    res.json({
+      allProgrammes: formattedAllProgrammes,
+      filteredProgrammes: formattedFilteredProgrammes,
+    })
   }
 )
 

services/backend/src/routes/studyProgramme.ts

@@ -12,10 +12,8 @@ import {
 } from '../services/analyticsService'
 import { getCreditsProduced } from '../services/providerCredits'
 import { getBasicStatsForStudytrack } from '../services/studyProgramme/studyProgrammeBasics'
-import {
-  getStudyProgrammeCoursesForStudyTrack,
-  getStudyProgrammeStatsForColorizedCoursesTable,
-} from '../services/studyProgramme/studyProgrammeCourses'
+import { getStudyProgrammeCoursesForStudyTrack } from '../services/studyProgramme/studyProgrammeCourses'
+import { getStudyProgrammeStatsForColorizedCoursesTable } from '../services/studyProgramme/studyProgrammeCoursesBySemester'
 import { getGraduationStatsForStudyTrack } from '../services/studyProgramme/studyProgrammeGraduations'
 import { updateBasicView, updateStudyTrackView } from '../services/studyProgramme/studyProgrammeUpdates'
 import { getStudyRightsInProgramme, getStudyTracksForProgramme } from '../services/studyProgramme/studyRightFinders'
@@ -135,13 +133,13 @@ interface GetStudyTrackStatsRequest extends Request {
 
 router.get('/:id/studytrackstats', async (req: GetStudyTrackStatsRequest, res: Response) => {
   const code = req.params.id
-  const { graduated, special_groups: specialGroups, combined_programme: combinedProgramme } = req.query
+  const { special_groups: specialGroups, combined_programme: combinedProgramme } = req.query
   if (!code) {
     return res.status(422).end()
   }
 
   void logInfoForGrafana(code, combinedProgramme)
-  const data = await getStudyTrackStats(code, combinedProgramme, graduated, specialGroups)
+  const data = await getStudyTrackStats(code, combinedProgramme, specialGroups)
   if (data) {
     return res.json(data)
   }
@@ -151,13 +149,12 @@ router.get('/:id/studytrackstats', async (req: GetStudyTrackStatsRequest, res: R
     studyProgramme: code,
     combinedProgramme,
     settings: {
-      graduated: graduated === 'GRADUATED_INCLUDED',
       specialGroups: specialGroups === 'SPECIAL_INCLUDED',
     },
     studyRightsOfProgramme,
   })
   if (updated) {
-    await setStudyTrackStats(updated, graduated, specialGroups)
+    await setStudyTrackStats(updated, specialGroups)
   }
   return res.json(updated)
 })

services/backend/src/routes/tags.ts

@@ -1,5 +1,5 @@
 import { Router } from 'express'
-import { difference } from 'lodash'
+import { difference } from 'lodash-es'
 
 import { CanError } from '@oodikone/shared/routes'
 import { NewTag, Role, StudentTag, Tag } from '@oodikone/shared/types'

services/backend/src/routes/teachers.ts

@@ -1,7 +1,6 @@
 import { Request, Response, Router } from 'express'
 
 import { mapToProviders, splitByEmptySpace } from '@oodikone/shared/util'
-import { serviceProvider } from '../config'
 import * as auth from '../middleware/auth'
 import { getProvidersOfFaculty, isFaculty } from '../services/organizations'
 import { getTeachersBySearchTerm, getTeacherStatistics, getYearlyStatistics } from '../services/teachers'
@@ -12,8 +11,7 @@ const router = Router()
 
 const iamGroupsGivingFullAccess = ['hy-dekaanit', 'hy-varadekaanit-opetus']
 
-const fullAccessAuth = () =>
-  serviceProvider === 'toska' ? auth.roles([], iamGroupsGivingFullAccess) : auth.roles(['teachers'])
+const fullAccessAuth = () => auth.roles([], iamGroupsGivingFullAccess)
 
 interface GetTeachersRequest extends Request {
   query: {

services/backend/src/routes/university.ts

@@ -1,8 +1,7 @@
 import { Request, Response, Router } from 'express'
-import { cloneDeep } from 'lodash'
+import { cloneDeep } from 'lodash-es'
 
 import { Graduated, NameWithCode } from '@oodikone/shared/types'
-import { serviceProvider } from '../config'
 import { magicFacultyCode } from '../config/organizationConstants'
 import { OrganizationModel } from '../models'
 import { getDegreeProgrammesOfFaculty } from '../services/faculty/faculty'
@@ -39,18 +38,17 @@ interface GetProgressStatsRequest extends Request {
 
 router.get('/allprogressstats', async (req: GetProgressStatsRequest, res: Response) => {
   const specialGroups = req.query?.specialsIncluded === 'true' ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED'
-  const graduated = req.query?.graduated
   const allFaculties = await getSortedFaculties()
   const facultyCodes = allFaculties.map(faculty => faculty.code)
   const codeToData: Record<string, FacultyProgressData> = {}
 
   for (const facultyCode of facultyCodes) {
-    let data = await getFacultyProgressStats(facultyCode, specialGroups, graduated)
+    let data = await getFacultyProgressStats(facultyCode, specialGroups)
     if (!data) {
       logger.info(`Data missing from server: Refreshing progress faculty data for faculty ${facultyCode}`)
       const programmes = await getDegreeProgrammesOfFaculty(facultyCode, true)
-      data = await combineFacultyStudentProgress(facultyCode, programmes, specialGroups, graduated)
-      await setFacultyProgressStats(data, specialGroups, graduated)
+      data = await combineFacultyStudentProgress(facultyCode, programmes, specialGroups)
+      await setFacultyProgressStats(data, specialGroups)
     }
     codeToData[facultyCode] = data
   }
@@ -71,6 +69,12 @@ router.get('/allprogressstats', async (req: GetProgressStatsRequest, res: Respon
       master: {} as Record<string, number[]>,
       doctor: {} as Record<string, number[]>,
     },
+    graduatedCount: {
+      bachelor: {} as Record<string, number>,
+      bachelorMaster: {} as Record<string, number>,
+      master: {} as Record<string, number>,
+      doctor: {} as Record<string, number>,
+    },
   }
 
   const unifyProgressStats = (progStats: number[][][]) => {
@@ -96,6 +100,14 @@ router.get('/allprogressstats', async (req: GetProgressStatsRequest, res: Respon
         if (!universityData.creditCounts[fieldName][year]) universityData.creditCounts[fieldName][year] = []
         universityData.creditCounts[fieldName][year].push(...facultyData.creditCounts[fieldName][year])
       }
+
+      for (const fieldName of degreeNames) {
+        if (!facultyData.graduatedCount[fieldName] || Object.keys(facultyData.graduatedCount[fieldName]).length === 0)
+          continue
+        if (!universityData.graduatedCount[fieldName][year]) universityData.graduatedCount[fieldName][year] = 0
+        universityData.graduatedCount[fieldName][year] += facultyData.graduatedCount[fieldName][year]
+      }
+
       const progStats = ['bachelorsProgStats', 'bcMsProgStats', 'mastersProgStats', 'doctoralProgStats'] as const
       for (const fieldName of progStats) {
         if (Object.keys(facultyData[fieldName] || {}).length === 0) continue
@@ -112,7 +124,7 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
   const allFaculties = await getSortedFaculties()
   const facultyCodes = allFaculties.map(faculty => faculty.code)
   const facultyData: Record<string, Awaited<ReturnType<typeof countGraduationTimes>>> = {}
-  const programmeFilter = serviceProvider === 'toska' ? 'NEW_DEGREE_PROGRAMMES' : 'ALL_PROGRAMMES'
+  const programmeFilter = 'NEW_DEGREE_PROGRAMMES'
   for (const facultyCode of facultyCodes) {
     let data: any = await getGraduationStats(facultyCode, programmeFilter, true)
     if (!data) {

services/backend/src/routes/updater.ts

@@ -57,7 +57,7 @@ router.post('/update/v2/customlist/:type', async (req: PostCustomListRequest, re
     programmes: refreshProgrammesByList,
   }
   if (typeof typeToJob[type] !== 'function') {
-    return res.status(400).json('Bad job type')
+    return res.status(400).json('Updater does not recognize this job type: ' + type)
   }
   const response = await typeToJob[type](list)
   if (response) {

services/backend/src/routes/usersFd.ts

@@ -1,68 +0,0 @@
-import { Request, Response, Router } from 'express'
-
-import { isDev } from '../config'
-import * as auth from '../middleware/auth'
-import * as userService from '../services/userService'
-
-const router = Router()
-
-interface NewUserEppnRequest extends Request {
-  params: {
-    newUserEppn: string
-  }
-}
-
-router.get('/from-sisu-by-eppn/:newUserEppn', auth.roles(['admin']), async (req: NewUserEppnRequest, res: Response) => {
-  let { username: requesterEppn } = req.user
-  const { newUserEppn } = req.params
-  // In order to test this feature in the dev environment we need to set an eppn that demo sisu will recognize
-  if (isDev && requesterEppn === 'mluukkai') {
-    requesterEppn = newUserEppn
-  }
-  const person = await userService.getUserFromSisuByEppn(requesterEppn, newUserEppn)
-  res.json(person)
-})
-
-interface User {
-  id: string
-  first_name: string
-  last_name: string
-  call_name: string
-  eppn: string
-  email_address: string
-  student_number: string
-  phone_number: string
-  document_state: string
-}
-
-interface AddUserRequest extends Request {
-  body: {
-    user: User
-  }
-}
-
-interface DeleteUserRequest extends Request {
-  body: {
-    userId: string
-  }
-}
-
-router.post('/add', auth.roles(['admin']), async (req: AddUserRequest, res: Response) => {
-  const { user } = req.body
-  const oldUser = await userService.findByUsername(user.eppn)
-  if (oldUser) return res.status(400).json({ error: 'Trying to add a user already present in the user database.' })
-  const success = await userService.addNewUser(user)
-  res.json(success)
-})
-
-router.delete('/delete', auth.roles(['admin']), async (req: DeleteUserRequest, res: Response) => {
-  const { userId } = req.body
-  try {
-    await userService.deleteUserById(userId)
-    res.json({ deleted: userId })
-  } catch (error) {
-    return res.status(400).json({ error: 'Deletion was unsuccessful with the user id included in the request' })
-  }
-})
-
-export default router

services/backend/src/services/analyticsService.ts

@@ -1,5 +1,5 @@
-import { Graduated, SpecialGroups, StudyTrackStats, YearType } from '@oodikone/shared/types'
-import { BasicStats, CreditStats } from '@oodikone/shared/types/studyProgramme'
+import { SpecialGroups, StudyTrackStats, YearType } from '@oodikone/shared/types'
+import { BasicStats, CreditStats, ProgrammeGraduationStats } from '@oodikone/shared/types/studyProgramme'
 import { facultyCodes, ignoredFacultyCodes } from '../config/organizationConstants'
 import { redisClient } from './redis'
 import { isRelevantProgramme } from './studyProgramme/studyProgrammeHelpers'
@@ -16,8 +16,8 @@ const createRedisKeyForCreditStats = (id: string, yearType: YearType, specialGro
 const createRedisKeyForGraduationStats = (id: string, yearType: YearType, specialGroups: SpecialGroups) => {
   return `GRADUATION_STATS_${id}_${yearType}_${specialGroups}`
 }
-const createRedisKeyForStudyTrackStats = (id: string, graduated: Graduated, specialGroups: SpecialGroups) => {
-  return `STUDYTRACK_STATS_${id}_${graduated}_${specialGroups}`
+const createRedisKeyForStudyTrackStats = (id: string, specialGroups: SpecialGroups) => {
+  return `STUDYTRACK_STATS_${id}_${specialGroups}`
 }
 
 // *Should* always be a string (previously unknown)
@@ -90,7 +90,7 @@ export const getGraduationStats = async (
   if (!dataFromRedis) {
     return null
   }
-  return JSON.parse(dataFromRedis)
+  return JSON.parse(dataFromRedis) as ProgrammeGraduationStats
 }
 
 export const setGraduationStats = async (data, yearType: YearType, specialGroups: SpecialGroups) => {
@@ -105,11 +105,10 @@ export const setGraduationStats = async (data, yearType: YearType, specialGroups
 export const getStudyTrackStats = async (
   id: string,
   combinedProgramme: string | null,
-  graduated: Graduated,
   specialGroups: SpecialGroups
 ) => {
   const searchKey = combinedProgramme ? `${id}-${combinedProgramme}` : id
-  const redisKey = createRedisKeyForStudyTrackStats(searchKey, graduated, specialGroups)
+  const redisKey = createRedisKeyForStudyTrackStats(searchKey, specialGroups)
   const dataFromRedis = await redisClient.get(redisKey)
   if (!dataFromRedis) {
     return null
@@ -117,11 +116,11 @@ export const getStudyTrackStats = async (
   return JSON.parse(dataFromRedis) as StudyTrackStats
 }
 
-export const setStudyTrackStats = async (data: StudyTrackStats, graduated: Graduated, specialGroups: SpecialGroups) => {
+export const setStudyTrackStats = async (data: StudyTrackStats, specialGroups: SpecialGroups) => {
   const { id } = data
   if (!shouldSaveToRedis(id)) {
     return
   }
-  const redisKey = createRedisKeyForStudyTrackStats(id, graduated, specialGroups)
+  const redisKey = createRedisKeyForStudyTrackStats(id, specialGroups)
   await redisClient.set(redisKey, JSON.stringify(data))
 }

services/backend/src/services/courses/courseFinders.ts

@@ -1,4 +1,4 @@
-import { orderBy } from 'lodash'
+import { orderBy } from 'lodash-es'
 import { Op } from 'sequelize'
 
 import { CourseWithSubsId } from '@oodikone/shared/types/course'
@@ -47,18 +47,19 @@ const getRawCourses = async (name: string, code: string, includeSpecial: boolean
       ...nameLikeTerm(name),
       ...codeLikeTerm(code),
     },
+    raw: true,
   })
 
 const getCourses = async (name: string, code: string, includeSpecial: boolean) => {
   const rawCourses = await getRawCourses(name, code, includeSpecial)
   return rawCourses
     .map(course => ({
-      ...course.toJSON(),
+      ...course,
       substitutions: orderBy(course.substitutions, [
         substitution => {
-          if (/^A/.exec(substitution)) return 4 // open university codes come last
-          if (/^\d/.exec(substitution)) return 2 // old numeric codes come second
-          if (/^[A-Za-z]/.exec(substitution)) return 1 // new letter based codes come first
+          if (/^A/.exec(substitution.at(0) ?? '')) return 4 // open university codes come last
+          if (/^\d/.exec(substitution.at(0) ?? '')) return 2 // old numeric codes come second
+          if (/^[A-Za-z]/.exec(substitution.at(0) ?? '')) return 1 // new letter based codes come first
           return 3 // unknown, comes before open uni?
         },
       ]),

services/backend/src/services/faculty/faculty.ts

@@ -1,9 +1,8 @@
-import { groupBy, orderBy } from 'lodash'
+import { groupBy, orderBy } from 'lodash-es'
 import { QueryTypes } from 'sequelize'
 
 import { Organization } from '@oodikone/shared/models'
 import { ProgrammeModuleWithRelevantAttributes } from '@oodikone/shared/types'
-import { serviceProvider } from '../../config'
 import { programmeCodes } from '../../config/programmeCodes'
 import { dbConnections } from '../../database/connection'
 import { OrganizationModel } from '../../models'
@@ -57,9 +56,7 @@ export const getDegreeProgrammesOfOrganization = async (organizationId: string,
   const programmesWithProgIds = programmesOfOrganization.map(programme => ({
     ...programme,
     progId:
-      programme.code in programmeCodes && serviceProvider !== 'fd'
-        ? programmeCodes[programme.code as keyof typeof programmeCodes]
-        : programme.code,
+      programme.code in programmeCodes ? programmeCodes[programme.code as keyof typeof programmeCodes] : programme.code,
   }))
   const programmesGroupedByCode = groupBy(orderBy(programmesWithProgIds, ['valid_from'], ['desc']), prog => prog.code)
   const curriculumPeriods = await getCurriculumPeriods()

services/backend/src/services/faculty/facultyBasics.ts

@@ -1,4 +1,4 @@
-import { cloneDeep } from 'lodash'
+import { cloneDeep } from 'lodash-es'
 
 import { NameWithCode, DegreeProgrammeType } from '@oodikone/shared/types'
 import { BasicStats } from '@oodikone/shared/types/studyProgramme'

services/backend/src/services/faculty/facultyGraduationTimes.ts

@@ -1,7 +1,9 @@
-import { cloneDeep, omit } from 'lodash'
+import { cloneDeep, omit } from 'lodash-es'
 
 import { Name, DegreeProgrammeType, Unarray } from '@oodikone/shared/types'
+import { ProgrammeGraduationStats } from '@oodikone/shared/types/studyProgramme'
 import { getGraduationStats, getStudyTrackStats, setGraduationStats, setStudyTrackStats } from '../analyticsService'
+import { GraduationTarget } from '../graduationHelpers'
 import { getGraduationStatsForStudyTrack, GraduationTimes } from '../studyProgramme/studyProgrammeGraduations'
 import { getMedian } from '../studyProgramme/studyProgrammeHelpers'
 import { getStudyRightsInProgramme } from '../studyProgramme/studyRightFinders'
@@ -22,7 +24,7 @@ export const programmeTypes = {
 } as const
 
 const getStatsByGraduationYear = async (facultyProgrammes: ProgrammesOfOrganization) => {
-  const newStats: Array<Awaited<ReturnType<typeof getGraduationStatsForStudyTrack>>> = []
+  const newStats: ProgrammeGraduationStats[] = []
   const medians: Record<string, LevelGraduationStats[]> = {}
   const programmes: { medians: Record<string, Record<string, ProgrammeStats>> } = { medians: {} }
 
@@ -133,7 +135,7 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
     if (programme.degreeProgrammeType == null || !(programme.degreeProgrammeType in programmeTypes)) {
       continue
     }
-    const statsFromRedis = await getStudyTrackStats(programme.code, '', 'GRADUATED_INCLUDED', 'SPECIAL_EXCLUDED')
+    const statsFromRedis = await getStudyTrackStats(programme.code, '', 'SPECIAL_EXCLUDED')
     if (statsFromRedis) {
       newStats.push(statsFromRedis)
       continue
@@ -144,12 +146,11 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
       studyProgramme: programme.code,
       combinedProgramme: '',
       settings: {
-        graduated: true,
         specialGroups: false,
       },
       studyRightsOfProgramme,
     })
-    await setStudyTrackStats(updatedStats, 'GRADUATED_INCLUDED', 'SPECIAL_EXCLUDED')
+    await setStudyTrackStats(updatedStats, 'SPECIAL_EXCLUDED')
     newStats.push(updatedStats)
   }
 
@@ -161,7 +162,7 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
     const basicStats = statsForProgramme.medians.basic
     const comboStats = statsForProgramme.medians.combo
     const classSizesByYear = basicStats.reduce<Record<string, number>>(
-      (acc, year) => ({ ...acc, [year.name]: year.classSize }),
+      (acc, year) => ({ ...acc, [year.name]: year.classSize ?? 0 }),
       {}
     )
     classSizes.programmes[code] = classSizesByYear
@@ -172,7 +173,8 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
     classSizes[level] ??= {}
     for (const statsForYear of basicStats) {
       classSizes[level][statsForYear.name] ??= 0
-      ;(classSizes[level][statsForYear.name] as number) += statsForYear.classSize
+      ;(classSizes[level][statsForYear.name] as number) += statsForYear.classSize ?? 0
+
       const correctYear = medians[level].find(entry => entry.name === statsForYear.name)
       if (!correctYear) {
         medians[level].push({ ...omit(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
@@ -193,7 +195,7 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
       classSizes.bcMsCombo ??= {}
       for (const statsForYear of comboStats) {
         classSizes.bcMsCombo[statsForYear.name] ??= 0
-        ;(classSizes.bcMsCombo[statsForYear.name] as number) += statsForYear.classSize
+        ;(classSizes.bcMsCombo[statsForYear.name] as number) += statsForYear.classSize ?? 0
         const correctYear = medians.bcMsCombo.find(entry => entry.name === statsForYear.name)
         if (!correctYear) {
           medians.bcMsCombo.push({ ...omit(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
@@ -252,22 +254,23 @@ export const countGraduationTimes = async (faculty: string, programmesOfFaculty:
   } = await getStatsByStartYear(programmesOfFaculty)
 
   const goals = {
-    bachelor: 36,
-    bcMsCombo: faculty === 'H90' ? 72 : 60,
-    master: 24,
-    doctor: 48,
+    bachelor: GraduationTarget.THREE_YEARS,
+    bcMsCombo: faculty === 'H90' ? GraduationTarget.SIX_YEARS : GraduationTarget.FIVE_YEARS,
+    master: GraduationTarget.TWO_YEARS,
+    doctor: GraduationTarget.FOUR_YEARS,
     exceptions: {
-      MH30_004: 6, // months more
-      '420420-ma': 6,
-      '420074-ma': 6,
-      '420119-ma': 6,
-      MH30_001: 48,
-      '320011-ma': 48,
-      '320001-ma': 48,
-      MH30_003: 42,
-      '320002-ma': 42,
-      '320009-ma': 42,
-    },
+      // Exceptions are additions to the base amount
+      MH30_004: GraduationTarget.HALF_YEAR,
+      '420420-ma': GraduationTarget.HALF_YEAR,
+      '420074-ma': GraduationTarget.HALF_YEAR,
+      '420119-ma': GraduationTarget.HALF_YEAR,
+      MH30_001: GraduationTarget.FOUR_YEARS,
+      '320011-ma': GraduationTarget.FOUR_YEARS,
+      '320001-ma': GraduationTarget.FOUR_YEARS,
+      MH30_003: GraduationTarget.THREE_POINT_FIVE_YEARS,
+      '320002-ma': GraduationTarget.THREE_POINT_FIVE_YEARS,
+      '320009-ma': GraduationTarget.THREE_POINT_FIVE_YEARS,
+    } as Record<string, number>,
   }
 
   const programmeNames = programmesOfFaculty.reduce<Record<string, Name>>((acc, { code, name }) => {

services/backend/src/services/faculty/facultyService.ts

@@ -1,5 +1,5 @@
-import { Graduated, Name, ProgrammeFilter, SpecialGroups, YearType } from '@oodikone/shared/types'
-import { DegreeProgramme } from 'src/routes/faculties'
+import { Name, ProgrammeFilter, SpecialGroups, YearType } from '@oodikone/shared/types'
+import { DegreeProgramme } from '../../routes/faculties'
 import { redisClient } from '../redis'
 import { FacultyProgressData } from './facultyStudentProgress'
 
@@ -22,11 +22,11 @@ const createRedisKeyForThesiswriters = (
 const createRedisKeyForGraduationTimeStats = (id: string, programmeFilter: ProgrammeFilter) => {
   return `FACULTY_GRADUATION_TIME_STATS_${id}_${programmeFilter}`
 }
-const createRedisKeyForFacultyProgress = (id: string, specialGroups: SpecialGroups, graduated: Graduated) => {
-  return `FACULTY_PROGRESS_STATS_${id}_${specialGroups}_${graduated}`
+const createRedisKeyForFacultyProgress = (id: string, specialGroups: SpecialGroups) => {
+  return `FACULTY_PROGRESS_STATS_${id}_${specialGroups}`
 }
-const createRedisKeyForFacultyStudents = (id: string, specialGroups: SpecialGroups, graduated: Graduated) => {
-  return `FACULTY_STUDENTS_STATS_${id}_${specialGroups}_${graduated}`
+const createRedisKeyForFacultyStudents = (id: string, specialGroups: SpecialGroups) => {
+  return `FACULTY_STUDENTS_STATS_${id}_${specialGroups}`
 }
 
 /*
@@ -177,18 +177,14 @@ export const getGraduationStats = async (id: string, programmeFilter: ProgrammeF
   return data
 }
 
-export const setFacultyProgressStats = async (
-  data: FacultyProgressData,
-  specialGroups: SpecialGroups,
-  graduated: Graduated
-) => {
+export const setFacultyProgressStats = async (data: FacultyProgressData, specialGroups: SpecialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForFacultyProgress(id, specialGroups, graduated)
+  const redisKey = createRedisKeyForFacultyProgress(id, specialGroups)
   await redisClient.set(redisKey, JSON.stringify(data))
 }
 
-export const getFacultyProgressStats = async (id: string, specialGroups: SpecialGroups, graduated: Graduated) => {
-  const redisKey = createRedisKeyForFacultyProgress(id, specialGroups, graduated)
+export const getFacultyProgressStats = async (id: string, specialGroups: SpecialGroups) => {
+  const redisKey = createRedisKeyForFacultyProgress(id, specialGroups)
   const dataFromRedis = await redisClient.get(redisKey)
 
   return dataFromRedis ? (JSON.parse(dataFromRedis) as FacultyProgressData) : null
@@ -204,18 +200,14 @@ type FacultyStudentsData = {
   programmeNames: Record<string, DegreeProgramme>
 }
 
-export const setFacultyStudentStats = async (
-  data: FacultyStudentsData,
-  specialGroups: SpecialGroups,
-  graduated: Graduated
-) => {
+export const setFacultyStudentStats = async (data: FacultyStudentsData, specialGroups: SpecialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForFacultyStudents(id, specialGroups, graduated)
+  const redisKey = createRedisKeyForFacultyStudents(id, specialGroups)
   await redisClient.set(redisKey, JSON.stringify(data))
 }
 
-export const getFacultyStudentStats = async (id: string, specialGroups: SpecialGroups, graduated: Graduated) => {
-  const redisKey = createRedisKeyForFacultyStudents(id, specialGroups, graduated)
+export const getFacultyStudentStats = async (id: string, specialGroups: SpecialGroups) => {
+  const redisKey = createRedisKeyForFacultyStudents(id, specialGroups)
   const dataFromRedis = await redisClient.get(redisKey)
 
   return dataFromRedis ? (JSON.parse(dataFromRedis) as FacultyStudentsData) : null

services/backend/src/services/faculty/facultyStudentProgress.ts

@@ -1,7 +1,7 @@
 import dayjs from 'dayjs'
-import { cloneDeep } from 'lodash'
+import { cloneDeep } from 'lodash-es'
 
-import { Graduated, SpecialGroups } from '@oodikone/shared/types'
+import { SpecialGroups } from '@oodikone/shared/types'
 import { rootOrgId } from '../../config'
 import { getStudyTrackStats, setStudyTrackStats } from '../analyticsService'
 import { getYearsArray } from '../studyProgramme/studyProgrammeHelpers'
@@ -15,7 +15,7 @@ type CreditLimits = Array<[number | null, number | null]>
 const createLimits = (
   months: number,
   startingCredits: number,
-  level: 'bachelor' | 'master' | 'doctor'
+  level: 'bachelor' | 'master' | 'bcMs' | 'doctor'
 ): CreditLimits => {
   if (level === 'doctor') {
     return [
@@ -28,11 +28,11 @@ const createLimits = (
   }
 
   return [
-    [Math.ceil(months * (60 / 12)), null],
-    [Math.ceil(months * (45 / 12)), Math.ceil(months * (60 / 12))],
-    [Math.ceil(months * (30 / 12)), Math.ceil(months * (45 / 12))],
-    [Math.ceil(months * (15 / 12)), Math.ceil(months * (30 / 12))],
-    [startingCredits + 1, Math.ceil(months * (15 / 12))],
+    [startingCredits + Math.ceil(months * (60 / 12)), null],
+    [startingCredits + Math.ceil(months * (45 / 12)), startingCredits + Math.ceil(months * (60 / 12))],
+    [startingCredits + Math.ceil(months * (30 / 12)), startingCredits + Math.ceil(months * (45 / 12))],
+    [startingCredits + Math.ceil(months * (15 / 12)), startingCredits + Math.ceil(months * (30 / 12))],
+    [startingCredits + 1, startingCredits + Math.ceil(months * (15 / 12))],
     [null, startingCredits],
   ]
 }
@@ -58,22 +58,18 @@ const calculateProgressStats = (
   progressStats: Record<string, Record<string, number[][]>>,
   progId: string
 ) => {
-  const goalMonthsForLevels = { bachelor: 36, master: 24, bcMs: 60, doctor: 48 } as const
+  // "bcMs" is the same as masters, as the students have graduated from bachelors.
+  // We need to add the 180cr to the starting point for compensation.
+  const goalMonthsForLevels = { bachelor: 36, master: 24, bcMs: 24, doctor: 48 } as const
 
   const allYears = Object.keys(creditCounts)
   const limitsForYears = allYears.reduce<Record<string, CreditLimits>>((acc, year) => {
     const startDate = new Date(`${year.slice(0, 4)}-08-01`)
     const lastDayOfMonth = dayjs().endOf('month')
-    const months =
-      Math.round(dayjs(lastDayOfMonth).diff(startDate, 'months', true)) +
-      Number(level === 'bcMs') * goalMonthsForLevels.bachelor
+    const months = Math.round(dayjs(lastDayOfMonth).diff(startDate, 'months', true))
     const goalMonths = goalMonthsForLevels[level]
 
-    acc[year] = createLimits(
-      Math.min(months, goalMonths),
-      Number(level === 'bcMs') * 180,
-      level === 'bcMs' ? 'master' : level
-    )
+    acc[year] = createLimits(Math.min(months, goalMonths), Number(level === 'bcMs') * 180, level)
     return acc
   }, {})
 
@@ -104,13 +100,13 @@ const calculateProgressStats = (
 export const combineFacultyStudentProgress = async (
   faculty: string,
   programmes: ProgrammesOfOrganization,
-  specialGroups: SpecialGroups,
-  graduated: Graduated
+  specialGroups: SpecialGroups
 ) => {
   const since = new Date('2017-08-01')
   const statsOfProgrammes: Array<Awaited<ReturnType<typeof getStudyTrackStatsForStudyProgramme>>> = []
   const allDegreeProgrammes = await getDegreeProgrammesOfOrganization(rootOrgId, false)
   const creditCounts: Record<string, Record<string, number[]>> = {}
+  const graduatedCount: Record<string, Record<string, number>> = {}
   const progressStats: Record<string, Record<string, number[][]>> = {}
   const yearlyTitles: Record<string, string[][]> = {}
 
@@ -123,7 +119,7 @@ export const combineFacultyStudentProgress = async (
     ) {
       continue
     }
-    const statsFromRedis = await getStudyTrackStats(studyProgramme, null, graduated, specialGroups)
+    const statsFromRedis = await getStudyTrackStats(studyProgramme, null, specialGroups)
     if (statsFromRedis) {
       statsOfProgrammes.push(statsFromRedis)
     } else {
@@ -131,22 +127,23 @@ export const combineFacultyStudentProgress = async (
       const updatedStats = await getStudyTrackStatsForStudyProgramme({
         studyProgramme,
         settings: {
-          graduated: graduated === 'GRADUATED_INCLUDED',
           specialGroups: specialGroups === 'SPECIAL_INCLUDED',
         },
         studyRightsOfProgramme,
       })
       statsOfProgrammes.push(updatedStats)
-      await setStudyTrackStats(updatedStats, graduated, specialGroups)
+      await setStudyTrackStats(updatedStats, specialGroups)
     }
   }
 
-  const updateCreditCounts = (
+  const updateCreditAndGraduatedCount = (
     level: 'bachelor' | 'bcMs' | 'master' | 'doctor',
-    creditCountsOfProgramme: Record<string, number[]>
+    creditCountsOfProgramme: Record<string, number[]>,
+    graduatedCountOfProgramme: Record<string, number>
   ) => {
     if (!(level in creditCounts)) {
       creditCounts[level] = cloneDeep(creditCountsOfProgramme)
+      graduatedCount[level] = cloneDeep(graduatedCountOfProgramme)
     } else {
       for (const year of Object.keys(creditCountsOfProgramme)) {
         if (!(year in creditCounts[level])) {
@@ -155,6 +152,10 @@ export const combineFacultyStudentProgress = async (
           creditCounts[level][year].push(...creditCountsOfProgramme[year])
         }
       }
+
+      for (const year of Object.keys(graduatedCountOfProgramme)) {
+        graduatedCount[level][year] += cloneDeep(graduatedCountOfProgramme[year])
+      }
     }
   }
 
@@ -167,11 +168,11 @@ export const combineFacultyStudentProgress = async (
     const { degreeProgrammeType, progId } = programmeInfo
     const level = programmeTypes[degreeProgrammeType]
 
-    updateCreditCounts(level, stats.creditCounts)
+    updateCreditAndGraduatedCount(level, stats.creditCounts, stats.graduatedCount)
     calculateProgressStats(level, stats.creditCounts, yearlyTitles, progressStats, progId)
 
     if (level === 'master' && Object.keys(stats.creditCountsCombo).length > 0) {
-      updateCreditCounts('bcMs', stats.creditCountsCombo)
+      updateCreditAndGraduatedCount('bcMs', stats.creditCountsCombo, stats.graduatedCount)
       calculateProgressStats('bcMs', stats.creditCountsCombo, yearlyTitles, progressStats, progId)
     }
   }
@@ -184,6 +185,7 @@ export const combineFacultyStudentProgress = async (
     bcMsProgStats: progressStats.bcMs,
     doctoralProgStats: progressStats.doctor,
     creditCounts: {} as Record<string, Record<string, number[]>>,
+    graduatedCount: {} as Record<string, Record<string, number>>,
     yearlyBachelorTitles: yearlyTitles.bachelor,
     yearlyBcMsTitles: yearlyTitles.bcMs,
     yearlyMasterTitles: yearlyTitles.master,
@@ -206,6 +208,14 @@ export const combineFacultyStudentProgress = async (
       }
       result.creditCounts[levelName][year] = creditCounts[level][year]
     }
+
+    for (const year of Object.keys(graduatedCount[level]).toSorted(sortByYear)) {
+      const levelName = level === 'bcMs' ? 'bachelorMaster' : level
+      if (!(levelName in result.graduatedCount)) {
+        result.graduatedCount[levelName] = {}
+      }
+      result.graduatedCount[levelName][year] = graduatedCount[level][year]
+    }
   }
 
   return result

services/backend/src/services/faculty/facultyStudents.ts

@@ -61,15 +61,14 @@ const calculateCombinedStats = (programmeCodes: string[], stats: StudyTrackStats
 export const combineFacultyStudents = async (
   code: string,
   programmes: ProgrammesOfOrganization,
-  specialGroups: 'SPECIAL_INCLUDED' | 'SPECIAL_EXCLUDED',
-  graduated: 'GRADUATED_INCLUDED' | 'GRADUATED_EXCLUDED'
+  specialGroups: 'SPECIAL_INCLUDED' | 'SPECIAL_EXCLUDED'
 ) => {
   let years: string[] = []
   const programmeCodes = programmes.map(programme => programme.code)
   const newStats: StudyTrackStats[] = []
 
   for (const studyProgrammeCode of programmeCodes) {
-    const statsFromRedis = await getStudyTrackStats(studyProgrammeCode, null, graduated, specialGroups)
+    const statsFromRedis = await getStudyTrackStats(studyProgrammeCode, null, specialGroups)
     if (statsFromRedis) {
       newStats.push(statsFromRedis)
       if (!years.length) {
@@ -81,12 +80,11 @@ export const combineFacultyStudents = async (
     const updatedStats = await getStudyTrackStatsForStudyProgramme({
       studyProgramme: studyProgrammeCode,
       settings: {
-        graduated: graduated === 'GRADUATED_INCLUDED',
         specialGroups: specialGroups === 'SPECIAL_INCLUDED',
       },
       studyRightsOfProgramme,
     })
-    await setStudyTrackStats(updatedStats, graduated, specialGroups)
+    await setStudyTrackStats(updatedStats, specialGroups)
     if (!years.length) {
       years = updatedStats.years
     }

services/backend/src/services/faculty/facultyThesisWriters.ts

@@ -1,4 +1,4 @@
-import { cloneDeep } from 'lodash'
+import { cloneDeep } from 'lodash-es'
 
 import { DegreeProgrammeType, type NameWithCode } from '@oodikone/shared/types'
 import { getGraduationStats, setGraduationStats } from '../analyticsService'

services/backend/src/services/faculty/facultyUpdates.ts

@@ -18,7 +18,6 @@ import { combineFacultyThesisWriters } from './facultyThesisWriters'
 const yearOptions = ['CALENDAR_YEAR', 'ACADEMIC_YEAR'] as const
 const specialGroupOptions = ['SPECIAL_INCLUDED', 'SPECIAL_EXCLUDED'] as const
 const programmeFilterOptions = ['ALL_PROGRAMMES', 'NEW_DEGREE_PROGRAMMES'] as const
-const graduatedOptions = ['GRADUATED_INCLUDED', 'GRADUATED_EXCLUDED'] as const
 
 export const updateFacultyOverview = async (facultyCode: string, statsType: StatsType) => {
   for (const programmeFilter of programmeFilterOptions) {
@@ -56,18 +55,11 @@ export const updateFacultyOverview = async (facultyCode: string, statsType: Stat
 export const updateFacultyProgressOverview = async (facultyCode: string) => {
   const onlyNew = await getDegreeProgrammesOfFaculty(facultyCode, true)
 
-  for (const graduated of graduatedOptions) {
-    for (const specialGroups of specialGroupOptions) {
-      const updateFacultyStudentStats = await combineFacultyStudents(facultyCode, onlyNew, specialGroups, graduated)
-      await setFacultyStudentStats(updateFacultyStudentStats, specialGroups, graduated)
-      const updateFacultyProgressStats = await combineFacultyStudentProgress(
-        facultyCode,
-        onlyNew,
-        specialGroups,
-        graduated
-      )
-      await setFacultyProgressStats(updateFacultyProgressStats, specialGroups, graduated)
-    }
+  for (const specialGroups of specialGroupOptions) {
+    const updateFacultyStudentStats = await combineFacultyStudents(facultyCode, onlyNew, specialGroups)
+    await setFacultyStudentStats(updateFacultyStudentStats, specialGroups)
+    const updateFacultyProgressStats = await combineFacultyStudentProgress(facultyCode, onlyNew, specialGroups)
+    await setFacultyProgressStats(updateFacultyProgressStats, specialGroups)
   }
 
   return 'OK'

services/backend/src/services/graduationHelpers.ts

@@ -1,13 +1,27 @@
-export const countTimeCategories = (times: number[], goal: number) => {
+/** Map years to semesters */
+export const GraduationTarget = {
+  HALF_YEAR: 1,
+  ONE_YEAR: 2,
+  TWO_YEARS: 4,
+  THREE_YEARS: 6,
+  THREE_POINT_FIVE_YEARS: 7,
+  FOUR_YEARS: 8,
+  FIVE_YEARS: 10,
+  SIX_YEARS: 12,
+} as const
+
+/** Uses semesters, not months */
+export const countTimeCategories = (times: number[], target: number) => {
   const statistics = { onTime: 0, yearOver: 0, wayOver: 0 }
-  times.forEach(time => {
-    if (time <= goal) {
-      statistics.onTime += 1
-    } else if (time <= goal + 12) {
-      statistics.yearOver += 1
+
+  for (const time of times) {
+    if (time <= target) {
+      statistics.onTime++
+    } else if (time <= target + GraduationTarget.ONE_YEAR) {
+      statistics.yearOver++
     } else {
-      statistics.wayOver += 1
+      statistics.wayOver++
     }
-  })
+  }
   return statistics
 }

services/backend/src/services/languageCenterData.js

@@ -1,10 +1,10 @@
-const { orderBy } = require('lodash')
-const { Op } = require('sequelize')
+import { orderBy } from 'lodash-es'
+import { Op } from 'sequelize'
 
-const { CourseModel, CreditModel, EnrollmentModel, SISStudyRightModel } = require('../models')
-const { redisClient } = require('./redis')
+import { CourseModel, CreditModel, EnrollmentModel, SISStudyRightModel } from '../models'
+import { redisClient } from './redis'
 
-const LANGUAGE_CENTER_REDIS_KEY = 'LANGUAGE_CENTER_DATA'
+export const LANGUAGE_CENTER_REDIS_KEY = 'LANGUAGE_CENTER_DATA'
 
 const isBetween = (start, date, end) => {
   return new Date(start).getTime() <= new Date(date).getTime() && new Date(date).getTime() <= new Date(end).getTime()
@@ -33,7 +33,7 @@ const getLanguageCenterCourses = async () => {
   return courses
 }
 
-const createArrayOfCourses = async (attempts, courses) => {
+export const createArrayOfCourses = async (attempts, courses) => {
   const fields = { completions: 0, enrollments: 0, difference: 0, rejected: 0 }
   const semesters = {}
   const faculties = {}
@@ -88,7 +88,7 @@ const createArrayOfCourses = async (attempts, courses) => {
   return courseList
 }
 
-const computeLanguageCenterData = async () => {
+export const computeLanguageCenterData = async () => {
   const courses = await getLanguageCenterCourses()
   const autumnSemester2017 = 135
 
@@ -218,17 +218,10 @@ const computeLanguageCenterData = async () => {
   return { tableData, faculties }
 }
 
-const getLanguageCenterData = async () => {
+export const getLanguageCenterData = async () => {
   const dataOnRedis = await redisClient.get(LANGUAGE_CENTER_REDIS_KEY)
   if (dataOnRedis) return JSON.parse(dataOnRedis)
   const freshData = await computeLanguageCenterData()
   await redisClient.set(LANGUAGE_CENTER_REDIS_KEY, JSON.stringify(freshData))
   return freshData
 }
-
-module.exports = {
-  computeLanguageCenterData,
-  createArrayOfCourses,
-  getLanguageCenterData,
-  LANGUAGE_CENTER_REDIS_KEY,
-}

services/backend/src/services/mailService.ts

@@ -1,11 +1,10 @@
-import axios from 'axios'
-
+import { Fetchios } from '@oodikone/shared/util/fetchios'
 import { isProduction, pateToken } from '../config'
 import { FormattedUser } from '../types'
-import { ApplicationError } from '../util/customErrors'
+import logger from '../util/logger'
 
-const pateClient = axios.create({
-  baseURL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/pate/',
+const pateClient = Fetchios.create({
+  baseUrl: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/pate/',
   params: {
     token: pateToken,
   },
@@ -22,18 +21,12 @@ const baseSettings = {
 }
 
 const sendEmail = async (options = {}) => {
-  if (!isProduction) throw new ApplicationError('Email sending is disabled in development mode.')
-  if (!pateToken) throw new ApplicationError('Email sending failed because pate token is missing.')
-  return await pateClient.post('/', options)
+  if (!isProduction) return logger.error('Email sending is disabled in development mode.')
+  if (!pateToken) return logger.error('Email sending failed because pate token is missing.')
+  return await pateClient.post('', options, {})
 }
 
-export const sendFeedbackToToska = async ({
-  feedbackContent,
-  user,
-}: {
-  feedbackContent: string
-  user: FormattedUser
-}) => {
+export const sendFeedback = async ({ feedbackContent, user }: { feedbackContent: string; user: FormattedUser }) => {
   const { name, userId, email } = user
   const userDetails = `Sent by ${name}, userid: ${userId}, email: ${email}`
   const text = [feedbackContent, userDetails].join('<br />')

services/backend/src/services/oriProvider.js

@@ -1,44 +1,45 @@
-const axios = require('axios')
+import { Fetchios } from '@oodikone/shared/util/fetchios'
 
 const { sisUrl, sisGrapqlAppAccount, sisGrapqlAppKey } = require('../config')
 const { ApplicationError } = require('../util/customErrors')
 
-const getSisuAccessToken = async eppn => {
+export const getSisuAccessToken = async eppn => {
   const oriAppAuthPath = `${sisUrl}/ori/application-auth`
-  let sisuResponse
-  try {
-    sisuResponse = await axios.post(
-      oriAppAuthPath,
-      {
-        username: sisGrapqlAppAccount,
-        key: sisGrapqlAppKey,
-        executingEppn: eppn,
-      },
-      { 'content-type': 'application/json' }
-    )
-  } catch (error) {
+
+  const { data: sisuResponse, error } = await Fetchios.post(
+    oriAppAuthPath,
+    {
+      username: sisGrapqlAppAccount,
+      key: sisGrapqlAppKey,
+      executingEppn: eppn,
+    },
+    { 'content-type': 'application/json' }
+  )
+
+  if (error) {
     throw new ApplicationError(
       'Could not find the user in Sisu, either with the eppn of the user or the new person.',
       404
     )
   }
-  return sisuResponse.data.authToken
+
+  return sisuResponse.authToken
 }
 
-const decodeJwtTokenPayloadToObject = token => {
+export const decodeJwtTokenPayloadToObject = token => {
   const payload = token.split('.')[1]
   const paddedPayload = `${payload}==`
   const bufferObj = Buffer.from(paddedPayload, 'base64')
   return JSON.parse(bufferObj.toString('utf-8'))
 }
 
-const getSisuAuthData = async eppn => {
+export const getSisuAuthData = async eppn => {
   const accessToken = await getSisuAccessToken(eppn)
   const tokenData = decodeJwtTokenPayloadToObject(accessToken)
   return { accessToken, tokenData }
 }
 
-const personSearchQuery = `query privatePerson($subjectUserId: ID!) {
+export const personSearchQuery = `query privatePerson($subjectUserId: ID!) {
       private_person(id: $subjectUserId) {
           id
           first_name:firstNames
@@ -52,32 +53,17 @@ const personSearchQuery = `query privatePerson($subjectUserId: ID!) {
       }
   }`
 
-const getGraphqlData = async (accessToken, queryObject) => {
-  const graphQlUrl = `${sisUrl}/api`
-  const authHeader = `Application ${accessToken}`
-  const config = {
-    url: graphQlUrl,
-    method: 'post',
-    headers: {
-      Authorization: authHeader,
-      'Accept-Charset': 'UTF-8',
-      'Content-Type': 'application/json',
-    },
-    data: queryObject,
-  }
-  let graphQlResponse
+export const getGraphqlData = async (accessToken, queryObject) => {
   try {
-    graphQlResponse = await axios(config)
+    const graphQlResponse = await Fetchios.post(`${sisUrl}/api`, queryObject, {
+      headers: {
+        Authorization: `Application ${accessToken}`,
+        'Accept-Charset': 'UTF-8',
+        'Content-Type': 'application/json',
+      },
+    })
+    return graphQlResponse.data.data.private_person
   } catch (error) {
     throw new ApplicationError('Graphql request failed for unclear reasons.', 500)
   }
-  return graphQlResponse.data.data.private_person
-}
-
-module.exports = {
-  getSisuAccessToken,
-  decodeJwtTokenPayloadToObject,
-  getSisuAuthData,
-  personSearchQuery,
-  getGraphqlData,
 }

services/backend/src/services/populations/formatStatisticsForApi.ts

@@ -69,6 +69,7 @@ export const formatStudentForAPI = (
     sis_person_id: student.sis_person_id,
     citizenships: student.citizenships,
     curriculumVersion: getCurriculumVersion(hops?.curriculum_period_id),
+    hasPersonalIdentityCode: student.hasPersonalIdentityCode,
 
     tags,
     transferredStudyright,

services/backend/src/services/populations/getStudentData.ts

@@ -79,6 +79,7 @@ type StudentPersonalData = Pick<
   | 'gender_code'
   | 'birthdate'
   | 'sis_person_id'
+  | 'hasPersonalIdentityCode'
 >
 
 type StudentStudyRightElement = Pick<
@@ -125,6 +126,7 @@ export const getStudents = (studentNumbers: string[]): Promise<Array<StudentData
       'gender_code',
       'birthdate',
       'sis_person_id',
+      'hasPersonalIdentityCode',
     ],
     include: [
       {

services/backend/src/services/populations/shared.ts

@@ -1,6 +1,7 @@
-import { orderBy } from 'lodash'
+import { orderBy } from 'lodash-es'
 import { Op } from 'sequelize'
 
+import { CourseStats } from '@oodikone/shared/routes/populations'
 import { Name, DegreeProgrammeType } from '@oodikone/shared/types'
 import { SISStudyRightElementModel, CourseModel } from '../../models'
 import { SemesterStart } from '../../util/semester'
@@ -75,9 +76,9 @@ export const getOptionsForStudents = (
   return optionMap
 }
 
-export const getCourses = (courses: string[]): Promise<Array<Pick<CourseModel, 'code' | 'name' | 'substitutions'>>> =>
+export const getCourses = (courses: string[]): Promise<Array<CourseStats>> =>
   CourseModel.findAll({
-    attributes: ['code', 'name', 'substitutions'],
+    attributes: ['code', 'name', 'substitutions', 'substitution_groups', 'is_study_module'],
     where: {
       code: { [Op.in]: courses },
     },

services/backend/src/services/providerCredits.ts

@@ -5,6 +5,7 @@ import { mapToProviders } from '@oodikone/shared/util'
 import { getCreditStats, setCreditStats } from './analyticsService'
 import { getCourseCodesOfProvider } from './providers'
 import { getCreditsForProvider, getTransferredCredits } from './studyProgramme/creditGetters'
+import { getStudentHetuStateMap } from './studyProgramme/studentGetters'
 import { defineYear } from './studyProgramme/studyProgrammeHelpers'
 import { getSISStudyRightsOfStudents } from './studyProgramme/studyRightFinders'
 
@@ -47,15 +48,17 @@ const basicDegreeExtentCodes = [ExtentCode.BACHELOR, ExtentCode.MASTER, ExtentCo
   This is probably very rare to cause differences, only one such was found in MH60_001
 */
 
-// TODO: How should licentiate and doctorate studies be categorized?
-const getCategory = (extentCode?: ExtentCode, degreeStudyRightExtentCode?: ExtentCode) => {
+const getCategory = (hasHetu: boolean, extentCode?: ExtentCode, degreeStudyRightExtentCode?: ExtentCode) => {
   if (degreeStudyRightExtentCode && basicDegreeExtentCodes.includes(degreeStudyRightExtentCode)) {
     return 'basic'
   }
   switch (extentCode) {
     case ExtentCode.OPEN_UNIVERSITY_STUDIES:
-    case ExtentCode.SUMMER_AND_WINTER_SCHOOL:
-      return 'open-uni'
+      if (hasHetu) {
+        return 'open-uni-hetu'
+      } else {
+        return 'open-uni-no-hetu'
+      }
     case ExtentCode.MASTER:
     case ExtentCode.BACHELOR_AND_MASTER:
     case ExtentCode.BACHELOR:
@@ -63,6 +66,7 @@ const getCategory = (extentCode?: ExtentCode, degreeStudyRightExtentCode?: Exten
     case ExtentCode.NON_DEGREE_STUDIES:
     case ExtentCode.NON_DEGREE_PEGAGOGICAL_STUDIES_FOR_TEACHERS:
     case ExtentCode.NON_DEGREE_PROGRAMME_FOR_SPECIAL_EDUCATION_TEACHERS:
+    case ExtentCode.SPECIALIZATION_STUDIES:
       return 'separate'
     case ExtentCode.EXCHANGE_STUDIES:
     case ExtentCode.EXCHANGE_STUDIES_POSTGRADUATE:
@@ -72,9 +76,9 @@ const getCategory = (extentCode?: ExtentCode, degreeStudyRightExtentCode?: Exten
     case ExtentCode.DOCTOR:
     case ExtentCode.LICENTIATE:
     case ExtentCode.CONTINUING_EDUCATION:
-    case ExtentCode.SPECIALIZATION_STUDIES:
     case ExtentCode.SPECIALIST_TRAINING_IN_MEDICINE_AND_DENTISTRY:
     case ExtentCode.STUDIES_FOR_SECONDARY_SCHOOL_STUDENTS:
+    case ExtentCode.SUMMER_AND_WINTER_SCHOOL:
     case undefined:
       return 'other'
     default:
@@ -122,38 +126,43 @@ export const computeCreditsProduced = async (providerCode: string, isAcademicYea
 
   const students = [...new Set(credits.map(({ studentNumber }) => studentNumber))]
 
-  const studyRights = await getSISStudyRightsOfStudents(students)
+  const [studyRights, studentToHetuStateMap] = await Promise.all([
+    getSISStudyRightsOfStudents(students),
+    getStudentHetuStateMap(students),
+  ])
 
-  const studyRightIdToStudyRightMap = studyRights.reduce<Record<string, (typeof studyRights)[number]>>((obj, cur) => {
-    obj[cur.id] = cur
-    return obj
-  }, {})
+  const studyRightIdToStudyRightMap = new Map(studyRights.map(sr => [sr.id, sr]))
 
-  const studentNumberToStudyRightsMap = studyRights.reduce<Record<string, typeof studyRights>>((obj, cur) => {
-    obj[cur.studentNumber] ??= []
-    obj[cur.studentNumber].push(cur)
-    return obj
-  }, {})
+  const studentNumberToStudyRightsMap = studyRights.reduce((map, cur) => {
+    // Node does not support getOrInsert as of writing :-(
+    if (map.has(cur.studentNumber)) {
+      map.get(cur.studentNumber)!.push(cur)
+    } else {
+      map.set(cur.studentNumber, [cur])
+    }
+    return map
+  }, new Map<string, [(typeof studyRights)[number]]>())
 
   const stats: Record<string, Record<string, number>> = {}
 
   for (const { attainmentDate, studentNumber, credits: numOfCredits, studyrightId, semestercode } of credits) {
-    const studyRightLinkedToAttainment = studyRightIdToStudyRightMap[studyrightId]
+    const studyRightLinkedToAttainment = studyRightIdToStudyRightMap.get(studyrightId)
     const attainmentYear = defineYear(attainmentDate, isAcademicYear)
     const degreeStudyRight = getBasicDegreeStudyRight(
-      studentNumberToStudyRightsMap[studentNumber],
+      studentNumberToStudyRightsMap.get(studentNumber),
       attainmentDate,
       semestercode
     )
+    const hasHetu = !!studentToHetuStateMap.get(studentNumber)
 
     if (!studyRightLinkedToAttainment && !degreeStudyRight) {
       continue
     }
 
-    const category = getCategory(studyRightLinkedToAttainment?.extentCode, degreeStudyRight?.extentCode)
+    const category = getCategory(hasHetu, studyRightLinkedToAttainment?.extentCode, degreeStudyRight?.extentCode)
     stats[attainmentYear] ??= {}
     stats[attainmentYear][category] ??= 0
-    stats[attainmentYear][category] += numOfCredits || 0
+    stats[attainmentYear][category] += numOfCredits
   }
 
   const transferredCredits = await getTransferredCredits(rapoFormattedProviderCode, since)
@@ -165,7 +174,7 @@ export const computeCreditsProduced = async (providerCode: string, isAcademicYea
     const attainmentYear = defineYear(createdate, isAcademicYear)
     stats[attainmentYear] ??= {}
     stats[attainmentYear].transferred ??= 0
-    stats[attainmentYear].transferred += credits || 0
+    stats[attainmentYear].transferred += credits
   })
 
   return { stats, id: providerCode }

services/backend/src/services/sisUpdaterService.ts

@@ -1,46 +1,48 @@
-import axios from 'axios'
 import { Op } from 'sequelize'
 
+import { Fetchios } from '@oodikone/shared/util/fetchios'
 import { SECRET_TOKEN, SIS_UPDATER_URL } from '../config'
 import { StudyplanModel } from '../models'
 
-const client = axios.create({ baseURL: SIS_UPDATER_URL })
-const params = { params: { token: SECRET_TOKEN } }
+const client = Fetchios.create({
+  baseUrl: SIS_UPDATER_URL,
+  params: { token: SECRET_TOKEN ?? '' },
+})
 
 export const abortUpdate = async () => {
-  const response = await client.get('/v1/abort', params)
+  const response = await client.get<{ message: string }>('/v1/abort', {})
   return response.data?.message
 }
 
 export const updateSISCoursesByCourseCode = async (coursecodes: string[]) => {
   const data = { coursecodes }
-  const response = await client.post('/v1/courses', data, params)
+  const response = await client.post('/v1/courses', data, {})
   return response.data
 }
 
 export const updateSISMetadata = async () => {
-  const response = await client.get('/v1/meta', params)
+  const response = await client.get('/v1/meta', {})
   return response.data
 }
 
 export const updateSISProgrammes = async () => {
-  const response = await client.get('/v1/programmes', params)
+  const response = await client.get('/v1/programmes', {})
   return response.data
 }
 
 export const updateSISRedisCache = async () => {
-  const response = await client.get('/v1/rediscache', params)
+  const response = await client.get('/v1/rediscache', {})
   return response.data
 }
 
 export const updateSISStudents = async () => {
-  const response = await client.get('/v1/students', params)
+  const response = await client.get('/v1/students', {})
   return response.data
 }
 
 export const updateSISStudentsByStudentNumber = async (studentnumbers: string[]) => {
   const data = { studentnumbers }
-  const response = await client.post('/v1/students', data, params)
+  const response = await client.post('/v1/students', data, {})
   return response.data
 }
 
@@ -54,6 +56,6 @@ export const updateSISStudyPlans = async (days: number) => {
     limit: 1000,
   })
   const studentnumbers = result.map(r => r.studentnumber)
-  const response = await client.post('v1/studyplans', { studentnumbers }, params)
+  const response = await client.post('v1/studyplans', { studentnumbers }, {})
   return response.data
 }

services/backend/src/services/studyGuidanceGroups.ts

@@ -1,6 +1,5 @@
 import * as Sentry from '@sentry/node'
-import { AxiosError } from 'axios'
-import { uniq } from 'lodash'
+import { uniq } from 'lodash-es'
 
 import type { StudyGuidanceGroup, Tags, TagsByGroupId, GroupsWithTags } from '@oodikone/shared/types/studyGuidanceGroup'
 import { StudyGuidanceGroupTagModel } from '../models/kone'
@@ -12,22 +11,23 @@ const importerClient = getImporterClient()
 const getGroupsFromImporter = async (sisPersonId: string): Promise<StudyGuidanceGroup[]> => {
   if (!importerClient || !sisPersonId) return []
 
-  try {
-    const response: { data: Record<string, StudyGuidanceGroup> } = await importerClient.get(
-      `/person-groups/person/${sisPersonId}`
-    )
+  const { data, error } = await importerClient.get<Record<string, StudyGuidanceGroup>>(
+    `/person-groups/person/${sisPersonId}`,
+    {}
+  )
 
-    return response?.data ? Object.values(response.data) : []
-  } catch (error) {
+  if (error) {
     logger.error('Could not fetch study guidance groups!')
-    if (error instanceof AxiosError) {
+    if (error instanceof Error) {
       logger.error(error.stack)
-      logger.error(JSON.stringify(error.response?.data))
+      logger.error(JSON.stringify(error.message))
       Sentry.captureException(error)
     }
+
+    return []
   }
 
-  return []
+  return data ? Object.values(data) : []
 }
 
 export const getAllGroupsAndStudents = async (sisPersonId: string) => {

services/backend/src/services/studyProgramme/index.ts

@@ -1,4 +1,4 @@
-import { memoize } from 'lodash'
+import { memoize } from 'lodash-es'
 import { Op, QueryTypes } from 'sequelize'
 
 import { Name, CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
@@ -52,15 +52,13 @@ export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, p
       },
     })
 
-    const allEnrollments = {} as Record<string, string[]>
+    const allEnrollments = new Map<string, Set<string>>()
     for (const { studentnumber, course_code: courseCode } of enrollmentsCourses) {
       const code = getCourseCode(courseCode)
-      if (!(code in allEnrollments)) {
-        allEnrollments[code] = []
-      }
-      if (!allEnrollments[code].includes(studentnumber)) {
-        allEnrollments[code].push(studentnumber)
+      if (!allEnrollments.has(code)) {
+        allEnrollments.set(code, new Set())
       }
+      allEnrollments.get(code)!.add(studentnumber)
     }
 
     const credits = await CreditModel.findAll({
@@ -75,7 +73,7 @@ export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, p
       },
     })
 
-    const courseNames = (
+    const courseCodeToName = (
       await CourseModel.findAll({
         attributes: ['code', 'name'],
         where: {
@@ -87,63 +85,50 @@ export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, p
       return acc
     }, new Map())
 
-    const creditCourses = credits.map(credit => {
-      return {
-        code: getCourseCode(credit.course_code),
-        studentNumber: credit.student_studentnumber,
-        creditTypeCode: credit.credittypecode,
-        courseName: courseNames.get(credit.course_code)!,
-        isStudyModule: credit.isStudyModule,
-      }
-    })
+    const creditCourses = credits.map(credit => ({
+      code: getCourseCode(credit.course_code),
+      studentNumber: credit.student_studentnumber,
+      creditTypeCode: credit.credittypecode,
+      courseName: courseCodeToName.get(credit.course_code)!,
+      isStudyModule: credit.isStudyModule,
+    }))
 
-    const passedByCourseCodes = {} as Record<string, string[]>
-    const notCompletedByCourseCodes = {} as Record<string, string[]>
-    const courses: Record<string, { code: string; name: Name; isStudyModule: boolean }> = {}
-    for (const course of creditCourses) {
-      if (!(course.code in courses)) {
-        courses[course.code] = {
-          code: course.code,
-          name: course.courseName,
-          isStudyModule: course.isStudyModule,
-        }
-        passedByCourseCodes[course.code] = []
-        notCompletedByCourseCodes[course.code] = []
+    const passedByCourseCodes = new Map<string, Set<string>>()
+    const notCompletedByCourseCodes = new Map<string, Set<string>>()
+    const courses = new Map<string, { name: Name; isStudyModule: boolean }>()
+
+    for (const { code, courseName, creditTypeCode, isStudyModule, studentNumber } of creditCourses) {
+      if (!courses.has(code)) {
+        courses.set(code, {
+          isStudyModule,
+          name: courseName,
+        })
+        passedByCourseCodes.set(code, new Set())
+        notCompletedByCourseCodes.set(code, new Set())
       }
-      if ([CreditTypeCode.PASSED, CreditTypeCode.IMPROVED, CreditTypeCode.APPROVED].includes(course.creditTypeCode)) {
-        passedByCourseCodes[course.code].push(course.studentNumber)
+      if ([CreditTypeCode.PASSED, CreditTypeCode.IMPROVED, CreditTypeCode.APPROVED].includes(creditTypeCode)) {
+        passedByCourseCodes.get(code)!.add(studentNumber)
       }
-      if (
-        course.creditTypeCode === CreditTypeCode.FAILED &&
-        !passedByCourseCodes[course.code].includes(course.studentNumber)
-      ) {
-        notCompletedByCourseCodes[course.code].push(course.studentNumber)
+      if (creditTypeCode === CreditTypeCode.FAILED && !passedByCourseCodes.get(code)!.has(studentNumber)) {
+        notCompletedByCourseCodes.get(code)!.add(studentNumber)
       }
     }
 
     // If student has enrollments, but no attainment for a particular course, they have no credit info.
     programmeCourses.forEach(courseCode => {
-      if (allEnrollments[courseCode]) {
-        allEnrollments[courseCode].forEach(studentnumber => {
-          if (passedByCourseCodes[courseCode] && !passedByCourseCodes[courseCode].includes(studentnumber)) {
-            notCompletedByCourseCodes[courseCode].push(studentnumber)
-          }
-        })
-      }
+      allEnrollments.get(courseCode)?.forEach(studentnumber => {
+        if (passedByCourseCodes.get(courseCode)?.has(studentnumber)) {
+          notCompletedByCourseCodes.get(courseCode)?.add(studentnumber)
+        }
+      })
     })
 
-    return Object.keys(courses)
-      .reduce<Array<{ code: string; name: Name; isStudyModule: boolean }>>(
-        (acc, val) => [...acc, { ...courses[val] }],
-        []
-      )
-      .map(course => ({
-        code: course.code,
-        name: course.name,
-        type: 'notCompleted',
-        isStudyModule: course.isStudyModule,
-        totalNotCompleted: [...new Set(notCompletedByCourseCodes[course.code])].length,
-      }))
+    return [...courses.entries()].map(([code, { isStudyModule, name }]) => ({
+      code,
+      name,
+      isStudyModule,
+      allNotPassed: notCompletedByCourseCodes.get(code)?.size ?? 0,
+    }))
   } catch (error) {
     logger.error(`getNotCompletedForProgrammeCourses failed ${error}`)
     return []

services/backend/src/services/studyProgramme/studentGetters.ts

@@ -1,348 +1,261 @@
-import { QueryTypes } from 'sequelize'
+import { Op, QueryTypes } from 'sequelize'
 
-import { Name, ExtentCode } from '@oodikone/shared/types'
+import { CreditTypeCode, ExtentCode, Name } from '@oodikone/shared/types'
 import { dbConnections } from '../../database/connection'
+import { StudentModel } from '../../models'
 import logger from '../../util/logger'
 
-const { sequelize } = dbConnections
+// Categories here are as per Linjaukset.md in internal docs.
+// Refer to them before making changes
+const degreeExtentCodes = [ExtentCode.BACHELOR, ExtentCode.MASTER, ExtentCode.BACHELOR_AND_MASTER]
 
-type QueryResult = {
-  totalStudents: string
-  totalCredits: number
-  code: string
+const exchangeStudentExtentCodes = [ExtentCode.EXCHANGE_STUDIES, ExtentCode.EXCHANGE_STUDIES_POSTGRADUATE]
+
+const otherUniversityExtentCodes = [ExtentCode.CONTRACT_TRAINING]
+
+const separateStudyExtentCodes = [
+  ExtentCode.NON_DEGREE_STUDIES,
+  ExtentCode.NON_DEGREE_PEGAGOGICAL_STUDIES_FOR_TEACHERS,
+  ExtentCode.NON_DEGREE_PROGRAMME_FOR_SPECIAL_EDUCATION_TEACHERS,
+  ExtentCode.SPECIALIZATION_STUDIES,
+]
+
+const otherExtentCodes = [
+  ExtentCode.CONTINUING_EDUCATION,
+  ExtentCode.DOCTOR,
+  ExtentCode.LICENTIATE,
+  ExtentCode.SPECIALIST_TRAINING_IN_MEDICINE_AND_DENTISTRY, // Includes veterinary
+  ExtentCode.STUDIES_FOR_SECONDARY_SCHOOL_STUDENTS,
+  ExtentCode.SUMMER_AND_WINTER_SCHOOL,
+]
+
+const openUniversityExtentCodes = [ExtentCode.OPEN_UNIVERSITY_STUDIES]
+
+export type ProgrammeCourseCreditRow = {
+  courseCode: string
   courseName: Name
+  studentNumber: string
+  attainmentDate: string
+  credits: number
+
+  variant: 'degree' | 'exchange' | 'separate' | 'otherUniversity' | 'other' | 'openUni' | 'none'
+
   isStudyModule: boolean
 }
 
-export const getStudentsForProgrammeCourses = async (from: Date, to: Date, programmeCourses: string[]) => {
-  if (!programmeCourses.length) {
-    return []
-  }
-  try {
-    const result: QueryResult[] = await sequelize.query(
-      `
-      WITH Dist AS (
-        SELECT DISTINCT
-          cr.student_studentnumber AS student,
-          cr.credits AS credits,
-          co.code AS code,
-          co.name AS "courseName",
-          cr."isStudyModule"
-        FROM
-          credit cr
-        INNER JOIN
-          course co
-          ON cr.course_code = co.code
-        WHERE
-          cr.attainment_date BETWEEN :from AND :to
-          AND cr.course_code IN (:programmeCourses)
-          AND cr.credittypecode = 4
-      )
-      SELECT
-        COUNT(student) AS "totalStudents",
-        SUM(credits) AS "totalCredits",
-        code,
-        "courseName",
-        "isStudyModule"
-      FROM
-        Dist
-      GROUP BY
-        dist.code,
-        dist."courseName",
-        dist."isStudyModule";
-      `,
-      {
-        type: QueryTypes.SELECT,
-        replacements: { from, to, programmeCourses },
-      }
-    )
-    return result.map(course => ({
-      code: course.code,
-      name: course.courseName,
-      type: 'passed',
-      isStudyModule: course.isStudyModule,
-      totalPassed: parseInt(course.totalStudents, 10),
-      totalAllCredits: course.totalCredits,
-    }))
-  } catch (error) {
-    logger.error(`getStudentsForProgrammeCourses() function failed ${error}`)
-    return []
-  }
+export type TransferCourseCreditRow = {
+  courseCode: string
+  courseName: Name
+  studentNumber: string
+  attainmentDate: string
+  credits: number
+  isStudyModule: boolean
 }
 
-export const getOwnStudentsForProgrammeCourses = async (
-  from: Date,
-  to: Date,
-  programmeCourses: string[],
-  studyProgramme?: string
-) => {
-  if (!programmeCourses.length) {
+const { sequelize } = dbConnections
+
+/**
+NOTE:
+ 1. There are some attainments that are linked to a valid study right, but the attainmentdate
+    is outside the study right's validity period. (maybe human error when the credits were registered)
+ 2. There are some attainments with no attached study right at all.
+ 3. Possibly some other edge cases.
+
+ Such cases get variant "none" and are not included in any category
+
+ Order of cases is somewhat important. refer to linjaukset.md, but TLDR;
+   degree student -> has any degree (bsc / msc) study right in the university
+   openuni -> only if no other study rights
+**/
+export const getProgrammeCourseAggregates = async (params: {
+  courseCodes: string[]
+  from: Date
+  to: Date
+}): Promise<ProgrammeCourseCreditRow[]> => {
+  const { courseCodes, from, to } = params
+
+  if (courseCodes.length === 0) {
     return []
   }
+
   try {
-    const result: QueryResult[] = await sequelize.query(
+    return await sequelize.query<ProgrammeCourseCreditRow>(
       `
-    WITH Dist AS (
-      SELECT DISTINCT
-        cr.student_studentnumber AS student,
-        cr.credits AS credits,
-        co.code AS code,
-        co.name AS "courseName",
-        cr."isStudyModule"
-      FROM
-        credit cr
-      INNER JOIN
-        sis_study_rights sr
-        ON sr.student_number = cr.student_studentnumber
-      INNER JOIN
-        sis_study_right_elements se
-        ON se.study_right_id = sr.id
-      INNER JOIN
-        course co
-        ON cr.course_code = co.code
-      WHERE
-        cr.attainment_date BETWEEN :from AND :to
-        AND cr.course_code IN (:programmeCourses)
-        AND cr.credittypecode = 4
-        AND (
-          se.code = :studyProgramme
-          AND cr.attainment_date BETWEEN se.start_date AND se.end_date
+        WITH filtered_credits AS (
+          SELECT
+            cr.course_code AS "courseCode",
+            co.name AS "courseName",
+            cr.student_studentnumber AS "studentNumber",
+            cr.attainment_date AS "attainmentDate",
+            cr.credits AS credits,
+            cr."isStudyModule" AS "isStudyModule"
+          FROM
+            credit cr
+            INNER JOIN course co ON co.code = cr.course_code
+          WHERE
+            cr.course_code IN (:courseCodes)
+            AND cr.attainment_date BETWEEN :from AND :to
+            AND cr.credittypecode = :creditType
         )
-    )
-    SELECT
-      COUNT(student) AS "totalStudents",
-      SUM(credits) AS "totalCredits",
-      code,
-      "courseName",
-      "isStudyModule"
-    FROM
-      Dist
-    GROUP BY
-      dist.code,
-      "courseName",
-      "isStudyModule";
-    `,
-      {
-        type: QueryTypes.SELECT,
-        replacements: { from, to, programmeCourses, studyProgramme },
-      }
-    )
-    return result.map(course => ({
-      code: course.code,
-      name: course.courseName,
-      type: 'ownProgramme',
-      totalProgrammeStudents: parseInt(course.totalStudents, 10),
-      totalProgrammeCredits: course.totalCredits,
-    }))
-  } catch (error) {
-    logger.error(`getOwnStudentsForProgrammeCourses() function failed ${error}`)
-    return []
-  }
-}
-
-export const getOtherStudentsForProgrammeCourses = async (
-  from: Date,
-  to: Date,
-  programmeCourses: string[],
-  studyProgramme?: string
-) => {
-  if (!programmeCourses.length) {
-    return []
-  }
-  try {
-    const result: QueryResult[] = await sequelize.query(
-      `
-      WITH Dist AS (
-        SELECT DISTINCT
-          cr.student_studentnumber AS student,
-          cr.credits AS credits,
-          co.code AS code,
-          co.name AS "courseName",
-          cr."isStudyModule"
+        SELECT
+          fc."courseCode",
+          fc."courseName",
+          fc."studentNumber",
+          fc."attainmentDate",
+          fc.credits,
+          fc."isStudyModule",
+          CASE
+            WHEN EXISTS (
+              SELECT
+                1
+              FROM
+                sis_study_rights sr
+              WHERE
+                sr.student_number = fc."studentNumber"
+                AND sr.extent_code IN (:degreeExtentCodes)
+                AND fc."attainmentDate" BETWEEN sr.start_date AND COALESCE(sr.end_date, 'infinity'::date)
+            ) THEN 'degree'
+            WHEN EXISTS (
+              SELECT
+                1
+              FROM
+                sis_study_rights sr
+              WHERE
+                sr.student_number = fc."studentNumber"
+                AND sr.extent_code IN (:exchangeStudentExtentCodes)
+                AND fc."attainmentDate" BETWEEN sr.start_date AND COALESCE(sr.end_date, 'infinity'::date)
+            ) THEN 'exchange'
+            WHEN EXISTS (
+              SELECT
+                1
+              FROM
+                sis_study_rights sr
+              WHERE
+                sr.student_number = fc."studentNumber"
+                AND sr.extent_code IN (:separateStudyExtentCodes)
+                AND fc."attainmentDate" BETWEEN sr.start_date AND COALESCE(sr.end_date, 'infinity'::date)
+            ) THEN 'separate'
+            WHEN EXISTS (
+              SELECT
+                1
+              FROM
+                sis_study_rights sr
+              WHERE
+                sr.student_number = fc."studentNumber"
+                AND sr.extent_code IN (:otherUniversityExtentCodes)
+                AND fc."attainmentDate" BETWEEN sr.start_date AND COALESCE(sr.end_date, 'infinity'::date)
+            ) THEN 'otherUniversity'
+            WHEN EXISTS (
+              SELECT
+                1
+              FROM
+                sis_study_rights sr
+              WHERE
+                sr.student_number = fc."studentNumber"
+                AND sr.extent_code IN (:otherExtentCodes)
+                AND fc."attainmentDate" BETWEEN sr.start_date AND COALESCE(sr.end_date, 'infinity'::date)
+            ) THEN 'other'
+            WHEN EXISTS (
+              SELECT
+                1
+              FROM
+                sis_study_rights sr
+              WHERE
+                sr.student_number = fc."studentNumber"
+                AND sr.extent_code IN (:openUniversityExtentCodes)
+                AND fc."attainmentDate" BETWEEN sr.start_date AND COALESCE(sr.end_date, 'infinity'::date)
+            ) THEN 'openUni'
+          ELSE 'none'
+          END AS variant
         FROM
-          credit cr
-        INNER JOIN
-          course co
-          ON cr.course_code = co.code
-        WHERE
-          cr.attainment_date BETWEEN :from AND :to
-          AND cr.course_code IN (:programmeCourses)
-          AND cr.credittypecode = 4
-          AND cr.student_studentnumber NOT IN (
-            SELECT
-              sr.student_number
-            FROM
-              sis_study_rights sr
-            INNER JOIN
-              sis_study_right_elements sre
-              ON sre.study_right_id = sr.id
-              AND sre.code = :studyProgramme
-              AND cr.attainment_date BETWEEN sre.start_date AND sre.end_date
-            WHERE
-              sr.student_number = cr.student_studentnumber
-          )
-      )
-      SELECT
-        COUNT(student) AS "totalStudents",
-        SUM(credits) AS "totalCredits",
-        code,
-        "courseName",
-        "isStudyModule"
-      FROM
-        Dist
-      GROUP BY
-        dist.code,
-        "courseName",
-        "isStudyModule";
-      `,
-      {
-        type: QueryTypes.SELECT,
-        replacements: { from, to, programmeCourses, studyProgramme },
-      }
-    )
-    return result.map(course => ({
-      code: course.code,
-      name: course.courseName,
-      type: 'otherProgramme',
-      isStudyModule: course.isStudyModule,
-      totalOtherProgrammeStudents: parseInt(course.totalStudents, 10),
-      totalOtherProgrammeCredits: course.totalCredits,
-    }))
-  } catch (error) {
-    logger.error(`getOtherStudentsForProgrammeCourses() function failed ${error}`)
-    return []
-  }
-}
-
-export const getTransferStudentsForProgrammeCourses = async (from: Date, to: Date, programmeCourses: string[]) => {
-  if (!programmeCourses.length) {
-    return []
-  }
-  try {
-    const result: QueryResult[] = await sequelize.query(
-      `
-      WITH Dist AS (
-        SELECT DISTINCT
-          cr.student_studentnumber AS student,
-          cr.credits AS credits,
-          co.code AS code,
-          co.name AS "courseName",
-          cr."isStudyModule"
-        FROM
-          credit cr
-        INNER JOIN
-          course co
-          ON cr.course_code = co.code
-        WHERE
-          cr.attainment_date BETWEEN :from AND :to
-          AND cr.course_code IN (:programmeCourses)
-          AND cr.credittypecode = 9
-      )
-      SELECT
-        COUNT(student) AS "totalStudents",
-        SUM(credits) AS "totalCredits",
-        code,
-        "courseName",
-        "isStudyModule"
-      FROM
-        Dist
-      GROUP BY
-        dist.code,
-        "courseName",
-        "isStudyModule";
-      `,
-      {
-        type: QueryTypes.SELECT,
-        replacements: { from, to, programmeCourses },
-      }
-    )
-    return result.map(course => ({
-      code: course.code,
-      name: course.courseName,
-      type: 'transfer',
-      isStudyModule: course.isStudyModule,
-      totalTransferStudents: parseInt(course.totalStudents, 10),
-      totalTransferCredits: course.totalCredits,
-    }))
-  } catch (error) {
-    logger.error(`getTransferStudentsForProgrammeCourses() function failed ${error}`)
-    return []
-  }
-}
-
-export const getStudentsWithoutStudyRightForProgrammeCourses = async (
-  from: Date,
-  to: Date,
-  programmeCourses: string[]
-) => {
-  if (!programmeCourses.length) {
-    return []
-  }
-  try {
-    const result: QueryResult[] = await sequelize.query(
-      `
-      WITH Dist AS (
-        SELECT DISTINCT
-          cr.student_studentnumber AS student,
-          cr.credits AS credits,
-          co.code AS code,
-          co.name AS "courseName",
-          cr."isStudyModule"
-        FROM
-          credit cr
-        INNER JOIN
-          course co
-          ON cr.course_code = co.code
-        LEFT JOIN
-          sis_study_rights sr
-          ON sr.student_number = cr.student_studentnumber
-          AND sr.extent_code IN (:degreeExtentCodes)
-          AND cr.attainment_date BETWEEN sr.start_date AND sr.end_date
-        WHERE
-          cr.attainment_date BETWEEN :from AND :to
-          AND cr.course_code IN (:programmeCourses)
-          AND cr.credittypecode = 4
-          AND sr.student_number IS NULL
-      )
-      SELECT
-        COUNT(student) AS "totalStudents",
-        SUM(credits) AS "totalCredits",
-        code,
-        "courseName",
-        "isStudyModule"
-      FROM
-        Dist
-      GROUP BY
-        dist.code,
-        "courseName",
-        "isStudyModule";
+          filtered_credits fc
       `,
       {
         type: QueryTypes.SELECT,
         replacements: {
+          courseCodes,
           from,
           to,
-          programmeCourses,
-          degreeExtentCodes: [
-            ExtentCode.BACHELOR,
-            ExtentCode.MASTER,
-            ExtentCode.BACHELOR_AND_MASTER,
-            ExtentCode.DOCTOR,
-            ExtentCode.LICENTIATE,
-            ExtentCode.SPECIALIST_TRAINING_IN_MEDICINE_AND_DENTISTRY,
-          ],
+          creditType: CreditTypeCode.PASSED,
+          degreeExtentCodes,
+          otherUniversityExtentCodes,
+          separateStudyExtentCodes,
+          exchangeStudentExtentCodes,
+          openUniversityExtentCodes,
+          otherExtentCodes,
         },
       }
     )
-    return result.map(course => ({
-      code: course.code,
-      name: course.courseName,
-      type: 'noStudyright',
-      totalWithoutStudyRightStudents: parseInt(course.totalStudents, 10),
-      totalWithoutStudyRightCredits: course.totalCredits,
-    }))
   } catch (error) {
-    logger.error(`getStudentsWithoutStudyRightForProgrammeCourses() function failed ${error}`)
+    logger.error('getProgrammeCourseAggregates failed', { error })
     return []
   }
 }
+
+export const getTransferCourseAggregates = async (params: {
+  courseCodes: string[]
+  from: Date
+  to: Date
+}): Promise<TransferCourseCreditRow[]> => {
+  const { courseCodes, from, to } = params
+
+  if (courseCodes.length === 0) {
+    return []
+  }
+
+  try {
+    return await sequelize.query<TransferCourseCreditRow>(
+      `
+        SELECT
+          cr.course_code AS "courseCode",
+          co.name AS "courseName",
+          cr.student_studentnumber AS "studentNumber",
+          cr.attainment_date AS "attainmentDate",
+          cr.credits AS credits,
+          cr."isStudyModule" AS "isStudyModule"
+        FROM
+          credit cr
+          INNER JOIN course co ON co.code = cr.course_code
+        WHERE
+          cr.course_code IN (:courseCodes)
+          AND cr.attainment_date BETWEEN :from AND :to
+          AND cr.credittypecode = :creditType
+      `,
+      {
+        type: QueryTypes.SELECT,
+        replacements: {
+          courseCodes,
+          from,
+          to,
+          creditType: CreditTypeCode.APPROVED,
+        },
+      }
+    )
+  } catch (error) {
+    logger.error('getTransferCourseAggregates failed', { error })
+    return []
+  }
+}
+
+export const getStudentHetuStateMap = async (studentNumbers: string[]): Promise<Map<string, boolean>> => {
+  if (!studentNumbers.length) return new Map()
+
+  try {
+    return new Map(
+      (
+        await StudentModel.findAll({
+          attributes: ['studentnumber', 'hasPersonalIdentityCode'],
+          where: {
+            studentnumber: {
+              [Op.in]: studentNumbers,
+            },
+          },
+          raw: true,
+        })
+      ).map(student => [student.studentnumber, student.hasPersonalIdentityCode])
+    )
+  } catch (error) {
+    logger.error('getOpenUniHetutonSplit failed', { error })
+    return new Map()
+  }
+}

services/backend/src/services/studyProgramme/studyProgrammeBasics.ts

@@ -1,4 +1,4 @@
-import { indexOf } from 'lodash'
+import { indexOf } from 'lodash-es'
 
 import { Name, EnrollmentType, SemesterEnrollment } from '@oodikone/shared/types'
 import { BasicStats } from '@oodikone/shared/types/studyProgramme'

services/backend/src/services/studyProgramme/studyProgrammeCourses.ts

@@ -1,390 +1,293 @@
-import { orderBy, range } from 'lodash'
-import { Op } from 'sequelize'
+import { range } from 'lodash-es'
 
-import { Name, StudyProgrammeCourse, CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
+import { Name, StudyProgrammeCourse, YearType } from '@oodikone/shared/types'
 import { mapToProviders } from '@oodikone/shared/util'
-import { CreditModel, EnrollmentModel } from '../../models'
-import { getOpenUniCourseCode } from '../../util'
-import { createArrayOfCourses } from '../languageCenterData'
-import { getCurrentStudyYearStartDate, getNotCompletedForProgrammeCourses, getAllProgrammeCourses } from '.'
-import {
-  getOtherStudentsForProgrammeCourses,
-  getOwnStudentsForProgrammeCourses,
-  getStudentsForProgrammeCourses,
-  getStudentsWithoutStudyRightForProgrammeCourses,
-  getTransferStudentsForProgrammeCourses,
-} from './studentGetters'
+import { createEmptyStats, YearStats } from '@oodikone/shared/util/studyProgramme'
 
-const getCurrentYearStartDate = () => {
-  return new Date(new Date().getFullYear(), 0, 1)
-}
+import { getAllProgrammeCourses, getCurrentStudyYearStartDate, getNotCompletedForProgrammeCourses } from '.'
+import { getStudentHetuStateMap, getProgrammeCourseAggregates, getTransferCourseAggregates } from './studentGetters'
+
+const START_YEAR = 2017
+const JULY = 6
+const AUGUST = 7
+
+const getCurrentYearStartDate = () => new Date(new Date().getFullYear(), 0, 1)
+
+const getFrom = (yearType: YearType, year: number) =>
+  yearType === 'ACADEMIC_YEAR' ? new Date(year, AUGUST, 1, 0, 0, 0) : new Date(year, 0, 1, 0, 0, 0)
+
+const getTo = (yearType: YearType, year: number) =>
+  yearType === 'ACADEMIC_YEAR' ? new Date(year + 1, JULY, 31, 23, 59, 59) : new Date(year, 11, 31, 23, 59, 59)
 
 const getAllStudyProgrammeCourses = async (studyProgramme: string) => {
   const providerCode = mapToProviders([studyProgramme])[0]
+  if (!providerCode) return []
+
   const normalCourses = await getAllProgrammeCourses(providerCode)
-  return normalCourses.reduce((acc, curr) => {
-    acc.push(curr.code)
-    if (curr.substitutions?.includes(`AY${curr.code}`)) {
-      acc.push(`AY${curr.code}`)
+  const courseCodes = new Set<string>()
+
+  for (const course of normalCourses) {
+    courseCodes.add(course.code)
+    if (course.substitutions?.includes(`AY${course.code}`)) {
+      courseCodes.add(`AY${course.code}`)
     }
-    return acc
-  }, [] as string[])
+  }
+
+  return [...courseCodes]
 }
 
-const getFrom = (academicYear: string, year: number) => {
-  return academicYear === 'ACADEMIC_YEAR' ? new Date(year, 7, 1, 0, 0, 0) : new Date(year, 0, 1, 0, 0, 0)
+const getYearKey = (date: Date, isAcademicYear: boolean) => {
+  const year = date.getFullYear()
+  if (!isAcademicYear) return year
+  return date.getMonth() >= 7 ? year : year - 1
 }
 
-const getTo = (academicYear: string, year: number) => {
-  return academicYear === 'ACADEMIC_YEAR' ? new Date(year + 1, 6, 31, 23, 59, 59) : new Date(year, 11, 31, 23, 59, 59)
+type YearAccumulator = {
+  stats: YearStats
+  allPassed: Set<string>
+
+  degreeStudents: Set<string>
+  exchangeStudents: Set<string>
+  otherUniversityStudents: Set<string>
+  separateStudents: Set<string>
+  otherStudents: Set<string>
+
+  openStudentsHasHetu: Set<string>
+  openStudentsNoHetu: Set<string>
+  transferStudents: Set<string>
 }
 
-// * Combined return type, should be split into smaller types
-type Students = {
+type CourseAccumulator = {
   code: string
   name: Name
-  type: string
-  isStudyModule?: boolean
-  year?: number
-  totalPassed?: number
-  totalAllCredits?: number
-  totalNotCompleted?: number
-  totalProgrammeStudents?: number
-  totalProgrammeCredits?: number
-  totalWithoutStudyRightStudents?: number
-  totalWithoutStudyRightCredits?: number
-  totalOtherProgrammeStudents?: number
-  totalOtherProgrammeCredits?: number
-  totalTransferStudents?: number
-  totalTransferCredits?: number
+  isStudyModule: boolean
+  years: Map<number, YearAccumulator>
 }
 
-// TODO: It would be a good idea to split this into smaller functions
-// The amount of different return types makes this very messy
-const makeYearlyPromises = (
-  years: number[],
-  academicYear: string,
-  type: 'passed' | 'notCompleted' | 'ownStudents' | 'withoutStudyRight' | 'otherStudents' | 'transfer',
-  programmeCourses: string[],
-  studyProgramme?: string
-) => {
-  return years.map(async year => {
-    const from = getFrom(academicYear, year)
-    const to = getTo(academicYear, year)
-    let result: Students[] | null = null
+const createYearAccumulator = (isStudyModule: boolean): YearAccumulator => ({
+  stats: createEmptyStats(isStudyModule),
+  allPassed: new Set(),
+  degreeStudents: new Set(),
+  exchangeStudents: new Set(),
+  otherUniversityStudents: new Set(),
+  separateStudents: new Set(),
+  otherStudents: new Set(),
 
-    switch (type) {
-      case 'passed':
-        result = await getStudentsForProgrammeCourses(from, to, programmeCourses)
-        break
-      case 'notCompleted':
-        result = await getNotCompletedForProgrammeCourses(from, to, programmeCourses)
-        break
-      case 'ownStudents':
-        result = await getOwnStudentsForProgrammeCourses(from, to, programmeCourses, studyProgramme)
-        break
-      case 'withoutStudyRight':
-        result = await getStudentsWithoutStudyRightForProgrammeCourses(from, to, programmeCourses)
-        break
-      case 'otherStudents':
-        result = await getOtherStudentsForProgrammeCourses(from, to, programmeCourses, studyProgramme)
-        break
-      case 'transfer':
-        result = await getTransferStudentsForProgrammeCourses(from, to, programmeCourses)
-        break
-      default:
-        result = await getStudentsForProgrammeCourses(from, to, programmeCourses)
-    }
+  openStudentsHasHetu: new Set(),
+  openStudentsNoHetu: new Set(),
+  transferStudents: new Set(),
+})
 
-    return result.map(course => {
-      course.year = year
-      return course
+const ensureCourse = (
+  courseMap: Map<string, CourseAccumulator>,
+  code: string,
+  name: Name,
+  isStudyModule: boolean
+): CourseAccumulator => {
+  if (!courseMap.has(code)) {
+    courseMap.set(code, {
+      code,
+      name,
+      isStudyModule,
+      years: new Map(),
     })
-  })
+  }
+
+  const course = courseMap.get(code)!
+  course.isStudyModule = course.isStudyModule || isStudyModule
+  course.name = course.name ?? name
+  return course
 }
 
-type Attempt = {
-  studentNumber: string
-  courseCode: string
-  completed: boolean
-  date: Date
-  semestercode: number
-  enrolled?: boolean
+const ensureYear = (course: CourseAccumulator, year: number, isStudyModule: boolean) => {
+  if (!course.years.has(year)) {
+    course.years.set(year, createYearAccumulator(isStudyModule))
+  }
+  const yearAccumulator = course.years.get(year)!
+  yearAccumulator.stats.isStudyModule ??= isStudyModule
+  return yearAccumulator
 }
 
-export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgramme: string) => {
-  const courses = await getAllProgrammeCourses(mapToProviders([studyProgramme])[0])
-  const autumnSemester2017 = 135
-  const courseCodes = courses.map(course => course.code)
+const finalizeYearStats = (yearAccumulator: YearAccumulator): YearStats => {
+  const { stats } = yearAccumulator
 
-  const credits = await CreditModel.findAll({
-    attributes: ['course_code', 'student_studentnumber', 'semestercode', 'attainment_date'],
-    where: {
-      course_code: { [Op.in]: courseCodes },
-      semestercode: { [Op.gte]: autumnSemester2017 },
-      credittypecode: CreditTypeCode.PASSED,
-    },
-    raw: true,
-  })
+  stats.allPassed = yearAccumulator.allPassed.size
 
-  const enrollments = await EnrollmentModel.findAll({
-    attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'state'],
-    where: {
-      course_code: { [Op.in]: courseCodes },
-      semestercode: { [Op.gte]: autumnSemester2017 },
-      state: { [Op.in]: [EnrollmentState.ENROLLED, EnrollmentState.REJECTED] },
-    },
-    raw: true,
-  })
+  stats.degreeStudents = yearAccumulator.degreeStudents.size
+  stats.exchangeStudents = yearAccumulator.exchangeStudents.size
+  stats.otherUniversityStudents = yearAccumulator.otherUniversityStudents.size
+  stats.separateStudents = yearAccumulator.separateStudents.size
+  stats.otherStudents = yearAccumulator.otherStudents.size
 
-  const studentList = new Set<string>()
-  const attemptsByStudents = {} as Record<string, Attempt[]>
+  stats.openStudentsHasHetu = yearAccumulator.openStudentsHasHetu.size
+  stats.openStudentsNoHetu = yearAccumulator.openStudentsNoHetu.size
+  stats.transferStudents = yearAccumulator.transferStudents.size
 
-  credits.forEach(credit => {
-    const studentNumber = credit.student_studentnumber
-    studentList.add(studentNumber)
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
-    attemptsByStudents[studentNumber].push({
-      studentNumber,
-      courseCode: credit.course_code,
-      completed: true,
-      date: credit.attainment_date,
-      semestercode: credit.semestercode,
-    })
-  })
+  stats.allStudents = stats.allPassed + stats.allNotPassed
 
-  enrollments.forEach(enrollment => {
-    const studentNumber = enrollment.studentnumber
-    if (!attemptsByStudents[studentNumber]) {
-      attemptsByStudents[studentNumber] = []
-    }
-    studentList.add(studentNumber)
-    if (
-      attemptsByStudents[studentNumber].find(
-        attempt =>
-          !attempt.completed &&
-          attempt.semestercode === enrollment.semestercode &&
-          attempt.courseCode === enrollment.course_code
-      )
-    ) {
-      return
-    }
-    attemptsByStudents[studentNumber].push({
-      studentNumber,
-      courseCode: enrollment.course_code,
-      completed: false,
-      date: enrollment.enrollment_date_time,
-      semestercode: enrollment.semestercode,
-      enrolled: enrollment.state === EnrollmentState.ENROLLED,
-    })
-  })
-
-  const attemptsArray = [] as Attempt[]
-  studentList.forEach(studentNumber => attemptsArray.push(...attemptsByStudents[studentNumber]))
-
-  const unorderedTableData = await createArrayOfCourses(attemptsArray, courses)
-  const tableData = orderBy(unorderedTableData, 'code')
-
-  return { tableData }
+  return stats
 }
 
+const getYearRange = async (unixMillis: number, isAcademicYear: boolean) => {
+  const startDate = isAcademicYear ? await getCurrentStudyYearStartDate(unixMillis) : getCurrentYearStartDate()
+  const lastYear = startDate.getFullYear()
+  return range(START_YEAR, lastYear + 1)
+}
+
+/**
+ * @endpoint "programme courses"
+ */
 export const getStudyProgrammeCoursesForStudyTrack = async (
   unixMillis: number,
   studyProgramme: string,
-  academicYear: string,
+  academicYear: YearType,
   combinedProgramme: string
 ) => {
-  const startDate =
-    academicYear === 'ACADEMIC_YEAR' ? await getCurrentStudyYearStartDate(unixMillis) : getCurrentYearStartDate()
-  const startYear = startDate.getFullYear()
-  const yearRange = range(2017, startYear + 1)
+  const isAcademicYear = academicYear === 'ACADEMIC_YEAR'
+  const yearRange = await getYearRange(unixMillis, isAcademicYear)
+  if (yearRange.length === 0) {
+    return []
+  }
+
   const mainProgrammeCourses = await getAllStudyProgrammeCourses(studyProgramme)
   const secondProgrammeCourses = combinedProgramme ? await getAllStudyProgrammeCourses(combinedProgramme) : []
-  const programmeCourses = [...mainProgrammeCourses, ...secondProgrammeCourses]
+  const programmeCourses = [...new Set([...mainProgrammeCourses, ...secondProgrammeCourses])]
 
-  const yearlyPassedStudentByCoursePromises = makeYearlyPromises(yearRange, academicYear, 'passed', programmeCourses)
-  const yearlyNotCompletedStudentByCoursePromises = makeYearlyPromises(
-    yearRange,
-    academicYear,
-    'notCompleted',
-    programmeCourses
-  )
-  const yearlyProgrammeStudentsPromises = makeYearlyPromises(
-    yearRange,
-    academicYear,
-    'ownStudents',
-    programmeCourses,
-    studyProgramme
-  )
-  const yearlyStudentsWithoutStudyRightPromises = makeYearlyPromises(
-    yearRange,
-    academicYear,
-    'withoutStudyRight',
-    programmeCourses
-  )
-  const yearlyOtherProgrammeStudentsPromises = makeYearlyPromises(
-    yearRange,
-    academicYear,
-    'otherStudents',
-    programmeCourses,
-    studyProgramme
-  )
-  const yearlyTransferStudentsPromises = makeYearlyPromises(
-    yearRange,
-    academicYear,
-    'transfer',
-    programmeCourses,
-    studyProgramme
-  )
+  if (programmeCourses.length === 0) {
+    return []
+  }
 
-  const [
-    yearlyPassedStudentByCourse,
-    yearlyNotCompletedStudentByCourse,
-    yearlyProgrammeStudents,
-    yearlyStudentsWithoutStudyRight,
-    yearlyOtherProgrammeStudents,
-    yearlyTransferStudents,
-  ] = await Promise.all([
-    Promise.all(yearlyPassedStudentByCoursePromises),
-    Promise.all(yearlyNotCompletedStudentByCoursePromises),
-    Promise.all(yearlyProgrammeStudentsPromises),
-    Promise.all(yearlyStudentsWithoutStudyRightPromises),
-    Promise.all(yearlyOtherProgrammeStudentsPromises),
-    Promise.all(yearlyTransferStudentsPromises),
+  const firstYear = yearRange[0]
+  const lastYear = yearRange[yearRange.length - 1]
+  const from = getFrom(academicYear, firstYear)
+  const to = getTo(academicYear, lastYear)
+
+  const [programmeCredits, transferCredits] = await Promise.all([
+    getProgrammeCourseAggregates({
+      courseCodes: programmeCourses,
+      from,
+      to,
+    }),
+    getTransferCourseAggregates({
+      courseCodes: programmeCourses,
+      from,
+      to,
+    }),
   ])
 
-  let maxYear = 0
-  const allCourses = [
-    ...yearlyPassedStudentByCourse.flat(),
-    ...yearlyNotCompletedStudentByCourse.flat(),
-    ...yearlyProgrammeStudents.flat(),
-    ...yearlyStudentsWithoutStudyRight.flat(),
-    ...yearlyOtherProgrammeStudents.flat(),
-    ...yearlyTransferStudents.flat(),
-  ].reduce(
-    (acc, curr) => {
-      if (curr.year! > maxYear) {
-        maxYear = curr.year!
-      }
-      if (!acc[curr.code]) {
-        acc[curr.code] = {
-          code: curr.code,
-          name: curr.name,
-          isStudyModule: curr.isStudyModule ?? false,
-          years: {},
-        }
-      }
-
-      if (!acc[curr.code].years[curr.year!]) {
-        acc[curr.code].years[curr.year!] = {
-          isStudyModule: curr.isStudyModule ?? false,
-          totalAllStudents: 0,
-          totalPassed: 0,
-          totalNotCompleted: 0,
-          totalAllCredits: 0,
-          totalProgrammeStudents: 0,
-          totalProgrammeCredits: 0,
-          totalOtherProgrammeStudents: 0,
-          totalOtherProgrammeCredits: 0,
-          totalWithoutStudyRightStudents: 0,
-          totalWithoutStudyRightCredits: 0,
-          totalTransferStudents: 0,
-          totalTransferCredits: 0,
-        }
-      }
-      switch (curr.type) {
-        case 'passed':
-          acc[curr.code].years[curr.year!].totalPassed += curr.totalPassed ?? 0
-          acc[curr.code].years[curr.year!].totalAllStudents += acc[curr.code].years[curr.year!].totalPassed
-          acc[curr.code].years[curr.year!].totalAllCredits += curr.totalAllCredits ?? 0
-          break
-        case 'notCompleted':
-          acc[curr.code].years[curr.year!].totalNotCompleted += curr.totalNotCompleted ?? 0
-          acc[curr.code].years[curr.year!].totalAllStudents += acc[curr.code].years[curr.year!].totalNotCompleted
-          break
-        case 'ownProgramme':
-          acc[curr.code].years[curr.year!].totalProgrammeStudents += curr.totalProgrammeStudents ?? 0
-          acc[curr.code].years[curr.year!].totalProgrammeCredits += curr.totalProgrammeCredits ?? 0
-          break
-        case 'otherProgramme':
-          acc[curr.code].years[curr.year!].totalOtherProgrammeStudents += curr.totalOtherProgrammeStudents ?? 0
-          acc[curr.code].years[curr.year!].totalOtherProgrammeCredits += curr.totalOtherProgrammeCredits ?? 0
-          break
-        case 'noStudyright':
-          acc[curr.code].years[curr.year!].totalWithoutStudyRightStudents += curr.totalWithoutStudyRightStudents ?? 0
-          acc[curr.code].years[curr.year!].totalWithoutStudyRightCredits += curr.totalWithoutStudyRightCredits ?? 0
-          break
-        case 'transfer':
-          acc[curr.code].years[curr.year!].totalTransferStudents += curr.totalTransferStudents ?? 0
-          acc[curr.code].years[curr.year!].totalTransferCredits += curr.totalTransferCredits ?? 0
-          break
-        default:
-          break
-      }
-      return acc
-    },
-    {} as Record<string, StudyProgrammeCourse>
+  const openUniStudentHetuMap = await getStudentHetuStateMap(
+    programmeCredits.filter(c => c.variant === 'openUni').map(c => c.studentNumber)
   )
-  const ayCourses = Object.keys(allCourses).filter(courseCode => courseCode.startsWith('AY'))
-  const properties = [
-    'totalAllStudents',
-    'totalPassed',
-    'totalNotCompleted',
-    'totalAllCredits',
-    'totalProgrammeStudents',
-    'totalProgrammeCredits',
-    'totalOtherProgrammeStudents',
-    'totalOtherProgrammeCredits',
-    'totalWithoutStudyrightStudents',
-    'totalWithoutStudyrightCredits',
-    'totalTransferCredits',
-    'totalTransferStudents',
-  ]
-  ayCourses.forEach(ayCourse => {
-    const openUniCourseCode = getOpenUniCourseCode(ayCourse)
-    if (!openUniCourseCode) {
-      return
+
+  const courseMap = new Map<string, CourseAccumulator>()
+  let maxYear = 0
+
+  for (const row of programmeCredits) {
+    const attainmentDate = new Date(row.attainmentDate)
+    const year = getYearKey(attainmentDate, isAcademicYear)
+
+    if (!yearRange.includes(year)) continue
+    maxYear = Math.max(maxYear, year)
+
+    const course = ensureCourse(courseMap, row.courseCode, row.courseName, row.isStudyModule)
+    const yearAccumulator = ensureYear(course, year, row.isStudyModule)
+    const { stats } = yearAccumulator
+
+    stats.allCredits += row.credits
+    yearAccumulator.allPassed.add(row.studentNumber)
+
+    switch (row.variant) {
+      case 'degree':
+        yearAccumulator.degreeStudents.add(row.studentNumber)
+        stats.degreeStudentsCredits += row.credits
+        break
+      case 'exchange':
+        yearAccumulator.exchangeStudents.add(row.studentNumber)
+        stats.exchangeStudentsCredits += row.credits
+        break
+      case 'separate':
+        yearAccumulator.separateStudents.add(row.studentNumber)
+        stats.separateStudentsCredits += row.credits
+        break
+      case 'otherUniversity':
+        yearAccumulator.otherUniversityStudents.add(row.studentNumber)
+        stats.otherUniversityCredits += row.credits
+        break
+      case 'other':
+        yearAccumulator.otherStudents.add(row.studentNumber)
+        stats.otherStudentsCredits += row.credits
+        break
+      case 'openUni':
+        if (openUniStudentHetuMap.get(row.studentNumber)) {
+          stats.openStudentsHasHetuCredits += row.credits
+          yearAccumulator.openStudentsHasHetu.add(row.studentNumber)
+        } else {
+          stats.openStudentsNoHetuCredits += row.credits
+          yearAccumulator.openStudentsNoHetu.add(row.studentNumber)
+        }
+        break
+      default:
+        break
     }
-    const normCode = openUniCourseCode[1]
+  }
 
-    if (allCourses[normCode]) {
-      const mergedCourse = {} as StudyProgrammeCourse
-      mergedCourse.code = allCourses[normCode].code
-      mergedCourse.name = allCourses[normCode].name
-      mergedCourse.years = {}
+  for (const row of transferCredits) {
+    const attainmentDate = new Date(row.attainmentDate)
+    const year = getYearKey(attainmentDate, isAcademicYear)
+    if (!yearRange.includes(year)) continue
+    maxYear = Math.max(maxYear, year)
 
-      yearRange
-        .filter(year => year <= maxYear)
-        .forEach(year => {
-          if (!allCourses[normCode].years[year]) {
-            mergedCourse.years[year] = {
-              totalAllStudents: 0,
-              totalPassed: 0,
-              totalNotCompleted: 0,
-              totalAllCredits: 0,
-              totalProgrammeStudents: 0,
-              totalProgrammeCredits: 0,
-              totalOtherProgrammeStudents: 0,
-              totalOtherProgrammeCredits: 0,
-              totalWithoutStudyRightStudents: 0,
-              totalWithoutStudyRightCredits: 0,
-              totalTransferCredits: 0,
-              totalTransferStudents: 0,
-            }
-          } else {
-            mergedCourse.years[year] = { ...allCourses[normCode].years[year] }
-          }
-          if (allCourses[ayCourse].years[year]) {
-            properties.forEach(prop => {
-              mergedCourse.years[year][prop] += allCourses[ayCourse].years[year][prop]
-            })
-          }
-        })
-      allCourses[normCode] = mergedCourse
-      delete allCourses[ayCourse]
+    const course = ensureCourse(courseMap, row.courseCode, row.courseName, row.isStudyModule)
+    const yearAccumulator = ensureYear(course, year, row.isStudyModule)
+    yearAccumulator.transferStudents.add(row.studentNumber)
+    yearAccumulator.stats.transferStudentsCredits += row.credits
+  }
+
+  const notCompletedResults = await Promise.all(
+    yearRange.map(year => {
+      const yearFrom = getFrom(academicYear, year)
+      const yearTo = getTo(academicYear, year)
+      return getNotCompletedForProgrammeCourses(yearFrom, yearTo, programmeCourses)
+    })
+  )
+
+  notCompletedResults.forEach((results, index) => {
+    const year = yearRange[index]
+    if (!results) return
+    if (results.length > 0) {
+      maxYear = Math.max(maxYear, year)
+    }
+    for (const result of results) {
+      const course = ensureCourse(courseMap, result.code, result.name, result.isStudyModule ?? false)
+      const yearAccumulator = ensureYear(course, year, result.isStudyModule ?? false)
+      yearAccumulator.stats.allNotPassed += result.allNotPassed
     }
   })
-  return Object.values(allCourses)
+
+  if (maxYear === 0) {
+    maxYear = lastYear
+  }
+
+  const coursesRecord: Record<string, StudyProgrammeCourse> = {}
+
+  for (const [code, course] of courseMap.entries()) {
+    const years: Record<number, YearStats> = {}
+    for (const [year, yearAccumulator] of course.years.entries()) {
+      if (year > maxYear) continue
+      years[year] = finalizeYearStats(yearAccumulator)
+    }
+
+    if (Object.keys(years).length === 0) {
+      continue
+    }
+
+    coursesRecord[code] = {
+      code,
+      name: course.name,
+      isStudyModule: course.isStudyModule,
+      years,
+    }
+  }
+
+  return Object.values(coursesRecord)
 }

services/backend/src/services/studyProgramme/studyProgrammeCoursesBySemester.ts

@@ -0,0 +1,95 @@
+import { orderBy } from 'lodash-es'
+import { Op } from 'sequelize'
+
+import { CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
+import { mapToProviders } from '@oodikone/shared/util'
+import { CreditModel, EnrollmentModel } from '../../models'
+import { createArrayOfCourses } from '../languageCenterData'
+import { getAllProgrammeCourses } from '.'
+
+type Attempt = {
+  studentNumber: string
+  courseCode: string
+  completed: boolean
+  date: Date
+  semestercode: number
+  enrolled?: boolean
+}
+
+export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgramme: string) => {
+  const courses = await getAllProgrammeCourses(mapToProviders([studyProgramme])[0])
+  const autumnSemester2017 = 135
+  const courseCodes = courses.map(course => course.code)
+
+  const credits = await CreditModel.findAll({
+    attributes: ['course_code', 'student_studentnumber', 'semestercode', 'attainment_date'],
+    where: {
+      course_code: { [Op.in]: courseCodes },
+      semestercode: { [Op.gte]: autumnSemester2017 },
+      credittypecode: CreditTypeCode.PASSED,
+    },
+    raw: true,
+  })
+
+  const enrollments = await EnrollmentModel.findAll({
+    attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'state'],
+    where: {
+      course_code: { [Op.in]: courseCodes },
+      semestercode: { [Op.gte]: autumnSemester2017 },
+      state: { [Op.in]: [EnrollmentState.ENROLLED, EnrollmentState.REJECTED] },
+    },
+    raw: true,
+  })
+
+  const studentList = new Set<string>()
+  const attemptsByStudents = {} as Record<string, Attempt[]>
+
+  credits.forEach(credit => {
+    const studentNumber = credit.student_studentnumber
+    studentList.add(studentNumber)
+    if (!attemptsByStudents[studentNumber]) {
+      attemptsByStudents[studentNumber] = []
+    }
+    attemptsByStudents[studentNumber].push({
+      studentNumber,
+      courseCode: credit.course_code,
+      completed: true,
+      date: credit.attainment_date,
+      semestercode: credit.semestercode,
+    })
+  })
+
+  enrollments.forEach(enrollment => {
+    const studentNumber = enrollment.studentnumber
+    if (!attemptsByStudents[studentNumber]) {
+      attemptsByStudents[studentNumber] = []
+    }
+    studentList.add(studentNumber)
+    if (
+      attemptsByStudents[studentNumber].find(
+        attempt =>
+          !attempt.completed &&
+          attempt.semestercode === enrollment.semestercode &&
+          attempt.courseCode === enrollment.course_code
+      )
+    ) {
+      return
+    }
+    attemptsByStudents[studentNumber].push({
+      studentNumber,
+      courseCode: enrollment.course_code,
+      completed: false,
+      date: enrollment.enrollment_date_time,
+      semestercode: enrollment.semestercode,
+      enrolled: enrollment.state === EnrollmentState.ENROLLED,
+    })
+  })
+
+  const attemptsArray = [] as Attempt[]
+  studentList.forEach(studentNumber => attemptsArray.push(...attemptsByStudents[studentNumber]))
+
+  const unorderedTableData = await createArrayOfCourses(attemptsArray, courses)
+  const tableData = orderBy(unorderedTableData, 'code')
+
+  return { tableData }
+}

services/backend/src/services/studyProgramme/studyProgrammeGraduations.ts

@@ -1,5 +1,6 @@
-import dayjs from 'dayjs'
-import { indexOf, orderBy } from 'lodash'
+import dayjs, { Dayjs } from 'dayjs'
+import isSameOrAfter from 'dayjs/plugin/isSameOrAfter'
+import { indexOf, orderBy } from 'lodash-es'
 
 import {
   GraduationStatistics,
@@ -9,10 +10,15 @@ import {
   DegreeProgrammeType,
   ExtentCode,
   Phase,
+  EnrollmentType,
+  TransferInfo,
 } from '@oodikone/shared/types'
+import { ChartGraduationTimes } from '@oodikone/shared/types/graduations'
+import { ProgrammeGraduationStats } from '@oodikone/shared/types/studyProgramme'
 import { mapToProviders } from '@oodikone/shared/util'
 import { getDegreeProgrammeType, getMinimumCreditsOfProgramme, sortByProgrammeCode } from '../../util'
-import { countTimeCategories } from '../graduationHelpers'
+import logger from '../../util/logger'
+import { countTimeCategories, GraduationTarget } from '../graduationHelpers'
 import { getSemestersAndYears } from '../semesters'
 import { getStudyRightThesisCredits, getOrganizationThesisCredits } from './creditGetters'
 import { getGraduatedStats } from './studyProgrammeBasics'
@@ -31,6 +37,8 @@ import {
 } from './studyProgrammeHelpers'
 import { getStudyRightsInProgramme } from './studyRightFinders'
 
+dayjs.extend(isSameOrAfter)
+
 /**
  * For bachelor + master study right combo. This rules out master's programmes that don't
  * follow the two-phase degree structure (e.g. degree programmes in medicine and dentistry)
@@ -44,33 +52,65 @@ export const shouldIncludeComboStats = async (studyProgramme: string) => {
   return minimumCredits !== null && minimumCredits <= 180
 }
 
-const calculateAbsenceInMonths = (
-  absence: Awaited<ReturnType<typeof getSemestersAndYears>>['semesters'][string],
-  startDate: Date,
-  endDate: Date
-) => {
-  if (endDate < absence.startdate || absence.enddate < startDate) {
-    return 0
-  }
-
-  // Without 'true' as the third argument, the result will be truncated, not rounded (e.g. 4.999 would be 4, not 5). This is why we use Math.round() instead
-  return Math.round(dayjs(absence.enddate).diff(absence.startdate, 'months', true))
-}
-
+/**
+ * Absence rules:
+ * 1. Student can have any number of statutory absences (lakisääteinen poissaolo)
+ * 2a. Student can have 2 non-statutory absences
+ * 2b. If study right started before 1.8.2015, student can have 4 non-statutory absences instead of 2
+ *
+ * Additional _non-statutory_ absences will consume semesters
+ *
+ * @returns duration of studies in SEMESTERS (not months!)
+ */
 export const calculateDurationOfStudies = (
   startDate: Date,
   graduationDate: Date,
   semesterEnrollments: SemesterEnrollment[],
-  semesters: Awaited<ReturnType<typeof getSemestersAndYears>>['semesters']
+  semesters: Awaited<ReturnType<typeof getSemestersAndYears>>['semesters'],
+  transferInfo?: TransferInfo | null
 ) => {
-  const semestersWithStatutoryAbsence = semesterEnrollments
-    .filter(enrollment => enrollment.statutoryAbsence)
-    .map(enrollment => enrollment.semester)
-  const monthsToSubtract = semestersWithStatutoryAbsence.reduce(
-    (acc, semester) => acc + calculateAbsenceInMonths(semesters[semester], startDate, graduationDate),
-    0
+  /** Transferred students: unclear whether this should be based on the original start date or not. Likely doesn't matter. */
+  const allowedNonStatutoryAbsences = dayjs(transferInfo?.originalStartDate ?? startDate).isBefore(
+    dayjs('2015-08-01'),
+    'day'
+  )
+    ? 4
+    : 2
+
+  /** A semester ending and the next one starting overlap, this takes care of that */
+  const getCorrectSemester = (date: Dayjs) =>
+    Object.values(semesters).find(
+      ({ startdate: semesterStart, enddate: semesterEnd }) =>
+        date.isSameOrAfter(semesterStart, 'day') && date.isBefore(semesterEnd, 'day')
+    )?.semestercode
+
+  const startSemester = getCorrectSemester(dayjs(startDate))
+  const endSemester = getCorrectSemester(dayjs(graduationDate))
+
+  if (!startSemester || !endSemester) {
+    logger.error(`calculateDurationOfStudies(): No semesters found! start={${startDate}}, end={${graduationDate}}`)
+    return -1
+  }
+
+  const validSemesters = semesterEnrollments.filter(se => se.semester <= endSemester && se.semester >= startSemester)
+
+  const semestersWithStatutoryAbsence =
+    validSemesters.filter(e => e.statutoryAbsence).length + (transferInfo?.usedStatutoryAbsenceTerms ?? 0)
+
+  const semestersWithNonStatutoryAbsence = Math.min(
+    validSemesters.filter(enrollment => enrollment.type === EnrollmentType.ABSENT && !enrollment.statutoryAbsence)
+      .length + (transferInfo?.usedAbsenceTerms ?? 0),
+    allowedNonStatutoryAbsences
+  )
+
+  return (
+    (transferInfo?.usedTerms ?? 0) +
+    endSemester -
+    startSemester -
+    semestersWithStatutoryAbsence -
+    semestersWithNonStatutoryAbsence +
+    1
   )
-  return Math.round(dayjs(graduationDate).subtract(monthsToSubtract, 'months').diff(startDate, 'months', true))
 }
 
 export type GraduationTimes = {
@@ -169,7 +209,13 @@ const getGraduationTimeAndThesisWriterStats = async ({
 
     const graduationDate = correctStudyRightElement.endDate
 
-    const duration = calculateDurationOfStudies(startDate, graduationDate, studyRight.semesterEnrollments, semesters)
+    const duration = calculateDurationOfStudies(
+      startDate,
+      graduationDate,
+      studyRight.semesterEnrollments,
+      semesters,
+      studyRight.transferInfo
+    )
     const graduationYear = defineYear(graduationDate, isAcademicYear)
 
     if (!graduationTimes[graduationYear] || !graduationTimesCombo[graduationYear]) {
@@ -184,8 +230,8 @@ const getGraduationTimeAndThesisWriterStats = async ({
   }
 
   const goal = await getGoal(studyProgramme)
-  const times: GraduationTimes = { medians: [], goal }
-  const comboTimes: GraduationTimes = { medians: [], goal: goal + 36 }
+  const times: ChartGraduationTimes = { medians: [], goal }
+  const comboTimes: ChartGraduationTimes = { medians: [], goal: goal + GraduationTarget.THREE_YEARS }
 
   for (const year of years.toReversed()) {
     const median = getMedian(graduationTimes[year])
@@ -193,18 +239,18 @@ const getGraduationTimeAndThesisWriterStats = async ({
     times.medians.push({
       y: median,
       amount: graduationTimes[year].length,
-      name: year,
+      name: year.toString(),
       statistics,
       times: [...graduationTimes[year]],
     })
 
     if (doCombo) {
       const median = getMedian(graduationTimesCombo[year])
-      const statistics = countTimeCategories(graduationTimesCombo[year], goal + 36)
+      const statistics = countTimeCategories(graduationTimesCombo[year], goal + GraduationTarget.THREE_YEARS)
       comboTimes.medians.push({
         y: median,
         amount: graduationTimesCombo[year].length,
-        name: year,
+        name: year.toString(),
         statistics,
         times: [...graduationTimesCombo[year]],
       })
@@ -217,12 +263,16 @@ const formatStats = (stats: Record<string, ProgrammeWithYears>, years: Array<str
   const tableStats = Object.values(stats)
     .filter(p => years.map(year => p[year]).find(started => started !== 0)) // Filter out programmes with no-one started between the selected years
     .map(p => [p.code, getId(p.code), p.name, ...years.map(year => p[year])])
-    .sort((a, b) => sortByProgrammeCode(a[0] as string, b[0] as string))
+    .sort((a, b) =>
+      sortByProgrammeCode(a[0] as string, b[0] as string)
+    ) as ProgrammeGraduationStats['programmesBeforeOrAfterTableStats']
 
   const graphStats = Object.values(stats)
     .filter(p => years.map(year => p[year]).find(started => started !== 0)) // Filter out programmes with no-one started between the selected years
     .map(p => ({ name: p.name, code: p.code, data: years.map(year => p[year]) }))
-    .sort((a, b) => sortByProgrammeCode(a.code, b.code))
+    .sort((a, b) =>
+      sortByProgrammeCode(a.code, b.code)
+    ) as ProgrammeGraduationStats['programmesBeforeOrAfterGraphStats']
 
   return { tableStats, graphStats }
 }
@@ -333,7 +383,7 @@ export const getGraduationStatsForStudyTrack = async ({
   studyProgramme: string
   combinedProgramme?: string
   settings: { isAcademicYear: boolean; includeAllSpecials: boolean }
-}) => {
+}): Promise<ProgrammeGraduationStats> => {
   const { isAcademicYear, includeAllSpecials } = settings
   const since = getStartDate(isAcademicYear)
   const years = getYearsArray(since.getFullYear(), isAcademicYear) as number[]
@@ -367,6 +417,7 @@ export const getGraduationStatsForStudyTrack = async ({
     'Graduated licentiate',
     'Wrote thesis licentiate',
   ]
+
   const programmesBeforeOrAfterTitles = ['Code', 'Id', 'Programme', ...years]
   const tableStatsDefault = combinedProgramme
     ? reversedYears.map(year => [

services/backend/src/services/studyProgramme/studyProgrammeHelpers.ts

@@ -1,12 +1,12 @@
-import { orderBy } from 'lodash'
+import { orderBy } from 'lodash-es'
 import { Op } from 'sequelize'
 
 import { SISStudyRight, SISStudyRightElement } from '@oodikone/shared/models'
 import { DegreeProgrammeType, Phase } from '@oodikone/shared/types'
-import { serviceProvider } from '../../config'
 import { programmeCodes } from '../../config/programmeCodes'
 import { ProgrammeModuleModel } from '../../models'
 import { getDegreeProgrammeType } from '../../util'
+import { GraduationTarget } from '../graduationHelpers'
 
 export function getYearsArray(since: number, isAcademicYear: true, yearsCombined?: boolean): string[]
 export function getYearsArray(since: number, isAcademicYear: false, yearsCombined: true): Array<'Total' | number>
@@ -45,15 +45,16 @@ export const getStatsBasis = (years: Array<string | number>) => {
 }
 
 export const getMedian = (values: number[]) => {
-  if (values.length === 0) {
-    return 0
+  if (!values.length) return 0
+
+  // compareFn mandatory
+  const sorted = values.toSorted((a, b) => a - b)
+
+  const half = Math.floor(sorted.length / 2)
+  if (sorted.length % 2) {
+    return sorted[half]
   }
-  values.sort((a, b) => a - b)
-  const half = Math.floor(values.length / 2)
-  if (values.length % 2) {
-    return values[half]
-  }
-  return (values[half - 1] + values[half]) / 2.0
+  return (sorted[half - 1] + sorted[half]) / 2.0
 }
 
 export function defineYear(date: Date, isAcademicYear: true): string
@@ -97,17 +98,11 @@ export const getThesisType = async (studyProgramme: string) => {
   const degreeProgrammeType = await getDegreeProgrammeType(studyProgramme)
   if (degreeProgrammeType === DegreeProgrammeType.MASTER) {
     const mastersThesisTypes = ['urn:code:course-unit-type:masters-thesis']
-    if (serviceProvider !== 'toska') {
-      mastersThesisTypes.push('urn:code:course-unit-type:amk-masters-thesis')
-    }
     return mastersThesisTypes
   }
 
   if (degreeProgrammeType === DegreeProgrammeType.BACHELOR) {
     const bachelorsThesisTypes = ['urn:code:course-unit-type:bachelors-thesis']
-    if (serviceProvider !== 'toska') {
-      bachelorsThesisTypes.push('urn:code:course-unit-type:amk-bachelors-thesis')
-    }
     return bachelorsThesisTypes
   }
 
@@ -167,8 +162,7 @@ export const tableTitles = {
 } as const
 
 export const getId = (code: string) => {
-  if (serviceProvider !== 'fd') return code in programmeCodes ? programmeCodes[code as keyof typeof programmeCodes] : ''
-  return code
+  return code in programmeCodes ? programmeCodes[code as keyof typeof programmeCodes] : ''
 }
 
 export const getGoal = async (programme?: string) => {
@@ -191,9 +185,9 @@ export const getGoal = async (programme?: string) => {
     return 0
   }
   if ([DegreeProgrammeType.DOCTOR, DegreeProgrammeType.LICENTIATE].includes(degreeProgrammeType)) {
-    return 48
+    return GraduationTarget.FOUR_YEARS
   }
-  return (minimumCredits / 60) * 12
+  return ((minimumCredits / 60) * 12) / 6 // 60 Credits per year divided to semesters
 }
 
 export const isRelevantProgramme = (code: string) => /^(KH|MH)\d{2}_\d{3}$/.test(code) || /^T\d{6}$/.test(code)

services/backend/src/services/studyProgramme/studyProgrammeUpdates.ts

@@ -42,23 +42,19 @@ export const updateBasicView = async (code: string, combinedProgramme: string) =
 }
 
 export const updateStudyTrackView = async (code: string, combinedProgramme: string) => {
-  const graduatedOptions = ['GRADUATED_INCLUDED', 'GRADUATED_EXCLUDED'] as const
   const specialGroupOptions = ['SPECIAL_INCLUDED', 'SPECIAL_EXCLUDED'] as const
   const studyRightsOfProgramme = await getStudyRightsInProgramme(code, false, true)
 
-  for (const graduated of graduatedOptions) {
-    for (const specialGroup of specialGroupOptions) {
-      const stats = await getStudyTrackStatsForStudyProgramme({
-        studyProgramme: code,
-        combinedProgramme,
-        settings: {
-          specialGroups: specialGroup === 'SPECIAL_INCLUDED',
-          graduated: graduated === 'GRADUATED_INCLUDED',
-        },
-        studyRightsOfProgramme,
-      })
-      await setStudyTrackStats(stats, graduated, specialGroup)
-    }
+  for (const specialGroup of specialGroupOptions) {
+    const stats = await getStudyTrackStatsForStudyProgramme({
+      studyProgramme: code,
+      combinedProgramme,
+      settings: {
+        specialGroups: specialGroup === 'SPECIAL_INCLUDED',
+      },
+      studyRightsOfProgramme,
+    })
+    await setStudyTrackStats(stats, specialGroup)
   }
 
   return 'OK'

services/backend/src/services/studyProgramme/studyRightFinders.ts

@@ -14,15 +14,18 @@ export const getStudyRightsInProgramme = async (
     where.graduated = true
   }
 
-  const studyRights = await SISStudyRightModel.findAll({
-    attributes: ['id'],
-    include: {
-      model: SISStudyRightElementModel,
-      as: 'studyRightElements',
-      attributes: [],
-      where,
-    },
-  })
+  const studyRightIds = (
+    await SISStudyRightModel.findAll({
+      attributes: ['id'],
+      include: {
+        model: SISStudyRightElementModel,
+        as: 'studyRightElements',
+        attributes: [],
+        where,
+      },
+      raw: true,
+    })
+  ).map(sr => sr.id)
 
   const include: Includeable[] = [
     {
@@ -54,11 +57,11 @@ export const getStudyRightsInProgramme = async (
 
   return (
     await SISStudyRightModel.findAll({
-      attributes: ['id', 'extentCode', 'semesterEnrollments', 'studentNumber'],
+      attributes: ['id', 'extentCode', 'semesterEnrollments', 'studentNumber', 'transferInfo'],
       include,
       where: {
         id: {
-          [Op.in]: studyRights.map(studyRight => studyRight.toJSON().id),
+          [Op.in]: studyRightIds,
         },
       },
     })

services/backend/src/services/studyProgramme/studyTrackStats.ts

@@ -10,7 +10,7 @@ import {
   SemesterEnrollment,
 } from '@oodikone/shared/types'
 import { createLocaleComparator, keysOf } from '../../util'
-import { countTimeCategories } from '../graduationHelpers'
+import { countTimeCategories, GraduationTarget } from '../graduationHelpers'
 import { getSemestersAndYears } from '../semesters'
 import { getDateOfFirstSemesterPresent } from './studyProgrammeBasics'
 import { calculateDurationOfStudies, shouldIncludeComboStats } from './studyProgrammeGraduations'
@@ -39,7 +39,7 @@ const getGraduationTimeStats = async (
   combinedProgramme?: string
 ) => {
   const goal = await getGoal(combinedProgramme ?? studyProgramme)
-  const finalGraduationTimes = { goals: { basic: goal, combo: goal + 36 } }
+  const finalGraduationTimes = { goals: { basic: goal, combo: goal + GraduationTarget.THREE_YEARS } }
 
   const calculateGraduationTimes = (
     graduationTimes: Record<string, Record<string, number[]>>,
@@ -171,7 +171,6 @@ const combineStats = (
 const getMainStatsByTrackAndYear = async (
   years: string[],
   studyProgramme: string,
-  includeGraduated: boolean,
   includeAllSpecials: boolean,
   doCombo: boolean,
   studyRightsOfProgramme: Array<SISStudyRight>,
@@ -277,7 +276,13 @@ const getMainStatsByTrackAndYear = async (
 
     const graduationDate = studyRightElement.endDate
 
-    const duration = calculateDurationOfStudies(startDate, graduationDate, studyRight.semesterEnrollments, semesters)
+    const duration = calculateDurationOfStudies(
+      startDate,
+      graduationDate,
+      studyRight.semesterEnrollments,
+      semesters,
+      studyRight.transferInfo
+    )
 
     if (countAsBachelorMaster) {
       if (!graduationTimesCombo[programmeOrStudyTrack]) {
@@ -314,7 +319,8 @@ const getMainStatsByTrackAndYear = async (
       bachelorStartDate,
       studyRight.studyRightElements.find(element => element.code === combinedProgramme)!.endDate,
       studyRight.semesterEnrollments,
-      semesters
+      semesters,
+      studyRight.transferInfo
     )
     if (graduationTimesCombinedProgrammeCombo[combinedProgramme!][startYearInNewBachelorProgramme]) {
       graduationTimesCombinedProgrammeCombo[combinedProgramme!][startYearInNewBachelorProgramme].push(combinedDuration)
@@ -338,9 +344,6 @@ const getMainStatsByTrackAndYear = async (
     if (!studyRightElement) {
       continue
     }
-    if (!includeGraduated && studyRightElement.graduated) {
-      continue
-    }
 
     const [hasTransferredFromProgramme, hasTransferredToProgramme] = hasTransferredFromOrToProgramme(
       studyRight,
@@ -378,15 +381,18 @@ const getMainStatsByTrackAndYear = async (
     }
 
     const countAsBachelorMaster = doCombo && studyRight.extentCode === ExtentCode.BACHELOR_AND_MASTER
-    if (countAsBachelorMaster) {
-      const startedInBachelor = getStudyRightElementsWithPhase(studyRight, 1)[0]?.startDate
-      if (startedInBachelor && years.includes(defineYear(startedInBachelor, true))) {
-        creditCountsCombo[defineYear(startedInBachelor, true)].push(
-          getCreditCount(studyRight.student.credits, startedInBachelor)
-        )
+    // TODO: FIXME: This could be written better
+    if (!studyRightElement.graduated) {
+      if (countAsBachelorMaster) {
+        const startedInBachelor = getStudyRightElementsWithPhase(studyRight, 1)[0]?.startDate
+        if (startedInBachelor && years.includes(defineYear(startedInBachelor, true))) {
+          creditCountsCombo[defineYear(startedInBachelor, true)].push(
+            getCreditCount(studyRight.student.credits, startedInBachelor)
+          )
+        }
+      } else {
+        creditCounts[startYear].push(getCreditCount(studyRight.student.credits, startedInProgramme))
       }
-    } else {
-      creditCounts[startYear].push(getCreditCount(studyRight.student.credits, startedInProgramme))
     }
   }
 
@@ -410,6 +416,11 @@ const getMainStatsByTrackAndYear = async (
     }
   }
 
+  const graduatedCount: Record<string, number> = {}
+  years.forEach(year => {
+    graduatedCount[year] = yearlyStats[year] ? yearlyStats[year][studyProgramme].graduated : 0
+  })
+
   const { mainStatsByYear, mainStatsByTrack, otherCountriesCount } = combineStats(
     years,
     yearlyStats,
@@ -438,6 +449,7 @@ const getMainStatsByTrackAndYear = async (
     otherCountriesCount,
     creditCounts,
     creditCountsCombo,
+    graduatedCount,
     graduationTimes: finalGraduationTimes,
     graduationTimesSecondProg: finalCombinedGraduationTimes,
   }
@@ -453,7 +465,7 @@ export const getStudyTrackStatsForStudyProgramme = async ({
 }: {
   studyProgramme: string
   combinedProgramme?: string
-  settings: { graduated: boolean; specialGroups: boolean }
+  settings: { specialGroups: boolean }
   studyRightsOfProgramme: SISStudyRight[]
 }) => {
   const isAcademicYear = true
@@ -467,7 +479,6 @@ export const getStudyTrackStatsForStudyProgramme = async ({
   const stats = await getMainStatsByTrackAndYear(
     years,
     studyProgramme,
-    settings.graduated,
     settings.specialGroups,
     doCombo,
     studyRightsOfProgramme,
@@ -484,10 +495,10 @@ export const getStudyTrackStatsForStudyProgramme = async ({
     creditCounts: stats.creditCounts,
     creditCountsCombo: stats.creditCountsCombo,
     doCombo,
+    graduatedCount: stats.graduatedCount,
     graduationTimes: stats.graduationTimes,
     graduationTimesSecondProg: stats.graduationTimesSecondProg,
     id: combinedProgramme ? `${studyProgramme}-${combinedProgramme}` : studyProgramme,
-    includeGraduated: settings.graduated,
     mainStatsByTrack: stats.mainStatsByTrack,
     mainStatsByYear: stats.mainStatsByYear,
     otherCountriesCount: stats.otherCountriesCount,

services/backend/src/services/userService.ts

@@ -1,9 +1,8 @@
-import { isEqual, keyBy, omit, uniq } from 'lodash'
+import { isEqual, keyBy, omit, uniq } from 'lodash-es'
 import { LRUCache } from 'lru-cache'
 
 import { User } from '@oodikone/shared/models/user'
 import { DetailedProgrammeRights, Role } from '@oodikone/shared/types'
-import { serviceProvider } from '../config'
 import { roles } from '../config/roles'
 import { sequelizeUser } from '../database/connection'
 import { UserModel } from '../models/user'
@@ -11,13 +10,12 @@ import { ExpandedUser, FormattedUser, IamAccess } from '../types'
 
 import { createLocaleComparator, getFullStudyProgrammeRights, hasFullAccessToStudentData } from '../util'
 import * as jami from '../util/jami'
-import mami from '../util/mami'
 import { sendNotificationAboutNewUser } from './mailService'
 import { getSisuAuthData, personSearchQuery, getGraphqlData } from './oriProvider'
 import { getStudentnumbersByElementdetails } from './students'
 import { checkStudyGuidanceGroupsAccess, getAllStudentsUserHasInGroups } from './studyGuidanceGroups'
 
-const userAccessUtils = serviceProvider === 'toska' ? jami : mami
+const userAccessUtils = jami
 const { getAllUserAccess, getUserIams, getUserIamAccess } = userAccessUtils
 
 const courseStatisticsGroup = 'grp-oodikone-basic-users'
@@ -68,7 +66,7 @@ export const updateUser = async (username: string, fields: Partial<User>) => {
   userDataCache.delete(username)
 }
 
-const getIamBasedRights = (iamAccess: IamAccess): DetailedProgrammeRights[] => {
+const getIamBasedRights = (iamAccess: IamAccess | undefined): DetailedProgrammeRights[] => {
   if (!iamAccess) {
     return []
   }
@@ -89,8 +87,8 @@ const getUserProgrammesRights = (userProgrammes: string[]): DetailedProgrammeRig
 }
 
 const getStudyProgrammeRights = (
-  iamAccess: IamAccess,
-  specialGroup: Record<string, boolean>,
+  iamAccess: IamAccess | undefined,
+  specialGroup: Record<string, boolean> | undefined,
   userProgrammes: string[]
 ): DetailedProgrammeRights[] => {
   const hasFullSisuAccess = specialGroup?.fullSisuAccess
@@ -233,15 +231,7 @@ export const getMockedUser = async ({ userToMock, mockedBy }: { userToMock: stri
   return mockedUser
 }
 
-export const getMockedUserFd = async ({ userToMock, mockedBy }: { userToMock: string; mockedBy: string }) => {
-  const mockedByFromDb = (await UserModel.findOne({ where: { username: mockedBy } }))?.toJSON()
-  if (!mockedByFromDb) {
-    return null
-  }
-  return await getMockedUser({ userToMock, mockedBy })
-}
-
-export const getUserToska = async ({
+export const getUser = async ({
   username,
   name,
   email,
@@ -283,28 +273,6 @@ export const getUserToska = async ({
   return user
 }
 
-export const getUserFd = async ({ username }: { username: string }) => {
-  if (userDataCache.has(username)) {
-    return userDataCache.get(username) as FormattedUser
-  }
-
-  const userFromDbOrm = await UserModel.findOne({ where: { username } })
-  if (!userFromDbOrm) {
-    return null
-  }
-
-  userFromDbOrm.lastLogin = new Date()
-  await userFromDbOrm.save()
-
-  const userFromDb = userFromDbOrm.toJSON()
-
-  const detailedProgrammeRights = getStudyProgrammeRights({}, {}, userFromDb.programmeRights)
-  const user = await formatUser({ ...userFromDb, iamGroups: [], detailedProgrammeRights })
-
-  userDataCache.set(username, user)
-  return user
-}
-
 export const addNewUser = async user => {
   const name = user.first_name.concat(' ', user.last_name)
   try {

services/backend/src/util/importerClient.ts

@@ -1,43 +1,17 @@
-import axios from 'axios'
-
-import { importerDbApiPassword, importerDbApiUser, importerToken, importerUrl, serviceProvider } from '../config'
+import { Fetchios } from '@oodikone/shared/util/fetchios'
+import { importerToken, importerUrl } from '../config'
 import logger from './logger'
 
-const createBasicAuthHeader = () => {
-  const auth = `${importerDbApiUser}:${importerDbApiPassword}`
-  const token = Buffer.from(auth).toString('base64')
-  return {
-    Authorization: `Basic ${token}`,
-  }
-}
-
-const toskaImporterClient = axios.create({
+const importerClient = Fetchios.create({
   headers: { token: importerToken },
-  baseURL: importerUrl,
+  baseUrl: importerUrl,
   timeout: 10000,
 })
 
-const fdImporterClient = axios.create({
-  headers: createBasicAuthHeader(),
-  baseURL: importerUrl,
-})
-
-const missingBasicAuthCredentials = !importerDbApiUser || !importerDbApiPassword
-
-const getFdImporterClient = () => {
-  if (missingBasicAuthCredentials) {
-    logger.error("Basic auth credentials not set, can't return client!")
-    return null
-  }
-  return fdImporterClient
-}
-
-const getToskaImporterClient = () => {
+export const getImporterClient = () => {
   if (!importerToken) {
     logger.error("Importer token not set, can't return client!")
     return null
   }
-  return toskaImporterClient
+  return importerClient
 }
-
-export const getImporterClient = serviceProvider === 'toska' ? getToskaImporterClient : getFdImporterClient

services/backend/src/util/jami.ts

@@ -1,12 +1,12 @@
 import * as Sentry from '@sentry/node'
-import axios from 'axios'
+import { Fetchios } from '@oodikone/shared/util/fetchios'
 
-import { importerToken, isStaging, jamiUrl, serviceProvider } from '../config'
+import { importerToken, isStaging, jamiUrl } from '../config'
 import { IamAccess } from '../types'
 import logger from './logger'
 
-const jamiClient = axios.create({
-  baseURL: jamiUrl,
+const jamiClient = Fetchios.create({
+  baseUrl: jamiUrl,
   params: {
     token: importerToken,
   },
@@ -24,29 +24,34 @@ export const getUserIamAccess = async (
 ): Promise<UserIamAccess> => {
   if (iamGroups.length === 0) return {}
 
-  try {
-    const { data } = await jamiClient.post('/', {
+  const { data, error } = await jamiClient.post<Pick<UserIamAccess, 'specialGroup'>>(
+    '',
+    {
       userId: sisPersonId,
       getSisuAccess: !isStaging && !!sisPersonId,
       iamGroups,
-    })
+    },
+    {}
+  )
 
+  if (error === null) {
     const { specialGroup, ...resetOfIamAccess } = data
     return { iamAccess: resetOfIamAccess, specialGroup }
-  } catch (error) {
-    if (attempt > 3) {
-      logger.error('[Jami] error: ', error)
-      Sentry.captureException(error)
-      return {}
-    }
-
-    return getUserIamAccess(sisPersonId, iamGroups, attempt + 1)
   }
+
+  if (attempt > 3) {
+    logger.error('[Jami] error: ', error)
+    Sentry.captureException(error)
+    return {}
+  }
+
+  return getUserIamAccess(sisPersonId, iamGroups, attempt + 1)
 }
 
 export const getUserIams = async (userId: string) => {
   try {
-    const { data } = await jamiClient.get(`/${userId}`)
+    const { data } = await jamiClient.get<any>(userId, {})
+
     return data.iamGroups as string[]
   } catch (error: any) {
     if (error.response.status !== 404) {
@@ -65,18 +70,16 @@ type Access = {
 }
 
 export const getAllUserAccess = async (userIds: string[]) => {
-  const { data } = await jamiClient.post('access-and-special-groups', { userIds })
+  const { data } = await jamiClient.post('access-and-special-groups', { userIds }, {})
   return data as Access[]
 }
 
-if (serviceProvider === 'toska') {
-  jamiClient
-    .get('/ping', { timeout: 4000 })
-    .then(() => {
-      logger.info('JAMI connected')
-    })
-    .catch(error => {
-      logger.error('JAMI not responding :(', { error })
-      logger.info('Are you sure you are using the latest JAMI image?')
-    })
-}
+jamiClient
+  .get('ping', { timeout: 4000 })
+  .then(() => {
+    logger.info('JAMI connected')
+  })
+  .catch(error => {
+    logger.error('JAMI not responding :(', { error })
+    logger.info('Are you sure you are using the latest JAMI image?')
+  })

services/backend/src/util/logger.js

@@ -1,58 +0,0 @@
-const os = require('os')
-const winston = require('winston')
-const { WinstonGelfTransporter } = require('winston-gelf-transporter')
-const LokiTransport = require('winston-loki')
-
-const { isProduction, serviceProvider } = require('../config')
-
-const { colorize, combine, timestamp, printf } = winston.format
-
-const transports = [new winston.transports.Console()]
-
-if (serviceProvider !== 'fd') {
-  transports.push(
-    new LokiTransport({
-      host: 'http://loki-svc.toska-lokki.svc.cluster.local:3100',
-      labels: { app: 'oodikone', environment: process.env.NODE_ENV ?? 'production' },
-    })
-  )
-}
-
-if (isProduction && serviceProvider !== 'fd') {
-  transports.push(
-    new WinstonGelfTransporter({
-      handleExceptions: true,
-      host: 'svm-116.cs.helsinki.fi',
-      port: 9503,
-      protocol: 'udp',
-      hostName: os.hostname(),
-      additional: {
-        app: 'oodikone',
-        environment: 'production',
-      },
-    })
-  )
-}
-
-const logger = winston.createLogger({
-  level: isProduction ? 'info' : 'debug',
-  format: combine(
-    colorize(),
-    timestamp({ format: isProduction ? 'D.M.YYYY,HH.mm.ss' : 'HH.mm.ss' }),
-    printf(({ level, message, timestamp, error, meta }) => {
-      let log = `${timestamp} ${level}: ${message}`
-      if (error?.stack) {
-        log = `${log}\n${error.stack}`
-      }
-      if (Object.keys(meta || {}).length > 0) {
-        log = `${log}\n${JSON.stringify(meta, null, 2)}`
-      }
-      return log
-    })
-  ),
-  transports,
-})
-
-logger.on('error', error => console.error('Logging failed! Reason: ', error)) // eslint-disable-line no-console
-
-module.exports = logger

services/backend/src/util/logger.ts

@@ -0,0 +1,50 @@
+import os from 'os'
+import * as winston from 'winston'
+import { WinstonGelfTransporter } from 'winston-gelf-transporter'
+import LokiTransport from 'winston-loki'
+
+import { isProduction } from '../config'
+
+const transports: winston.transport[] = [new winston.transports.Console()]
+
+transports.push(
+  new LokiTransport({
+    host: 'http://loki-svc.toska-lokki.svc.cluster.local:3100',
+    labels: { app: 'oodikone', environment: process.env.NODE_ENV ?? 'production' },
+  }),
+  new WinstonGelfTransporter({
+    handleExceptions: true,
+    host: 'svm-116.cs.helsinki.fi',
+    port: 9503,
+    protocol: 'udp',
+    hostName: os.hostname(),
+    additional: {
+      app: 'oodikone',
+      environment: 'production',
+    },
+  }) as winston.transport
+)
+
+const { colorize, combine, timestamp, printf } = winston.format
+const logger = winston.createLogger({
+  level: isProduction ? 'info' : 'debug',
+  format: combine(
+    colorize(),
+    timestamp({ format: isProduction ? 'D.M.YYYY,HH.mm.ss' : 'HH.mm.ss' }),
+    printf(({ level, message, timestamp, error, meta }) => {
+      let log = `${timestamp} ${level}: ${message}`
+      if (error instanceof Error) {
+        log = `${log}\n${error.stack}`
+      }
+      if (Object.keys(meta ?? {}).length > 0) {
+        log = `${log}\n${JSON.stringify(meta, null, 2)}`
+      }
+      return log
+    })
+  ),
+  transports,
+})
+
+logger.on('error', error => console.error('Logging failed! Reason: ', error)) // eslint-disable-line no-console
+
+export default logger

services/backend/src/util/mami.js

@@ -1,17 +0,0 @@
-const getUserIamAccess = async (_, __, ___) => {
-  return {}
-}
-
-const getUserIams = async _ => {
-  return []
-}
-
-const getAllUserAccess = async _ => {
-  return []
-}
-
-module.exports = {
-  getUserIamAccess,
-  getUserIams,
-  getAllUserAccess,
-}

services/backend/src/worker/worker.js

@@ -3,7 +3,7 @@ const { Worker: BullMQWorker } = require('bullmq')
 const dayjs = require('dayjs')
 
 const { redis, concurrentWorkers } = require('../config')
-const logger = require('../util/logger')
+const logger = require('../util/logger').default
 const { queueName } = require('./queue')
 
 const connection = {

services/backend/tests/changelog.test.ts

@@ -0,0 +1,24 @@
+import { Express } from 'express'
+import assert from 'node:assert/strict'
+import { describe, it, before } from 'node:test'
+import request from 'supertest'
+
+import { initTests } from './utils'
+
+let app: Express
+before(async () => {
+  app = await initTests()
+})
+
+void describe('Changelog', async () => {
+  await it('should return dev data in dev env', async () => {
+    const res = await request(app)
+      .get('/changelog')
+      .set('shib-session-id', 'test')
+      .set('uid', 'mluukkai')
+      .set('hygroupcn', 'grp-oodikone-basic-users')
+
+    assert.strictEqual(res.status, 200)
+    assert.strictEqual(res.body[0].title, 'Release 3')
+  })
+})

services/backend/tests/population.test.ts

@@ -0,0 +1,200 @@
+import { Express } from 'express'
+import assert from 'node:assert/strict'
+import { describe, it, before } from 'node:test'
+import { Sequelize } from 'sequelize'
+import request, { Response } from 'supertest'
+
+import { PopulationstatisticsResBody } from '@oodikone/shared/routes/populations'
+import { dbConnections } from '../src/database/connection'
+import { initTests } from './utils'
+
+let app: Express
+let connection: Sequelize
+before(async () => {
+  app = await initTests()
+  connection = dbConnections.sequelize
+})
+
+// Override Supertest's Response body with our own type
+type ResponseWithBody<T> = Omit<Response, 'body'> & { body: T }
+
+const populationUrl = (
+  programme = '',
+  years: string[] = [],
+  semesters: ('SPRING' | 'FALL')[] = [],
+  students: string[] = []
+) => {
+  return `/v3/populationstatistics?${programme && 'programme=' + programme}${years.map(year => '&years=' + year).join('')}${semesters.map(sem => '&semesters=' + sem).join('')}${students.map(s => '&studentStatuses=' + s).join('')}`
+}
+
+void describe('Population statistics', async () => {
+  await it('should fail when all fields not defined', async () => {
+    const resAllMissing = await request(app)
+      .get(populationUrl())
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')
+
+    const resProgramme = await request(app)
+      .get(populationUrl('KH50_001'))
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')
+
+    const resProgrammeAndYears = await request(app)
+      .get(populationUrl('KH50_001', ['2021', '2022']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')
+
+    assert.strictEqual(resAllMissing.status, 400)
+    assert.strictEqual(resProgramme.status, 400)
+    assert.strictEqual(resProgrammeAndYears.status, 400)
+  })
+
+  await it('should not return any data to unauthorized user', async () => {
+    const res = (await request(app)
+      .get(populationUrl('KH50_001', ['2021', '2022'], ['SPRING', 'FALL']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'norights')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    assert.deepStrictEqual(res.body, { error: 'Trying to request unauthorized students data' })
+    assert.strictEqual(res.status, 403)
+  })
+
+  await it('should fail when trying to access a programme that user has no permissions to', async () => {
+    const res = (await request(app)
+      .get(populationUrl('MH50_001', ['2021', '2022'], ['SPRING', 'FALL']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'onlyiamrights')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    assert.strictEqual(res.status, 403)
+    assert.deepStrictEqual(res.body, { error: 'Trying to request unauthorized students data' })
+  })
+
+  await it('should return students in the programme for authorized user', async () => {
+    const res = (await request(app)
+      .get(populationUrl('KH50_001', ['2021', '2022'], ['SPRING', 'FALL']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    assert.strictEqual(res.status, 200)
+    assert.notStrictEqual(res.body.students.length, 0)
+  })
+
+  await it('should work with the programme flag', async () => {
+    const resBachelor = (await request(app)
+      .get(populationUrl('KH50_001', ['2021'], ['SPRING', 'FALL']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    const resMaster = (await request(app)
+      .get(populationUrl('MH50_001', ['2021'], ['SPRING', 'FALL']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    assert.strictEqual(resBachelor.status, 200)
+    assert.strictEqual(resBachelor.body.students.length, 38)
+
+    assert.strictEqual(resMaster.status, 200)
+    assert.strictEqual(resMaster.body.students.length, 36)
+  })
+
+  await it('should work with semester flag', async () => {
+    const resFall = (await request(app)
+      .get(populationUrl('KH50_001', ['2021'], ['FALL']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    const resSpring = (await request(app)
+      .get(populationUrl('KH50_001', ['2021'], ['SPRING']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    assert.strictEqual(resFall.status, 200)
+    assert.strictEqual(resSpring.status, 200)
+
+    assert.strictEqual(resFall.body.students.length, 34)
+    assert.strictEqual(resSpring.body.students.length, 4)
+  })
+
+  await it('should work with year flag correctly', async () => {
+    const res2021 = (await request(app)
+      .get(populationUrl('KH50_001', ['2021'], ['FALL', 'SPRING']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+    const res2022 = (await request(app)
+      .get(populationUrl('KH50_001', ['2022'], ['FALL', 'SPRING']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    const res2021_2022 = (await request(app)
+      .get(populationUrl('KH50_001', ['2021', '2022'], ['FALL', 'SPRING']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    const dbCount2021 = await connection.query(
+      `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2021-07-31' and "sis_study_right_elements".start_date < '2022-08-01'`,
+      { raw: true, plain: true }
+    )
+    const dbCount2022 = await connection.query(
+      `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2022-07-31' and "sis_study_right_elements".start_date < '2023-08-01'`,
+      { raw: true, plain: true }
+    )
+    const dbCount2021_2022 = await connection.query(
+      `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2021-07-31' and "sis_study_right_elements".start_date < '2023-08-01'`,
+      { raw: true, plain: true }
+    )
+
+    assert.strictEqual(res2021.status, 200)
+    assert.strictEqual(res2022.status, 200)
+
+    assert.strictEqual(res2021.body.students.length, dbCount2021?.count)
+    assert.strictEqual(res2022.body.students.length, dbCount2022?.count)
+
+    assert.strictEqual(res2021_2022.status, 200)
+    assert.strictEqual(res2021_2022.body.students.length, dbCount2021_2022?.count)
+  })
+
+  await it.skip('should return only students who have transferred out of the programme when TRANSFERRED flag is set', async () => {
+    // TODO: This doesn't work correctly...
+    const resWithTransOut = (await request(app)
+      .get(populationUrl('KH50_001', ['2019'], ['FALL', 'SPRING'], ['']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'mluukkai')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    const resWithoutTransOut = (await request(app)
+      .get(populationUrl('KH50_001', ['2019'], ['FALL', 'SPRING'], ['TRANSFERRED']))
+      .set('shib-session-id', 'test')
+      .set('uid', 'mluukkai')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+
+    const dbCountWithoutTransferredOut = await connection.query(
+      `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2019-07-31' and "sis_study_right_elements".start_date < '2020-08-01'`,
+      { raw: true, plain: true }
+    )
+    const dbCountWithTransferredOut = await connection.query(
+      `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2019-07-31' and "sis_study_right_elements".start_date < '2020-08-01'`,
+      { raw: true, plain: true }
+    )
+
+    assert.strictEqual(resWithTransOut.status, 200)
+    assert.strictEqual(resWithoutTransOut.status, 200)
+
+    assert.strictEqual(resWithTransOut.body.students.length, dbCountWithTransferredOut?.count)
+    assert.strictEqual(resWithoutTransOut.body.students.length, dbCountWithoutTransferredOut?.count)
+  })
+  await it.todo('should return only degree-students and non-degree students when NONDEGREE flag is set')
+  await it.todo('should return only degree-students and exchange students when EXCHANGE flag is set')
+})

services/backend/tests/utils.ts

@@ -0,0 +1,24 @@
+import express from 'express'
+
+import { baseUrl } from '../src/config'
+import { initializeDatabaseConnection, dbConnections } from '../src/database/connection'
+import routes from '../src/routes'
+
+export const initTests = async () => {
+  // Copied from "app.ts"
+  await initializeDatabaseConnection()
+
+  void (await dbConnections.connect())
+
+  dbConnections.on('connect', () => {
+    console.info('Connected to sis db successfully')
+  })
+  dbConnections.on('error', () => {
+    console.error('Failed to connect to sis db!')
+  })
+
+  const app = express()
+  routes(app, baseUrl)
+
+  return app
+}

services/backend/tsconfig.json

@@ -17,8 +17,9 @@
     "sourceMap": true,
     "sourceRoot": "/",
     "inlineSources": true,
-    "resolveJsonModule": true
+    "resolveJsonModule": true,
+    "types": ["node"]
   },
-  "include": ["index.ts", "src"],
+  "include": ["index.ts", "src", "tests"],
   "exclude": ["../frontend"]
 }

services/frontend/.dockerignore

@@ -1,6 +0,0 @@
-.dockerignore
-.eslintrc.json
-.gitignore
-*Dockerfile
-build/
-node_modules/

services/frontend/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi9/nodejs-22-minimal
+FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 
@@ -22,15 +22,14 @@ ENV REACT_APP_SENTRY_RELEASE=$SENTRY_RELEASE
 ARG SENTRY_DSN=''
 ENV REACT_APP_SENTRY_DSN=$SENTRY_DSN
 
-ARG SERVICE_PROVIDER='toska'
-ENV REACT_APP_SERVICE_PROVIDER=$SERVICE_PROVIDER
-
 ARG LANGUAGE_CENTER_VIEW_ENABLED
 ENV REACT_APP_LANGUAGE_CENTER_VIEW_ENABLED=$LANGUAGE_CENTER_VIEW_ENABLED
 
 ARG SIS_URL='https://sisu.helsinki.fi'
 ENV REACT_APP_SIS_URL=$SIS_URL
 
+RUN npm install -g serve@^14
+
 WORKDIR /opt/app-root/shared
 COPY ./shared .
 
@@ -39,7 +38,7 @@ COPY ./frontend/package*.json .
 RUN npm ci --include=dev
 COPY ./frontend .
 
-RUN npm run build && npm install -g serve@^14
+RUN npm run build
 
 EXPOSE 5000
 CMD ["serve", "--single", "--listen", "5000", "build"]

services/frontend/dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM node:22-alpine
+FROM node:24-alpine
 
 ENV TZ="Europe/Helsinki"