[Update][Studyrights] prevent creating phase 2 elements before if there is no phase 1 graduation

Further clarification possibly needed

.github/workflows/pull_request.yaml

@@ -15,17 +15,22 @@ jobs:
   lint_packages:
     name: 'Lint packages'
     runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
+
     steps:
       - uses: actions/checkout@v6
       - name: Lint backend, frontend and shared
         uses: ./.github/actions/lint
 
   test_api:
+    name: 'Test backend api'
+    runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
     needs:
       - lint_packages
     strategy:
       fail-fast: false
-    runs-on: ubuntu-24.04
+
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -49,12 +54,15 @@ jobs:
         run: npm run oodikone:ci:api
 
   test_end_to_end:
+    name: ${{ matrix.name }}
     runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'renovate[bot]' }}
     needs:
       - lint_packages
       - test_api
     strategy:
       fail-fast: false
+
       matrix:
         include:
           - name: 'Test course statistics'
@@ -83,7 +91,7 @@ jobs:
             spec: cypress/e2e/Study_guidance_groups.js
           - name: 'Test university'
             spec: cypress/e2e/University.js
-    name: ${{ matrix.name }}
+
     steps:
       - uses: actions/checkout@v6
       - name: 'Run tests'

.github/workflows/staging.yaml

@@ -13,6 +13,7 @@ on:
       - 'services/**'
       - 'updater/sis-updater-worker/src/db/migrations/**'
       - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
       - 'package*.json'
 
 jobs:
@@ -91,12 +92,11 @@ jobs:
   report_success:
     name: 'Report deployment success to the "oodikone_notifications" channel'
     runs-on: ubuntu-latest
+    if: success()
     needs:
       - release_frontend
       - release_backend
 
-    if: success()
-
     steps:
       - uses: actions/checkout@v6
       - name: 'Send notification to Slack'
@@ -110,12 +110,11 @@ jobs:
   report_failure:
     name: 'Report deployment failure to the "oodikone_notifications" channel'
     runs-on: ubuntu-latest
+    if: failure()
     needs:
       - release_frontend
       - release_backend
 
-    if: failure()
-
     steps:
       - uses: actions/checkout@v6
       - name: 'Send notification to Slack'

.github/workflows/test.yaml

@@ -123,6 +123,7 @@ jobs:
       kone_db_migrations: ${{ steps.check_migrations.outputs.kone_db_migrations }}
       user_db_migrations: ${{ steps.check_migrations.outputs.user_db_migrations }}
       sis_db_migrations: ${{ steps.check_migrations.outputs.sis_db_migrations }}
+
     steps:
       - uses: actions/checkout@v6
       - name: Check if new migrations have been added
@@ -142,11 +143,12 @@ jobs:
 
   build_and_push_anon_sis_db:
     name: 'Build anon sis-db and push to toska hub if new migrations have been added'
+    runs-on: ubuntu-latest
+    if: needs.check_migration_files.outputs.sis_db_migrations == 'true'
     needs:
       - pull_docker_images
       - check_migration_files
-    runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.sis_db_migrations == 'true'
+
     steps:
       - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'
@@ -173,13 +175,14 @@ jobs:
 
   build_and_push_anon_kone_db:
     name: 'Build anon kone-db and push to toska hub if new migrations have been added'
+    runs-on: ubuntu-latest
+    if: needs.check_migration_files.outputs.kone_db_migrations == 'true'
     needs:
       - pull_docker_images
       - build_backend
       - build_frontend
       - check_migration_files
-    runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.kone_db_migrations == 'true'
+
     steps:
       - uses: actions/checkout@v6
 
@@ -213,13 +216,13 @@ jobs:
 
   build_and_push_anon_user_db:
     name: 'Build anon user-db and push to toska hub if new migrations have been added'
+    runs-on: ubuntu-latest
+    if: needs.check_migration_files.outputs.user_db_migrations == 'true'
     needs:
       - pull_docker_images
       - build_backend
       - build_frontend
       - check_migration_files
-    runs-on: ubuntu-latest
-    if: needs.check_migration_files.outputs.user_db_migrations == 'true'
     steps:
       - uses: actions/checkout@v6
       - uses: dorny/paths-filter@v3
@@ -260,16 +263,18 @@ jobs:
 
   # === Backend api-tests ===
   test_api:
+    name: 'Run backend api tests'
+    runs-on: ubuntu-latest
+    if: always() && !cancelled() && needs.lint_packages.result == 'success'
     needs:
       - pull_docker_images
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
       - lint_packages
-    if: always() && !cancelled() && needs.lint_packages.result == 'success'
     strategy:
       fail-fast: false
-    runs-on: ubuntu-24.04
+
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -301,15 +306,85 @@ jobs:
       - name: Run api-tests
         run: npm run oodikone:ci:api
 
-  # === End-to-end tests ===
-  test_end_to_end:
+  # === Frontend unit testing ===
+  test_unit_frontend:
+    name: 'Run frontend unit tests'
+    runs-on: ubuntu-latest
+    if: always() && !cancelled() && needs.lint_packages.result == 'success'
     needs:
       - pull_docker_images
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
       - lint_packages
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install node-24
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Install dependencies
+        run: npm ci
+        working-directory: ./services/frontend
+
+      - name: Run unit tests
+        run: npm run test:unit
+        working-directory: ./services/frontend
+
+  # === Frontend component testing ===
+  test_components:
+    name: 'Test frontend components'
+    runs-on: ubuntu-latest
     if: always() && !cancelled() && needs.lint_packages.result == 'success'
+    needs:
+      - pull_docker_images
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
+      - lint_packages
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install node-24
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Install dependencies
+        run: npm ci --prefix shared && npm ci --prefix frontend
+        working-directory: ./services
+
+      - name: Install Playwright Browsers
+        run: npm exec -- playwright install --with-deps
+        working-directory: ./services/frontend
+
+      - name: Run Playwright tests
+        run: npm run test:component
+        working-directory: ./services/frontend
+
+      - uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
+
+  # === End-to-end tests ===
+  test_end_to_end:
+    runs-on: ubuntu-latest
+    if: always() && !cancelled() && needs.lint_packages.result == 'success'
+    needs:
+      - pull_docker_images
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
+      - lint_packages
     strategy:
       fail-fast: false
       matrix:
@@ -340,9 +415,7 @@ jobs:
             spec: cypress/e2e/Study_guidance_groups.js
           - name: 'Test university'
             spec: cypress/e2e/University.js
-
     name: ${{ matrix.name }}
-    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -384,14 +457,15 @@ jobs:
   # === Other tests ===
   test_service_exit_statuses:
     name: 'Check services run correctly'
+    runs-on: ubuntu-latest
+    if: always() && !cancelled() && needs.lint_packages.result == 'success'
     needs:
       - pull_docker_images
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
       - lint_packages
-    if: always() && !cancelled() && needs.lint_packages.result == 'success'
-    runs-on: ubuntu-latest
+
     steps:
       - uses: actions/checkout@v6
       - name: 'Login to toska docker registry'

README.md

@@ -149,8 +149,8 @@ It is recommended to spend some time to become familiar with `docker` and `docke
 - `docker compose ps`: view the containers in the running environment
 - `docker compose logs frontend`: print logs for just frontend
 - `docker compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
-- `docker exec -it backend sh`: open bash terminal inside backend container
-- `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database
+- `docker exec -it oodikone-backend sh`: open bash terminal inside backend container
+- `docker exec -it oodikone-sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database
 
 ### User types for development
 
@@ -238,6 +238,20 @@ npm run cypress:run
 
 Remember to revert the `baseUrl` in [cypress.config.js](./cypress.config.js) to its original value when you're done.
 
+## How to read clients minds
+
+Search for client queries in grafana using a variation of this query:
+
+```
+{app="oodikone"} | regexp `\s(?P<payload>\{.*\}$)` | line_format `{{.payload}}` | json userId, mockedBy | userId = `client` | mockedBy != `admin`
+```
+
+For quick access to _any and all params_ under certain path:
+
+```
+{app="oodikone"} |= `path` | regexp `\s(?P<payload>\{.*\}$)` | line_format `{{.payload}}` | json userId, mockedBy | userId = `client` | mockedBy != `admin`
+```
+
 ## ✌🏼 Maintainers and contribution
 
 [Toska](https://toska.dev) - University of Helsinki

anonyymioodi/commit_and_push_to_toska_hub.sh

@@ -43,6 +43,6 @@ fi
 
 infomsg "Committing and pushing to registry"
 database=${args[0]}
-docker commit "$database" "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
+docker commit "oodikone-$database" "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
 docker push "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
 successmsg "Succesfully pushed"

anonyymioodi/create_container_from_dump.sh

@@ -52,13 +52,13 @@ fi
 
 
 infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
-database=${args[0]}
-docker-compose down --remove-orphans
-docker-compose up -d "$database"
+database="${args[0]}"
+docker compose down --remove-orphans
+docker compose up -d "$database"
 
 database_dump="$database.sqz"
 infomsg "Attempting to restore database $database from dump $database_dump"
-retry docker-compose exec "$database" pg_isready --dbname="$database"
-docker exec -i "$database" /bin/bash -c "pg_restore --username=postgres \
+retry docker compose exec "$database" pg_isready --dbname="$database"
+docker exec -i "oodikone-$database" /bin/bash -c "pg_restore --username=postgres \
 --format=custom --dbname=$database --no-owner" < "$database_dump"
 successmsg "Database $database succesfully created"

anonyymioodi/docker-compose.yml

@@ -4,7 +4,7 @@ x-defaults: &defaults
 services:
   kone-db:
     <<: *defaults
-    container_name: kone-db
+    container_name: oodikone-kone-db
     environment:
       PGDATA: /data
       POSTGRES_DB: kone-db
@@ -13,7 +13,7 @@ services:
 
   sis-db:
     <<: *defaults
-    container_name: sis-db
+    container_name: oodikone-sis-db
     environment:
       PGDATA: /data
       POSTGRES_DB: sis-db
@@ -23,7 +23,7 @@ services:
 
   sis-importer-db:
     <<: *defaults
-    container_name: sis-importer-db
+    container_name: oodikone-sis-importer-db
     environment:
       PGDATA: /data
       POSTGRES_DB: sis-importer-db
@@ -32,7 +32,7 @@ services:
 
   user-db:
     <<: *defaults
-    container_name: user-db
+    container_name: oodikone-user-db
     environment:
       PGDATA: /data
       POSTGRES_DB: user-db

anonyymioodi/update_staging_db.sh

@@ -56,7 +56,7 @@ function create_dumps() {
 function create_dump_from_db() {
   echo "Dumping db $1..."
 
-  docker exec "$1" pg_dump -Fc -U postgres "$1" > "$dump_directory/$1.sqz" || die "Failed to dump db $1"
+  docker exec "oodikone-$1" pg_dump -Fc -U postgres "$1" > "$dump_directory/$1.sqz" || die "Failed to dump db $1"
 }
 
 

cli.sh

@@ -18,11 +18,11 @@ source "$PROJECT_ROOT"/scripts/functions_for_setup.sh
 # Try to run cleanup function if things fail.
 trap cleanup SIGINT SIGTERM ERR EXIT
 
-# Run docker-compose down on cleanup
+# Run docker compose down on cleanup
 cleanup() {
   trap - SIGINT SIGTERM ERR EXIT
-  warningmsg "Trying to run docker-compose down and remove orphans"
-  docker-compose --profile oodikone --profile updater down --remove-orphans
+  warningmsg "Trying to run docker compose down and remove orphans"
+  docker compose --profile oodikone --profile updater down --remove-orphans
 }
 
 # === CLI ===

cypress/e2e/Course_statistics.js

@@ -537,7 +537,8 @@ describe('Basic user', () => {
         cy.contains('Show population').should('be.enabled')
       })
 
-      describe('Students tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Students tab', () => {
         describe('Info boxes', () => {
           it('Student statistics table', () => {
             cy.cs('StudentStatistics-info-box-button').click()
@@ -639,7 +640,8 @@ describe('Basic user', () => {
         })
       })
 
-      describe('Attempts tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Attempts tab', () => {
         beforeEach(() => {
           openAttemptsTab()
         })
@@ -774,7 +776,8 @@ describe('Basic user', () => {
         cy.contains('Show population').should('be.enabled')
       })
 
-      describe('Students tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Students tab', () => {
         it('Show grades off, Separate by semesters off', () => {
           const tableContents = [
             // [Time, Total students, Passed, Failed, Enrolled no grade, Pass rate, Fail rate]
@@ -960,7 +963,8 @@ describe('Basic user', () => {
         })
       })
 
-      describe('Attempts tab', () => {
+      // TODO: Writing these as backend api tests
+      describe.skip('Attempts tab', () => {
         beforeEach(() => {
           openAttemptsTab()
         })
@@ -1206,7 +1210,8 @@ describe('Basic user', () => {
   })
 })
 
-describe('Only course statistics', () => {
+// TODO: Writing these as backend api tests
+describe.skip('Only course statistics', () => {
   beforeEach(() => {
     cy.init('/coursestatistics', 'onlycoursestatistics')
     cy.url().should('include', '/coursestatistics')

cypress/e2e/Degree_programme_overview.js

@@ -32,36 +32,21 @@ const deleteTag = tagName => {
 
 describe('Degree programme overview', () => {
   describe('Degree programme selector', () => {
-    beforeEach(() => {
-      cy.init('/study-programme', 'admin')
-    })
-
-    it('Correct categories', () => {
-      cy.contains('Bachelor programmes')
-      cy.contains('Master programmes')
-      cy.contains('Combined programmes')
-      cy.contains('Doctoral programmes')
-    })
-
     it('Degree programme search filter', () => {
+      cy.init('/study-programme', 'admin')
       cy.contains('Tietojenkäsittelytieteen kandiohjelma').should('exist')
       cy.contains('Matemaattisten tieteiden kandiohjelma').should('exist')
       cy.cs('study-programme-filter').type('Tietojenkäsittelytieteen')
       cy.contains('Tietojenkäsittelytieteen kandiohjelma').should('exist')
       cy.contains('Matemaattisten tieteiden kandiohjelma').should('not.exist')
     })
-
-    it('Old and specialized programmes filter', () => {
-      cy.contains('Biotekniikka (ylempi)').should('not.exist')
-      cy.cs('filter-old-programmes-toggle').click()
-      cy.contains('Biotekniikka (ylempi)').should('exist')
-    })
   })
 
   describe('Basic information tab works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
       cy.contains('a', 'Matemaattisten tieteiden kandiohjelma').click({ force: true })
+      cy.cs('year-toggle').click()
     })
 
     // If the backend breaks for one of the sections, the section header is not rendered and this will fail
@@ -235,6 +220,7 @@ describe('Degree programme overview', () => {
     it('are split into two graphs', () => {
       cy.init('/study-programme')
       cy.contains('a', 'Matematiikan ja tilastotieteen maisteriohjelma').click({ force: true })
+      cy.cs('year-toggle').click() // Tests are written for calendar year
 
       cy.cs('unset-breakdown-bar-chart')
       cy.cs('unset-breakdown-bar-chart')
@@ -530,6 +516,8 @@ describe('Degree programme overview', () => {
 
     describe('By credit type tab', () => {
       beforeEach(() => {
+        cy.cs('show-credits-students-toggle').click()
+        cy.cs('year-toggle').click()
         cy.get('thead').within(() => {
           cy.get('tr')
             .eq(0)
@@ -544,7 +532,7 @@ describe('Degree programme overview', () => {
               cy.get('td').eq(0).contains('Course')
               cy.get('td').eq(1).contains('MAT11004')
               cy.get('td').eq(2).contains('Differentiaalilaskenta')
-              cy.get('td').eq(3).contains('1170')
+              cy.get('td').eq(3).contains('1240')
             })
         })
       })
@@ -568,7 +556,7 @@ describe('Degree programme overview', () => {
       })
 
       it('year toggle works', () => {
-        cy.cs('year-toggle').click()
+        cy.cs('year-toggle').click() // Default view now Academic years
 
         cy.cs('from-year-select').click()
         cy.cs('from-year-select-option-2017').click()
@@ -582,7 +570,7 @@ describe('Degree programme overview', () => {
               cy.get('td').eq(0).contains('Course')
               cy.get('td').eq(1).contains('MAT11001')
               cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
-              cy.get('td').eq(3).contains('1220')
+              cy.get('td').eq(3).contains('1265')
             })
         })
       })
@@ -641,6 +629,8 @@ describe('Degree programme overview', () => {
       })
 
       it("'Show credits/Show students' toggle works", () => {
+        cy.cs('year-toggle').click()
+        cy.wait(500) // So that the toggle has time to load
         cy.get('thead tr')
           .eq(0)
           .within(() => {
@@ -715,7 +705,7 @@ describe('Degree programme overview', () => {
               cy.get('td').eq(0).contains('Course')
               cy.get('td').eq(1).contains('MAT11001')
               cy.get('td').eq(2).contains('Johdatus yliopistomatematiikkaan')
-              cy.get('td').eq(3).contains('375')
+              cy.get('td').eq(3).contains('277')
             })
         })
       })

cypress/e2e/Faculty_statistics.js

@@ -110,15 +110,15 @@ describe('Faculty statistics', () => {
       })
 
       it('Toggle years works', () => {
-        cy.cs('credits-produced-by-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('thesis-writers-of-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('students-of-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('graduated-of-the-faculty-interactive-data-table').should('contain', '2022')
-        cy.cs('year-toggle').click()
         cy.cs('credits-produced-by-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
         cy.cs('thesis-writers-of-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
         cy.cs('students-of-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
         cy.cs('graduated-of-the-faculty-interactive-data-table').should('contain', '2022 - 2023')
+        cy.cs('year-toggle').click()
+        cy.cs('credits-produced-by-the-faculty-interactive-data-table').should('contain', '2022')
+        cy.cs('thesis-writers-of-the-faculty-interactive-data-table').should('contain', '2022')
+        cy.cs('students-of-the-faculty-interactive-data-table').should('contain', '2022')
+        cy.cs('graduated-of-the-faculty-interactive-data-table').should('contain', '2022')
       })
 
       it('Toggle programmes works', () => {

docker-compose.ci.yml

@@ -1,10 +1,12 @@
+name: oodikone
+
 x-defaults: &defaults
   restart: unless-stopped
 
 services:
   backend:
     <<: *defaults
-    container_name: backend
+    container_name: oodikone-backend
     depends_on:
       - kone-db
       - redis
@@ -31,7 +33,7 @@ services:
 
   frontend:
     <<: *defaults
-    container_name: frontend
+    container_name: oodikone-frontend
     build:
       context: services
       dockerfile: frontend/Dockerfile
@@ -41,7 +43,7 @@ services:
 
   kone-db:
     <<: *defaults
-    container_name: kone-db
+    container_name: oodikone-kone-db
     environment:
       PGDATA: /data
       POSTGRES_DB: kone-db
@@ -50,16 +52,16 @@ services:
 
   redis:
     <<: *defaults
+    container_name: oodikone-redis
     command: >
       sh -c 'redis-server --appendonly yes && redis-cli flushall'
     image: redis:7.2.6
-    container_name: redis
     ports:
       - 6379:6379
 
   sis-db:
     <<: *defaults
-    container_name: sis-db
+    container_name: oodikone-sis-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
     environment:
       PGDATA: /data
@@ -69,7 +71,7 @@ services:
 
   user-db:
     <<: *defaults
-    container_name: user-db
+    container_name: oodikone-user-db
     environment:
       PGDATA: /data
       POSTGRES_DB: user-db
@@ -78,6 +80,7 @@ services:
 
   jami:
     <<: *defaults
+    container_name: oodikone-jami
     image: quay.io/toska/jami:production
     environment:
       - npm_config_cache=/tmp
@@ -86,21 +89,20 @@ services:
       - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
     ports:
       - 3003:3003
-    container_name: jami
 
   jami-db:
     <<: *defaults
+    container_name: oodikone-jami-db
     image: postgres:15.10
     environment:
       - PGDATA=/data
       - POSTGRES_PASSWORD=postgres
-    container_name: jami-db
     ports:
       - 5433:5432
 
   nginx:
+    container_name: oodikone-nginx
     image: nginx:latest
-    container_name: nginx
     depends_on:
       - frontend
       - backend
@@ -114,16 +116,16 @@ services:
   # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
   sis-importer-db:
     <<: *defaults
-    container_name: sis-importer-db
+    container_name: oodikone-sis-importer-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
     profiles:
       - updater
 
   sis-updater-worker:
     <<: *defaults
+    container_name: oodikone-updater-worker
     build:
       context: updater/sis-updater-worker
-    container_name: sis-updater-worker
     depends_on:
       - redis
       - sis-db
@@ -142,9 +144,9 @@ services:
 
   sis-updater-scheduler:
     <<: *defaults
+    container_name: oodikone-updater-scheduler
     build:
       context: updater/sis-updater-scheduler
-    container_name: sis-updater-scheduler
     depends_on:
       - redis
       - sis-importer-db

docker-compose.real.yml

@@ -1,3 +1,5 @@
+name: oodikone
+
 services:
   backend:
     environment:
@@ -20,3 +22,4 @@ services:
 
 volumes:
   redis-data-real:
+    name: oodikone-redis-data-real

docker-compose.test.yml

@@ -1,9 +1,14 @@
 services:
   api-tests:
+    container_name: oodikone-api-tests
+    build:
+      context: services
+      dockerfile: backend/dev.Dockerfile
     volumes:
-      - backend-tests-nodemod:/opt/app-root/backend/node_modules
       - ./services/backend/:/opt/app-root/backend
       - ./services/shared/:/opt/app-root/shared
+      - backend-tests-nodemod:/opt/app-root/backend/node_modules
+      - shared-tests-nodemod:/opt/app-root/shared/node_modules
     environment:
       CI: 'true'
       NODE_ENV: 'test'
@@ -17,10 +22,10 @@ services:
       JAMI_URL: 'http://jami:3003/'
       IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
       CRYPT_KEY: 'e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'
-    build:
-      context: services/backend
-      dockerfile: dev.Dockerfile
     command: npm run test
 
 volumes:
   backend-tests-nodemod:
+    name: oodikone-backend-tests-nodemod
+  shared-tests-nodemod:
+    name: oodikone-shared-tests-nodemod

docker-compose.yml

@@ -1,3 +1,5 @@
+name: oodikone
+
 x-defaults: &defaults
   restart: unless-stopped
 
@@ -7,7 +9,7 @@ x-build_defaults: &build_defaults
 services:
   adminer:
     <<: *defaults
-    container_name: adminer
+    container_name: oodikone-adminer
     environment:
       ADMINER_DESIGN: pepa-linha
     image: adminer:4.8.1
@@ -18,10 +20,10 @@ services:
 
   backend:
     <<: *defaults
+    container_name: oodikone-backend
     build:
-      <<: *build_defaults
-      context: services/backend
-    container_name: backend
+      dockerfile: backend/dev.Dockerfile
+      context: services
     depends_on:
       - kone-db
       - redis
@@ -47,14 +49,15 @@ services:
     volumes:
       - ./services/backend/:/opt/app-root/backend
       - ./services/shared/:/opt/app-root/shared
+      - shared-nodemod:/opt/app-root/shared/node_modules
       - backend-nodemod:/opt/app-root/backend/node_modules
 
   frontend:
     <<: *defaults
+    container_name: oodikone-frontend
     build:
-      <<: *build_defaults
-      context: services/frontend
-    container_name: frontend
+      dockerfile: frontend/dev.Dockerfile
+      context: services
     ports:
       - 3000:3000
     profiles:
@@ -62,11 +65,12 @@ services:
     volumes:
       - ./services/frontend/:/opt/app-root/frontend
       - ./services/shared/:/opt/app-root/shared
+      - shared-nodemod:/opt/app-root/shared/node_modules
       - frontend-nodemod:/opt/app-root/frontend/node_modules
 
   kone-db:
     <<: *defaults
-    container_name: kone-db
+    container_name: oodikone-kone-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
     profiles:
       - oodikone
@@ -75,8 +79,8 @@ services:
 
   redis:
     <<: *defaults
+    container_name: oodikone-redis
     command: 'redis-server --appendonly yes'
-    container_name: redis
     image: redis:7.2.6
     ports:
       - '6379:6379'
@@ -85,7 +89,7 @@ services:
 
   sis-db:
     <<: *defaults
-    container_name: sis-db
+    container_name: oodikone-sis-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
     volumes:
       - sis-db-data:/data
@@ -95,7 +99,7 @@ services:
 
   sis-importer-db:
     <<: *defaults
-    container_name: sis-importer-db
+    container_name: oodikone-sis-importer-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
     profiles:
       - updater
@@ -106,10 +110,10 @@ services:
 
   sis-updater-scheduler:
     <<: *defaults
+    container_name: oodikone-updater-scheduler
     build:
       <<: *build_defaults
       context: updater/sis-updater-scheduler
-    container_name: sis-updater-scheduler
     depends_on:
       - redis
       - sis-importer-db
@@ -130,10 +134,10 @@ services:
 
   sis-updater-worker:
     <<: *defaults
+    container_name: oodikone-updater-worker
     build:
       <<: *build_defaults
       context: updater/sis-updater-worker
-    container_name: sis-updater-worker
     depends_on:
       - redis
       - sis-db
@@ -154,7 +158,7 @@ services:
 
   user-db:
     <<: *defaults
-    container_name: user-db
+    container_name: oodikone-user-db
     image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
     profiles:
       - oodikone
@@ -163,6 +167,7 @@ services:
 
   jami:
     <<: *defaults
+    container_name: oodikone-jami
     image: quay.io/toska/jami:production
     env_file:
       - .env
@@ -173,15 +178,14 @@ services:
       - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/jami-db-real
     ports:
       - 3003:3003
-    container_name: jami
 
   jami-db:
     <<: *defaults
+    container_name: oodikone-jami-db
     image: postgres:15.10
     environment:
       - PGDATA=/data
       - POSTGRES_PASSWORD=postgres
-    container_name: jami-db
     ports:
       - 5433:5432
     volumes:
@@ -189,12 +193,24 @@ services:
 
 volumes:
   backend-nodemod:
+    name: oodikone-backend-nodemod
   frontend-nodemod:
+    name: oodikone-frontend-nodemod
+  shared-nodemod:
+    name: oodikone-shared-nodemod
   kone-db-data:
+    name: oodikone-kone-db-data
   redis-data:
+    name: oodikone-redis-data
   sis-db-data:
+    name: oodikone-sis-db-data
   sis-importer-db-data:
+    name: oodikone-sis-importer-db-data
   sis-updater-scheduler-nodemod:
+    name: oodikone-updater-scheduler-nodemod
   sis-updater-worker-nodemod:
+    name: oodikone-updater-worker-nodemod
   user-db-data:
+    name: oodikone-user-db-data
   jami-data:
+    name: oodikone-jami-data

documentation/migrations.md

@@ -8,7 +8,7 @@ These instructions explain how to update the schemas for the following databases
 
 The steps to **add, remove, or modify columns** are mostly the same for each database, but the file paths and container names differ. Always **double-check** that you have the correct function name, column name, data type, and other relevant details in your migration file.
 
-> [!NOTE]  
+> [!NOTE]
 > If you need to change the `sis-importer-db` schema, see the instructions in the [sis-importer repo](https://github.com/UniversityOfHelsinkiCS/sis-importer/blob/master/README.md#tricks--tips).
 
 ## General guidelines
@@ -25,7 +25,7 @@ The steps to **add, remove, or modify columns** are mostly the same for each dat
 - Use **snake_case** in the actual database columns.
 - If a model has `underscored: true`, Sequelize automatically converts camelCase to snake_case in all queries. Otherwise, specify the column name explicitly.
 
-> [!IMPORTANT]  
+> [!IMPORTANT]
 > The column name **in the database** (snake_case) must be used in the migration file, not the camelCase name from your model.
 
 ### Verifying the migration locally
@@ -73,7 +73,7 @@ If you need to **revert a migration locally**, follow these steps:
 
 ## `sis-db`
 
-> [!NOTE]  
+> [!NOTE]
 > These instructions assume that needed data is already in the `sis-importer-db`. If not, you must add the data there first. See the [sis-importer repo](https://github.com/UniversityOfHelsinkiCS/sis-importer/blob/master/README.md#tricks--tips) for more details.
 
 1. **Update the updater model:**

eslint.config.mjs

@@ -263,5 +263,17 @@ export default tseslint.config(
       'no-promise-executor-return': 'off',
     },
   },
+
+  // Frontend tests
+  {
+    files: ['services/frontend/tests/**/*.{ts,tsx}', "services/frontend/playwright/**/*.{ts,tsx}", 'services/frontend/playwright-ct.config.ts'],
+    rules: {
+      'import-x/no-extraneous-dependencies': 'off',
+      'import-x/no-unused-modules': 'off',
+      'import-x/no-default-export': 'off',
+      'react/jsx-filename-extension': 'off',
+    },
+  },
+
   eslintPluginPrettierRecommended // This must be the last plugin so it can override other configs
 )

package.json

@@ -12,7 +12,8 @@
     "stylelint": "stylelint",
     "prettier": "prettier",
     "cli": "./cli.sh",
-    "oodikone": "./run.sh oodikone anon up --detach",
+    "logs": "docker compose --file docker-compose.yml logs backend frontend sis-updater-worker sis-updater-scheduler jami -f",
+    "oodikone": "npm run docker:down && ./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
     "oodikone:ci:setup": "docker compose --file docker-compose.ci.yml --profile tests up --detach",
     "oodikone:ci:api": "docker compose --file docker-compose.test.yml up --exit-code-from api-tests",
@@ -21,14 +22,14 @@
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
     "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
-    "testupdater": "docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start",
-    "flushredis": "docker exec -it redis redis-cli flushall",
+    "testupdater": "docker compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start",
+    "flushredis": "docker exec -it oodikone-redis redis-cli flushall",
     "prepare": "husky",
     "install:local": "npm i && npm i --prefix services/shared && npm i --prefix services/frontend && npm i --prefix services/backend && npm i --prefix updater/sis-updater-scheduler && npm i --prefix updater/sis-updater-worker",
-    "install:docker": "docker exec frontend npm i && docker exec backend npm i && docker exec sis-updater-scheduler npm i && docker exec sis-updater-worker npm i",
+    "install:docker": "docker exec oodikone-frontend npm i && docker exec oodikone-backend npm i && docker exec oodikone-updater-scheduler npm i && docker exec oodikone-updater-worker npm i",
     "install:both": "npm run install:local && npm run install:docker",
     "ci:local": "npm ci && npm ci --prefix services/shared && npm ci --prefix services/frontend && npm ci --prefix services/backend && npm ci --prefix updater/sis-updater-scheduler && npm ci --prefix updater/sis-updater-worker",
-    "ci:docker": "docker exec frontend npm ci && docker exec backend npm ci && docker exec sis-updater-scheduler npm ci && docker exec sis-updater-worker npm ci",
+    "ci:docker": "docker exec oodikone-frontend npm ci && docker exec oodikone-backend npm ci && docker exec oodikone-updater-scheduler npm ci && docker exec oodikone-updater-worker npm ci",
     "ci:both": "npm run ci:local && npm run ci:docker"
   },
   "repository": {

run.sh

@@ -23,7 +23,7 @@ Usage: $(basename "${BASH_SOURCE[0]}") option [version] command --flag
 Parameters:
 * Option: oodikone/updater/both
 * Version: anon/real. Not necessary when running down/logs/pull.
-* Command: will be passed to docker-compose.
+* Command: will be passed to docker compose.
 EOF
 }
 

scripts/functions_for_setup.sh

@@ -12,7 +12,7 @@ DUMP_DIR="$PROJECT_ROOT/.databasedumps"
 S3_CONFIG_FILE=~/.s3cfg
 S3_BUCKET="s3://psyduck"
 
-## Following the naming convention in docker-compose, these are names for database
+## Following the naming convention in docker compose, these are names for database
 ## services. The real data databases inside service will have names suffixed by "-real".
 KONE_DB_NAME="kone-db"
 SIS_DB_NAME="sis-db"
@@ -112,11 +112,11 @@ reset_databases() {
   infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
 
   npm run docker:down
-  docker-compose up -d "${databases[@]}"
+  docker compose up -d "${databases[@]}"
 
   for database in "${databases[@]}"; do
     local database_dump="$database_dump_dir/$database.sql.gz"
-    local database_container="$database"
+    local database_container="oodikone-$database"
     local database_name="$database-real"
 
     infomsg "Attempting to create database $database_name from dump $database_dump inside container $database_container"
@@ -215,6 +215,7 @@ set_up_oodikone_from_scratch() {
     "$PROJECT_ROOT"
     "$PROJECT_ROOT/services/frontend"
     "$PROJECT_ROOT/services/backend"
+    "$PROJECT_ROOT/services/shared"
     "$PROJECT_ROOT/updater/sis-updater-scheduler"
     "$PROJECT_ROOT/updater/sis-updater-worker"
   )
@@ -223,7 +224,7 @@ set_up_oodikone_from_scratch() {
   "$PROJECT_ROOT"/run.sh both down --remove-orphans --volumes --rmi all || infomsg "Cleaning errored, but will continue"
   for folder in "${folders_to_set_up[@]}"; do
     cd "$folder" || die "Couldn't change directory to folder $folder"
-    rm -rf node_modules|| die "Couldn't remove node_modules in folder $folder"
+    rm -rf node_modules || die "Couldn't remove node_modules in folder $folder"
   done
 
   infomsg "Installing npm packages locally to root and each subfolder to enable linting and formatting"

services/backend/Dockerfile

@@ -50,6 +50,9 @@ ENV SENTRY_RELEASE=$SENTRY_RELEASE
 
 WORKDIR /opt/app-root/shared
 COPY --from=build /opt/app-root/shared/dist .
+COPY --from=build /opt/app-root/shared/.npmrc .
+COPY --from=build /opt/app-root/shared/package*.json ./
+RUN npm ci --omit=dev
 
 WORKDIR /opt/app-root/backend
 COPY --from=build /opt/app-root/backend/.npmrc .

services/backend/dev.Dockerfile

@@ -5,10 +5,16 @@ ENV TZ="Europe/Helsinki"
 ENV NODE_ENV=development
 ENV NODE_OPTIONS=--max-old-space-size=4096
 
-WORKDIR /opt/app-root/backend
-COPY --chown=1001:1001 . .
-
 USER 1001
+
+WORKDIR /opt/app-root/shared
+COPY --chown=1001:1001 ./shared .
+
+RUN npm ci
+
+WORKDIR /opt/app-root/backend
+COPY --chown=1001:1001 ./backend .
+
 RUN npm ci
 
 EXPOSE 8080

services/backend/package-lock.json

@@ -27,7 +27,6 @@
         "umzug": "^3.8.2",
         "unfuck-utf8-headers-middleware": "^1.0.1",
         "winston": "^3.17.0",
-        "winston-gelf-transporter": "^1.0.2",
         "winston-loki": "^6.1.3"
       },
       "devDependencies": {
@@ -2658,15 +2657,6 @@
         "simple-swizzle": "^0.2.2"
       }
     },
-    "node_modules/colors": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz",
-      "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=0.1.90"
-      }
-    },
     "node_modules/colorspace": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz",
@@ -2782,12 +2772,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/core-util-is": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
-      "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==",
-      "license": "MIT"
-    },
     "node_modules/cors": {
       "version": "2.8.5",
       "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
@@ -3151,6 +3135,7 @@
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz",
       "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==",
+      "dev": true,
       "license": "MIT"
     },
     "node_modules/fastq": {
@@ -3352,15 +3337,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/gelf-pro": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/gelf-pro/-/gelf-pro-1.4.0.tgz",
-      "integrity": "sha512-VXQ1DHQjQr8/Xil83hHgozqeCcVrggGAA6A8hgjojQSatiF6PErR+nnYv/DCBxFBolO6licomD7Abaj0Iec7Qg==",
-      "license": "MIT",
-      "dependencies": {
-        "lodash": "~4.17.21"
-      }
-    },
     "node_modules/generic-pool": {
       "version": "3.9.0",
       "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
@@ -3769,12 +3745,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/isarray": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-      "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
-      "license": "MIT"
-    },
     "node_modules/jju": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/jju/-/jju-1.4.0.tgz",
@@ -4477,12 +4447,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/process-nextick-args": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
-      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
-      "license": "MIT"
-    },
     "node_modules/protobufjs": {
       "version": "7.5.4",
       "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz",
@@ -5631,88 +5595,6 @@
         "node": ">= 12.0.0"
       }
     },
-    "node_modules/winston-gelf-transporter": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/winston-gelf-transporter/-/winston-gelf-transporter-1.0.2.tgz",
-      "integrity": "sha512-N1+D778Ju5q6u7jIKgwwyYsfND8SKhw4QLtUXAS83BYgkQvRtjvNs8PWS2oY4U4voPKh3U6DcjWkPtpfgKN5dw==",
-      "license": "MIT",
-      "dependencies": {
-        "gelf-pro": "1.3.3",
-        "logform": "2.1.2",
-        "winston-transport": "4.3.0"
-      },
-      "engines": {
-        "node": ">= 8.x.x"
-      }
-    },
-    "node_modules/winston-gelf-transporter/node_modules/fecha": {
-      "version": "2.3.3",
-      "resolved": "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz",
-      "integrity": "sha512-lUGBnIamTAwk4znq5BcqsDaxSmZ9nDVJaij6NvRt/Tg4R69gERA+otPKbS86ROw9nxVMw2/mp1fnaiWqbs6Sdg==",
-      "license": "MIT"
-    },
-    "node_modules/winston-gelf-transporter/node_modules/logform": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/logform/-/logform-2.1.2.tgz",
-      "integrity": "sha512-+lZh4OpERDBLqjiwDLpAWNQu6KMjnlXH2ByZwCuSqVPJletw0kTWJf5CgSNAUKn1KUkv3m2cUz/LK8zyEy7wzQ==",
-      "license": "MIT",
-      "dependencies": {
-        "colors": "^1.2.1",
-        "fast-safe-stringify": "^2.0.4",
-        "fecha": "^2.3.3",
-        "ms": "^2.1.1",
-        "triple-beam": "^1.3.0"
-      }
-    },
-    "node_modules/winston-gelf-transporter/node_modules/ms": {
-      "version": "2.1.3",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
-      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
-      "license": "MIT"
-    },
-    "node_modules/winston-gelf-transporter/node_modules/readable-stream": {
-      "version": "2.3.8",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
-      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
-      "license": "MIT",
-      "dependencies": {
-        "core-util-is": "~1.0.0",
-        "inherits": "~2.0.3",
-        "isarray": "~1.0.0",
-        "process-nextick-args": "~2.0.0",
-        "safe-buffer": "~5.1.1",
-        "string_decoder": "~1.1.1",
-        "util-deprecate": "~1.0.1"
-      }
-    },
-    "node_modules/winston-gelf-transporter/node_modules/safe-buffer": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
-      "license": "MIT"
-    },
-    "node_modules/winston-gelf-transporter/node_modules/string_decoder": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-      "license": "MIT",
-      "dependencies": {
-        "safe-buffer": "~5.1.0"
-      }
-    },
-    "node_modules/winston-gelf-transporter/node_modules/winston-transport": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.3.0.tgz",
-      "integrity": "sha512-B2wPuwUi3vhzn/51Uukcao4dIduEiPOcOt9HJ3QeaXgkJ5Z7UwpBzxS4ZGNHtrxrUvTwemsQiSys0ihOf8Mp1A==",
-      "license": "MIT",
-      "dependencies": {
-        "readable-stream": "^2.3.6",
-        "triple-beam": "^1.2.0"
-      },
-      "engines": {
-        "node": ">= 6.4.0"
-      }
-    },
     "node_modules/winston-loki": {
       "version": "6.1.3",
       "resolved": "https://registry.npmjs.org/winston-loki/-/winston-loki-6.1.3.tgz",

services/backend/package.json

@@ -3,7 +3,7 @@
   "private": true,
   "scripts": {
     "build": "tsc",
-    "test": "node --import tsx --test --test-reporter spec --test-force-exit"
+    "test": "node --import=tsx --test --test-force-exit --test-reporter spec tests/**/*.ts"
   },
   "dependencies": {
     "@oodikone/shared": "file:../shared",
@@ -27,7 +27,6 @@
     "umzug": "^3.8.2",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "winston": "^3.17.0",
-    "winston-gelf-transporter": "^1.0.2",
     "winston-loki": "^6.1.3"
   },
   "devDependencies": {
@@ -43,10 +42,5 @@
     "supertest": "^7.2.2",
     "tsx": "^4.19.4",
     "typescript": "^5.9.2"
-  },
-  "overrides": {
-    "winston-gelf-transporter": {
-      "gelf-pro": "^1.4.0"
-    }
   }
 }

services/backend/src/events.ts

@@ -13,7 +13,7 @@ import { updateBasicView, updateStudyTrackView } from './services/studyProgramme
 import { findAndSaveTeachers } from './services/teachers/top'
 import { deleteOutdatedUsers } from './services/userService'
 import logger from './util/logger'
-import { jobMaker, addToFlow } from './worker/queue'
+import { jobQueue } from './worker/queue'
 
 const schedule = (cronTime: string, onTick: () => void) => {
   const onComplete = null
@@ -41,7 +41,7 @@ export const refreshFaculties = async () => {
   logger.info('Adding jobs to refresh all faculties')
   const faculties = await getFaculties()
   for (const faculty of faculties) {
-    jobMaker.faculty(faculty.code)
+    jobQueue.refreshFaculty(faculty.code)
   }
 }
 
@@ -53,12 +53,13 @@ export const refreshLanguageCenterData = async () => {
 }
 
 const refreshProgrammesAndFaculties = async () => {
+  logger.info('Refreshing all degree programmes and faculties')
   const facultyCodes = (await getFaculties()).map(faculty => faculty.code)
   for (const faculty of facultyCodes) {
     const programmeCodes = (await getDegreeProgrammesOfFaculty(faculty, true))
       .map(programme => programme.code)
       .filter(code => isRelevantProgramme(code))
-    await addToFlow(faculty, programmeCodes)
+    await jobQueue.addToFlow(faculty, programmeCodes)
   }
 }
 
@@ -82,7 +83,7 @@ export const refreshProgrammes = async () => {
   }
   for (const code of programmeCodes) {
     // If combined programme is given, this updates only the bachelor programme
-    jobMaker.programme(code)
+    jobQueue.refreshProgramme(code)
   }
 }
 
@@ -96,8 +97,12 @@ const dailyJobs = async () => {
   try {
     await refreshTeacherLeaderboard()
     await refreshProgrammesAndFaculties()
-    if (languageCenterViewEnabled) jobMaker.languagecenter()
-    jobMaker.closeToGraduation()
+
+    if (languageCenterViewEnabled) {
+      jobQueue.refreshLanguagecenter()
+    }
+
+    jobQueue.refreshCloseToGraduation()
   } catch (error) {
     logger.error('Daily jobs failed', error)
   }
@@ -118,11 +123,11 @@ export const startCron = () => {
     })
     schedule('0 19 * * 1', () => {
       logger.info('Updating students whose studyplans have not been updated recently')
-      jobMaker.studyplansUpdate(4)
+      jobQueue.refreshStudyplans(4)
     })
     schedule('0 10 * * 2', () => {
       logger.info('Updating students whose studyplans have not been updated recently')
-      jobMaker.studyplansUpdate(5)
+      jobQueue.refreshStudyplans(5)
     })
   }
 }

services/backend/src/routes/completedCoursesSearch.ts

@@ -2,8 +2,9 @@ import { Router } from 'express'
 import { difference, intersection } from 'lodash-es'
 
 import { CanError } from '@oodikone/shared/routes'
+import type { CompletedCoursesStudent, CompletedCoursesCourse } from '@oodikone/shared/types'
 import { tryCatch } from '@oodikone/shared/util'
-import { Courses, FormattedStudent, getCompletedCourses } from '../services/completedCoursesSearch'
+import { getCompletedCourses } from '../services/completedCoursesSearch'
 import {
   getUserSearches,
   createNewUserSearch,
@@ -19,11 +20,14 @@ const router = Router()
 
 const importerClient = getImporterClient()
 
-export type SearchResBody = CanError<
-  { discardedStudentNumbers: string[] } & { students: Omit<FormattedStudent, 'allEnrollments'>[]; courses: Courses }
+type SearchResBody = CanError<
+  { discardedStudentNumbers: string[] } & {
+    students: Omit<CompletedCoursesStudent, 'allEnrollments'>[]
+    courses: CompletedCoursesCourse[]
+  }
 >
-export type SearchReqBody = never
-export type SearchQuery = {
+type SearchReqBody = never
+type SearchQuery = {
   studentlist: string
   courselist: string
 }

services/backend/src/routes/courses.ts

@@ -51,7 +51,7 @@ router.get<never, CanError<CoursesMultiResBody>, CoursesMultiReqBody, CoursesMul
   }
 )
 
-type CourseYearlyStatsResBody = Awaited<ReturnType<typeof getCourseYearlyStats>>
+export type CourseYearlyStatsResBody = Awaited<ReturnType<typeof getCourseYearlyStats>>
 
 router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody, CourseYearlyStatsQuery>(
   '/courseyearlystats',
@@ -83,7 +83,7 @@ router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody,
     const useSeparate = separate === 'true'
 
     const results = await getCourseYearlyStats(courseCodes, useSeparate, anonymizationSalt, useCombineSubstitutions)
-    res.json(results)
+    return res.json(results)
   }
 )
 

services/backend/src/routes/faculties.ts

@@ -2,6 +2,7 @@ import { Router } from 'express'
 
 import type { CanError, GenericApplicationError } from '@oodikone/shared/routes'
 import type {
+  FacultyGraduationStatistics,
   Graduated,
   Name,
   NameWithCode,
@@ -22,7 +23,6 @@ import {
   setBasicStats,
   getThesisWritersStats,
   setThesisWritersStats,
-  getGraduationStats,
   setGraduationStats,
   getFacultyStudentStats,
   setFacultyStudentStats,
@@ -30,7 +30,7 @@ import {
   setFacultyProgressStats,
   type BasicData,
   type ThesisWriterData,
-  GraduationData,
+  getGraduationStats,
 } from '../services/faculty/facultyService'
 import { combineFacultyStudentProgress } from '../services/faculty/facultyStudentProgress'
 import { combineFacultyStudents } from '../services/faculty/facultyStudents'
@@ -135,17 +135,29 @@ router.get<
   return res.json(await getFacultyCredits(faculty.code, yearType === 'ACADEMIC_YEAR'))
 })
 
-type GetGraduationStatsResBody = GraduationData
 type GetGraduationStatsQuery = {
   programme_filter: ProgrammeFilter
 }
 
 router.get<
   GetStatsParams,
-  CanError<GetGraduationStatsResBody, GenericApplicationError>,
+  CanError<FacultyGraduationStatistics, GenericApplicationError>,
   GetStatsReqBody,
   GetGraduationStatsQuery
 >('/:id/graduationtimes', auth.roles(['facultyStatistics']), async (req, res) => {
+  // Recursively clears graduation time arrays "times" from result.
+  // We don't need to send them to client,
+  // but university view (uses same data) needs it to compute medians.
+  const clearTimes = (data: any) => {
+    if (Array.isArray(data)) {
+      data.forEach(clearTimes)
+    } else if (!!data && typeof data === 'object') {
+      if (data.times) data.times = []
+      Object.values(data).forEach(clearTimes)
+    }
+    return data
+  }
+
   const { id: facultyId } = req.params
   const { programme_filter: programmeFilter } = req.query
 
@@ -153,7 +165,7 @@ router.get<
   if (!faculty) throw new ApplicationError(`The organization with the id ${facultyId} was not found.`, 422)
 
   const data = await getGraduationStats(faculty.code, programmeFilter)
-  if (data) return res.json(data)
+  if (data) return res.json(clearTimes(data))
 
   const programmes = await getDegreeProgrammesOfFaculty(faculty.code, programmeFilter === 'NEW_DEGREE_PROGRAMMES')
   if (!programmes.length) throw new ApplicationError('Unprocessable request', 422)
@@ -161,7 +173,7 @@ router.get<
   const updatedStats = await countGraduationTimes(faculty.code, programmes)
   if (updatedStats) await setGraduationStats(updatedStats, programmeFilter)
 
-  return res.json(updatedStats)
+  return res.json(clearTimes(updatedStats))
 })
 
 type GetProgressStatsResBody = {

services/backend/src/routes/population.ts

@@ -112,7 +112,7 @@ router.get<never, CanError<PopulationstatisticsResBody>, PopulationstatisticsReq
     const studentStatuses = handleQueryArrays(req.query.studentStatuses) // NOTE: TRANSFERRED means transferred out of the program
 
     const requiredFields = [years, semesters, programme]
-    if (requiredFields.some(field => !field)) {
+    if (requiredFields.some(field => !field.length)) {
       return res.status(400).json({ error: 'The query should have years, semesters and a programme defined' })
     }
 
@@ -223,14 +223,18 @@ router.get<
 })
 
 // Used in custom population and single study guidance groups
-router.post<never, GetCustomPopulationResBody, CustomPopulationQuery>(
+router.post<never, CanError<GetCustomPopulationResBody>, CustomPopulationQuery>(
   '/populationstatisticsbystudentnumbers',
   async (req, res) => {
-    const { studentNumbers, tags } = req.body
+    const { studentNumbers, tags } = req.body ?? { studentNumbers: [] }
     const { id: userId, roles, studentsUserCanAccess } = req.user
 
     const students = handleQueryArrays(studentNumbers)
 
+    if (!students.length) {
+      return res.status(422).json({ error: 'Body should include student numbers' })
+    }
+
     const filteredStudentNumbers = hasFullAccessToStudentData(roles)
       ? students
       : intersection(students, studentsUserCanAccess)

services/backend/src/routes/university.ts

@@ -1,11 +1,12 @@
 import { Request, Response, Router } from 'express'
 import { cloneDeep } from 'lodash-es'
 
-import { Graduated, NameWithCode } from '@oodikone/shared/types'
+import { FacultyGraduationStatistics, Graduated, NameWithCode } from '@oodikone/shared/types'
+import { MediansByCategory, UniversityGraduationStatistics } from '@oodikone/shared/types/graduations'
 import { magicFacultyCode } from '../config/organizationConstants'
 import { OrganizationModel } from '../models'
 import { getDegreeProgrammesOfFaculty } from '../services/faculty/faculty'
-import { countGraduationTimes, LevelGraduationStats } from '../services/faculty/facultyGraduationTimes'
+import { countGraduationTimes } from '../services/faculty/facultyGraduationTimes'
 import { getSortedFaculties } from '../services/faculty/facultyHelpers'
 import {
   getFacultyProgressStats,
@@ -22,13 +23,12 @@ const router = Router()
 
 const degreeNames = ['bachelor', 'bachelorMaster', 'master', 'doctor'] as const
 
-const getProgrammeNames = (faculties: OrganizationModel[]) => {
-  return faculties.reduce<Record<string, NameWithCode>>((obj, faculty) => {
+const getProgrammeNames = (faculties: OrganizationModel[]) =>
+  faculties.reduce<Record<string, NameWithCode>>((obj, faculty) => {
     const { name, code } = faculty.toJSON()
     obj[faculty.code] = { code, ...name }
     return obj
   }, {})
-}
 
 interface GetProgressStatsRequest extends Request {
   query: {
@@ -120,16 +120,20 @@ router.get('/allprogressstats', async (req: GetProgressStatsRequest, res: Respon
   res.status(200).json(universityData)
 })
 
-router.get('/allgraduationstats', async (_req: Request, res: Response) => {
-  const degreeNames = ['bachelor', 'bcMsCombo', 'master', 'doctor'] as const
+router.get('/allgraduationstats', async (_req: Request, res: Response<UniversityGraduationStatistics>) => {
+  const degreeTypes = ['bachelor', 'bcMsCombo', 'master', 'doctor'] as const
+  /** University view does not support old degree programmes */
+  const programmeFilter = 'NEW_DEGREE_PROGRAMMES' as const
+
   const allFaculties = await getSortedFaculties()
   const facultyCodes = allFaculties.map(faculty => faculty.code)
-  const facultyData: Record<string, Awaited<ReturnType<typeof countGraduationTimes>>> = {}
-  const programmeFilter = 'NEW_DEGREE_PROGRAMMES'
+  const facultyData: Record<string, FacultyGraduationStatistics> = {}
+
   for (const facultyCode of facultyCodes) {
-    let data: any = await getGraduationStats(facultyCode, programmeFilter, true)
+    let data = await getGraduationStats(facultyCode, programmeFilter, true)
+
     if (!data) {
-      logger.info(`Data missing from server: Refreshing graduation faculty data for faculty ${facultyCode}`)
+      logger.info(`Data missing from redis: Refreshing graduation faculty data for faculty ${facultyCode}`)
       const programmes = await getDegreeProgrammesOfFaculty(facultyCode, true)
       data = await countGraduationTimes(facultyCode, programmes)
       await setGraduationStats(data, programmeFilter)
@@ -137,48 +141,50 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
     facultyData[facultyCode] = data
   }
 
-  const universityData = {
-    goals: {
-      bachelor: GraduationTarget.THREE_YEARS,
-      bcMsCombo: GraduationTarget.FIVE_YEARS,
-      master: GraduationTarget.TWO_YEARS,
-      doctor: GraduationTarget.FOUR_YEARS,
+  const goals: UniversityGraduationStatistics['goals'] = {
+    bachelor: GraduationTarget.THREE_YEARS,
+    bcMsCombo: GraduationTarget.FIVE_YEARS,
+    master: GraduationTarget.TWO_YEARS,
+    doctor: GraduationTarget.FOUR_YEARS,
+  }
+  const programmeNames = getProgrammeNames(allFaculties)
+
+  const byGradYear: UniversityGraduationStatistics['byGradYear'] = {
+    medians: {
+      bachelor: [],
+      bcMsCombo: [],
+      master: [],
+      doctor: [],
     },
-    programmeNames: getProgrammeNames(allFaculties),
-    byGradYear: {
-      medians: {} as Record<string, LevelGraduationStats[]>,
-      programmes: {
-        medians: {} as Record<
-          string,
-          Record<string, { programmes: string[]; data: Array<LevelGraduationStats & { name: string; code: string }> }>
-        >,
+    programmes: {
+      medians: {
+        bachelor: {},
+        bcMsCombo: {},
+        master: {},
+        doctor: {},
       },
     },
-    classSizes: {
-      bachelor: null as Record<string, number> | null,
-      bcMsCombo: null as Record<string, number> | null,
-      master: null as Record<string, number> | null,
-      doctor: null as Record<string, number> | null,
-      programmes: {} as Record<string, Record<string, Record<string, number>>>,
-    },
+  }
+  const classSizes: UniversityGraduationStatistics['classSizes'] = {
+    programmes: {},
   }
 
-  const unifyTotals = (facultyData: Record<string, LevelGraduationStats[]>, isLast: boolean) => {
-    const mediansForUniversity = universityData.byGradYear.medians
-    for (const degree of degreeNames) {
-      if (!mediansForUniversity[degree]) mediansForUniversity[degree] = []
-      if (!facultyData[degree]) continue
-      for (const yearStats of facultyData[degree]) {
-        const universityStats = mediansForUniversity[degree]
-        const universityYearStats = mediansForUniversity[degree].find(stats => stats.name === yearStats.name)
+  /** Computes university level stats */
+  const unifyTotals = (facultyData: MediansByCategory, isLast: boolean) => {
+    const mediansForUniversity = byGradYear.medians
+    for (const degreeType of degreeTypes) {
+      if (!facultyData[degreeType]) continue
+
+      for (const facultyYearStats of facultyData[degreeType]) {
+        const universityYearStats = mediansForUniversity[degreeType].find(stats => stats.name === facultyYearStats.name)
         if (!universityYearStats) {
-          universityStats.push(yearStats)
+          mediansForUniversity[degreeType].push(facultyYearStats)
         } else {
-          universityYearStats.times.push(...yearStats.times)
-          universityYearStats.amount += yearStats.amount
-          universityYearStats.statistics.onTime += yearStats.statistics.onTime
-          universityYearStats.statistics.yearOver += yearStats.statistics.yearOver
-          universityYearStats.statistics.wayOver += yearStats.statistics.wayOver
+          universityYearStats.times.push(...facultyYearStats.times)
+          universityYearStats.amount += facultyYearStats.amount
+          universityYearStats.statistics.onTime += facultyYearStats.statistics.onTime
+          universityYearStats.statistics.yearOver += facultyYearStats.statistics.yearOver
+          universityYearStats.statistics.wayOver += facultyYearStats.statistics.wayOver
           if (isLast) {
             universityYearStats.median = getMedian(universityYearStats.times)
           }
@@ -187,29 +193,30 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
     }
   }
 
-  const unifyProgrammeStats = (facultyData: Record<string, LevelGraduationStats[]>, facultyCode: string) => {
-    const mediansForUniversity = universityData.byGradYear.programmes.medians
-    for (const degree of degreeNames) {
-      if (!facultyData[degree]) continue
-      if (!mediansForUniversity[degree]) mediansForUniversity[degree] = {}
-      for (const yearData of facultyData[degree]) {
-        if (!mediansForUniversity[degree][yearData.name]) {
-          mediansForUniversity[degree][yearData.name] = { programmes: [], data: [] }
+  /** Computes faculty breakdown */
+  const unifyFacultyStats = (facultyData: MediansByCategory, facultyCode: string) => {
+    const mediansForUniversity = byGradYear.programmes.medians
+    for (const degreeType of degreeTypes) {
+      if (!facultyData[degreeType]) continue
+
+      for (const facultyYearStats of facultyData[degreeType]) {
+        if (!mediansForUniversity[degreeType][facultyYearStats.name]) {
+          mediansForUniversity[degreeType][facultyYearStats.name] = { programmes: [], data: [] }
         }
-        const uniYearStats = mediansForUniversity[degree][yearData.name]
+        const uniYearStats = mediansForUniversity[degreeType][facultyYearStats.name]
         if (!uniYearStats.programmes.find(prog => prog === facultyCode)) {
           uniYearStats.programmes.push(facultyCode)
         }
         const uniYearFacultyStats = uniYearStats.data.find(item => item.code === facultyCode)
         if (!uniYearFacultyStats) {
-          uniYearStats.data.push({ ...cloneDeep(yearData), name: facultyCode, code: facultyCode })
+          uniYearStats.data.push({ ...cloneDeep(facultyYearStats), name: facultyCode, code: facultyCode })
         } else {
           uniYearFacultyStats.times.push(...uniYearFacultyStats.times)
           uniYearFacultyStats.median = getMedian(uniYearFacultyStats.times)
-          uniYearFacultyStats.amount += yearData.amount
-          uniYearFacultyStats.statistics.onTime += yearData.statistics.onTime
-          uniYearFacultyStats.statistics.yearOver += yearData.statistics.yearOver
-          uniYearFacultyStats.statistics.wayOver += yearData.statistics.wayOver
+          uniYearFacultyStats.amount += facultyYearStats.amount
+          uniYearFacultyStats.statistics.onTime += facultyYearStats.statistics.onTime
+          uniYearFacultyStats.statistics.yearOver += facultyYearStats.statistics.yearOver
+          uniYearFacultyStats.statistics.wayOver += facultyYearStats.statistics.wayOver
         }
       }
     }
@@ -218,27 +225,20 @@ router.get('/allgraduationstats', async (_req: Request, res: Response) => {
   for (let i = 0; i < facultyCodes.length; i++) {
     const facultyCode = facultyCodes[i]
     const data = facultyData[facultyCode]
-    unifyTotals(data.byGradYear.medians, i === facultyCodes.length - 1)
-    unifyProgrammeStats(data.byGradYear.medians, facultyCode)
-    for (const degree of degreeNames) {
-      if (!data.classSizes[degree]) {
-        continue
-      }
-      const facultyClassSizes = data.classSizes[degree] as Record<string, number>
-      if (!universityData.classSizes[degree]) {
-        universityData.classSizes[degree] = facultyClassSizes
-      } else {
-        Object.entries(facultyClassSizes).forEach(([key, value]) => {
-          universityData.classSizes[degree]![key] += value
-        })
-      }
-    }
 
-    const { programmes, ...rest } = data.classSizes
-    universityData.classSizes.programmes[facultyCode] = rest as Record<string, Record<string, number>>
+    unifyTotals(data.byGradYear.medians, i === facultyCodes.length - 1)
+    unifyFacultyStats(data.byGradYear.medians, facultyCode)
+
+    const { programmes: _, ...rest } = data.classSizes
+    classSizes.programmes[facultyCode] = rest
   }
 
-  res.status(200).json(universityData)
+  res.status(200).json({
+    goals,
+    programmeNames,
+    byGradYear,
+    classSizes,
+  })
 })
 
 export default router

services/backend/src/routes/updater.ts

@@ -4,6 +4,7 @@ import { languageCenterViewEnabled } from '../config'
 import { refreshFaculties, refreshProgrammes } from '../events'
 import {
   abortUpdate,
+  nukeRedis,
   updateSISCoursesByCourseCode,
   updateSISMetadata,
   updateSISProgrammes,
@@ -12,17 +13,17 @@ import {
   updateSISStudentsByStudentNumber,
 } from '../services/sisUpdaterService'
 import logger from '../util/logger'
-import { getJobs, jobMaker, removeWaitingJobs } from '../worker/queue'
+import { jobQueue } from '../worker/queue'
 
 const router = Router()
 
 const refreshFacultiesByList = (facultyCodes: string[]) => {
-  facultyCodes.forEach(code => jobMaker.faculty(code))
+  facultyCodes.forEach(code => jobQueue.refreshFaculty(code))
   return 'Added jobs for refreshing faculties'
 }
 
 const refreshProgrammesByList = (programmeCodes: string[]) => {
-  programmeCodes.forEach(code => jobMaker.programme(code))
+  programmeCodes.forEach(code => jobQueue.refreshProgramme(code))
   return 'Added jobs for refreshing programme'
 }
 
@@ -80,11 +81,17 @@ router.get('/refresh_redis_cache', async (req: Request, res: Response) => {
   }
 })
 
+router.get('/nuke_redis', async (req: Request, res: Response) => {
+  logger.info(`${req.user.username} requested complete wipe of redis`)
+  const response = await nukeRedis()
+  if (response) res.status(200).json('Complete flush of redis scheduled')
+})
+
 router.post('/refresh-teacher-leaderboard', (req: Request, res: Response) => {
   logger.info(
     `${req.user.username} requested refresh of teacher leaderboard for the current and previous academic year`
   )
-  jobMaker.teacherLeaderboard()
+  jobQueue.refreshTeacherLeaderboard()
   res.status(200).json('Teacher leaderboard for the current and previous academic year refreshed')
 })
 
@@ -104,13 +111,13 @@ router.post('/refresh_language_center_data', (req: Request, res: Response) => {
   logger.info(`${req.user.username} requested refresh of language center data`)
   if (!languageCenterViewEnabled)
     res.status(418).json({ error: 'The language center functionality is not activated in your environment.' })
-  jobMaker.languagecenter()
+  jobQueue.refreshLanguagecenter()
   res.status(200).json('Added job for refreshing language center data')
 })
 
 router.post('/refresh-close-to-graduation', (req: Request, res: Response) => {
   logger.info(`${req.user.username} requested refresh of close to graduation data`)
-  jobMaker.closeToGraduation()
+  jobQueue.refreshCloseToGraduation()
   res.status(200).json('Added job for refreshing close to graduation data')
 })
 
@@ -120,13 +127,12 @@ router.get('/abort', async (_req: Request, res: Response) => {
 })
 
 router.get('/jobs', async (_req: Request, res: Response) => {
-  const waiting = await getJobs('waiting')
-  const active = await getJobs('active')
-  res.status(200).json({ waiting, active })
+  const jobs = await jobQueue.getJobs()
+  res.status(200).json(jobs)
 })
 
 router.delete('/jobs', async (_req: Request, res: Response) => {
-  await removeWaitingJobs()
+  await jobQueue.clearWaitingJobs()
   res.status(200).end()
 })
 

services/backend/src/services/completedCoursesSearch.ts

@@ -1,62 +1,28 @@
 import { col, fn, Op, where } from 'sequelize'
 
-import { CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
-import { omitKeys } from '@oodikone/shared/util'
+import {
+  CompletedCoursesCourse,
+  CompletedCoursesStudent,
+  CreditTypeCode,
+  EnrollmentState,
+  StudentCredits,
+} from '@oodikone/shared/types'
+import { enrollmentTimeDateThreshold, omitKeys } from '@oodikone/shared/util'
 import { CourseModel, CreditModel, EnrollmentModel, StudentModel, StudyplanModel } from '../models'
 
-export type Courses = Array<Pick<CourseModel, 'code' | 'name' | 'substitution_groups'>>
-
-interface StudentCredit {
-  date: Date
-  courseCode: string
-  creditType: CreditTypeCode
-  substitution: string[] | null
-}
-
-interface StudentEnrollment {
-  date: Date
-  courseCode: string
-  substitution: string[] | null
-}
-
-interface StudentInfo {
-  firstnames: string
-  lastname: string
-  email: string
-  sis_person_id: string
-  coursesInStudyPlan: string[]
-  secondaryEmail: string | null
-  credits: StudentCredit[]
-  enrollments: Record<string, StudentEnrollment>
-}
-
-type StudentCredits = Record<
-  string,
-  Omit<StudentInfo, 'credits' | 'enrollments'> & {
-    credits: StudentCredit[]
-    enrollments: StudentEnrollment[]
-  }
->
-
-export interface FormattedStudent extends StudentInfo {
-  studentNumber: string
-  allEnrollments: StudentEnrollment[]
-}
-
-interface StudentWithStudyplanNested
-  extends Pick<
-    StudentModel,
-    'studentnumber' | 'firstnames' | 'lastname' | 'email' | 'sis_person_id' | 'secondary_email'
-  > {
+type StudentWithStudyplanNested = Pick<
+  StudentModel,
+  'studentnumber' | 'firstnames' | 'lastname' | 'email' | 'sis_person_id' | 'secondary_email'
+> & {
   studyplans: { included_courses: string[] }[]
 }
 
-interface StudentWithCourses extends Omit<StudentWithStudyplanNested, 'studyplans'> {
+type StudentWithCourses = Omit<StudentWithStudyplanNested, 'studyplans'> & {
   coursesInStudyPlan: string[]
 }
 
 const getCourses = async (courseCodes: string[]) => {
-  const courses: Courses = (
+  const courses: CompletedCoursesCourse[] = (
     await CourseModel.findAll({
       attributes: ['code', 'name', 'substitution_groups'],
       where: where(fn('LOWER', col('code')), {
@@ -67,25 +33,28 @@ const getCourses = async (courseCodes: string[]) => {
   return courses
 }
 
-const getPassedCredits = async (courses: Courses, fullCourseCodes: string[], studentNumbers: string[]) => {
-  const credits: Array<
-    Pick<CreditModel, 'course_code' | 'student_studentnumber' | 'credittypecode' | 'attainment_date'>
-  > = await CreditModel.findAll({
-    raw: true,
-    attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'attainment_date'],
-    order: [['attainment_date', 'DESC']],
-    where: {
-      course_code: {
-        [Op.in]: fullCourseCodes,
+const getPassedCredits = async (
+  courses: CompletedCoursesCourse[],
+  fullCourseCodes: string[],
+  studentNumbers: string[]
+) => {
+  const credits: Pick<CreditModel, 'course_code' | 'student_studentnumber' | 'credittypecode' | 'attainment_date'>[] =
+    await CreditModel.findAll({
+      raw: true,
+      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'attainment_date'],
+      order: [['attainment_date', 'DESC']],
+      where: {
+        course_code: {
+          [Op.in]: fullCourseCodes,
+        },
+        student_studentnumber: {
+          [Op.in]: studentNumbers,
+        },
+        credittypecode: {
+          [Op.not]: CreditTypeCode.FAILED,
+        },
       },
-      student_studentnumber: {
-        [Op.in]: studentNumbers,
-      },
-      credittypecode: {
-        [Op.not]: CreditTypeCode.FAILED,
-      },
-    },
-  })
+    })
 
   const creditsByStudentNumber = Object.groupBy(credits, ({ student_studentnumber }) => student_studentnumber)
 
@@ -125,7 +94,11 @@ const getPassedCredits = async (courses: Courses, fullCourseCodes: string[], stu
   return formattedCredits
 }
 
-const getEnrollments = async (courses: Courses, fullCourseCodes: string[], studentNumbers: string[]) => {
+const getEnrollments = async (
+  courses: CompletedCoursesCourse[],
+  fullCourseCodes: string[],
+  studentNumbers: string[]
+) => {
   const enrollments: Array<Pick<EnrollmentModel, 'course_code' | 'enrollment_date_time' | 'studentnumber'>> =
     await EnrollmentModel.findAll({
       attributes: ['course_code', 'enrollment_date_time', 'studentnumber'],
@@ -140,6 +113,7 @@ const getEnrollments = async (courses: Courses, fullCourseCodes: string[], stude
         state: {
           [Op.eq]: EnrollmentState.ENROLLED,
         },
+        enrollment_date_time: { [Op.gte]: enrollmentTimeDateThreshold },
       },
     })
 
@@ -208,7 +182,7 @@ const getStudents = async (studentNumbers: string[]): Promise<StudentWithCourses
 export const getCompletedCourses = async (
   studentNumbers: string[],
   courseCodes: string[]
-): Promise<{ students: Omit<FormattedStudent, 'allEnrollments'>[]; courses: Courses }> => {
+): Promise<{ students: Omit<CompletedCoursesStudent, 'allEnrollments'>[]; courses: CompletedCoursesCourse[] }> => {
   const courses = await getCourses(courseCodes)
   const courseCodesSet = new Set(courseCodes)
 
@@ -280,22 +254,25 @@ export const getCompletedCourses = async (
     })
   })
 
-  const students = Object.keys(studentCredits).reduce((acc: FormattedStudent[], studentNumber) => {
-    const student = studentCredits[studentNumber]
-    acc.push({
-      studentNumber,
-      sis_person_id: student.sis_person_id,
-      credits: student.credits,
-      enrollments: {},
-      allEnrollments: student.enrollments,
-      firstnames: student.firstnames,
-      lastname: student.lastname,
-      email: student.email,
-      secondaryEmail: student.secondaryEmail,
-      coursesInStudyPlan: student.coursesInStudyPlan,
-    })
-    return acc
-  }, [] as FormattedStudent[])
+  const students = Object.keys(studentCredits).reduce<CompletedCoursesStudent[]>(
+    (acc: CompletedCoursesStudent[], studentNumber) => {
+      const student = studentCredits[studentNumber]
+      acc.push({
+        studentNumber,
+        sis_person_id: student.sis_person_id,
+        credits: student.credits,
+        enrollments: {},
+        allEnrollments: student.enrollments,
+        firstnames: student.firstnames,
+        lastname: student.lastname,
+        email: student.email,
+        secondaryEmail: student.secondaryEmail,
+        coursesInStudyPlan: student.coursesInStudyPlan,
+      })
+      return acc
+    },
+    []
+  )
 
   students.forEach(student => {
     courseCodes.forEach(courseCode => {

services/backend/src/services/courses/creditsAndEnrollmentsOfCourse.ts

@@ -2,6 +2,7 @@ import { Op } from 'sequelize'
 
 import { Credit, Enrollment } from '@oodikone/shared/models'
 import { EnrollmentState, Unification } from '@oodikone/shared/types'
+import { enrollmentTimeDateThreshold } from '@oodikone/shared/util'
 import {
   StudentModel,
   CreditModel,
@@ -142,7 +143,7 @@ export const getEnrollmentsForCourses = async (codeGroups: string[][], unificati
     include: [
       {
         model: EnrollmentModel,
-        attributes: ['studentnumber', 'enrollment_date_time', 'course_code'],
+        attributes: ['studentnumber', 'enrollment_date_time', 'course_code', 'studyright_id'],
         include: [
           {
             model: SemesterModel,
@@ -158,7 +159,8 @@ export const getEnrollmentsForCourses = async (codeGroups: string[][], unificati
           course_code: {
             [Op.in]: allCourseCodes,
           },
-          enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
+          // Date when OK changed from Oodi to Sisu data, studyright_id is null before that date
+          enrollment_date_time: { [Op.gte]: enrollmentTimeDateThreshold },
           state: EnrollmentState.ENROLLED,
           is_open: getIsOpen(unification),
         },

services/backend/src/services/courses/index.ts

@@ -3,10 +3,12 @@ import { Op, fn as dbFn, col as dbCol } from 'sequelize'
 
 import { Credit, Enrollment } from '@oodikone/shared/models'
 import { Name, EnrollmentState, Unification } from '@oodikone/shared/types'
+import { enrollmentTimeDateThreshold, getSemesterCodeAt } from '@oodikone/shared/util'
 import { dateIsBetween } from '@oodikone/shared/util/datetime'
 import logger from '../../../src/util/logger'
 import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, SISStudyRightElementModel } from '../../models'
 import { isOpenUniCourseCode } from '../../util'
+import { getSemestersAndYears, SemestersAndYears } from '../semesters'
 import { CourseYearlyStatsCounter } from './courseYearlyStatsCounter'
 import {
   getCreditsForCourses,
@@ -52,26 +54,25 @@ const parseCredit = (
   studyRightElements: Array<SISStudyRightElementModel>
 ): FormattedCredit => {
   const singleCourse = isSingleCourse(creditGroup)
-  // Latest attainment for a course
-  const credit =
+
+  // TODO: Calculate substitutions correctly once course_unit_realisation hits
+  const attainment =
     creditGroup
-      .sort((a, b) => b.attainment_date.getTime() - a.attainment_date.getTime())
-      .find(credit => !CreditModel.failed(credit)) ?? creditGroup.at(0)!
+      .sort((a, b) => b.attainment_date.getTime() - a.attainment_date.getTime()) // desc
+      .find(credit => CreditModel.passed(credit)) ??
+    creditGroup.find(credit => CreditModel.failed(credit)) ??
+    creditGroup.at(0)!
 
-  const courseCode = singleCourse ? credit.course_code : mainCourseCode
-  const grade = singleCourse ? credit.grade : 'substituted'
-  const credits = singleCourse ? credit.credits : creditGroup.reduce((acc, credit) => acc + credit.credits, 0)
+  const courseCode = singleCourse ? attainment.course_code : mainCourseCode
+  const grade = singleCourse ? attainment.grade : 'substituted'
+  const credits = singleCourse ? attainment.credits : creditGroup.reduce((acc, credit) => acc + credit.credits, 0)
 
-  // Take the first attainment for the course (see Student statistics info-box)
-  // TODO: Substitution groups should be marked for the semester when
-  // the group was first finished eg. when was the first time the last missing
-  // course of the group was passed
   const {
     semester,
     attainment_date: attainmentDate,
     student_studentnumber: studentNumber,
     studyright_id: studyRightId,
-  } = creditGroup.at(-1)!
+  } = attainment
 
   const { yearcode: yearCode, yearname: yearName, semestercode: semesterCode, name: semesterName } = semester
 
@@ -99,7 +100,7 @@ const parseCredit = (
     attainmentDate,
     courseCode,
     grade,
-    passed: singleCourse ? !CreditModel.failed(credit) : true,
+    passed: singleCourse ? !CreditModel.failed(attainment) : true,
     studentNumber: anonymizationSalt ? anonymizeStudentNumber(studentNumber, anonymizationSalt) : studentNumber,
     programme,
     credits,
@@ -107,31 +108,44 @@ const parseCredit = (
 }
 
 type FormattedEnrollment = {
-  yearCode: number
-  yearName: string
-  semesterCode: number
-  semesterName: Name
   courseCode: string
   enrollmentDateTime: Date
+  programme: FormattedProgramme
   studentNumber: string
 }
 
-const parseEnrollment = (enrollment: Enrollment, anonymizationSalt: string | null): FormattedEnrollment => {
+const parseEnrollment = (
+  enrollment: Enrollment,
+  anonymizationSalt: string | null,
+  studyRightElements: Array<SISStudyRightElementModel>
+): FormattedEnrollment => {
   const {
     studentnumber: studentNumber,
-    semester,
     enrollment_date_time: enrollmentDateTime,
     course_code: courseCode,
+    studyright_id: studyRightId,
   } = enrollment
-  const { yearcode: yearCode, yearname: yearName, semestercode: semesterCode, name: semesterName } = semester
+
+  const programmeOfCredit: SISStudyRightElementModel | undefined =
+    studyRightElements.find(studyRightElement => studyRightElement.studyRightId === studyRightId) ??
+    studyRightElements
+      .filter(({ startDate, endDate }) => dateIsBetween(enrollmentDateTime, startDate, endDate))
+      .sort((a, b) => b.startDate.getTime() - a.startDate.getTime())
+      .at(0) // The newest studyRightElement
+
+  const programme = programmeOfCredit
+    ? formatStudyRightElement(programmeOfCredit)
+    : {
+        code: 'OTHER',
+        name: { en: 'Other', fi: 'Muu', sv: 'Andra' },
+        facultyCode: 'OTHER',
+        organization: { name: { en: 'Other', fi: 'Muu', sv: 'Andra' } },
+      }
 
   return {
-    yearCode,
-    yearName,
-    semesterCode,
-    semesterName,
     courseCode,
     enrollmentDateTime,
+    programme,
     studentNumber: anonymizationSalt ? anonymizeStudentNumber(studentNumber, anonymizationSalt) : studentNumber,
   }
 }
@@ -150,6 +164,19 @@ const getSubstitutionGroupDetails = async (codeGroups: string[][]) => {
   )
 }
 
+const getSemesterAndYearByDate = (
+  date: Date,
+  semesters: SemestersAndYears['semesters'],
+  years: SemestersAndYears['years']
+) => {
+  const semesterCode = getSemesterCodeAt(semesters, date) ?? getSemesterCodeAt(semesters, new Date())!
+
+  const semester = semesters[semesterCode]
+  const year = years[semester.yearcode]
+
+  return { year, semester }
+}
+
 const getYearlyStatsOfNew = async (
   course: Pick<CourseModel, 'code' | 'name' | 'substitution_groups'> | null,
   courseCode: string,
@@ -179,6 +206,8 @@ const getYearlyStatsOfNew = async (
       break
   }
 
+  const { semesters, years } = await getSemestersAndYears()
+
   const [creditGroups, enrollmentGroups] = await Promise.all([
     getCreditsForCourses(filteredCreditGroupCodes, unification),
     getEnrollmentsForCourses(filteredCreditGroupCodes, unification),
@@ -219,6 +248,7 @@ const getYearlyStatsOfNew = async (
       programme.organization
     )
 
+    // Credits/attainments have quaranteed matching attainment_date and semesters/years
     const groupCode = separate ? semesterCode : yearCode
     const groupName = separate ? semesterName : yearName
     counter.markCreditToGroup(studentNumber, passed, grade, groupCode, groupName, creditCourseCode, yearCode)
@@ -231,16 +261,32 @@ const getYearlyStatsOfNew = async (
     for (const enrollment of enrollments) {
       const {
         studentNumber,
-        semesterCode,
-        semesterName,
-        yearCode,
-        yearName,
         courseCode: enrollmentCourseCode,
         enrollmentDateTime,
-      } = parseEnrollment(enrollment, anonymizationSalt)
+        programme,
+      } = parseEnrollment(
+        enrollment,
+        anonymizationSalt,
+        studentNumberToSrElementsMap[enrollment.studentnumber ?? 0] ?? []
+      )
 
-      const groupCode = separate ? semesterCode : yearCode
-      const groupName = separate ? semesterName : yearName
+      // Enrollments can have conflicting enrollment_date_time and semestercode, so we need to manually select
+      // semestercode matching enrollment_date_time
+      const { semester, year } = getSemesterAndYearByDate(enrollment.enrollment_date_time, semesters, years)
+
+      counter.markStudyProgramme(
+        studentNumber,
+        year.yearcode,
+        false, // passed
+        0, // credits
+        programme.code,
+        programme.name,
+        programme.facultyCode,
+        programme.organization
+      )
+
+      const groupCode = separate ? semester.semestercode : year.yearcode
+      const groupName = separate ? semester.name : year.yearname
 
       counter.markEnrollmentToGroup(
         studentNumber,
@@ -248,7 +294,7 @@ const getYearlyStatsOfNew = async (
         groupCode,
         groupName,
         enrollmentCourseCode,
-        yearCode
+        year.yearcode
       )
     }
   }
@@ -313,6 +359,7 @@ export const getCourseYearlyStats = async (
           [Op.in]: courseCodes,
         },
         state: EnrollmentState.ENROLLED,
+        enrollment_date_time: { [Op.gte]: enrollmentTimeDateThreshold },
       },
     }),
   ])
@@ -334,11 +381,12 @@ export const getCourseYearlyStats = async (
       const course: Pick<CourseModel, 'code' | 'name' | 'substitution_groups'> | null = await CourseModel.findOne({
         attributes: ['code', 'name', 'substitution_groups'],
         where: { code: courseCode },
+        raw: true,
       })
 
       if (!course) {
         logger.error('Course for course stats not found with code' + courseCode)
-        return []
+        return {}
       }
 
       const [openStats, regularStats, unifyStats] = await Promise.all([

services/backend/src/services/faculty/facultyGraduationTimes.ts

@@ -1,36 +1,78 @@
-import { cloneDeep } from 'lodash-es'
-
-import { Name, DegreeProgrammeType, Unarray } from '@oodikone/shared/types'
-import { ProgrammeGraduationStats } from '@oodikone/shared/types/studyProgramme'
+import { Name, DegreeProgrammeType, FacultyGraduationStatistics, ClassSizes } from '@oodikone/shared/types'
+import type {
+  GraduationStatistics,
+  MedianEntry,
+  MediansByCategory,
+  MediansByProgrammes,
+  ProgrammeGraduationStats,
+  ProgrammeMedians,
+} from '@oodikone/shared/types'
 import { omitKeys } from '@oodikone/shared/util'
 import { getGraduationStats, getStudyTrackStats, setGraduationStats, setStudyTrackStats } from '../analyticsService'
 import { GraduationTarget } from '../graduationHelpers'
-import { getGraduationStatsForStudyTrack, GraduationTimes } from '../studyProgramme/studyProgrammeGraduations'
+import { getGraduationStatsForStudyTrack } from '../studyProgramme/studyProgrammeGraduations'
 import { getMedian } from '../studyProgramme/studyProgrammeHelpers'
 import { getStudyRightsInProgramme } from '../studyProgramme/studyRightFinders'
 import { getStudyTrackStatsForStudyProgramme } from '../studyProgramme/studyTrackStats'
 import type { ProgrammesOfOrganization } from './faculty'
 
-export type LevelGraduationStats = Omit<Unarray<GraduationTimes['medians']>, 'y'> & { median: number }
-
-type ProgrammeStats = {
-  data: Array<Omit<LevelGraduationStats, 'times'> & { code: string }>
-  programmes: string[]
-}
-
 export const programmeTypes = {
   [DegreeProgrammeType.BACHELOR]: 'bachelor',
   [DegreeProgrammeType.MASTER]: 'master',
   [DegreeProgrammeType.DOCTOR]: 'doctor',
 } as const
 
-const getStatsByGraduationYear = async (facultyProgrammes: ProgrammesOfOrganization) => {
-  const newStats: ProgrammeGraduationStats[] = []
-  const medians: Record<string, LevelGraduationStats[]> = {}
-  const programmes: { medians: Record<string, Record<string, ProgrammeStats>> } = { medians: {} }
+const setYearlyBreakdown = (code: string, progId: string, level: ProgrammeMedians, year: MedianEntry) => {
+  if (year.y === 0) return
+
+  level[year.name] ??= { data: [], programmes: [] }
+  level[year.name].programmes.push(code)
+  level[year.name].data.push(
+    omitKeys({ ...structuredClone(year), name: progId, code, median: year.y }, ['y', 'classSize'])
+  )
+}
+
+const mergeStats = (
+  medians: MediansByCategory,
+  level: keyof MediansByCategory,
+  statsForYear: MedianEntry,
+  classSizes?: ClassSizes
+) => {
+  if (classSizes) {
+    classSizes[level][statsForYear.name] ??= 0
+    classSizes[level][statsForYear.name] += statsForYear.classSize ?? 0
+  }
+
+  const gradStatsForYear = medians[level].find(entry => entry.name === statsForYear.name)
+
+  if (!gradStatsForYear) {
+    medians[level].push({ ...omitKeys(structuredClone(statsForYear), ['y']), median: statsForYear.y })
+  } else {
+    gradStatsForYear.times.push(...statsForYear.times)
+    gradStatsForYear.median = getMedian(gradStatsForYear.times)
+    gradStatsForYear.amount += statsForYear.amount
+    gradStatsForYear.statistics.onTime += statsForYear.statistics.onTime
+    gradStatsForYear.statistics.yearOver += statsForYear.statistics.yearOver
+    gradStatsForYear.statistics.wayOver += statsForYear.statistics.wayOver
+  }
+}
+
+const getStatsByGraduationYear = async (
+  facultyProgrammes: ProgrammesOfOrganization
+): Promise<GraduationStatistics['byGradYear']> => {
+  const newStats: Pick<ProgrammeGraduationStats, 'id' | 'graduationTimes' | 'comboTimes'>[] = []
+  const medians: MediansByCategory = { bachelor: [], master: [], bcMsCombo: [], doctor: [] }
+  const programmes: { medians: MediansByProgrammes } = {
+    medians: {
+      bachelor: {},
+      master: {},
+      bcMsCombo: {},
+      doctor: {},
+    },
+  }
 
   for (const programme of facultyProgrammes) {
-    if (programme.degreeProgrammeType == null || !(programme.degreeProgrammeType in programmeTypes)) {
+    if (!programme.degreeProgrammeType || !(programme.degreeProgrammeType in programmeTypes)) {
       continue
     }
     const statsFromRedis = await getGraduationStats(programme.code, '', 'CALENDAR_YEAR', 'SPECIAL_INCLUDED')
@@ -51,92 +93,44 @@ const getStatsByGraduationYear = async (facultyProgrammes: ProgrammesOfOrganizat
   }
 
   for (const { id: programmeCode, graduationTimes, comboTimes } of newStats) {
-    const { degreeProgrammeType, progId, code } = facultyProgrammes.find(({ code }) => code === programmeCode)!
+    const { degreeProgrammeType, progId } = facultyProgrammes.find(({ code }) => code === programmeCode)!
     const level = programmeTypes[degreeProgrammeType as keyof typeof programmeTypes]
 
-    if (!medians[level]) {
-      medians[level] = cloneDeep(graduationTimes.medians).map(year => ({ ...omitKeys(year, ['y']), median: year.y }))
-    } else {
-      for (const statsForYear of graduationTimes.medians) {
-        const correctYear = medians[level].find(entry => entry.name === statsForYear.name)
-        if (!correctYear) {
-          medians[level].push({ ...omitKeys(cloneDeep(statsForYear), ['y']), median: statsForYear.y })
-          continue
-        }
-        correctYear.times.push(...statsForYear.times)
-        correctYear.median = getMedian(correctYear.times)
-        correctYear.amount += statsForYear.amount
-        correctYear.statistics.onTime += statsForYear.statistics.onTime
-        correctYear.statistics.yearOver += statsForYear.statistics.yearOver
-        correctYear.statistics.wayOver += statsForYear.statistics.wayOver
-      }
-    }
-
-    if (level === 'master' && comboTimes.medians.length) {
-      if (!medians.bcMsCombo) {
-        medians.bcMsCombo = cloneDeep(comboTimes.medians).map(year => ({ ...omitKeys(year, ['y']), median: year.y }))
-      } else {
-        for (const statsForYear of comboTimes.medians) {
-          const correctYear = medians.bcMsCombo.find(entry => entry.name === statsForYear.name)
-          if (!correctYear) {
-            medians.bcMsCombo.push({ ...omitKeys(cloneDeep(statsForYear), ['y']), median: statsForYear.y })
-            continue
-          }
-          correctYear.times.push(...statsForYear.times)
-          correctYear.median = getMedian(correctYear.times)
-          correctYear.amount += statsForYear.amount
-          correctYear.statistics.onTime += statsForYear.statistics.onTime
-          correctYear.statistics.yearOver += statsForYear.statistics.yearOver
-          correctYear.statistics.wayOver += statsForYear.statistics.wayOver
-        }
-      }
-    }
-
-    if (!programmes.medians[level]) {
-      programmes.medians[level] = {}
-    }
-
     for (const year of graduationTimes.medians) {
-      if (year.y === 0) continue
-      if (!programmes.medians[level][year.name]) {
-        programmes.medians[level][year.name] = { data: [], programmes: [] }
-      }
-      programmes.medians[level][year.name].data.push(
-        omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y'])
-      )
-      programmes.medians[level][year.name].programmes.push(code)
+      mergeStats(medians, level, year)
+      setYearlyBreakdown(programmeCode, progId, programmes.medians[level], year)
     }
 
     if (level === 'master' && comboTimes.medians.length) {
-      if (!programmes.medians.bcMsCombo) {
-        programmes.medians.bcMsCombo = {}
-      }
       for (const year of comboTimes.medians) {
-        if (year.y === 0) continue
-        if (!programmes.medians.bcMsCombo[year.name]) {
-          programmes.medians.bcMsCombo[year.name] = { data: [], programmes: [] }
-        }
-        programmes.medians.bcMsCombo[year.name].data.push(
-          omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y'])
-        )
-        programmes.medians.bcMsCombo[year.name].programmes.push(code)
+        mergeStats(medians, 'bcMsCombo', year)
+        setYearlyBreakdown(programmeCode, progId, programmes.medians.bcMsCombo, year)
       }
     }
   }
   return { medians, programmes }
 }
 
-const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization) => {
-  const newStats: Array<Awaited<ReturnType<typeof getStudyTrackStatsForStudyProgramme>>> = []
-  const medians: Record<string, LevelGraduationStats[]> = {}
-  const programmes: { medians: Record<string, Record<string, ProgrammeStats>> } = { medians: {} }
-  const classSizes: Record<string, Record<string, number | Record<string, number>>> = { programmes: {} }
+const getStatsByStartYear = async (
+  facultyProgrammes: ProgrammesOfOrganization
+): Promise<GraduationStatistics['byStartYear'] & Pick<GraduationStatistics, 'classSizes'>> => {
+  const classSizes: ClassSizes = { bachelor: {}, master: {}, bcMsCombo: {}, doctor: {}, programmes: {} }
+  const newStats: Awaited<ReturnType<typeof getStudyTrackStatsForStudyProgramme>>[] = []
+  const medians: MediansByCategory = { bachelor: [], master: [], bcMsCombo: [], doctor: [] }
+  const programmes: { medians: MediansByProgrammes } = {
+    medians: {
+      bachelor: {},
+      master: {},
+      bcMsCombo: {},
+      doctor: {},
+    },
+  }
 
   for (const programme of facultyProgrammes) {
-    if (programme.degreeProgrammeType == null || !(programme.degreeProgrammeType in programmeTypes)) {
+    if (!programme.degreeProgrammeType || !(programme.degreeProgrammeType in programmeTypes)) {
       continue
     }
-    const statsFromRedis = await getStudyTrackStats(programme.code, '', 'SPECIAL_EXCLUDED')
+    const statsFromRedis = await getStudyTrackStats(programme.code, null, 'SPECIAL_EXCLUDED')
     if (statsFromRedis) {
       newStats.push(statsFromRedis)
       continue
@@ -156,9 +150,13 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
   }
 
   for (const { id: programmeCode, graduationTimes } of newStats) {
-    const { degreeProgrammeType, progId, code } = facultyProgrammes.find(({ code }) => code === programmeCode)!
+    const { degreeProgrammeType, progId } = facultyProgrammes.find(({ code }) => code === programmeCode)!
     const level = programmeTypes[degreeProgrammeType as keyof typeof programmeTypes]
-    if (!(programmeCode in graduationTimes)) continue
+
+    if (!(programmeCode in graduationTimes)) {
+      continue
+    }
+
     const statsForProgramme = graduationTimes[programmeCode]
     const basicStats = statsForProgramme.medians.basic
     const comboStats = statsForProgramme.medians.combo
@@ -166,91 +164,31 @@ const getStatsByStartYear = async (facultyProgrammes: ProgrammesOfOrganization)
       (acc, year) => ({ ...acc, [year.name]: year.classSize ?? 0 }),
       {}
     )
-    classSizes.programmes[code] = classSizesByYear
+    classSizes.programmes[programmeCode] = classSizesByYear
 
-    if (!medians[level]) {
-      medians[level] = cloneDeep(basicStats).map(year => ({ ...omitKeys(year, ['y', 'classSize']), median: year.y }))
-    }
-    classSizes[level] ??= {}
     for (const statsForYear of basicStats) {
-      classSizes[level][statsForYear.name] ??= 0
-      ;(classSizes[level][statsForYear.name] as number) += statsForYear.classSize ?? 0
-
-      const correctYear = medians[level].find(entry => entry.name === statsForYear.name)
-      if (!correctYear) {
-        medians[level].push({ ...omitKeys(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
-        continue
-      }
-      correctYear.times.push(...statsForYear.times)
-      correctYear.median = getMedian(correctYear.times)
-      correctYear.amount += statsForYear.amount
-      correctYear.statistics.onTime += statsForYear.statistics.onTime
-      correctYear.statistics.yearOver += statsForYear.statistics.yearOver
-      correctYear.statistics.wayOver += statsForYear.statistics.wayOver
+      mergeStats(medians, level, statsForYear, classSizes)
+      setYearlyBreakdown(programmeCode, progId, programmes.medians[level], statsForYear)
     }
 
     if (level === 'master' && comboStats.length) {
-      if (!medians.bcMsCombo) {
-        medians.bcMsCombo = cloneDeep(comboStats).map(year => ({
-          ...omitKeys(year, ['y', 'classSize']),
-          median: year.y,
-        }))
-      }
-      classSizes.bcMsCombo ??= {}
       for (const statsForYear of comboStats) {
-        classSizes.bcMsCombo[statsForYear.name] ??= 0
-        ;(classSizes.bcMsCombo[statsForYear.name] as number) += statsForYear.classSize ?? 0
-        const correctYear = medians.bcMsCombo.find(entry => entry.name === statsForYear.name)
-        if (!correctYear) {
-          medians.bcMsCombo.push({ ...omitKeys(cloneDeep(statsForYear), ['y', 'classSize']), median: statsForYear.y })
-          continue
-        }
-        correctYear.times.push(...statsForYear.times)
-        correctYear.median = getMedian(correctYear.times)
-        correctYear.amount += statsForYear.amount
-        correctYear.statistics.onTime += statsForYear.statistics.onTime
-        correctYear.statistics.yearOver += statsForYear.statistics.yearOver
-        correctYear.statistics.wayOver += statsForYear.statistics.wayOver
-      }
-    }
-
-    if (!programmes.medians[level]) {
-      programmes.medians[level] = {}
-    }
-
-    for (const year of basicStats) {
-      if (year.y === 0) continue
-      if (!programmes.medians[level][year.name]) {
-        programmes.medians[level][year.name] = { data: [], programmes: [] }
-      }
-      programmes.medians[level][year.name].data.push(
-        omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y', 'classSize'])
-      )
-      programmes.medians[level][year.name].programmes.push(code)
-    }
-
-    if (level === 'master' && comboStats.length) {
-      if (!programmes.medians.bcMsCombo) {
-        programmes.medians.bcMsCombo = {}
-      }
-      for (const year of comboStats) {
-        if (year.y === 0) continue
-        if (!programmes.medians.bcMsCombo[year.name]) {
-          programmes.medians.bcMsCombo[year.name] = { data: [], programmes: [] }
-        }
-        programmes.medians.bcMsCombo[year.name].data.push(
-          omitKeys({ ...cloneDeep(year), name: progId, code, median: year.y }, ['times', 'y', 'classSize'])
-        )
-        programmes.medians.bcMsCombo[year.name].programmes.push(code)
+        mergeStats(medians, 'bcMsCombo', statsForYear, classSizes)
+        setYearlyBreakdown(programmeCode, progId, programmes.medians.bcMsCombo, statsForYear)
       }
     }
   }
+
   return { medians, programmes, classSizes }
 }
 
-export const countGraduationTimes = async (faculty: string, programmesOfFaculty: ProgrammesOfOrganization) => {
+export const countGraduationTimes = async (
+  facultyCode: string,
+  programmesOfFaculty: ProgrammesOfOrganization
+): Promise<FacultyGraduationStatistics> => {
   const { medians: mediansByGraduationYear, programmes: programmesByGraduationYear } =
     await getStatsByGraduationYear(programmesOfFaculty)
+
   const {
     medians: mediansByStartYear,
     programmes: programmesByStartYear,
@@ -259,7 +197,7 @@ export const countGraduationTimes = async (faculty: string, programmesOfFaculty:
 
   const goals = {
     bachelor: GraduationTarget.THREE_YEARS,
-    bcMsCombo: faculty === 'H90' ? GraduationTarget.SIX_YEARS : GraduationTarget.FIVE_YEARS,
+    bcMsCombo: facultyCode === 'H90' ? GraduationTarget.SIX_YEARS : GraduationTarget.FIVE_YEARS,
     master: GraduationTarget.TWO_YEARS,
     doctor: GraduationTarget.FOUR_YEARS,
     exceptions: {
@@ -283,7 +221,7 @@ export const countGraduationTimes = async (faculty: string, programmesOfFaculty:
   }, {})
 
   return {
-    id: faculty,
+    id: facultyCode,
     goals,
     byGradYear: {
       medians: mediansByGraduationYear,

services/backend/src/services/faculty/facultyService.ts

@@ -1,4 +1,4 @@
-import { Name, ProgrammeFilter, SpecialGroups, YearType } from '@oodikone/shared/types'
+import { FacultyGraduationStatistics, Name, ProgrammeFilter, SpecialGroups, YearType } from '@oodikone/shared/types'
 import { DegreeProgramme } from '../../routes/faculties'
 import { redisClient } from '../redis'
 import { FacultyProgressData } from './facultyStudentProgress'
@@ -29,16 +29,16 @@ const createRedisKeyForFacultyStudents = (id: string, specialGroups: SpecialGrou
   return `FACULTY_STUDENTS_STATS_${id}_${specialGroups}`
 }
 
-/*
+/**
   Faculty data objects have graduation times left in as arrays, so that
-  university-level evaluation overview can count median-times by itself.
+  university-level evaluation overview can count median-times by itself (in backend).
   Faculties, however, don't need this, and it isn't needed in frontend
-  for universityview either. This removes them from the object.
+  for university view either. This clears the array.
 */
-const removeGraduationTimes = (data: GraduationData) => {
+const removeGraduationTimes = (data: FacultyGraduationStatistics) => {
   Object.values(data.byGradYear.medians).forEach(array =>
     array.forEach(yearStat => {
-      yearStat.times = null
+      yearStat.times = []
     })
   )
 }
@@ -158,7 +158,7 @@ export type GraduationData = {
   classSizes: Record<string, Record<string, number | Record<string, number>>>
 }
 
-export const setGraduationStats = async (data: GraduationData, programmeFilter: ProgrammeFilter) => {
+export const setGraduationStats = async (data: FacultyGraduationStatistics, programmeFilter: ProgrammeFilter) => {
   const { id } = data
   const redisKey = createRedisKeyForGraduationTimeStats(id, programmeFilter)
   await redisClient.set(redisKey, JSON.stringify(data))
@@ -170,7 +170,7 @@ export const getGraduationStats = async (id: string, programmeFilter: ProgrammeF
   if (!dataFromRedis) {
     return null
   }
-  const data = JSON.parse(dataFromRedis) as GraduationData
+  const data = JSON.parse(dataFromRedis) as FacultyGraduationStatistics
   if (!keepGraduationTimes) {
     removeGraduationTimes(data)
   }

services/backend/src/services/graduationHelpers.ts

@@ -8,7 +8,7 @@ export const GraduationTarget = {
   FOUR_YEARS: 8,
   FIVE_YEARS: 10,
   SIX_YEARS: 12,
-} as const
+}
 
 /** Uses semesters, not months */
 export const countTimeCategories = (times: number[], target: number) => {

services/backend/src/services/languageCenterData.js

@@ -106,8 +106,8 @@ export const computeLanguageCenterData = async () => {
     attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'studyright_id', 'state'],
     where: {
       [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
-      semestercode: { [Op.gte]: autumnSemester2017 },
       state: { [Op.in]: ['ENROLLED', 'REJECTED'] },
+      // This doesn't need the `enrollmentTimeDateThreshold`, ColorizedDataTable doesn't use studyright_ids
     },
     raw: true,
   })

services/backend/src/services/openUni/format.ts

@@ -1,33 +0,0 @@
-import { CourseModel, CreditModel, EnrollmentModel, StudentModel } from '../../models'
-
-export const formatCourseInfo = (course: CourseModel) => {
-  return {
-    label: course.code,
-    name: course.name,
-  }
-}
-
-export const formatOpenCredits = (credit: CreditModel) => {
-  return {
-    attainmentCourseCode: credit.course_code,
-    attainmentDate: credit.attainment_date,
-    attainmentGrade: credit.grade,
-    attainmentStudentNumber: credit.student_studentnumber,
-  }
-}
-
-export const formatOpenEnrollments = (enrollment: EnrollmentModel) => {
-  return {
-    enrollmentCourseCode: enrollment.course_code,
-    enrollmentDateTime: enrollment.enrollment_date_time,
-    enrollmentStudentNumber: enrollment.studentnumber,
-  }
-}
-
-export const formatStudentInfo = (student: StudentModel) => {
-  return {
-    studentNumber: student.studentnumber,
-    email: student.email,
-    secondaryEmail: student.secondary_email,
-  }
-}

services/backend/src/services/openUni/openUniSearches.ts

@@ -1,94 +1,4 @@
-import { Op, WhereOptions } from 'sequelize'
-
-import { SISStudyRight } from '@oodikone/shared/models'
-import { EnrollmentState, ExtentCode } from '@oodikone/shared/types'
-import { CourseModel, CreditModel, EnrollmentModel, StudentModel, SISStudyRightModel } from '../../models'
 import { OpenUniPopulationSearchModel } from '../../models/kone'
-import { formatCourseInfo, formatOpenCredits, formatOpenEnrollments, formatStudentInfo } from './format'
-
-export const getCredits = async (courseCodes: string[], startdate: Date) =>
-  (
-    await CreditModel.findAll({
-      attributes: ['attainment_date', 'course_code', 'grade', 'student_studentnumber'],
-      where: {
-        course_code: {
-          [Op.in]: courseCodes,
-        },
-        attainment_date: {
-          [Op.gte]: startdate,
-        },
-      },
-    })
-  ).map(formatOpenCredits)
-
-export const getStudentInfo = async (studentNumbers: string[]) =>
-  (
-    await StudentModel.findAll({
-      attributes: ['studentnumber', 'email', 'secondary_email'],
-      where: {
-        studentnumber: {
-          [Op.in]: studentNumbers,
-        },
-      },
-    })
-  ).map(formatStudentInfo)
-
-export const getEnrollments = async (courseCodes: string[], startDate: Date, endDate: Date) =>
-  (
-    await EnrollmentModel.findAll({
-      attributes: ['course_code', 'enrollment_date_time', 'studentnumber'],
-      where: {
-        course_code: {
-          [Op.in]: courseCodes,
-        },
-        is_open: true,
-        enrollment_date_time: {
-          [Op.and]: {
-            [Op.lte]: endDate,
-            [Op.gte]: startDate,
-          },
-        },
-        state: EnrollmentState.ENROLLED,
-      },
-    })
-  ).map(formatOpenEnrollments)
-
-export const getCourseNames = async (courseCodes: string[]) =>
-  (
-    await CourseModel.findAll({
-      attributes: ['code', 'name'],
-      where: {
-        code: {
-          [Op.in]: courseCodes,
-        },
-      },
-    })
-  ).map(formatCourseInfo)
-
-export const getStudyRights = async (studentNumbers: string[]) => {
-  const where: WhereOptions<SISStudyRight> = {
-    extentCode: {
-      [Op.in]: [
-        ExtentCode.BACHELOR,
-        ExtentCode.BACHELOR_AND_MASTER,
-        ExtentCode.MASTER,
-        ExtentCode.LICENTIATE,
-        ExtentCode.DOCTOR,
-      ],
-    },
-  }
-
-  if (studentNumbers.length > 0) {
-    where.studentNumber = {
-      [Op.in]: studentNumbers,
-    }
-  }
-
-  return await SISStudyRightModel.findAll({
-    attributes: ['startDate', 'endDate', 'studentNumber'],
-    where,
-  })
-}
 
 export const getOpenUniSearchesByUser = async (userId: string) => {
   return await OpenUniPopulationSearchModel.findAll({

services/backend/src/services/populations/getStudentData.ts

@@ -3,6 +3,8 @@ import { Op } from 'sequelize'
 import { Credit, Enrollment, SISStudyRight, SISStudyRightElement, Student, Studyplan } from '@oodikone/shared/models'
 import { Tag, TagStudent } from '@oodikone/shared/models/kone'
 import { EnrollmentState } from '@oodikone/shared/types'
+import { enrollmentTimeDateThreshold } from '@oodikone/shared/util'
+import { dateMaxFromList } from '@oodikone/shared/util/datetime'
 import {
   CreditModel,
   EnrollmentModel,
@@ -48,8 +50,8 @@ export type StudentEnrollment = Pick<
   Enrollment,
   'course_code' | 'state' | 'enrollment_date_time' | 'semestercode' | 'studentnumber' | 'studyright_id'
 >
-export const getEnrollments = (studentNumbers: string[], startDate: string): Promise<Array<StudentEnrollment>> =>
-  EnrollmentModel.findAll({
+export const getEnrollments = (studentNumbers: string[], startDate: string): Promise<Array<StudentEnrollment>> => {
+  return EnrollmentModel.findAll({
     attributes: ['course_code', 'state', 'enrollment_date_time', 'semestercode', 'studentnumber', 'studyright_id'],
     where: {
       studentnumber: {
@@ -57,11 +59,12 @@ export const getEnrollments = (studentNumbers: string[], startDate: string): Pro
       },
       state: EnrollmentState.ENROLLED,
       enrollment_date_time: {
-        [Op.gte]: startDate,
+        [Op.gte]: dateMaxFromList(new Date(startDate), enrollmentTimeDateThreshold),
       },
     },
     raw: true,
   })
+}
 
 type StudentPersonalData = Pick<
   Student,

services/backend/src/services/semesters.ts

@@ -4,7 +4,7 @@ import { Name } from '@oodikone/shared/types'
 import { dateMinFromList, dateMaxFromList } from '@oodikone/shared/util/datetime'
 import { SemesterModel } from '../models'
 
-type SemestersAndYears = {
+export type SemestersAndYears = {
   years: Record<
     string,
     {

services/backend/src/services/sisUpdaterService.ts

@@ -44,6 +44,11 @@ export const updateSISStudentsByStudentNumber = async (studentnumbers: string[])
   return response.data
 }
 
+export const nukeRedis = async () => {
+  const response = await client.get('/nuke_redis', params)
+  return response.data
+}
+
 export const updateSISStudyPlans = async (days: number) => {
   const limitDate = new Date()
   limitDate.setDate(limitDate.getDate() - days)

services/backend/src/services/students.ts

@@ -1,9 +1,10 @@
+import dayjs from 'dayjs'
 import { Op, QueryTypes } from 'sequelize'
 
 import type { Credit, Student } from '@oodikone/shared/models'
 import { EnrollmentState, UnifyStatus } from '@oodikone/shared/types'
 import { FormattedStudentForSearch, StudentPageStudent } from '@oodikone/shared/types/studentData'
-import { splitByEmptySpace } from '@oodikone/shared/util'
+import { enrollmentTimeDateThreshold, splitByEmptySpace } from '@oodikone/shared/util'
 import { dbConnections } from '../database/connection'
 import {
   StudentModel,
@@ -182,7 +183,7 @@ export const findByCourseAndSemesters = async (
           e.studentnumber = s.studentnumber
           AND e.course_code IN (:codes)
           AND e.semestercode BETWEEN ${fromSemester} AND ${toSemester}
-          AND e.enrollment_date_time >= '2021-05-31'
+          AND e.enrollment_date_time >= '${dayjs(enrollmentTimeDateThreshold).format('YYYY-MM-DD')}'
           AND e.state = :enrollmentState
       );
     `,

services/backend/src/services/studyProgramme/index.ts

@@ -2,6 +2,8 @@ import { memoize } from 'lodash-es'
 import { Op, QueryTypes } from 'sequelize'
 
 import { Name, CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
+import { enrollmentTimeDateThreshold } from '@oodikone/shared/util'
+import { dateIsBetween } from '@oodikone/shared/util/datetime'
 import { dbConnections } from '../../database/connection'
 import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, ProgrammeModuleModel } from '../../models'
 import logger from '../../util/logger'
@@ -27,7 +29,7 @@ export const getAllProgrammeCourses = async (providerCode: string) => {
   return courses
 }
 
-const getCourseCode = (code: string) => {
+const removeOpenUniCodePrefix = (code: string) => {
   if (code.startsWith('AY')) {
     return code.replace('AY', '')
   }
@@ -39,14 +41,23 @@ const getCourseCode = (code: string) => {
 
 export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, programmeCourses: string[]) => {
   try {
-    const enrollmentsCourses = await EnrollmentModel.findAll({
-      attributes: ['studentnumber', 'course_code'],
+    const enrollmentsCourses: Array<
+      Pick<EnrollmentModel, 'studentnumber' | 'course_code' | 'enrollment_date_time' | 'course'>
+    > = await EnrollmentModel.findAll({
+      raw: true,
+      nest: true,
+      attributes: ['studentnumber', 'course_code', 'enrollment_date_time'],
+      include: {
+        model: CourseModel,
+        attributes: ['is_study_module'],
+      },
       where: {
         course_code: {
           [Op.in]: programmeCourses,
         },
         enrollment_date_time: {
           [Op.between]: [from, to],
+          [Op.gte]: enrollmentTimeDateThreshold,
         },
         state: EnrollmentState.ENROLLED,
       },
@@ -54,27 +65,36 @@ export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, p
 
     const allEnrollments = new Map<string, Set<string>>()
     for (const { studentnumber, course_code: courseCode } of enrollmentsCourses) {
-      const code = getCourseCode(courseCode)
+      const code = removeOpenUniCodePrefix(courseCode)
       if (!allEnrollments.has(code)) {
         allEnrollments.set(code, new Set())
       }
       allEnrollments.get(code)!.add(studentnumber)
     }
 
-    const credits = await CreditModel.findAll({
-      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'isStudyModule'],
+    // TODO: This is being called for each year, could be optimized to call only once
+    const credits: Array<
+      Pick<
+        CreditModel,
+        'course_code' | 'student_studentnumber' | 'credittypecode' | 'isStudyModule' | 'attainment_date'
+      >
+    > = await CreditModel.findAll({
+      raw: true,
+      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'isStudyModule', 'attainment_date'],
       where: {
         course_code: {
           [Op.in]: programmeCourses,
         },
-        attainment_date: {
-          [Op.between]: [from, to],
-        },
       },
+      order: [['credittypecode', 'ASC']], // Passed credits for a student have to be handled first
     })
 
+    // Pick only this (between to, from) year's credits
+    const filteredCredits = credits.filter(credit => dateIsBetween(credit.attainment_date, from, to))
+
     const courseCodeToName = (
       await CourseModel.findAll({
+        raw: true,
         attributes: ['code', 'name'],
         where: {
           code: programmeCourses,
@@ -85,8 +105,8 @@ export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, p
       return acc
     }, new Map())
 
-    const creditCourses = credits.map(credit => ({
-      code: getCourseCode(credit.course_code),
+    const creditCourses = filteredCredits.map(credit => ({
+      code: removeOpenUniCodePrefix(credit.course_code),
       studentNumber: credit.student_studentnumber,
       creditTypeCode: credit.credittypecode,
       courseName: courseCodeToName.get(credit.course_code)!,
@@ -97,6 +117,15 @@ export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, p
     const notCompletedByCourseCodes = new Map<string, Set<string>>()
     const courses = new Map<string, { name: Name; isStudyModule: boolean }>()
 
+    const studentHasPassedCourse = (studentNumber: string, courseCode: string) =>
+      !!credits.find(
+        credit =>
+          credit.student_studentnumber === studentNumber &&
+          credit.course_code === courseCode &&
+          CreditModel.passed(credit)
+      ) ||
+      (passedByCourseCodes.get(courseCode)?.has(studentNumber) ?? false)
+
     for (const { code, courseName, creditTypeCode, isStudyModule, studentNumber } of creditCourses) {
       if (!courses.has(code)) {
         courses.set(code, {
@@ -106,29 +135,38 @@ export const getNotCompletedForProgrammeCourses = async (from: Date, to: Date, p
         passedByCourseCodes.set(code, new Set())
         notCompletedByCourseCodes.set(code, new Set())
       }
-      if ([CreditTypeCode.PASSED, CreditTypeCode.IMPROVED, CreditTypeCode.APPROVED].includes(creditTypeCode)) {
+
+      if (CreditModel.passed({ credittypecode: creditTypeCode })) {
         passedByCourseCodes.get(code)!.add(studentNumber)
       }
-      if (creditTypeCode === CreditTypeCode.FAILED && !passedByCourseCodes.get(code)!.has(studentNumber)) {
+      if (creditTypeCode === CreditTypeCode.FAILED && !studentHasPassedCourse(studentNumber, code)) {
         notCompletedByCourseCodes.get(code)!.add(studentNumber)
       }
     }
 
-    // If student has enrollments, but no attainment for a particular course, they have no credit info.
-    programmeCourses.forEach(courseCode => {
-      allEnrollments.get(courseCode)?.forEach(studentnumber => {
-        if (passedByCourseCodes.get(courseCode)?.has(studentnumber)) {
-          notCompletedByCourseCodes.get(courseCode)?.add(studentnumber)
-        }
-      })
-    })
+    // Add course details to courseMap if there are only enrollments
+    for (const { studentnumber: studentNumber, course_code: courseCode, course } of enrollmentsCourses) {
+      if (!courses.has(courseCode)) {
+        courses.set(courseCode, {
+          isStudyModule: course.is_study_module,
+          name: courseCodeToName.get(courseCode)!,
+        })
+        passedByCourseCodes.set(courseCode, new Set())
+        notCompletedByCourseCodes.set(courseCode, new Set())
+      }
 
-    return [...courses.entries()].map(([code, { isStudyModule, name }]) => ({
-      code,
-      name,
-      isStudyModule,
-      allNotPassed: notCompletedByCourseCodes.get(code)?.size ?? 0,
-    }))
+      if (!studentHasPassedCourse(studentNumber, courseCode)) {
+        notCompletedByCourseCodes.get(courseCode)!.add(studentNumber)
+      }
+    }
+    return [...courses.entries()].map(([code, { isStudyModule, name }]) => {
+      return {
+        code,
+        name,
+        isStudyModule,
+        allNotPassed: notCompletedByCourseCodes.get(code)?.size ?? 0,
+      }
+    })
   } catch (error) {
     logger.error(`getNotCompletedForProgrammeCourses failed ${error}`)
     return []

services/backend/src/services/studyProgramme/studentGetters.ts

@@ -95,7 +95,7 @@ export const getProgrammeCourseAggregates = async (params: {
           WHERE
             cr.course_code IN (:courseCodes)
             AND cr.attainment_date BETWEEN :from AND :to
-            AND cr.credittypecode = :creditType
+            AND cr.credittypecode IN (:creditTypes)
         )
         SELECT
           fc."courseCode",
@@ -176,7 +176,7 @@ export const getProgrammeCourseAggregates = async (params: {
           courseCodes,
           from,
           to,
-          creditType: CreditTypeCode.PASSED,
+          creditTypes: [CreditTypeCode.PASSED, CreditTypeCode.APPROVED],
           degreeExtentCodes,
           otherUniversityExtentCodes,
           separateStudyExtentCodes,

services/backend/src/services/studyProgramme/studyProgrammeCoursesBySemester.ts

@@ -35,7 +35,7 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
     attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'state'],
     where: {
       course_code: { [Op.in]: courseCodes },
-      semestercode: { [Op.gte]: autumnSemester2017 },
+      semestercode: { [Op.gte]: autumnSemester2017 }, // It is ok to use enrollments before OK sisu-migration, ColorizedTableData doesn't need it
       state: { [Op.in]: [EnrollmentState.ENROLLED, EnrollmentState.REJECTED] },
     },
     raw: true,

services/backend/src/services/studyProgramme/studyProgrammeGraduations.ts

@@ -3,7 +3,7 @@ import isSameOrAfter from 'dayjs/plugin/isSameOrAfter'
 import { indexOf, orderBy } from 'lodash-es'
 
 import {
-  GraduationStatistics,
+  GraduationTimeCategories,
   Name,
   SemesterEnrollment,
   StudyTrack,
@@ -118,7 +118,7 @@ export type GraduationTimes = {
     y: number
     amount: number
     name: string | number
-    statistics: GraduationStatistics
+    statistics: GraduationTimeCategories
     times: number[]
   }>
   goal: number

services/backend/src/services/studyProgramme/studyRightFinders.ts

@@ -65,7 +65,7 @@ export const getStudyRightsInProgramme = async (
         },
       },
     })
-  ).map(studyRight => studyRight.toJSON())
+  ).map(studyRight => studyRight.toJSON<SISStudyRight>())
 }
 
 export const getStudyTracksForProgramme = async (studyProgramme: string) => {

services/backend/src/util/index.ts

@@ -106,5 +106,5 @@ export const safeJSONParse = async <T>(json: string): Promise<T | null> => {
 
 // NOTE: Make sure we have only arrays, "/...&years=2021" is parsed by express as years: 2021, not [2021]
 export const handleQueryArrays = <T>(requestField: T | T[]): T[] => {
-  return Array.isArray(requestField) ? requestField : [requestField]
+  return requestField ? (Array.isArray(requestField) ? requestField : [requestField]) : []
 }

services/backend/src/util/logger.ts

@@ -1,6 +1,4 @@
-import os from 'os'
 import * as winston from 'winston'
-import { WinstonGelfTransporter } from 'winston-gelf-transporter'
 import LokiTransport from 'winston-loki'
 
 import { isProduction } from '../config'
@@ -11,18 +9,7 @@ transports.push(
   new LokiTransport({
     host: 'http://loki-svc.toska-lokki.svc.cluster.local:3100',
     labels: { app: 'oodikone', environment: process.env.NODE_ENV ?? 'production' },
-  }),
-  new WinstonGelfTransporter({
-    handleExceptions: true,
-    host: 'svm-116.cs.helsinki.fi',
-    port: 9503,
-    protocol: 'udp',
-    hostName: os.hostname(),
-    additional: {
-      app: 'oodikone',
-      environment: 'production',
-    },
-  }) as winston.transport
+  })
 )
 
 const { colorize, combine, timestamp, printf } = winston.format

services/backend/src/worker/queue.js

@@ -1,92 +0,0 @@
-const { Queue, FlowProducer } = require('bullmq')
-
-const { redis } = require('../config')
-
-const connection = {
-  host: redis,
-  port: 6379,
-}
-
-const defaultJobOptions = {
-  attempts: 3,
-  removeOnComplete: true,
-  removeOnFail: true,
-  backoff: {
-    type: 'exponential',
-    delay: 60 * 1000,
-  },
-}
-
-const queueName = 'refresh-redis-data'
-
-const queue = new Queue(queueName, {
-  connection,
-  defaultJobOptions,
-})
-
-const flowProducer = new FlowProducer({
-  connection,
-})
-
-const addJob = (type, data, keep) => {
-  // job name (first arg) makes the job unique, and we want unique based on faculty/programme code
-  const name = data?.code ? `${type}-${data.code}` : type
-  queue.add(name, { ...data }, { jobId: name, keepJobs: keep ?? 0 })
-}
-
-const getJobs = async type => {
-  let jobs = []
-  if (type === 'waiting') {
-    jobs = await queue.getWaiting()
-  } else if (type === 'active') {
-    jobs = await queue.getActive()
-  }
-  return jobs
-}
-
-const removeWaitingJobs = async () => {
-  await queue.drain()
-}
-
-const jobMaker = {
-  faculty: code => addJob('faculty', { code }),
-  programme: code => addJob('programme', { code }),
-  teacherLeaderboard: () => addJob('teacherLeaderboard'),
-  languagecenter: () => addJob('languagecenter'),
-  studyplansUpdate: days => addJob('studyplansUpdate', { days }, 7200),
-  closeToGraduation: () => addJob('closeToGraduation'),
-}
-
-const addToFlow = async (facultyCode, programmeCodes) => {
-  await flowProducer.add({
-    name: `faculty-${facultyCode}`,
-    data: {
-      code: facultyCode,
-    },
-    queueName,
-    children: programmeCodes.map(programmeCode => ({
-      name: `programme-${programmeCode}`,
-      data: {
-        code: programmeCode,
-      },
-      queueName,
-      opts: {
-        jobId: `programme-${programmeCode}`,
-        ...defaultJobOptions,
-      },
-    })),
-    opts: {
-      jobId: `faculty-${facultyCode}`,
-      ...defaultJobOptions,
-    },
-  })
-}
-
-module.exports = {
-  jobMaker,
-  getJobs,
-  removeWaitingJobs,
-  defaultJobOptions,
-  queueName,
-  addToFlow,
-}

services/backend/src/worker/queue.ts

@@ -0,0 +1,98 @@
+import { Queue, FlowProducer, BaseJobOptions } from 'bullmq'
+import { redis } from '../config'
+
+const connection = {
+  host: redis,
+  port: 6379,
+}
+
+const defaultJobOptions = {
+  attempts: 3,
+  removeOnComplete: true,
+  removeOnFail: true,
+  backoff: {
+    type: 'exponential',
+    delay: 60 * 1000,
+  },
+}
+
+/**
+ * Pushes jobs to redis for a worker to pick them up
+ */
+class JobQueue {
+  #queue: Queue
+  #flowProducer: FlowProducer
+  name = 'refresh-redis-data'
+
+  constructor() {
+    this.#queue = new Queue(this.name, {
+      connection,
+      defaultJobOptions,
+    })
+    this.#flowProducer = new FlowProducer({
+      connection,
+    })
+  }
+
+  #add(type: string, data?: any, jobOpts?: Partial<BaseJobOptions>) {
+    // name makes the job unique, and we want uniqueness based on faculty/programme code
+    const name = data?.code ? `${type}-${data.code}` : type
+    void this.#queue.add(name, { ...data }, { ...jobOpts, jobId: name })
+  }
+
+  async clearWaitingJobs() {
+    await this.#queue.drain()
+  }
+
+  async getJobs() {
+    const [waiting, active] = await Promise.all([this.#queue.getWaiting(), this.#queue.getActive()])
+    return { waiting, active }
+  }
+
+  refreshFaculty(code: string) {
+    this.#add('faculty', { code })
+  }
+  refreshProgramme(code: string) {
+    this.#add('programme', { code })
+  }
+  refreshTeacherLeaderboard() {
+    this.#add('teacherLeaderboard')
+  }
+  refreshLanguagecenter() {
+    this.#add('languagecenter')
+  }
+  refreshStudyplans(days: number) {
+    this.#add('studyplansUpdate', { days })
+  }
+  refreshCloseToGraduation() {
+    this.#add('closeToGraduation')
+  }
+
+  /** Flow ensures all child jobs (programmes) are processed before the parent (faculty) */
+  async addToFlow(facultyCode: string, programmeCodes: string[]) {
+    await this.#flowProducer.add({
+      name: `faculty-${facultyCode}`,
+      data: {
+        code: facultyCode,
+      },
+      queueName: this.name,
+      children: programmeCodes.map(programmeCode => ({
+        name: `programme-${programmeCode}`,
+        data: {
+          code: programmeCode,
+        },
+        queueName: this.name,
+        opts: {
+          jobId: `programme-${programmeCode}`,
+          ...defaultJobOptions,
+        },
+      })),
+      opts: {
+        jobId: `faculty-${facultyCode}`,
+        ...defaultJobOptions,
+      },
+    })
+  }
+}
+
+export const jobQueue = new JobQueue()

services/backend/src/worker/worker.ts

@@ -1,10 +1,10 @@
-const Sentry = require('@sentry/node')
-const { Worker: BullMQWorker } = require('bullmq')
-const dayjs = require('dayjs')
+import * as Sentry from '@sentry/node'
+import { Worker as BullMQWorker } from 'bullmq'
+import dayjs from 'dayjs'
 
-const { redis, concurrentWorkers } = require('../config')
-const logger = require('../util/logger').default
-const { queueName } = require('./queue')
+import { redis, concurrentWorkers } from '../config'
+import logger from '../util/logger'
+import { jobQueue } from './queue'
 
 const connection = {
   host: redis,
@@ -14,7 +14,7 @@ const connection = {
 // https://github.com/taskforcesh/bullmq/issues/2075#issuecomment-1646079335
 process.execArgv = process.execArgv.filter(arg => !arg.includes('--max_old_space_size='))
 
-const worker = new BullMQWorker(queueName, `${__dirname}/processor.js`, {
+const worker = new BullMQWorker(jobQueue.name, `${__dirname}/processor.js`, {
   connection,
   useWorkerThreads: true,
   concurrency: concurrentWorkers,
@@ -38,7 +38,7 @@ worker.on('error', error => {
 })
 
 worker.on('failed', (job, error) => {
-  logger.error(`Job ${job.id} failed. ${error.stack ?? ''}`)
+  logger.error(`Job ${job?.id ?? 'unknown id'} failed. ${error.stack ?? ''}`)
   Sentry.captureException(error)
 })
 

services/backend/tests/courses/courseyearlystats.test.ts

@@ -0,0 +1,215 @@
+import { Express } from 'express'
+import assert from 'node:assert/strict'
+import { describe, it, before, after } from 'node:test'
+import { Sequelize } from 'sequelize'
+import request from 'supertest'
+
+import { dbConnections } from '../../src/database/connection'
+import { CourseYearlyStatsResBody } from '../../src/routes/courses'
+import { initTests, ResponseWithBody } from '../utils'
+
+let app: Express
+let connection: Sequelize
+before(async () => {
+  app = await initTests()
+  connection = dbConnections.sequelize
+})
+
+after(async () => {
+  await connection.close()
+})
+
+void describe('Course yearly statistics', async () => {
+  await it('should return nothing with missing parameters', async () => {
+    const res = await request(app)
+      .get('/courseyearlystats')
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')
+
+    assert.strictEqual(res.status, 422)
+    assert.strictEqual(res.body?.error, 'Missing required query parameters')
+  })
+
+  await it('should return correct amount of values for a course without substitutions', async t => {
+    const res = (await request(app)
+      .get('/courseyearlystats?codes=TKT10002&combineSubstitutions=false')
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
+
+    assert.strictEqual(res.status, 200)
+    assert.strictEqual(res.body.length, 1, 'Query to return anything')
+    const body = res.body.at(0)!
+    assert(
+      'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
+      'All keys of courseyearlystats not defined'
+    )
+
+    await t.test('(statistic years)', () => {
+      assert.strictEqual(body.unifyStats!.statistics.length, 8, 'unifyStats years')
+      assert.strictEqual(body.regularStats!.statistics.length, 8, 'regularStats years')
+      assert.strictEqual(body.openStats!.statistics.length, 4, 'openStats years')
+
+      // Faculties should have the same amount of years as normal statistics (above)
+      assert.strictEqual(Object.keys(body.unifyStats!.facultyStats).length, 8, 'unifyStats years')
+      assert.strictEqual(Object.keys(body.regularStats!.facultyStats).length, 8, 'regularStats years')
+      assert.strictEqual(Object.keys(body.openStats!.facultyStats).length, 4, 'openStats years')
+    })
+
+    await t.test('(statistic contents)', async tt => {
+      await tt.test('- 2016-2017', () => {
+        const stats2016 = body.unifyStats!.statistics.find(year => year.name === '2016-2017')!
+        assert.deepStrictEqual(stats2016.students.studentNumbers, ['484144'], 'Incorrect students marked to 2016 stats')
+
+        assert.deepStrictEqual(stats2016.attempts.grades['4'], ['484144'], 'Incorrect grades marked to 2016 stats')
+        assert.strictEqual(
+          stats2016.attempts.categories.passed.length,
+          1,
+          'Incorrect students marked as passed to 2016 stats'
+        )
+        assert.strictEqual(
+          stats2016.attempts.categories.failed.length,
+          0,
+          'Incorrect students marked as failed to 2016 stats'
+        )
+
+        assert('enrollments' in stats2016, 'Missing field enrollment in statsitics')
+        assert.strictEqual(stats2016.enrollments.length, 0, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2016.allEnrollments.length, 0, 'Enrollments found when none should exist')
+      })
+
+      await tt.test('- 2017-2018', () => {
+        const stats2017 = body.unifyStats!.statistics.find(year => year.name === '2017-2018')!
+        const { studentNumbers } = stats2017.students
+        assert(
+          !(
+            '509165' in studentNumbers ||
+            '455478' in studentNumbers ||
+            '457686' in studentNumbers ||
+            '547994' in studentNumbers
+          ),
+          "Students that have completed course later should not be included in the previous year's stats"
+        )
+        assert.strictEqual(stats2017.students.studentNumbers.length, 23, 'Incorrect amount of total students 2017')
+
+        assert.strictEqual(stats2017.attempts.grades['5'].length, 11, 'Incorrect grades marked to 2017 stats (5)')
+        assert.strictEqual(stats2017.attempts.grades['4'].length, 5, 'Incorrect grades marked to 2017 stats, (4)')
+        assert.strictEqual(stats2017.attempts.grades['3'].length, 2, 'Incorrect grades marked to 2017 stats (3)')
+        assert.strictEqual(stats2017.attempts.grades['2'].length, 1, 'Incorrect grades marked to 2017 stats (2)')
+        assert(!('1' in stats2017.attempts.grades), 'Incorrect grades marked to 2017 stats (1)')
+        assert.strictEqual(stats2017.attempts.grades['Hyl.'].length, 4, 'Incorrect grades marked to 2017 stats (Hyl.)')
+
+        assert(
+          !stats2017.attempts.grades['Hyl.'].includes('455478'),
+          "Specific student's failed grade is not marked to the correct year (should be only in passed 2020-2021)"
+        )
+
+        assert.strictEqual(
+          stats2017.attempts.categories.passed.length,
+          11 + 5 + 2 + 1 + 0,
+          'Incorrect students marked as passed to 2017 stats'
+        )
+        assert.strictEqual(
+          stats2017.attempts.categories.failed.length,
+          4,
+          'Incorrect students marked as failed to 2017 stats'
+        )
+
+        // These should not exist before 21-22
+        assert('enrollments' in stats2017, 'Missing field enrollment in statsitics')
+        assert.strictEqual(stats2017.enrollments.length, 0, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2017.allEnrollments.length, 0, 'Enrollments found when none should exist')
+      })
+
+      await tt.test('- 2018-2019', () => {
+        const stats2018 = body.unifyStats!.statistics.find(year => year.name === '2018-2019')!
+
+        // There are more students that have passed this course later even though they have failed the course during 2018-2019
+        assert.deepStrictEqual(
+          stats2018.attempts.categories.failed,
+          ['454128', '546124'],
+          'Incorrect students marked as failed 2018'
+        )
+
+        assert.strictEqual(stats2018.attempts.grades['0'].length, 2)
+        assert.strictEqual((stats2018.attempts.grades['Hyl.'] ?? []).length, 0)
+        assert.strictEqual(stats2018.attempts.categories.failed.length, 2, 'Incorrect amount of failed students 2018')
+      })
+
+      await tt.test('- 2020-2021', () => {
+        const stats2020 = body.unifyStats!.statistics.find(year => year.name === '2020-2021')!
+        assert(
+          stats2020.attempts.categories.passed.includes('455478'),
+          "Specific student's passed course is not marked to the correct year (passed)"
+        )
+        assert(
+          stats2020.attempts.grades['2'].includes('455478'),
+          "Specific student's passed course is not marked to the correct year (grade)"
+        )
+
+        // These should not exist before 21-22
+        assert('enrollments' in stats2020, 'Missing field enrollment in statsitics')
+        assert.strictEqual(stats2020.enrollments.length, 0, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2020.allEnrollments.length, 0, 'Enrollments found when none should exist')
+      })
+
+      // TODO: This should be fixed by course_unit_realisation
+      await tt.test('- 2022-2023', () => {
+        const stats2022 = body.unifyStats!.statistics.find(year => year.name === '2022-2023')!
+        assert.strictEqual(stats2022.students.studentNumbers.length, 27, 'Incorrect amount of total students 2022')
+
+        assert('enrollments' in stats2022, 'Missing field enrollment in statsitics')
+        // Student with enrollment_date_time outside of semestert start and end dates
+        assert(
+          !!stats2022.enrollments.find(enrollment => enrollment.studentNumber === '455129'),
+          'Student with incorrect semester (by updater) but a correct enrollment_date was not found in the correct year (by enrollment_date)'
+        )
+        assert.strictEqual(stats2022.enrollments.length, 34, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2022.allEnrollments.length, 43, 'Enrollments found when none should exist')
+      })
+
+      await tt.test('- 2023-2024', () => {
+        const stats2023 = body.unifyStats!.statistics.find(year => year.name === '2023-2024')!
+        assert.strictEqual(stats2023.students.studentNumbers.length, 1, 'Incorrect amount of total students 2023')
+
+        assert.deepStrictEqual(
+          stats2023.attempts.categories.passed,
+          ['521757'],
+          'Incorrect students marked as passed in 2023 stats'
+        )
+        assert.deepStrictEqual(
+          stats2023.attempts.grades['5'],
+          ['521757'],
+          "Correct student doesn't exist under correct grade"
+        )
+
+        assert('enrollments' in stats2023, 'Missing field enrollment in statsitics')
+        assert.strictEqual(stats2023.enrollments.length, 7, 'Enrollments found when none should exist')
+        assert.strictEqual(stats2023.allEnrollments.length, 7, 'Enrollments found when none should exist')
+      })
+    })
+
+    await it.todo('should return correct amount of students for a course with substitutions')
+    await it.todo('should return correct amount of students for multiple coursecodes without substitutions')
+    await it.todo('should return correct amount of students for multiple coursecodes with substitutions')
+
+    await it('should return correct amontu of students for specific problematic courses', async () => {
+      const res = (await request(app)
+        .get('/courseyearlystats?codes=MAT11003&combineSubstitutions=false')
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
+
+      assert.strictEqual(res.status, 200)
+      assert.strictEqual(res.body.length, 1, 'Query to return anything')
+      const body = res.body.at(0)!
+      assert(
+        'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
+        'All keys of courseyearlystats not defined'
+      )
+
+      // assert.strictEqual()
+    })
+  })
+})

services/backend/tests/population/populationStatistics.test.ts

@@ -2,11 +2,11 @@ import { Express } from 'express'
 import assert from 'node:assert/strict'
 import { describe, it, before } from 'node:test'
 import { Sequelize } from 'sequelize'
-import request, { Response } from 'supertest'
+import request from 'supertest'
 
 import { PopulationstatisticsResBody } from '@oodikone/shared/routes/populations'
 import { dbConnections } from '../../src/database/connection'
-import { initTests } from '../utils'
+import { initTests, ResponseWithBody } from '../utils'
 
 let app: Express
 let connection: Sequelize
@@ -15,9 +15,6 @@ before(async () => {
   connection = dbConnections.sequelize
 })
 
-// Override Supertest's Response body with our own type
-type ResponseWithBody<T> = Omit<Response, 'body'> & { body: T }
-
 const populationUrl = (
   programme = '',
   years: string[] = [],
@@ -28,28 +25,35 @@ const populationUrl = (
 }
 
 void describe('Population statistics', async () => {
-  await it('should fail when all fields not defined', async () => {
-    const resAllMissing = await request(app)
-      .get(populationUrl())
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')
+  await it('should fail when all fields not defined', async t => {
+    await t.test('(all)', async () => {
+      const resAllMissing = await request(app)
+        .get(populationUrl())
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')
+      assert.strictEqual(resAllMissing.status, 400)
+    })
 
-    const resProgramme = await request(app)
-      .get(populationUrl('KH50_001'))
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')
+    await t.test('(all but code)', async () => {
+      const resProgramme = await request(app)
+        .get(populationUrl('KH50_001'))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')
 
-    const resProgrammeAndYears = await request(app)
-      .get(populationUrl('KH50_001', ['2021', '2022']))
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')
+      assert.strictEqual(resProgramme.status, 400)
+    })
 
-    assert.strictEqual(resAllMissing.status, 400)
-    assert.strictEqual(resProgramme.status, 400)
-    assert.strictEqual(resProgrammeAndYears.status, 400)
+    await t.test('(all but code and years)', async () => {
+      const resProgrammeAndYears = await request(app)
+        .get(populationUrl('KH50_001', ['2021', '2022']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')
+
+      assert.strictEqual(resProgrammeAndYears.status, 400)
+    })
   })
 
   await it('should not return any data to unauthorized user', async () => {
@@ -85,85 +89,99 @@ void describe('Population statistics', async () => {
     assert.notStrictEqual(res.body.students.length, 0)
   })
 
-  await it('should work with the programme flag', async () => {
-    const resBachelor = (await request(app)
-      .get(populationUrl('KH50_001', ['2021'], ['SPRING', 'FALL']))
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+  await it('should work with the programme flag', async t => {
+    await t.test('(bachelor)', async () => {
+      const resBachelor = (await request(app)
+        .get(populationUrl('KH50_001', ['2021'], ['SPRING', 'FALL']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
 
-    const resMaster = (await request(app)
-      .get(populationUrl('MH50_001', ['2021'], ['SPRING', 'FALL']))
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+      assert.strictEqual(resBachelor.status, 200)
+      assert.strictEqual(resBachelor.body.students.length, 38)
+    })
 
-    assert.strictEqual(resBachelor.status, 200)
-    assert.strictEqual(resBachelor.body.students.length, 38)
+    await t.test('(master)', async () => {
+      const resMaster = (await request(app)
+        .get(populationUrl('MH50_001', ['2021'], ['SPRING', 'FALL']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
 
-    assert.strictEqual(resMaster.status, 200)
-    assert.strictEqual(resMaster.body.students.length, 36)
+      assert.strictEqual(resMaster.status, 200)
+      assert.strictEqual(resMaster.body.students.length, 36)
+    })
   })
 
-  await it('should work with semester flag', async () => {
-    const resFall = (await request(app)
-      .get(populationUrl('KH50_001', ['2021'], ['FALL']))
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+  await it('should work with semester flag', async t => {
+    await t.test('(FALL)', async () => {
+      const resFall = (await request(app)
+        .get(populationUrl('KH50_001', ['2021'], ['FALL']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
 
-    const resSpring = (await request(app)
-      .get(populationUrl('KH50_001', ['2021'], ['SPRING']))
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+      assert.strictEqual(resFall.status, 200)
+      assert.strictEqual(resFall.body.students.length, 34)
+    })
 
-    assert.strictEqual(resFall.status, 200)
-    assert.strictEqual(resSpring.status, 200)
+    await t.test('(SPRING)', async () => {
+      const resSpring = (await request(app)
+        .get(populationUrl('KH50_001', ['2021'], ['SPRING']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
 
-    assert.strictEqual(resFall.body.students.length, 34)
-    assert.strictEqual(resSpring.body.students.length, 4)
+      assert.strictEqual(resSpring.status, 200)
+      assert.strictEqual(resSpring.body.students.length, 4)
+    })
   })
 
-  await it('should work with year flag correctly', async () => {
-    const res2021 = (await request(app)
-      .get(populationUrl('KH50_001', ['2021'], ['FALL', 'SPRING']))
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
-    const res2022 = (await request(app)
-      .get(populationUrl('KH50_001', ['2022'], ['FALL', 'SPRING']))
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+  await it('should work with year flag correctly', async t => {
+    await t.test('(2021)', async () => {
+      const res2021 = (await request(app)
+        .get(populationUrl('KH50_001', ['2021'], ['FALL', 'SPRING']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
 
-    const res2021_2022 = (await request(app)
-      .get(populationUrl('KH50_001', ['2021', '2022'], ['FALL', 'SPRING']))
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+      const dbCount2021 = await connection.query(
+        `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2021-07-31' and "sis_study_right_elements".start_date < '2022-08-01'`,
+        { raw: true, plain: true }
+      )
+      assert.strictEqual(res2021.status, 200)
+      assert.strictEqual(res2021.body.students.length, dbCount2021?.count)
+    })
 
-    const dbCount2021 = await connection.query(
-      `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2021-07-31' and "sis_study_right_elements".start_date < '2022-08-01'`,
-      { raw: true, plain: true }
-    )
-    const dbCount2022 = await connection.query(
-      `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2022-07-31' and "sis_study_right_elements".start_date < '2023-08-01'`,
-      { raw: true, plain: true }
-    )
-    const dbCount2021_2022 = await connection.query(
-      `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2021-07-31' and "sis_study_right_elements".start_date < '2023-08-01'`,
-      { raw: true, plain: true }
-    )
+    await t.test('(2022)', async () => {
+      const res2022 = (await request(app)
+        .get(populationUrl('KH50_001', ['2022'], ['FALL', 'SPRING']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
+      const dbCount2022 = await connection.query(
+        `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2022-07-31' and "sis_study_right_elements".start_date < '2023-08-01'`,
+        { raw: true, plain: true }
+      )
 
-    assert.strictEqual(res2021.status, 200)
-    assert.strictEqual(res2022.status, 200)
+      assert.strictEqual(res2022.status, 200)
+      assert.strictEqual(res2022.body.students.length, dbCount2022?.count)
+    })
 
-    assert.strictEqual(res2021.body.students.length, dbCount2021?.count)
-    assert.strictEqual(res2022.body.students.length, dbCount2022?.count)
+    await t.test('(2021-2022)', async () => {
+      const res2021_2022 = (await request(app)
+        .get(populationUrl('KH50_001', ['2021', '2022'], ['FALL', 'SPRING']))
+        .set('shib-session-id', 'test')
+        .set('uid', 'basic')
+        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<PopulationstatisticsResBody>
 
-    assert.strictEqual(res2021_2022.status, 200)
-    assert.strictEqual(res2021_2022.body.students.length, dbCount2021_2022?.count)
+      const dbCount2021_2022 = await connection.query(
+        `select count(student_number)::integer from "sis_study_right_elements" join "sis_study_rights" on study_right_id = "sis_study_rights".id  where code = 'KH50_001' and "sis_study_right_elements".start_date > '2021-07-31' and "sis_study_right_elements".start_date < '2023-08-01'`,
+        { raw: true, plain: true }
+      )
+      assert.strictEqual(res2021_2022.status, 200)
+      assert.strictEqual(res2021_2022.body.students.length, dbCount2021_2022?.count)
+    })
   })
 
   await it.skip('should return only students who have transferred out of the programme when TRANSFERRED flag is set', async () => {

services/backend/tests/population/populationStatisticsByStudentNumbers.ts

@@ -13,11 +13,12 @@ before(async () => {
 void describe('Population statistics by student number', async () => {
   await it('should not return anything with missing parameters', async () => {
     const res = await request(app)
-      .get('/populationstatisticsbystudentnumbers')
+      .post('/populationstatisticsbystudentnumbers')
       .set('shib-session-id', 'test')
       .set('uid', 'basic')
       .set('hygroupcn', 'grp-oodikone-basic-users')
 
-    assert.deepEqual(res.status, 400)
+    assert.deepEqual(res.status, 422)
+    assert.deepEqual(res.body.error, 'Body should include student numbers')
   })
 })

services/backend/tests/studyProgramme/studyProgrammeCourseStats.test.ts

@@ -0,0 +1,127 @@
+import { Express } from 'express'
+import assert from 'node:assert/strict'
+import { describe, it, before, after } from 'node:test'
+import { Sequelize } from 'sequelize'
+import request from 'supertest'
+
+import { StudyProgrammeCourse } from '@oodikone/shared/types'
+import { dbConnections } from '../../src/database/connection'
+import { initTests, ResponseWithBody } from '../utils'
+
+let app: Express
+let connection: Sequelize
+before(async () => {
+  app = await initTests()
+  connection = dbConnections.sequelize
+})
+
+after(async () => {
+  await connection.close()
+})
+
+void describe("Study programme's course stats", async () => {
+  await it('should return data for correct courses', async () => {
+    const res = (await request(app)
+      .get('/studyprogrammes/KH50_001/coursestats?yearType=ACADEMIC_YEAR')
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<StudyProgrammeCourse[]>
+
+    assert.strictEqual(res.status, 200)
+    assert.strictEqual(res.body.length, 102, 'Response included incorrect amount of courses') // Not checked
+
+    const courseCodes = res.body.map(({ code }) => code)
+    assert(
+      ['MAT11003', 'MAT11001', 'MAT11004', 'AYMAT11003', 'AYMAT11001', 'AYMAT11004'].every(code =>
+        courseCodes.includes(code)
+      ),
+      'Required courses or their open uni variants missing'
+    )
+
+    assert(courseCodes.includes('MAT11015'), 'Course with no passing grades missing from statistics')
+  })
+
+  await it('should return correct data for a course', async t => {
+    const res = (await request(app)
+      .get('/studyprogrammes/KH50_001/coursestats?yearType=ACADEMIC_YEAR')
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<StudyProgrammeCourse[]>
+
+    // Raja-arvot, MAT11003
+    assert.strictEqual(res.status, 200)
+    assert(!!res.body.find(course => course.code === 'MAT11003'), "Didn't find required course")
+    const course = res.body.find(course => course.code === 'MAT11003')!
+
+    assert.deepStrictEqual(
+      Object.keys(course.years),
+      ['2017', '2018', '2019', '2020', '2021', '2022', '2023'],
+      'Years should also include years where no credits were completed but enrollments were made'
+    )
+    assert.strictEqual(Object.keys(course.years).length, 7, 'Incorrect amount of years')
+
+    await t.test('- 2017-2018', () => {
+      const year = course.years['2017']
+      assert.strictEqual(year.allPassed, 44, 'Incorrect amount of passed students (2017)')
+      assert.strictEqual(year.allNotPassed, 0, 'Stats should not include failed or not enrolled students (2017)')
+    })
+
+    await t.test('- 2018-2019', () => {
+      const year = course.years['2018']
+      assert.strictEqual(year.allPassed, 70, 'Incorrect amount of passed students (2018)')
+      assert.strictEqual(year.allNotPassed, 0, 'Stats should not include failed or not enrolled students (2018)')
+    })
+
+    await t.test('- 2019-2020', () => {
+      const year = course.years['2019']
+      assert.strictEqual(year.allPassed, 42, 'Incorrect amount of passed students (2019)')
+      assert.strictEqual(year.allNotPassed, 1, 'Stats should include one failed student (2019)')
+    })
+
+    await t.test('- 2020-2021', () => {
+      const year = course.years['2020']
+      assert.strictEqual(year.allPassed, 33, 'Incorrect amount of passed students (2020)')
+      assert.strictEqual(year.allNotPassed, 0, 'Stats should include one failed student (2020)')
+    })
+
+    await t.test('- 2021-2022', () => {
+      const year = course.years['2021']
+      assert.strictEqual(year.allPassed, 40, 'Incorrect amount of passed students (2021)')
+      assert.strictEqual(year.allNotPassed, 6, 'Stats should include one failed student (2021)')
+    })
+
+    await t.test('- 2022-2023', () => {
+      const year = course.years['2022']
+      assert.strictEqual(year.allPassed, 20, 'Incorrect amount of passed students (2022)')
+      assert.strictEqual(year.allNotPassed, 5, 'Stats should include one failed student (2022)')
+    })
+
+    await t.test('- 2023-2024', () => {
+      const year = course.years['2023']
+      assert.strictEqual(year.allPassed, 0, 'Incorrect amount of passed students (2023)')
+      assert.strictEqual(year.allNotPassed, 10, 'Stats should include one failed student (2023)')
+    })
+
+    await t.test('- total', () => {
+      const years = Object.values(course.years)
+      assert.strictEqual(
+        years.reduce((acc, year) => acc + year.allPassed, 0),
+        249,
+        'Total number of passed students incorrect (total)'
+      )
+      assert.strictEqual(
+        years.reduce((acc, year) => acc + year.allNotPassed, 0),
+        21 + 1,
+        'Total number of failed or not enrolled students incorrect (total)'
+      )
+      assert.strictEqual(
+        years.reduce((acc, year) => acc + year.allStudents, 0),
+        271,
+        'Total number of students incorrect (total)'
+      )
+    })
+  })
+
+  // TODO: Move somewhere else
+  await it.todo('should return matching data to /courseyearlystats')
+})

services/backend/tests/utils.ts

@@ -1,9 +1,13 @@
 import express from 'express'
+import { Response } from 'supertest'
 
 import { baseUrl } from '../src/config'
 import { initializeDatabaseConnection, dbConnections } from '../src/database/connection'
 import routes from '../src/routes'
 
+/** Override Supertest's Response body with our own type */
+export type ResponseWithBody<T> = Omit<Response, 'body'> & { body: T }
+
 export const initTests = async () => {
   // Copied from "app.ts"
   await initializeDatabaseConnection()
@@ -11,10 +15,13 @@ export const initTests = async () => {
   void (await dbConnections.connect())
 
   dbConnections.on('connect', () => {
-    console.info('Connected to sis db successfully')
+    console.info('Connected to sis db successfully (TEST)')
   })
   dbConnections.on('error', () => {
-    console.error('Failed to connect to sis db!')
+    console.error('Failed to connect to sis db! (TEST)')
+  })
+  dbConnections.on('close', () => {
+    console.info('Closing connection... (TEST)')
   })
 
   const app = express()

services/frontend/.gitignore

@@ -3,4 +3,12 @@
 .pnp.js
 
 # production
-/build
+/build
+
+# Playwright
+node_modules/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
+/playwright/.auth/

services/frontend/Dockerfile

@@ -32,6 +32,7 @@ RUN npm install -g serve@^14
 
 WORKDIR /opt/app-root/shared
 COPY ./shared .
+RUN npm ci --include=dev
 
 WORKDIR /opt/app-root/frontend
 COPY ./frontend/.npmrc .

services/frontend/dev.Dockerfile

@@ -2,10 +2,16 @@ FROM registry.access.redhat.com/ubi9/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
 
-WORKDIR /opt/app-root/frontend
-COPY --chown=1001:1001 . .
-
 USER 1001
+
+WORKDIR /opt/app-root/shared
+COPY --chown=1001:1001 ./shared .
+
+RUN npm ci
+
+WORKDIR /opt/app-root/frontend
+COPY --chown=1001:1001 ./frontend .
+
 RUN npm ci
 
 EXPOSE 3000

services/frontend/package.json

@@ -6,7 +6,9 @@
     "start": "vite",
     "build": "tsc --noEmit && NODE_OPTIONS=\"--max-old-space-size=4096\" vite build",
     "preview": "vite preview",
-    "tsc": "tsc"
+    "tsc": "tsc",
+    "test:unit": "node --import tsx --test './tests/unit/**/*test.ts' --test-reporter spec --test-force-exit",
+    "test:component": "playwright test -c playwright-ct.config.ts"
   },
   "dependencies": {
     "@emotion/react": "^11.14.0",
@@ -37,11 +39,13 @@
     "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz"
   },
   "devDependencies": {
+    "@playwright/experimental-ct-react": "1.61.0",
     "@types/lodash-es": "^4.17.12",
     "@types/node": "^24.10.1",
     "@types/react": "^18.3.23",
     "@types/react-dom": "^18.3.7",
     "@vitejs/plugin-react-swc": "^4.2.3",
+    "tsx": "4.22.4",
     "typescript": "^5.9.2",
     "vite": "^7.3.1"
   }

services/frontend/playwright-ct.config.ts

@@ -0,0 +1,60 @@
+import { defineConfig, devices } from '@playwright/experimental-ct-react'
+import path from 'path'
+import { fileURLToPath } from 'url'
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './tests',
+  /* The base directory, relative to the config file, for snapshot files created with toMatchSnapshot and toHaveScreenshot. */
+  snapshotDir: './tests/__snapshots__',
+  /* Maximum time one test can run for. */
+  timeout: 10 * 1000,
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: process.env.CI ? 'dot' : [['list', { printSteps: true }]],
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+
+    /* Port to use for Playwright component endpoint. */
+    ctPort: 3100,
+
+    /* Fix import from "@/xyz"*/
+    ctViteConfig: {
+      resolve: {
+        alias: {
+          '@': path.resolve(fileURLToPath(import.meta.url), '../src'),
+        },
+      },
+    },
+  },
+
+  // Don't run unit tests with playwright
+  testIgnore: '*.test.ts',
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] },
+    },
+  ],
+})

services/frontend/playwright/index.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Testing Page</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="./index.tsx"></script>
+  </body>
+</html>

services/frontend/playwright/index.tsx

@@ -0,0 +1,2 @@
+// Import styles, initialize component theme here.
+// import '../src/common.css';

services/frontend/src/common/InfoToolTips/faculty.ts

@@ -22,6 +22,7 @@ Mukana ovat oletusarvoisesti myös ohjelmaan siirtyneet.
 `,
     programmeToggle: `
 **New degree programmes**: Näyttää kaikissa luvuissa ja kuvaajissa pelkästään uuden opetussuunnitelman ohjelmat.
+
 **All degree programmes**: Näyttää kaikissa luvuissa ja kuvaajissa kuvaajissa myös vanhan opetussuunnitelman ohjelmat.
 `,
     averageGraduationTimes: `
@@ -33,14 +34,12 @@ Opiskelijoiden keskimääräiset valmistumisajat tutkintotasoittain.
 - **Doctor**: Tohtorin tutkinnon suorittaneet opiskelijat
 
 Valmistumisajoista on **vähennetty lakisääteiset poissaolot**.
-Luvuissa ei ole mukana opiskelijoita, jotka ovat vaihtaneet koulutuohjelmaa saman opiskeluoikeuden sisällä (pois lukien kandiohjelmasta maisteriohjelmaan siirtyneet).
-Toisin sanoen luvut vastaavat *Degree programme overview* -näkymän lukuja, kun näkymässä on valittuna *Special study rights excluded*.
 
 - **Breakdown**/**Median study times**:
   - **Breakdown**: Näyttää, kuinka moni opiskelija valmistui tavoiteajassa (vihreä palkki), kahden lukukauden sisällä tavoiteajasta (keltainen palkki) tai tätä myöhemmin (punainen palkki).
   - **Median study times**: Näyttää kyseisenä kalenteri- tai lukuvuonna aloittaneiden/valmistuneiden opiskelijoiden opintojen keston **mediaanin** lukukausina. Vihreä **katkoviiva** kuvastaa tavoiteaikaa. Keltainen katkoviiva on tavoiteaika + 2 lukukautta.
 - **Graduation year**/**Starting year**:
-  - **Graduation year**: Ryhmittelee opiskelijat valmistumisvuosittain (kalenterivuosi).
+  - **Graduation year**: Ryhmittelee opiskelijat valmistumisvuosittain (kalenterivuosi). Sisältää ohjelmaan siirtyneet opiskelijat.
   - **Starting year**: Ryhmittelee opiskelijat opiskeluoikeuden alkamisvuoden (lukuvuosi) perusteella. Bachelor + Master -kuvaajassa aloitusvuosi on aloitusvuosi kandiohjelmassa ja koulutusohjelma on maisteriohjelma, johon opiskelija on siirtynyt kandidaatiksi valmistumisen jälkeen.
 `,
   },

services/frontend/src/common/InfoToolTips/populationStudents.tsx

@@ -34,7 +34,7 @@ Credits and courses that
 Time passed since starting in the programme until graduation, excluding allowed absences (unlimited statutory and 2 non-statutory absences). Each unique calendar month increments the amount.
 
 **Example:**  
-from 31st of January to 1st of March = 3 months
+from 31st of January to 1st of March = 3 months  
 from 1st of January to 30th of March = 3 months`,
     programmes:
       'If a student has more than one programme, hover your mouse on the cell to view the rest. They are also displayed in the exported Excel file.',

services/frontend/src/common/InfoToolTips/studyProgramme.ts

@@ -142,7 +142,7 @@ Jos sivun yläosassa on valittuna ”All study rights” (oletus), mukana ovat m
       fi: `
 Näyttää valittuna akateemisena vuonna aloittaneiden opiskelijoiden kumulatiiviset opintopistokertymät persentiileittäin.
 
-Kuvaajiin piirtyy viivoja kuvaamaan kyseisen ohjelman tavoitevalmistumisaikoja ja opintopistetavoitteita:
+Kuvaajiin piirtyy viivoja kuvaamaan kyseisen ohjelman tavoitevalmistumisaikoja, opintopistetavoitteita ja vuosittaista opintopistekertymätavoitetta (60 op per lukuvuosi):
 
 - **Kandiohjelmat**: yleensä 3 vuotta ja 180 op
 - **Maisteriohjelmat**: yleensä 2 vuotta ja 120 op
@@ -157,7 +157,7 @@ Maisteriohjelmia tarkastellessa näytetään lisäksi kandi+maisteri -oikeudella
       en: `
 Shows the cumulative credit totals of students who began their studies in the selected academic year, broken down by percentile.
 
-Lines are drawn on the graphs to illustrate the target completion times and credit targets for the respective programs:
+Lines are drawn on the graphs to illustrate the target completion times, credit targets for the respective programs, and the yearly credit gain target (60 cr per academic year):
 
 - **Bachelor’s programs**: typically 3 years and 180 credits
 - **Master’s programs**: typically 2 years and 120 credits

services/frontend/src/common/index.ts

@@ -1,12 +1,8 @@
 import { Theme, useTheme } from '@mui/material/styles'
-import dayjs, { Dayjs, extend as dayjsExtend } from 'dayjs'
-import isBetween from 'dayjs/plugin/isBetween'
-import isSameOrAfter from 'dayjs/plugin/isSameOrAfter'
-import isSameOrBefore from 'dayjs/plugin/isSameOrBefore'
+import dayjs from 'dayjs'
 import { maxBy, orderBy } from 'lodash-es'
 
 import { GetTextIn } from '@/components/LanguagePicker/useLanguage'
-import type { SemestersData } from '@/hooks/useSemesters'
 import type { Absence } from '@/types/students'
 import type { SISStudyRightElement } from '@oodikone/shared/models'
 import {
@@ -21,10 +17,6 @@ import {
 import { StudentStudyRight } from '@oodikone/shared/types/studentData'
 import { range } from '@oodikone/shared/util'
 
-dayjsExtend(isBetween)
-dayjsExtend(isSameOrAfter)
-dayjsExtend(isSameOrBefore)
-
 export const isFall = (semester: number) => semester % 2 === 1
 
 /**
@@ -333,38 +325,6 @@ export const getCreditCategories = (
 
 export const validateInputLength = (input: string, minLength: number) => input?.trim().length >= minLength
 
-export const getCurrentSemester = (allSemesters: SemestersData['semesters']) => {
-  if (!allSemesters) return null
-  return Object.values(allSemesters).find(
-    semester => new Date(semester.startdate) <= new Date() && new Date(semester.enddate) >= new Date()
-  )
-}
-
-/**
- * @returns semestercode that was active during the targetDate
- */
-export const getSemesterCodeAt = (allSemesters?: SemestersData['semesters'], targetDate?: Date | string) => {
-  if (!targetDate) return undefined
-
-  return Object.values(allSemesters ?? {}).find(
-    semester =>
-      new Date(semester.startdate) <= new Date(targetDate) && new Date(semester.enddate) >= new Date(targetDate)
-  )?.semestercode
-}
-
-/**
- * @returns all semesters from semesterEnrollments that occurred partially or fully during a given period
- */
-export const getSemestersBetweenRange = (start: Dayjs, end: Dayjs, allSemesters?: SemestersData['semesters']) =>
-  Object.values(allSemesters ?? {}).filter(semester => {
-    const semesterStart = dayjs(semester.startdate)
-    const semesterEnd = dayjs(semester.enddate)
-
-    return dayjs(semesterStart).isSameOrBefore(end)
-      ? semesterEnd.isSameOrAfter(start)
-      : semesterStart.isSameOrBefore(end) && semesterEnd.isSameOrAfter(start)
-  })
-
 /**
  * Extracts items from a string separated by commas, semicolons, spaces, or newlines.
  * @param {string} input - The input string containing items.

services/frontend/src/components/Charts/MedianGraduations.tsx

@@ -3,14 +3,7 @@ import useTheme from '@mui/material/styles/useTheme'
 import ReactECharts from 'echarts-for-react'
 
 import { useLanguage } from '@/components/LanguagePicker/useLanguage'
-import {
-  FacultyClassSizes,
-  GraduationStatistics,
-  GraduationStats,
-  Name,
-  NameWithCode,
-  ProgrammeClassSizes,
-} from '@oodikone/shared/types'
+import type { ClassSizes, GraduationTimeCategories, GraduationStats, Name, NameWithCode } from '@oodikone/shared/types'
 import { getOnEvents, getSeriesLabel } from './util'
 
 type GraduationPoint = GraduationStats & { color?: string; realGoal?: number }
@@ -30,7 +23,7 @@ export const MedianGraduations = ({
   title,
   yearLabel,
 }: {
-  classSizes?: FacultyClassSizes['programmes'] | ProgrammeClassSizes['studyTracks' | 'programme']
+  classSizes?: ClassSizes['programmes']
   cypress: string
   data: GraduationStats[]
   expandKey?: string
@@ -106,7 +99,7 @@ export const MedianGraduations = ({
     code: string,
     amount: number,
     median: number,
-    statistics: GraduationStatistics | undefined,
+    statistics: GraduationTimeCategories | undefined,
     realGoal: number | undefined
   ) => {
     const sortingText =
@@ -164,7 +157,7 @@ export const MedianGraduations = ({
         data?: {
           code?: string
           amount?: number
-          statistics?: GraduationStatistics
+          statistics?: GraduationTimeCategories
           realGoal?: number
           name?: string
         }
@@ -231,6 +224,7 @@ export const MedianGraduations = ({
         },
         markLine: {
           symbol: 'none',
+          silent: true,
           label: {
             show: false,
           },

services/frontend/src/components/Charts/Percentiles.tsx

@@ -11,27 +11,56 @@ type PercentileProps = {
 }
 
 export const PercentileGraph = (props: PercentileProps) => {
-  const series = Object.entries(props.data).map(([percentile, values]) => ({
-    name: `${percentile}th`,
-    type: 'line',
-    showSymbol: false,
-    symbol: 'circle',
-    smooth: 0.2,
-    data: values,
-  }))
+  const colors = [
+    '#800080', // purple
+    '#3ba272', // green
+    '#ffa500', // orange
+    '#ee3333', // red
+    '#333333', // black
+  ] as const
+  const series = Object.entries(props.data)
+    .sort(([a, _], [b, __]) => Number(b) - Number(a))
+    .map(([percentile, values], idx) => {
+      return {
+        name: `${percentile}th`,
+        type: 'line',
+        color: colors[idx],
+        colorBy: 'series',
+        showSymbol: false,
+        symbol: 'circle',
+        smooth: 0.1,
+        z: 10,
+        data: values,
+      }
+    })
 
-  /** Credit and graduation goals */
+  /** Goal is 5 credits a month (60 per year).
+  Monthly incrementation to ensure the line is always drawn
+  (one continuous line would not if either end is not visible). */
+  const getCreditGainDataPoints = (data?: [string, number][]) =>
+    data?.map(([date, _], idx) => {
+      if (idx === 0) {
+        return [{ coord: [date, 0] }, { coord: [date, 0] }]
+      }
+
+      const [prevDate] = data[idx - 1]
+
+      return [{ coord: [prevDate, (idx - 1) * 5] }, { coord: [date, idx * 5] }]
+    }) ?? []
+
+  /** Credit and graduation targets */
   const markLines = [
     {
       type: 'line',
       name: '',
       markLine: {
+        z: 0,
         silent: true,
         symbol: 'none',
         lineStyle: {
-          color: '#888888',
+          color: 'grey',
           type: 'solid',
-          width: 1,
+          width: 1.5,
         },
         data: props.goalLines?.credits?.map(goal => ({ yAxis: goal })) ?? [],
       },
@@ -40,16 +69,34 @@ export const PercentileGraph = (props: PercentileProps) => {
       type: 'line',
       name: '',
       markLine: {
+        z: 0,
         silent: true,
         symbol: 'none',
         lineStyle: {
-          color: '#ff0000',
+          color: '#ee3333',
           type: 'solid',
-          width: 1,
+          width: 1.5,
         },
         data: props.goalLines?.dates?.map(goal => ({ xAxis: goal })) ?? [],
       },
     },
+    {
+      type: 'line',
+      name: '',
+      silent: true,
+      filtermode: 'none',
+      animation: false,
+      markLine: {
+        z: 1,
+        symbol: 'none',
+        lineStyle: {
+          color: 'grey',
+          type: 'solid',
+          width: 1.5,
+        },
+        data: getCreditGainDataPoints(props.data['90']),
+      },
+    },
   ]
 
   const option = {

services/frontend/src/components/CustomPopulation/CustomPopulationProgrammeDist/util.ts

@@ -2,9 +2,10 @@ import dayjs, { extend as dayjsExtend } from 'dayjs'
 import isBetween from 'dayjs/plugin/isBetween'
 import isSameOrAfter from 'dayjs/plugin/isSameOrAfter'
 import isSameOrBefore from 'dayjs/plugin/isSameOrBefore'
-import { getNewestProgrammeOfStudentAt, getSemesterCodeAt } from '@/common'
+import { getNewestProgrammeOfStudentAt } from '@/common'
 import type { SemestersData } from '@/hooks/useSemesters'
 import { CreditTypeCode, FormattedStudent } from '@oodikone/shared/types'
+import { getSemesterCodeAt } from '@oodikone/shared/util'
 
 dayjsExtend(isBetween)
 dayjsExtend(isSameOrBefore)

services/frontend/src/components/GraduationTimes/BreakdownDisplay/index.tsx

@@ -3,7 +3,8 @@ import Stack from '@mui/material/Stack'
 import Typography from '@mui/material/Typography'
 
 import { GraduationBreakdown } from '@/components/Charts/GraduationBreakdown'
-import { GraduationStats, Name, NameWithCode, ProgrammeMedians } from '@oodikone/shared/types'
+import { StyledMessage } from '@/components/common/StyledMessage'
+import type { GraduationStats, Name, NameWithCode, ProgrammeMedians } from '@oodikone/shared/types'
 
 export const BreakdownDisplay = ({
   data,
@@ -28,6 +29,14 @@ export const BreakdownDisplay = ({
   programmeDataVisible: boolean
   yearLabel: 'Graduation year' | 'Start year'
 }) => {
+  if (!data.length) {
+    return (
+      <Box sx={{ display: 'flex', mb: 1 }}>
+        <StyledMessage showIcon>No data</StyledMessage>
+      </Box>
+    )
+  }
+
   if (!allowExpand) {
     return (
       <Box>
@@ -57,7 +66,15 @@ export const BreakdownDisplay = ({
             names={names}
             yearLabel={yearLabel}
           />
-        ) : null}
+        ) : (
+          <Box
+            sx={{ display: 'flex', flex: '1', justifyContent: 'center', mx: '10%', my: 'auto', textAlign: 'center' }}
+          >
+            <Typography fontSize="1.2em" fontWeight="light" variant="subtitle1">
+              Click on a bar on the chart to open a {mode} level breakdown for that year
+            </Typography>
+          </Box>
+        )}
       </Stack>
     </Box>
   )

services/frontend/src/components/GraduationTimes/FacultyGraduations.tsx

@@ -1,8 +1,21 @@
 import Stack from '@mui/material/Stack'
 
 import { GraduationTimes, GraduationTimesProps } from '@/components/GraduationTimes'
-import { GetAllGraduationStatsResponse } from '@/types/api/university'
+import { ClassSizes, UniversityGraduationStatistics } from '@oodikone/shared/types/graduations'
 
+type UniFacultyGraduationsProps = {
+  isError: boolean
+  isLoading: boolean
+  showMedian: boolean
+  faculty?: string
+  universityMode: true
+  data: UniversityGraduationStatistics | undefined
+}
+
+/**
+ * This component should be merged with the one used in faculties tab,
+ * but is left as an exercise to a future oodikone developer.
+ */
 export const FacultyGraduations = ({
   data,
   faculty,
@@ -10,20 +23,13 @@ export const FacultyGraduations = ({
   isLoading,
   showMedian,
   universityMode,
-}: {
-  data: GetAllGraduationStatsResponse | undefined
-  isError: boolean
-  isLoading: boolean
-  faculty?: string
-  showMedian: boolean
-  universityMode?: boolean
-}) => {
+}: UniFacultyGraduationsProps) => {
   const medians = data?.byGradYear.medians
   const goals = data?.goals
   const goalExceptions = { ...goals?.exceptions, needed: faculty === 'H30' }
   const programmeData = data?.byGradYear.programmes.medians
   const facultyNames = data?.programmeNames
-  const classSizes = data?.classSizes
+  const classSizes = data?.classSizes as ClassSizes // illegal override, underlying type is acually correct
 
   const commonProps: Omit<GraduationTimesProps, 'data' | 'goal' | 'level' | 'title'> = {
     allowExpand: true,

services/frontend/src/components/GraduationTimes/MedianDisplay/index.tsx

@@ -3,8 +3,8 @@ import Stack from '@mui/material/Stack'
 import Typography from '@mui/material/Typography'
 
 import { MedianGraduations } from '@/components/Charts/MedianGraduations'
-import {
-  FacultyClassSizes,
+import type {
+  ClassSizes,
   GraduationStats,
   Name,
   NameWithCode,
@@ -14,7 +14,7 @@ import {
 
 type MedianDisplayProps = {
   mode: 'faculty' | 'programme' | 'study track'
-  classSizes: FacultyClassSizes | ProgrammeClassSizes | undefined
+  classSizes: ClassSizes | ProgrammeClassSizes | undefined
   allowExpand: boolean
   names: Record<string, string | Name> | Record<string, Name | NameWithCode> | undefined
   data: GraduationStats[]
@@ -114,7 +114,15 @@ export const MedianDisplay = ({
               title={title}
               yearLabel={yearLabel}
             />
-          ) : null}
+          ) : (
+            <Box
+              sx={{ display: 'flex', flex: '1', justifyContent: 'center', mx: '10%', my: 'auto', textAlign: 'center' }}
+            >
+              <Typography fontSize="1.2em" fontWeight="light" variant="subtitle1">
+                Click on a bar on the chart to open a {mode} level breakdown for that year
+              </Typography>
+            </Box>
+          )}
         </Stack>
       )}
     </Box>

services/frontend/src/components/GraduationTimes/index.tsx

@@ -1,10 +1,10 @@
-import { useState } from 'react'
+import { useMemo, useState } from 'react'
 
 import { BreakdownDisplay } from '@/components/GraduationTimes/BreakdownDisplay'
 import { MedianDisplay } from '@/components/GraduationTimes/MedianDisplay'
 import { Section } from '@/components/Section'
 import {
-  FacultyClassSizes,
+  ClassSizes,
   GraduationStats,
   Name,
   NameWithCode,
@@ -14,7 +14,7 @@ import {
 
 export type GraduationTimesProps = {
   mode: 'faculty' | 'programme' | 'study track' // "Mode" is one step below the current view in hierarchy e.g. viewing a programme overview -> mode should be study track
-  classSizes?: FacultyClassSizes | ProgrammeClassSizes | undefined
+  classSizes?: ClassSizes | ProgrammeClassSizes | undefined
   data: GraduationStats[] | undefined
   goal?: number | undefined
   goalExceptions?: Record<string, number> | { needed: boolean }
@@ -60,12 +60,18 @@ export const GraduationTimes = ({
     }
   }
 
+  // Keys are either years e.g. "2025" or year ranges "2025 - 2026"
+  const sortedData = useMemo(
+    () => data?.toSorted((a, b) => Number(b.name.slice(0, 4)) - Number(a.name.slice(0, 4))),
+    [data]
+  )
+
   return (
     <Section cypress={`${level}-graduation-times`} isError={isError} isLoading={isLoading} title={title}>
       {!showMedian ? (
         <BreakdownDisplay
           allowExpand={allowExpand}
-          data={data!}
+          data={sortedData!}
           expandKey={expandKey}
           handleClick={handleClick}
           level={level}
@@ -79,7 +85,7 @@ export const GraduationTimes = ({
         <MedianDisplay
           allowExpand={allowExpand}
           classSizes={classSizes}
-          data={data!}
+          data={sortedData!}
           expandKey={expandKey}
           goal={goal}
           goalExceptions={goalExceptions}

services/frontend/src/components/NavigationBar/OodikoneLogo.tsx

@@ -1,11 +1,33 @@
 import Chip from '@mui/material/Chip'
+import IconButton from '@mui/material/IconButton'
 import Stack from '@mui/material/Stack'
+import Tooltip from '@mui/material/Tooltip'
 import Typography from '@mui/material/Typography'
 
+import { useLocation } from 'react-router'
 import { Link } from '@/components/common/Link'
-import { isDev } from '@/conf'
+import { isDev, prodBasePath } from '@/conf'
+import { useGetAuthorizedUserQuery } from '@/redux/auth'
+
+const DevChip = ({ prod }: { prod?: true }) => {
+  const location = useLocation()
+  const url = `${prod ? 'http://localhost:3000' : prodBasePath}${location.pathname}${location.search}${location.hash}`
+
+  return (
+    <Tooltip title={`Open in ${prod ? 'development (visible for admins)' : 'production'}`}>
+      <IconButton
+        onClick={() => {
+          window.open(url, '_blank', 'noopener,noreferrer')
+        }}
+      >
+        <Chip color={prod ? 'success' : 'error'} label={prod ? 'prod' : 'dev'} size="small" />
+      </IconButton>
+    </Tooltip>
+  )
+}
 
 export const OodikoneLogo = () => {
+  const { isAdmin } = useGetAuthorizedUserQuery()
   return (
     <Stack alignItems="center" direction="row" gap={1}>
       <Typography
@@ -24,7 +46,7 @@ export const OodikoneLogo = () => {
       >
         oodikone
       </Typography>
-      {isDev ? <Chip color="error" label="dev" size="small" /> : null}
+      {isDev ? <DevChip /> : isAdmin ? <DevChip prod /> : null}
     </Stack>
   )
 }

services/frontend/src/components/PopulationDetails/CreditGainStats/CreditDistributionDevelopment.tsx

@@ -5,7 +5,7 @@ import FormLabel from '@mui/material/FormLabel'
 import Paper from '@mui/material/Paper'
 import RadioMui from '@mui/material/Radio'
 import RadioGroup from '@mui/material/RadioGroup'
-import dayjs, { Dayjs } from 'dayjs'
+import dayjs from 'dayjs'
 import ReactECharts from 'echarts-for-react'
 
 import { groupBy } from 'lodash-es'
@@ -19,11 +19,18 @@ import { useFilters } from '@/components/FilterView/useFilters'
 import { useLanguage } from '@/components/LanguagePicker/useLanguage'
 import { useDeepMemo } from '@/hooks/deepMemo'
 import { useSemesters } from '@/hooks/useSemesters'
+import { theme } from '@/theme'
 import { generateGradientColors } from '@/util/color'
 import { FormattedStudent } from '@oodikone/shared/types'
 import { range } from '@oodikone/shared/util'
 
-const splitStudentCredits = (student: FormattedStudent, timeSlots: any[], cumulative: boolean) => {
+type TimeSlot = {
+  start: dayjs.Dayjs
+  end: dayjs.Dayjs
+  label: string
+}
+
+const splitStudentCredits = (student: FormattedStudent, timeSlots: TimeSlot[], cumulative: boolean) => {
   if (!timeSlots.length) return []
 
   let timeSlotN = 0
@@ -31,8 +38,9 @@ const splitStudentCredits = (student: FormattedStudent, timeSlots: any[], cumula
 
   student.courses
     .filter(course => course.passed && !course.isStudyModuleCredit && dayjs(course.date).isAfter(timeSlots[0].start))
-    .sort((a, b) => Number(dayjs(a.date) > dayjs(b.date)))
+    .sort((a, b) => dayjs(a.date).unix() - dayjs(b.date).unix())
     .forEach(course => {
+      // Find correct timeslot
       while (timeSlotN < timeSlots.length && dayjs(course.date).isAfter(timeSlots[timeSlotN].end)) timeSlotN++
       if (timeSlots.length <= timeSlotN) return
 
@@ -47,18 +55,31 @@ const splitStudentCredits = (student: FormattedStudent, timeSlots: any[], cumula
   return results
 }
 
-const hasGraduatedBeforeDate = (student: FormattedStudent, programme: string, date: Dayjs) => {
+const hasGraduatedBetweenDates = (
+  student: FormattedStudent,
+  programme: string,
+  slot: TimeSlot,
+  cumulative: boolean
+) => {
+  const { start, end } = slot
   const correctStudyRight = student.studyRights.find(studyRight =>
     studyRight.studyRightElements.some(el => el.code === programme)
   )
   if (!correctStudyRight) return false
+
   const studyRightElement = correctStudyRight.studyRightElements.find(el => el.code === programme)
-  return Boolean(studyRightElement?.graduated && date.isAfter(studyRightElement?.endDate, 'day'))
+
+  // If we want cumulative graduations, we only want to check for correct end date. If not, check also the start date
+  return Boolean(
+    studyRightElement?.graduated &&
+      (start.isBefore(studyRightElement?.endDate, 'day') || cumulative) &&
+      end.isAfter(studyRightElement?.endDate, 'day')
+  )
 }
 
 const getChartData = (
   students: FormattedStudent[],
-  timeSlots: any[],
+  timeSlots: TimeSlot[],
   programme: string,
   timeDivision: TimeDivision,
   cumulative: boolean,
@@ -73,8 +94,9 @@ const getChartData = (
     timeSlots.length,
     6
   )
-  const colors = generateGradientColors(limits.length - 1)
-  colors.push('#ddd') // Color for graduated (grey)
+
+  // Also add color for graduated
+  const colors = generateGradientColors(limits.length - 1).concat(theme.palette.grey[300])
 
   const data: { value: number; students: Array<FormattedStudent['studentNumber']> }[][] = limits.map(() =>
     timeSlots.map(() => ({
@@ -83,12 +105,10 @@ const getChartData = (
     }))
   )
 
-  const studentCredits = students.map(student => splitStudentCredits(student, timeSlots, cumulative))
-
   timeSlots.forEach((slot, timeSlotIndex) => {
-    students.forEach((student, studentIndex) => {
-      const hasGraduated = !!programme && hasGraduatedBeforeDate(student, programme, dayjs(slot.end))
-      const credits = studentCredits[studentIndex][timeSlotIndex]
+    students.forEach(student => {
+      const hasGraduated = !!programme && hasGraduatedBetweenDates(student, programme, slot, cumulative)
+      const credits = splitStudentCredits(student, timeSlots, cumulative)[timeSlotIndex]
 
       const rangeIndex = hasGraduated
         ? limits.indexOf('Graduated')
@@ -117,7 +137,7 @@ const getChartData = (
     const color = colors[limitN]
 
     const limit = limits[limitN]
-    let name
+    let name: string
 
     if (limit === 'Graduated') {
       name = 'Graduated'
@@ -167,38 +187,42 @@ export const CreditDistributionDevelopment = ({
   const timeSlots = (() => {
     const startDate = dayjs().year(year).endOf('year')
 
-    if (timeDivision === TimeDivision.CALENDAR_YEAR) {
-      return range(year, dayjs().year() + 1).map(year => ({
-        start: dayjs().year(year),
-        end: dayjs().year(year).endOf('year'),
-        label: year.toString(),
-      }))
-    }
-
-    if (timeDivision === TimeDivision.ACADEMIC_YEAR) {
-      return Object.values(groupBy(semesters, 'yearcode'))
-        .map(([a, b]) => [dayjs(a.startdate), dayjs(b.enddate)])
-        .filter(([a, b]) => startDate.isBefore(b) && dayjs().isAfter(a))
-        .map(([start, end]) => ({
-          start,
-          end,
-          label: `${start.year()}-${end.year()}`,
+    switch (timeDivision) {
+      case TimeDivision.CALENDAR_YEAR: {
+        return range(year, dayjs().year() + 1).map(year => ({
+          start: dayjs().year(year).startOf('year'),
+          end: dayjs().year(year).endOf('year'),
+          label: year.toString(),
         }))
-    }
+      }
 
-    if (timeDivision === TimeDivision.SEMESTER) {
-      return Object.values(semesters)
-        .filter(semester => startDate.isBefore(semester.enddate) && dayjs().isAfter(semester.startdate))
-        .map(semester => ({
-          start: dayjs(semester.startdate),
-          end: dayjs(semester.enddate),
-          label: getTextIn(semester.name) ?? '',
-        }))
+      case TimeDivision.ACADEMIC_YEAR: {
+        return Object.values(groupBy(semesters, 'yearcode'))
+          .map(([a, b]) => [dayjs(a.startdate), dayjs(b.enddate)])
+          .filter(([a, b]) => startDate.isBefore(b) && dayjs().isAfter(a))
+          .map(([start, end]) => ({
+            start,
+            end,
+            label: `${start.year()}-${end.year()}`,
+          }))
+      }
+
+      case TimeDivision.SEMESTER: {
+        return Object.values(semesters)
+          .filter(semester => startDate.isBefore(semester.enddate) && dayjs().isAfter(semester.startdate))
+          .map(semester => ({
+            start: dayjs(semester.startdate),
+            end: dayjs(semester.enddate),
+            label: getTextIn(semester.name) ?? '',
+          }))
+      }
+
+      default:
+        return []
     }
-    return []
   })()
 
-  const series = useDeepMemo(
+  const series: ReturnType<typeof getChartData> = useDeepMemo(
     () => getChartData(filteredStudents, timeSlots, programme, timeDivision, cumulative, combinedProgramme),
     [filteredStudents, timeSlots, programme, timeDivision, cumulative, combinedProgramme]
   )

services/frontend/src/components/PopulationStudents/StudentTable/CoursesTab/index.tsx

@@ -72,8 +72,8 @@ const studentMapper = (
     return acc
   }, {})
 
-  // curriculumCourseCode -> passedSubstGroup
-  const passedSubstitutionsToCurriculumCourses = Object.keys(substitutionsToCurriculumCourses).reduce<
+  /** curriculumCourseCode -> passedSubstGroup */
+  const curriculumCoursesToPassedSubstitutionGroups = Object.keys(substitutionsToCurriculumCourses).reduce<
     Record<string, StudentCourse[]>
   >((acc, code) => {
     const passedSubstitutionGroups = substitutionsToCurriculumCourses[code].filter(substGroup =>
@@ -125,7 +125,7 @@ const studentMapper = (
     {}
   )
 
-  const mapSubstitutionCourses = (coursesToAdd: typeof passedSubstitutionsToCurriculumCourses) => {
+  const mapSubstitutionCourses = (coursesToAdd: typeof curriculumCoursesToPassedSubstitutionGroups) => {
     for (const [code, substitutionGroup] of Object.entries(coursesToAdd)) {
       courseMap[code] ??= {
         substitutedBy: substitutionGroup,
@@ -199,7 +199,7 @@ const studentMapper = (
   }
 
   mapCourses(passedCourses)
-  if (includeSubstitutions) mapSubstitutionCourses(passedSubstitutionsToCurriculumCourses)
+  if (includeSubstitutions) mapSubstitutionCourses(curriculumCoursesToPassedSubstitutionGroups)
 
   mapEnrollments(enrollments)
   if (includeSubstitutions) mapSubstitutionEnrollments(enrollmentsWithSubstitutions)
@@ -207,7 +207,10 @@ const studentMapper = (
   mapHopsSelections(hopsItems)
   if (includeSubstitutions) mapSubstitutionHopsSelections(hopsItemsWithSubstitutions)
 
-  const totalPassed = Object.keys(courseMap).length
+  const totalPassed = Object.values(courseMap).reduce(
+    (acc: number, course: any) => (course.substitutedBy?.length > 0 || !!course?.passed ? acc + 1 : acc),
+    0
+  )
   const totalPlanned = Object.keys(courseMap).length - totalPassed
 
   return {

services/frontend/src/components/PopulationStudents/StudentTable/GeneralTab/primitives.ts

@@ -2,7 +2,6 @@ import dayjs from 'dayjs'
 import {
   getAllProgrammesOfStudent,
   getHighestGradeOfCourseBetweenRange,
-  getSemestersBetweenRange,
   getStudentTotalCredits,
   getStudyRightStatusText,
 } from '@/common'
@@ -19,6 +18,7 @@ import type {
 } from '@oodikone/shared/types'
 import { GenderCodeToText } from '@oodikone/shared/types/genderCode'
 import type { StudentStudyPlan, StudentStudyRight, StudentStudyRightElement } from '@oodikone/shared/types/studentData'
+import { getSemestersBetweenRange } from '@oodikone/shared/util'
 import { joinProgrammes } from './util'
 
 export type Variant = 'population' | 'coursePopulation' | 'customPopulation' | 'studyGuidanceGroupPopulation'

services/frontend/src/components/PopulationStudents/StudentTable/ProgressTab/index.tsx

@@ -21,7 +21,14 @@ import { useGetProgressCriteriaQuery } from '@/redux/progressCriteria'
 import { CheckIcon, CloseIcon, RemoveIcon, SwapHorizIcon } from '@/theme'
 import { isMedicalProgramme } from '@/util/studyProgramme'
 import { formatDate } from '@/util/timeAndDate'
-import { CreditTypeCode, FormattedStudent, Name, ProgressCriteria, SemesterEnrollment } from '@oodikone/shared/types'
+import {
+  CreditTypeCode,
+  EnrollmentState,
+  FormattedStudent,
+  Name,
+  ProgressCriteria,
+  SemesterEnrollment,
+} from '@oodikone/shared/types'
 import { StudentCourse } from '@oodikone/shared/types/studentData'
 import { keyBy, range } from '@oodikone/shared/util'
 import { TableInfo } from './info'
@@ -254,7 +261,7 @@ export const ProgressTable = ({
     return { fall, spring }
   }
 
-  const getSemesterEnrollmentVal = (student: FormattedStudent, semesters: number[]) => {
+  const getSemesterEnrollmentVal = (student: FormattedStudent, semesters: number[]): [string, string] => {
     const { fall, spring } = getSemesterEnrollmentStatus(student, semesters)
     return [helpTexts[fall], helpTexts[spring]]
   }
@@ -406,19 +413,48 @@ export const ProgressTable = ({
                 .map(semester => semester.semestercode)
 
               return (
-                labelCriteria[criteriaHeaders[year].label]?.flatMap(label =>
-                  label.code === 'Criteria'
-                    ? [[getTextIn(label.name), student.criteriaProgress[`year${year + 1}`]?.totalSatisfied ?? 0]]
-                    : label.code === 'Enrollment'
-                      ? (() => {
-                          const [fall, spring] = getSemesterEnrollmentVal(student, semesters)
-                          return [
-                            [`${getTextIn(label.name)} - FALL`, fall],
-                            [`${getTextIn(label.name)} - SPRING`, spring],
-                          ]
-                        })()
-                      : [[getTextIn(label.name), student.criteriaProgress[`year${year + 1}`]?.credits ? 'Passed' : '']]
-                ) ?? []
+                labelCriteria[criteriaHeaders[year].label]?.flatMap(label => {
+                  let result: (string | number | null | undefined)[][] = []
+                  switch (label.code) {
+                    case 'Criteria': {
+                      result = [
+                        [getTextIn(label.name), student.criteriaProgress[`year${year + 1}`]?.totalSatisfied ?? 0],
+                      ]
+                      break
+                    }
+
+                    case 'Enrollment': {
+                      result = (() => {
+                        const [fall, spring] = getSemesterEnrollmentVal(student, semesters)
+                        return [
+                          [`${getTextIn(label.name)} - FALL`, fall],
+                          [`${getTextIn(label.name)} - SPRING`, spring],
+                        ]
+                      })()
+                      break
+                    }
+
+                    case 'Credits': {
+                      result = [
+                        [getTextIn(label.name), student.criteriaProgress[`year${year + 1}`]?.credits ? 'Passed' : ''],
+                      ]
+                      break
+                    }
+
+                    default: {
+                      const course = student.courses.find(course => course.course_code === label.code)
+                      const enrollment = student.enrollments.find(enrollment => enrollment.course_code === label.code)
+                      result = [
+                        [
+                          getTextIn(label.name),
+                          course?.passed ? 'Passed' : enrollment?.state === EnrollmentState.ENROLLED ? 'Enrolled' : '',
+                        ],
+                      ]
+                      break
+                    }
+                  }
+                  return result
+                }) ?? []
               )
             })
             .filter(([key, _]) => !!key)

services/frontend/src/components/Teachers/TeacherStatistics/index.jsx

@@ -11,7 +11,6 @@ import Typography from '@mui/material/Typography'
 
 import { useState } from 'react'
 
-import { getCurrentSemester } from '@/common'
 import { useLanguage } from '@/components/LanguagePicker/useLanguage'
 import { TeacherStatisticsTable } from '@/components/Teachers/TeacherStatisticsTable'
 import { useSemesters } from '@/hooks/useSemesters'
@@ -20,7 +19,7 @@ import { useGetProvidersQuery } from '@/redux/providers'
 import { useLazyGetTeacherStatisticsQuery } from '@/redux/teachers'
 import { getFullStudyProgrammeRights, hasFullAccessToTeacherData } from '@/util/access'
 import { createLocaleComparator } from '@/util/comparator'
-import { mapToProviders } from '@oodikone/shared/util'
+import { getCurrentSemester, mapToProviders } from '@oodikone/shared/util'
 
 export const TeacherStatistics = () => {
   const { getTextIn } = useLanguage()

services/frontend/src/conf.ts

@@ -13,6 +13,7 @@ interface CypressWindow extends Window {
 export const runningInCypress = typeof window !== 'undefined' && !!(window as CypressWindow).Cypress
 
 // Base paths
+export const prodBasePath = 'https://oodikone.helsinki.fi'
 export const basePath = process.env.PUBLIC_URL ?? ''
 export const apiBasePath = `${basePath}/api`
 

services/frontend/src/pages/CompletedCoursesSearch/SearchResults.tsx

@@ -17,20 +17,18 @@ import { useGetCompletedCoursesQuery } from '@/redux/completedCoursesSearch'
 import { CropSquareIcon, DoneIcon, RemoveIcon } from '@/theme'
 import { getDefaultMRTOptions } from '@/util/getDefaultMRTOptions'
 import { formatDate, isWithinSixMonths } from '@/util/timeAndDate'
-import { CreditTypeCode } from '@oodikone/shared/types'
+import { CompletedCoursesStudent, CreditTypeCode } from '@oodikone/shared/types'
+import { SearchValues } from '.'
 
-const isPassed = credit => [CreditTypeCode.PASSED, CreditTypeCode.APPROVED, CreditTypeCode.IMPROVED].includes(credit)
+const isPassed = (creditType: CreditTypeCode) =>
+  [CreditTypeCode.PASSED, CreditTypeCode.APPROVED, CreditTypeCode.IMPROVED].includes(creditType)
 
-const getTotalPassed = student => student.credits.filter(credit => isPassed(credit)).length
-
-const getTotalUnfinished = student => Object.values(student.enrollments).length
-
-const getCompletion = (student, courseCode, { icon }) => {
+const getCompletion = (student: CompletedCoursesStudent, courseCode: string, icon: boolean) => {
   const completion = student.credits.find(credit => credit.courseCode === courseCode && isPassed(credit.creditType))
   const enrollment = student.enrollments[courseCode]
   const isInStudyPlan = student.coursesInStudyPlan.includes(courseCode)
 
-  if (completion === undefined) {
+  if (!completion) {
     if (!enrollment) {
       if (isInStudyPlan) {
         return <CropSquareIcon fontSize="small" style={{ color: grey[500] }} />
@@ -47,9 +45,8 @@ const getCompletion = (student, courseCode, { icon }) => {
     return `Latest enrollment: ${formatDate(enrollment.date, DateFormat.ISO_DATE)}`
   }
 
-  const substituted = !!completion.substitution?.length
-  const substitutionString = substituted ? ` as ${completion.substitution.join(', ')}` : ''
-
+  const substituted = Array.isArray(completion.substitution) && !!completion.substitution.length
+  const substitutionString = substituted ? ` as ${completion.substitution!.join(', ')}` : ''
   return icon ? (
     <DoneIcon fontSize="small" style={{ color: substituted ? grey[700] : green[700] }} />
   ) : (
@@ -57,13 +54,13 @@ const getCompletion = (student, courseCode, { icon }) => {
   )
 }
 
-const getCellTitle = (student, courseCode) => {
+const getCellTitle = (student: CompletedCoursesStudent, courseCode: string) => {
   const credit = student.credits.find(credit => credit.courseCode === courseCode)
   const enrollment = student.enrollments[courseCode]
   if (!credit && !enrollment) {
     return 'Student has the course in their primary study plan'
   }
-  let title
+  let title: string
   if (credit) {
     const substitutionString = credit.substitution?.length ? `Substituted by: ${credit.substitution?.join(', ')}` : ''
     title = `Passed on ${formatDate(credit.date, DateFormat.ISO_DATE)}\n${substitutionString}`
@@ -98,7 +95,7 @@ const RightsNotification = ({ discardedStudentNumbers }: { discardedStudentNumbe
   )
 }
 
-export const SearchResults = ({ searchValues }) => {
+export const SearchResults = ({ searchValues }: { searchValues: SearchValues }) => {
   const { courseList, studentList } = searchValues
   const { visible: namesVisible } = useStudentNameVisibility()
   const { data, isFetching } = useGetCompletedCoursesQuery({ courseList, studentList })
@@ -116,8 +113,12 @@ export const SearchResults = ({ searchValues }) => {
     })
   }, [namesVisible])
 
-  const staticColumns = useMemo<MRT_ColumnDef<any>[]>(
-    () => [
+  const staticColumns = useMemo<MRT_ColumnDef<CompletedCoursesStudent>[]>(() => {
+    const getTotalPassed = (student: CompletedCoursesStudent) =>
+      student.credits.filter(credit => isPassed(credit.creditType)).length
+    const getTotalUnfinished = (student: CompletedCoursesStudent) =>
+      (data?.courses?.length ?? 0) - student.credits.length
+    return [
       {
         accessorKey: 'lastname',
         header: 'Last name',
@@ -139,27 +140,27 @@ export const SearchResults = ({ searchValues }) => {
         header: 'Email',
       },
       {
-        accessorFn: student => getTotalPassed(student),
+        accessorFn: getTotalPassed,
         id: 'passed',
         header: 'Passed',
       },
       {
-        accessorFn: student => getTotalUnfinished(student),
+        accessorFn: getTotalUnfinished,
         id: 'unfinished',
         header: 'Unfinished',
       },
-    ],
-    []
-  )
+    ]
+  }, [data])
 
-  const dynamicColumns = useMemo(() => {
+  const dynamicColumns = useMemo<MRT_ColumnDef<CompletedCoursesStudent>[]>(() => {
+    if (!data) return []
     const courseOrder = courseList ?? []
 
     return (
-      data?.courses
-        .toSorted(({ code: a }, { code: b }) => courseOrder.indexOf(b) < courseOrder.indexOf(a))
+      data.courses
+        .toSorted((a, b) => courseOrder.indexOf(a.code) - courseOrder.indexOf(b.code))
         .map(course => ({
-          accessorFn: student => getCompletion(student, course.code, { icon: false }),
+          accessorFn: (student: CompletedCoursesStudent) => getCompletion(student, course.code, false),
           id: course.code,
           header: `${course.code} – ${getTextIn(course.name)}`,
           Header: () => (
@@ -173,14 +174,17 @@ export const SearchResults = ({ searchValues }) => {
               sx={{ display: 'flex', justifyContent: 'center' }}
               title={getCellTitle(cell.row.original, course.code)}
             >
-              {getCompletion(cell.row.original, course.code, { icon: true })}
+              {getCompletion(cell.row.original, course.code, true)}
             </Box>
           ),
         })) ?? []
     )
   }, [data, getTextIn])
 
-  const columns = useMemo(() => [...staticColumns, ...dynamicColumns], [staticColumns, dynamicColumns])
+  const columns: any[] /* MRT getting replaced anyway */ = useMemo(
+    () => [...staticColumns, ...dynamicColumns],
+    [staticColumns, dynamicColumns]
+  )
 
   const defaultOptions = getDefaultMRTOptions(setExportData, setExportModalOpen, language)
 
@@ -204,7 +208,7 @@ export const SearchResults = ({ searchValues }) => {
         onClose={() => setExportModalOpen(false)}
         open={exportModalOpen}
       />
-      {data?.discardedStudentNumbers?.length > 0 && (
+      {!!data?.discardedStudentNumbers?.length && (
         <div style={{ width: '75%' }}>
           <RightsNotification discardedStudentNumbers={data.discardedStudentNumbers} />
         </div>

services/frontend/src/pages/CompletedCoursesSearch/index.tsx

@@ -14,9 +14,11 @@ import { CropSquareIcon, DoneIcon, RemoveIcon } from '@/theme'
 import { SearchModal } from './SearchModal'
 import { SearchResults } from './SearchResults'
 
+export type SearchValues = { courseList: string[]; studentList: string[] }
+
 export const CompletedCourses = () => {
   useTitle('Completed courses of students')
-  const [searchValues, setValues] = useState({ courseList: [], studentList: [] })
+  const [searchValues, setValues] = useState<SearchValues>({ courseList: [], studentList: [] })
 
   return (
     <PageLayout maxWidth="lg">
@@ -33,6 +35,10 @@ export const CompletedCourses = () => {
             <DoneIcon fontSize="small" sx={{ color: green[700] }} />: Student has completed the course with a passing
             grade.
           </Stack>
+          <Stack direction="row">
+            <DoneIcon fontSize="small" sx={{ color: grey[700] }} />: Student has completed another course/set of courses
+            that substitute the original course. Hover for details.
+          </Stack>
           <Stack direction="row">
             <RemoveIcon fontSize="small" sx={{ color: yellow[800] }} />: Student has not completed the course, but has
             an active enrollment from less than 6 months ago.
@@ -45,7 +51,7 @@ export const CompletedCourses = () => {
             <CropSquareIcon fontSize="small" sx={{ color: grey[500] }} />: Student has the course in their primary study
             plan, but has not enrolled to, or completed it.
           </Stack>
-          <b>Empty cell</b>: Student has no completion or enrollment for the course.
+          <b>Empty cell</b>: None apply.
         </Alert>
         <SearchModal setValues={setValues} />
         {searchValues.courseList.length > 0 && searchValues.studentList.length > 0 && (

services/frontend/src/pages/CourseStatistics/CourseTab/SingleCourseStats/index.tsx

@@ -29,6 +29,7 @@ import {
 import { DropdownOption } from '@/types/dropdownOption'
 import { parseQueryParams, queryParamsToString } from '@/util/queryparams'
 import { Name } from '@oodikone/shared/types'
+import { enrollmentTimeDateThresholdYearCode } from '@oodikone/shared/util'
 import { countTotalStats } from './countTotalStats'
 import { ResultTabs } from './ResultTabs'
 import { YearFilter } from './YearFilter'
@@ -273,16 +274,19 @@ export const SingleCourseStats = ({
   const countStudentEnrollmentStats = (
     allAttempts: Attempts,
     filteredEnrollments: Enrollment[],
+    allPassedStudents: Set<string>,
     displayEnrollments: boolean
   ) => {
-    const enrolledStudentsWithNoGrade = filteredEnrollments.filter(({ studentNumber }) => {
-      const hasFailed = allAttempts.categories.failed ? allAttempts.categories.failed.includes(studentNumber) : false
-      const hasPassed = allAttempts.categories.passed ? allAttempts.categories.passed.includes(studentNumber) : false
-      return !hasFailed && !hasPassed
-    })
     if (!displayEnrollments) {
       return { enrolledStudentsWithNoGrade: undefined, totalEnrollments: undefined }
     }
+
+    const enrolledStudentsWithNoGrade = filteredEnrollments.filter(({ studentNumber }) => {
+      const hasFailed = allAttempts.categories.failed?.includes(studentNumber) ?? false
+      const hasPassed = allPassedStudents.has(studentNumber)
+      return !hasFailed && !hasPassed
+    })
+
     return {
       enrolledStudentsWithNoGrade: enrolledStudentsWithNoGrade.length,
       totalEnrollments: filteredEnrollments.length,
@@ -295,6 +299,13 @@ export const SingleCourseStats = ({
     }
     const { statistics } = stats
     const filter = belongsToAtLeastOneProgramme(programmeCodes)
+
+    const allPassedStudents = statistics.filter(isStatInYearRange).reduce((acc, stats) => {
+      stats.attempts.categories.passed.forEach(studentNumber => acc.add(studentNumber))
+
+      return acc
+    }, new Set<string>())
+
     const formattedStats: FormattedStats[] = statistics
       .filter(isStatInYearRange)
       .map(
@@ -309,12 +320,17 @@ export const SingleCourseStats = ({
           allEnrollments = [],
           yearCode,
         }) => {
-          const displayEnrollments = yearCode >= 72 // Display enrollments only for Sisu era
+          const displayEnrollments = yearCode >= enrollmentTimeDateThresholdYearCode // Display enrollments only for Sisu era
           const filteredEnrollments = enrollments.filter(({ studentNumber }) => filter(studentNumber))
           const filteredAllEnrollments = allEnrollments.filter(({ studentNumber }) => filter(studentNumber))
           const totalEnrollments = displayEnrollments ? filteredAllEnrollments.length : undefined
 
-          const studentsEnrollments = countStudentEnrollmentStats(allAttempts, filteredEnrollments, displayEnrollments)
+          const studentsEnrollments = countStudentEnrollmentStats(
+            allAttempts,
+            filteredEnrollments,
+            allPassedStudents,
+            displayEnrollments
+          )
           const attempts = countAttemptStats(allAttempts, totalEnrollments, filter)
           const students = countStudentStats(allStudents, studentsEnrollments.enrolledStudentsWithNoGrade, filter)
           const parsedName = separate ? getTextIn(name as Name)! : name
@@ -379,6 +395,7 @@ export const SingleCourseStats = ({
     const excludedProgrammes = getExcluded()
     const primaryProgrammes = primary
     const comparisonProgrammes = comparison.filter(code => isValidProgrammeCode(code))
+
     if (comparison.includes('EXCLUDED')) {
       comparisonProgrammes.push(...excludedProgrammes)
     }

services/frontend/src/pages/Faculties/BasicInformationTab/index.tsx

@@ -128,7 +128,7 @@ export const BasicInformationTab = ({
   specialGroups: boolean
   studyProgrammes: boolean
 }) => {
-  const [academicYear, setAcademicYear] = useState(false)
+  const [academicYear, setAcademicYear] = useState(true)
 
   const yearType = academicYear ? 'ACADEMIC_YEAR' : 'CALENDAR_YEAR'
   const studyProgrammeFilter = studyProgrammes ? 'ALL_PROGRAMMES' : 'NEW_DEGREE_PROGRAMMES'

services/frontend/src/pages/Faculties/GraduationTimesTab/index.tsx

@@ -10,9 +10,8 @@ import { GraduationTimes, GraduationTimesProps } from '@/components/GraduationTi
 import { useLanguage } from '@/components/LanguagePicker/useLanguage'
 import { Section } from '@/components/Section'
 import { useGetFacultyGraduationTimesQuery } from '@/redux/facultyStats'
-import { GetFacultiesResponse, GetFacultyGraduationTimesResponse } from '@/types/api/faculty'
+import { GetFacultiesResponse } from '@/types/api/faculty'
 import { getTimestamp } from '@/util/timeAndDate'
-import { Name } from '@oodikone/shared/types'
 
 export const GraduationTimesTab = ({
   faculty,
@@ -23,22 +22,28 @@ export const GraduationTimesTab = ({
   setStudyProgrammes: (value: boolean) => void
   studyProgrammes: boolean
 }) => {
+  const { getTextIn } = useLanguage()
   const [showMedian, setShowMedian] = useState(false)
   const [groupByStartYear, setGroupByStartYear] = useState(false)
   const studyProgrammeFilter = studyProgrammes ? 'ALL_PROGRAMMES' : 'NEW_DEGREE_PROGRAMMES'
-  const graduationStats = useGetFacultyGraduationTimesQuery({ id: faculty?.id, studyProgrammeFilter })
-  const { getTextIn } = useLanguage()
+
+  const {
+    data,
+    isError: queryError,
+    isSuccess: querySuccess,
+    isFetching,
+  } = useGetFacultyGraduationTimesQuery({ id: faculty?.id, studyProgrammeFilter })
 
   const groupBy = groupByStartYear ? ('byStartYear' as const) : ('byGradYear' as const)
   const yearLabel = groupByStartYear ? ('Start year' as const) : ('Graduation year' as const)
-  const data = graduationStats?.data?.[groupBy].medians
-  const goals = graduationStats?.data?.goals
+
+  const groupedData = data?.[groupBy].medians
+  const goals = data?.goals
   const goalExceptions = { ...goals?.exceptions, needed: faculty?.code === 'H30' }
-  const programmeData = graduationStats?.data?.[groupBy].programmes.medians
-  const programmeNames = graduationStats?.data?.programmeNames
-  const classSizes = graduationStats?.data?.classSizes
-  const isError = graduationStats.isError || (graduationStats.isSuccess && !graduationStats.data)
-  const isLoading = graduationStats.isFetching
+  const programmeData = data?.[groupBy].programmes.medians
+  const classSizes = data?.classSizes
+
+  const isError = queryError || (querySuccess && !data)
 
   const commonProps: Omit<GraduationTimesProps, 'data' | 'goal' | 'level' | 'title'> = {
     allowExpand: true,
@@ -46,19 +51,17 @@ export const GraduationTimesTab = ({
     goalExceptions,
     groupBy,
     isError,
-    isLoading,
+    isLoading: isFetching,
     mode: 'programme',
-    names: programmeNames,
+    names: data?.programmeNames,
     showMedian,
     yearLabel,
   } as const
 
-  const exportToExcel = (data?: GetFacultyGraduationTimesResponse, programmeNames?: Record<string, Name>) => {
-    if (!data || !programmeNames) {
-      return
-    }
+  const exportToExcel = () => {
+    if (!data) return
+    const { programmeNames } = data
 
-    const educationLevels = ['bachelor', 'bcMsCombo', 'master', 'doctor', 'licentiate'] as const
     const sheetTitles = {
       bachelor: 'Bachelor',
       bcMsCombo: 'Bachelor + Master',
@@ -69,20 +72,17 @@ export const GraduationTimesTab = ({
 
     const book = utils.book_new()
 
-    educationLevels.forEach(level => {
+    Object.keys(sheetTitles).forEach(level => {
       const levelData = data[groupBy]?.programmes?.medians[level]
-      if (!levelData) {
-        return
-      }
+      if (!levelData) return
 
       const sheetData: Record<string, any>[] = []
 
       Object.keys(levelData).forEach(year => {
         const yearData = levelData[year]
         const { programmes } = yearData
-        const { data } = yearData
 
-        data.forEach((item, index) => {
+        yearData.data.forEach((item, index) => {
           sheetData.push({
             Code: programmes[index],
             Abbreviation: item.name,
@@ -91,7 +91,7 @@ export const GraduationTimesTab = ({
             'On time': item.statistics.onTime ?? 0,
             'Max. year overtime': item.statistics.yearOver ?? 0,
             Overtime: item.statistics.wayOver ?? 0,
-            'Median study time (months)': item.median ?? 0,
+            'Median study time (semesters)': item.median ?? 0,
           })
         })
       })
@@ -107,14 +107,14 @@ export const GraduationTimesTab = ({
     <Stack gap={2}>
       <Section
         cypress="average-graduation-times"
-        exportOnClick={() => exportToExcel(graduationStats.data, programmeNames)}
+        exportOnClick={() => exportToExcel()}
         infoBoxContent={facultyToolTips.common.averageGraduationTimes}
         title="Average graduation times by education level"
       >
         <ToggleContainer>
           <Toggle
             cypress="graduation-time-toggle"
-            disabled={isError || isLoading}
+            disabled={isError || isFetching}
             firstLabel="Breakdown"
             secondLabel="Median study times"
             setValue={setShowMedian}
@@ -122,7 +122,7 @@ export const GraduationTimesTab = ({
           />
           <Toggle
             cypress="group-by-toggle"
-            disabled={isError || isLoading}
+            disabled={isError || isFetching}
             firstLabel="Group by: Graduation year"
             secondLabel="Starting year"
             setValue={setGroupByStartYear}
@@ -130,7 +130,7 @@ export const GraduationTimesTab = ({
           />
           <Toggle
             cypress="programme-toggle"
-            disabled={isError || isLoading}
+            disabled={isError || isFetching}
             firstLabel="New degree programmes"
             infoBoxContent={facultyToolTips.common.programmeToggle}
             secondLabel="All degree programmes"
@@ -141,7 +141,7 @@ export const GraduationTimesTab = ({
       </Section>
       <Stack gap={2}>
         <GraduationTimes
-          data={data?.bachelor}
+          data={groupedData?.bachelor}
           goal={goals?.bachelor}
           level="bachelor"
           levelProgrammeData={programmeData?.bachelor}
@@ -149,7 +149,7 @@ export const GraduationTimesTab = ({
           {...commonProps}
         />
         <GraduationTimes
-          data={data?.bcMsCombo}
+          data={groupedData?.bcMsCombo}
           goal={goals?.bcMsCombo}
           level="bcMsCombo"
           levelProgrammeData={programmeData?.bcMsCombo}
@@ -157,7 +157,7 @@ export const GraduationTimesTab = ({
           {...commonProps}
         />
         <GraduationTimes
-          data={data?.master}
+          data={groupedData?.master}
           goal={goals?.master}
           level="master"
           levelProgrammeData={programmeData?.master}
@@ -165,7 +165,7 @@ export const GraduationTimesTab = ({
           {...commonProps}
         />
         <GraduationTimes
-          data={data?.doctor}
+          data={groupedData?.doctor}
           goal={goals?.doctor}
           level="doctor"
           levelProgrammeData={programmeData?.doctor}

services/frontend/src/pages/StudyProgramme/ProgrammeCoursesTab/ByCreditTypeTab/OverallStatsTable/CodeCell.tsx

@@ -11,7 +11,7 @@ export const CodeCell = ({ code }: { code: string }) => {
       <Typography variant="body2">{code}</Typography>
       <Link
         title={`Open course statistics for ${code}`}
-        to={`/coursestatistics?courseCodes=["${encodeURIComponent(code)}"]&separate=false`}
+        to={`/coursestatistics?courseCodes=["${encodeURIComponent(code)}"]&separate=false&combineSubstitutions=false`}
       >
         <IconButton color="primary" data-cy={`${code.toLowerCase()}-course-statistics-link-button`}>
           <NorthEastIcon fontSize="small" />

services/frontend/src/pages/StudyProgramme/ProgrammeCoursesTab/ByCreditTypeTab/index.tsx

@@ -37,7 +37,7 @@ export const ByCreditTypeTab = ({
   const [fromYear, setFromYear] = useState<number | null>(null)
   const [toYear, setToYear] = useState<number | null>(null)
   const [years, setYears] = useState<{ academic: YearOption[]; calendar: YearOption[] } | null>(null)
-  const [showStudents, setShowStudents] = useState(false)
+  const [showStudents, setShowStudents] = useState(true)
 
   useEffect(() => {
     if (!data) {

services/frontend/src/pages/StudyProgramme/StudyTracksAndClassStatisticsTab/index.tsx

@@ -137,10 +137,10 @@ export const StudyTracksAndClassStatisticsTab = ({
     ) as [string, ProgrammeOrStudyTrackGraduationStats][]
 
     const studyTrackStatsGraduationStats = graduationStats.reduce<ProgrammeMedians>((acc, [programme, { medians }]) => {
-      for (const { name, amount, statistics, y } of Object.values(combo ? medians.combo : medians.basic)) {
+      for (const { name, amount, statistics, y, times } of Object.values(combo ? medians.combo : medians.basic)) {
         acc[name] ??= { data: [], programmes: [] }
         acc[name].programmes.push(programme)
-        acc[name].data.push({ amount, name: programme, statistics, code: programme, median: y })
+        acc[name].data.push({ amount, name: programme, statistics, code: programme, median: y, times })
       }
 
       return acc
@@ -177,7 +177,7 @@ export const StudyTracksAndClassStatisticsTab = ({
     basic: gradStats
     combo: gradStats
   } =
-    hasStudyTracks && Object.keys(studyTrackStats?.graduationTimes ?? {}).length > 1
+    Object.keys(studyTrackStats?.graduationTimes ?? {}).length > 1
       ? {
           basic: calculateStudyTrackStats(),
           combo: studyTrackStats?.doCombo ? calculateStudyTrackStats(true) : {},

services/frontend/src/pages/StudyProgramme/index.tsx

@@ -61,7 +61,7 @@ export const StudyProgramme = () => {
   const { studyProgrammeId } = useParams()
 
   const [currentTab, setCurrentTab] = useTabs(5 + Number(isAdmin))
-  const [academicYear, setAcademicYear] = useState(false)
+  const [academicYear, setAcademicYear] = useState(true)
   const [specialGroupsExcluded, setSpecialGroupsExcluded] = useState(false)
 
   const { data } = useGetProgrammesQuery()

services/frontend/src/pages/Updater/index.tsx

@@ -2,6 +2,11 @@
 import Button from '@mui/material/Button'
 import Card from '@mui/material/Card'
 import CardContent from '@mui/material/CardContent'
+import Dialog from '@mui/material/Dialog'
+import DialogActions from '@mui/material/DialogActions'
+import DialogContent from '@mui/material/DialogContent'
+import DialogContentText from '@mui/material/DialogContentText'
+import DialogTitle from '@mui/material/DialogTitle'
 import FormControl from '@mui/material/FormControl'
 import FormControlLabel from '@mui/material/FormControlLabel'
 import Grid from '@mui/material/Grid2'
@@ -25,12 +30,11 @@ import { RefreshIcon } from '@/theme'
 
 const useTicker = () => {
   const [now, setNow] = useState(() => Date.now())
-  const intervalMs = 1000
 
   useEffect(() => {
-    const id = setInterval(() => setNow(Date.now()), intervalMs)
+    const id = setInterval(() => setNow(Date.now()), 1000)
     return () => clearInterval(id)
-  }, [intervalMs])
+  }, [])
 
   return now
 }
@@ -77,6 +81,7 @@ export const Updater = () => {
   const [type, setType] = useState('students')
   const [jobs, setJobs] = useState<Jobs | null>(null)
   const [syntheticError, setSyntheticError] = useState(false)
+  const [dialogOpen, setDialogOpen] = useState(false)
   const customListRef = useRef<HTMLInputElement>(null!)
   const jsonDataRef = useRef('')
 
@@ -106,6 +111,10 @@ export const Updater = () => {
   const refreshLanguageCenterData = () => void apiCall('/updater/refresh_language_center_data', 'post')
   const refreshCloseToGraduationData = () => void apiCall('/updater/refresh-close-to-graduation', 'post')
   const removeWaitingJobs = () => void callApi('/updater/jobs', 'delete')
+  const flushRedis = () => {
+    void apiCall('/updater/nuke_redis')
+    setDialogOpen(false)
+  }
 
   useEffect(() => {
     const updateJobs = async () => {
@@ -118,10 +127,7 @@ export const Updater = () => {
     }
     void updateJobs()
 
-    /* eslint-disable prefer-const */
-    let interval
-    /* eslint-disable @typescript-eslint/no-misused-promises */
-    interval = setInterval(updateJobs, 5000)
+    const interval = setInterval(() => void updateJobs(), 5000)
 
     return () => clearInterval(interval)
   }, [])
@@ -132,9 +138,36 @@ export const Updater = () => {
     <PageLayout maxWidth="lg">
       <PageTitle title="Updater" />
       <Stack spacing={2}>
-        <Button color="error" onClick={() => setSyntheticError(true)} variant="contained">
-          Cause frontend crash
-        </Button>
+        <Grid container>
+          <Grid paddingRight={1} size={6}>
+            <Button color="error" fullWidth onClick={() => setSyntheticError(true)} variant="contained">
+              Cause frontend crash
+            </Button>
+          </Grid>
+          <Grid paddingLeft={1} size={6}>
+            <Button color="error" fullWidth onClick={() => setDialogOpen(true)} variant="contained">
+              Destroy redis
+            </Button>
+          </Grid>
+        </Grid>
+        <Dialog onClose={() => setDialogOpen(false)} open={dialogOpen}>
+          <DialogTitle>Completely flush redis?</DialogTitle>
+          <DialogContent>
+            <DialogContentText>
+              Fully clears redis. This includes all cached data in updater/backend and anything BullMQ might have
+              stored. Equivalent to `redis-cli flushall` in container or `npm run flushredis` in development. 99% of
+              errors will be fixed by just refreshing the data for the relevant feature, e.g. faculty/study programme.
+            </DialogContentText>
+            <DialogActions>
+              <Button color="error" onClick={() => void flushRedis()} variant="contained">
+                I know what I'm doing, flush redis
+              </Button>
+              <Button onClick={() => setDialogOpen(false)} variant="contained">
+                Go back
+              </Button>
+            </DialogActions>
+          </DialogContent>
+        </Dialog>
         <Section
           infoBoxContent={updaterToolTips.updaterSection}
           title="Updater (data pulled from sis-importer-db and brought to sis-db)"

services/frontend/src/redux/completedCoursesSearch.ts

@@ -1,8 +1,9 @@
 import { RTKApi } from '@/apiConnection'
+import type { CompletedCoursesRes } from '@oodikone/shared/routes/completedCoursesOfStudents'
 
 const completedCoursesSearchApi = RTKApi.injectEndpoints({
   endpoints: builder => ({
-    getCompletedCourses: builder.query<any, { courseList: string[]; studentList: string[] }>({
+    getCompletedCourses: builder.query<CompletedCoursesRes, { courseList: string[]; studentList: string[] }>({
       query: ({ courseList, studentList }) =>
         `completedcoursessearch?courselist=${JSON.stringify(courseList)}&studentlist=${JSON.stringify(studentList)}`,
     }),

services/frontend/src/redux/facultyStats.ts

@@ -1,5 +1,5 @@
 import { RTKApi } from '@/apiConnection'
-import {
+import type {
   GetFacultiesResponse,
   GetFacultyBasicStatsResponse,
   GetFacultyBasicStatsRequest,
@@ -7,7 +7,6 @@ import {
   GetFacultyCreditStatsRequest,
   GetFacultyThesisStatsResponse,
   GetFacultyThesisStatsRequest,
-  GetFacultyGraduationTimesResponse,
   GetFacultyGraduationTimesRequest,
   GetFacultyProgressStatsResponse,
   GetFacultyProgressStatsRequest,
@@ -20,7 +19,7 @@ import {
   UpdateFacultyProgressTabResponse,
   UpdateFacultyProgressTabRequest,
 } from '@/types/api/faculty'
-import { GetAllGraduationStatsResponse } from '@/types/api/university'
+import type { FacultyGraduationStatistics, UniversityGraduationStatistics } from '@oodikone/shared/types/graduations'
 
 const facultystatsApi = RTKApi.injectEndpoints({
   endpoints: builder => ({
@@ -39,7 +38,7 @@ const facultystatsApi = RTKApi.injectEndpoints({
       query: ({ id, yearType, studyProgrammeFilter, specialGroups }) =>
         `/faculties/${id}/thesisstats?year_type=${yearType}&programme_filter=${studyProgrammeFilter}&special_groups=${specialGroups}`,
     }),
-    getFacultyGraduationTimes: builder.query<GetFacultyGraduationTimesResponse, GetFacultyGraduationTimesRequest>({
+    getFacultyGraduationTimes: builder.query<FacultyGraduationStatistics, GetFacultyGraduationTimesRequest>({
       query: ({ id, studyProgrammeFilter }) =>
         `/faculties/${id}/graduationtimes?programme_filter=${studyProgrammeFilter}`,
     }),
@@ -49,7 +48,7 @@ const facultystatsApi = RTKApi.injectEndpoints({
     getAllFacultiesProgressStats: builder.query<GetAllProgressStatsResponse, GetAllProgressStatsRequest>({
       query: ({ includeSpecials }) => `/university/allprogressstats?&specialsIncluded=${includeSpecials}`,
     }),
-    getAllFacultiesGraduationStats: builder.query<GetAllGraduationStatsResponse, void>({
+    getAllFacultiesGraduationStats: builder.query<UniversityGraduationStatistics, void>({
       query: () => '/university/allgraduationstats',
     }),
     getFacultyStudentStats: builder.query<GetFacultyStudentStatsResponse, GetFacultyStudentStatsRequest>({

services/frontend/src/types/api/faculty.ts

@@ -1,7 +1,6 @@
 import { GraphStat } from '@/types/graphStat'
 import { DegreeProgrammeType, Name, NameWithCode } from '@oodikone/shared/types'
 import { ProgrammeFilter, SpecialGroups, YearType } from '@oodikone/shared/types/faculty'
-import { GetAllGraduationStatsResponse } from './university'
 
 type Info = {
   graphStats: GraphStat[]
@@ -55,12 +54,6 @@ export type GetFacultyThesisStatsRequest = {
   specialGroups: SpecialGroups
 }
 
-export type GetFacultyGraduationTimesResponse = GetAllGraduationStatsResponse & {
-  // NOTE: This is NameWithCode in GetAllGraduationStatsResponse
-  programmeNames: Record<string, Name>
-  id: string
-}
-
 export type GetFacultyGraduationTimesRequest = {
   id: string
   studyProgrammeFilter: ProgrammeFilter

services/frontend/src/types/api/university.ts

@@ -1,30 +0,0 @@
-import { FacultyClassSizes, GraduationStats, NameWithCode, ProgrammeMedians } from '@oodikone/shared/types'
-
-// ? Almost identical to GetFacultyGraduationTimesResponse, could be combined?
-export interface GetAllGraduationStatsResponse {
-  byGradYear: {
-    medians: {
-      bachelor: GraduationStats[]
-      bcMsCombo: GraduationStats[]
-      doctor: GraduationStats[]
-      master: GraduationStats[]
-    }
-    programmes: {
-      medians: {
-        bachelor: ProgrammeMedians
-        bcMsCombo: ProgrammeMedians
-        doctor: ProgrammeMedians
-        master: ProgrammeMedians
-      }
-    }
-  }
-  classSizes: FacultyClassSizes
-  goals: {
-    bachelor: number
-    bcMsCombo: number
-    doctor: number
-    master: number
-    exceptions?: Record<string, number> // ? Is this used or needed?
-  }
-  programmeNames: Record<string, NameWithCode>
-}