[Frontend][OT] -UNSAFE- Add basic filtering capabilities

- Remove old fields from db
- Not null enforced
- Remove dead code

.claude/settings.json

@@ -5,7 +5,8 @@
       "Read(./.databasedumps/**)",
       "Read(**/*.sqz)",
       "Read(**/*.sql)",
-      "Bash(docker exec *-db *)"
+      "Bash(docker exec *-db *)",
+      "Bash(*psql *)"
     ]
   }
 }

.env.template

@@ -3,4 +3,8 @@
 # `node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"`
 IMPORTER_DB_API_TOKEN=
 PATE_TOKEN=
-CRYPT_KEY=
+CRYPT_KEY=
+# Changes the user and group docker runs as. Use "id -u" and "id -g" to run containers as current user+group.
+# Fixes errors with permissions when mounting.
+DOCKER_UID=
+DOCKER_GID=

.github/actions/test/updater-unit/action.yaml

@@ -0,0 +1,18 @@
+name: 'Run updater unit tests'
+description: 'Runs unit tests in sis-updater-worker'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./updater/sis-updater-worker
+
+    - name: Run updater unit tests
+      shell: bash
+      run: npm run test:unit --prefix ./updater/sis-updater-worker

.github/workflows/production.yaml

@@ -133,7 +133,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: 'Send notification to Slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.6.1
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -155,7 +155,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: 'Send notification to Slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.6.1
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure

.github/workflows/pull_request.yaml

@@ -26,3 +26,5 @@ jobs:
     secrets:
       username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
       password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      s3_access_key: ${{ secrets.S3_ACCESS_KEY }}
+      s3_secret_key: ${{ secrets.S3_SECRET_KEY }}

.github/workflows/staging.yaml

@@ -23,6 +23,8 @@ jobs:
     secrets:
       username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
       password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      s3_access_key: ${{ secrets.S3_ACCESS_KEY }}
+      s3_secret_key: ${{ secrets.S3_SECRET_KEY }}
 
   release_backend:
     name: 'Build and release backend'
@@ -110,7 +112,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: 'Send notification to Slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.6.1
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -131,7 +133,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: 'Send notification to Slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.6.1
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure

.github/workflows/test.yaml

@@ -15,6 +15,12 @@ on:
       password:
         required: true
         description: 'Toska registry password'
+      s3_access_key:
+        required: true
+        description: 'S3 access key for the test database dumps'
+      s3_secret_key:
+        required: true
+        description: 'S3 secret key for the test database dumps'
 
 jobs:
   # === Static tests / linting ===
@@ -95,6 +101,7 @@ jobs:
       - build_frontend
       - lint_packages
       - test_unit
+      - test_updater_unit
 
     steps:
       - uses: actions/checkout@v7
@@ -106,11 +113,14 @@ jobs:
           password: ${{ secrets.password }}
 
       - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
+        env:
+          ACCESS_KEY: ${{ secrets.s3_access_key }}
+          SECRET_KEY: ${{ secrets.s3_secret_key }}
         run: |
           docker compose --file docker-compose.ci.yml up --detach --quiet-pull sis-updater-worker
           sleep 30 # wait for sis-updater-worker to start and run migrations
-          # TODO: Disable this until fixed
-          # ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
+          sudo apt update -y && sudo apt install s3cmd -y
+          ./anonyymioodi/manage_test_dbs.sh release sis-db
 
       - name: Teardown containers started by compose
         if: always()
@@ -130,6 +140,7 @@ jobs:
       - build_frontend
       - lint_packages
       - test_unit
+      - test_updater_unit
 
     steps:
       - uses: actions/checkout@v7
@@ -142,11 +153,14 @@ jobs:
           password: ${{ secrets.password }}
 
       - name: 'Let backend run migrations, then push new version of image to toska hub'
+        env:
+          ACCESS_KEY: ${{ secrets.s3_access_key }}
+          SECRET_KEY: ${{ secrets.s3_secret_key }}
         run: |
           docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
           sleep 30 # wait for backend to start and run migrations
-          # TODO: Disable this until fixed
-          # ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
+          sudo apt update -y && sudo apt install s3cmd -y
+          ./anonyymioodi/manage_test_dbs.sh release kone-db
 
       - name: Teardown containers started by compose
         if: always()
@@ -166,6 +180,7 @@ jobs:
       - build_frontend
       - lint_packages
       - test_unit
+      - test_updater_unit
 
     steps:
       - uses: actions/checkout@v7
@@ -178,11 +193,14 @@ jobs:
           password: ${{ secrets.password }}
 
       - name: 'Let backend run migrations, then push new version of image to toska hub'
+        env:
+          ACCESS_KEY: ${{ secrets.s3_access_key }}
+          SECRET_KEY: ${{ secrets.s3_secret_key }}
         run: |
           docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
           sleep 30 # wait for backend to start and run migrations
-          # TODO: Disable this until fixed
-          # ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
+          sudo apt update -y && sudo apt install s3cmd -y
+          ./anonyymioodi/manage_test_dbs.sh release user-db
 
       - name: Teardown containers started by compose
         if: always()
@@ -249,10 +267,15 @@ jobs:
           EXIT_AFTER_IMMEDIATES: yes
 
       # TODO: Push the updated sis-db to toska-hub when updater is deterministic with test-data
-      # - name: Push newly build db to toska hub
-      #   if: |
-      #   success() && inputs.skip_db_update != 'true'
-      #   run: ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
+      - name: Push newly build db to toska hub
+        if: |
+          success() && inputs.skip_db_update != 'true'
+        env:
+          ACCESS_KEY: ${{ secrets.s3_access_key }}
+          SECRET_KEY: ${{ secrets.s3_secret_key }}
+        run: |
+          sudo apt update -y && sudo apt install s3cmd -y
+          ./anonyymioodi/manage_test_dbs.sh release sis-db
 
       - name: Dump logs on failure
         if: failure()
@@ -315,6 +338,19 @@ jobs:
       - name: Run unit tests (frontend and backend)
         uses: ./.github/actions/test/unit
 
+  # === Updater unit testing ===
+  test_updater_unit:
+    name: 'Run updater unit tests'
+    runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
+    timeout-minutes: 15
+    if: |
+      !cancelled() && !failure()
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Run updater unit tests
+        uses: ./.github/actions/test/updater-unit
+
   # === Frontend component testing ===
   test_components:
     name: 'Test frontend components'
@@ -346,35 +382,35 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: 'Test population statistics'
+          - name: 'Population statistics'
             spec: e2e/Population_statistics.spec.ts
-          - name: 'Test course statistics'
+          - name: 'Course statistics'
             spec: e2e/Course_statistics.spec.ts
-          - name: 'Test degree programme overview'
+          - name: 'Degree programme overview'
             spec: e2e/Degree_programme_overview.spec.ts
-          - name: 'Test Faculty statistics'
+          - name: 'faculty statistics'
             spec: e2e/Faculty_statistics.spec.ts
-          - name: 'Test teachers'
+          - name: 'Teachers'
             spec: e2e/Teachers.spec.ts
-          - name: 'Test users'
+          - name: 'Users'
             spec: e2e/Users.spec.ts
-          - name: 'Test access rights'
+          - name: 'Access rights'
             spec: e2e/Routes.spec.ts
-          - name: 'Test university'
+          - name: 'University'
             spec: e2e/University.spec.ts
-          - name: 'Test language center view'
+          - name: 'Language center view'
             spec: e2e/Language_center.spec.ts
-          - name: 'Test study guidance groups'
+          - name: 'Study guidance groups'
             spec: e2e/Study_guidance_groups.spec.ts
-          - name: 'Test custom population'
+          - name: 'Custom population'
             spec: e2e/Custom_population.spec.ts
-          - name: 'Test filters'
+          - name: 'Filters'
             spec: e2e/Filters.spec.ts
-          - name: 'Test students'
+          - name: 'Students'
             spec: e2e/Students.spec.ts
-          - name: 'Test completed courses'
+          - name: 'Completed courses'
             spec: e2e/Completed_courses.spec.ts
-          - name: 'Test numbers match'
+          - name: 'Numbers match'
             spec: e2e/Numbers_match.spec.ts
     name: ${{ matrix.name }}
     steps:

.github/workflows/updater.yaml

@@ -23,6 +23,15 @@ jobs:
       - name: 'Run linter'
         run: npm run lint -- updater
 
+  test_updater_unit:
+    name: 'Run updater unit tests'
+    runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v7
+      - name: Run updater unit tests
+        uses: ./.github/actions/test/updater-unit
+
   build_worker:
     name: 'Build Updater worker'
     runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
@@ -79,6 +88,7 @@ jobs:
     timeout-minutes: 15
     needs:
       - lint_scheduler_and_worker
+      - test_updater_unit
       - build_worker
       - build_scheduler
 

AGENTS.md

@@ -6,7 +6,7 @@ Oodikone is a Docker-based Node.js/TypeScript application for analysing Universi
 - `services/backend`: Express, Sequelize, PostgreSQL, Redis, and BullMQ.
 - `services/shared`: shared types and utilities.
 - `updater/`: Sisu data workers and scheduler.
-- `e2e/`: Playwright tests; `cypress/` contains legacy tests.
+- `e2e/`: Playwright e2e tests
 
 Use Node.js 24 or newer. Packages have separate dependencies and lockfiles. Install locally with
 `npm run install:local`, or use `npm run ci:local` for lockfile-exact installs. Docker uses host-mounted locks:
@@ -24,13 +24,14 @@ update dependencies locally, then run `npm run ci:docker`; do not install in con
   `services/backend/src/database/migrations_user`, and `updater/sis-updater-worker/src/db/migrations`; test migration
   changes against the relevant database service.
 - Do not access the database directly (e.g. `docker exec` into a database container, `psql`, Adminer, or similar).
-  Go through the application's models or services instead.
+  Go through the application's models or services instead or ask if there is questions about the data.
 - Add tests, especially unit tests, when adding or changing behavior.
 
 ## Local Commands
 
 ```text
 npm run oodikone       # start the anonymized environment
+npm run oodikone:ci    # start the test mode
 npm run both           # start Oodikone and the updater
 npm run docker:down    # stop containers
 docker compose ps      # show service status

anonyymioodi/README.md

@@ -4,6 +4,30 @@ This folder contains a guide and scripts to modify or create anonymous database
 
 The quick guide should be enough for most purposes: updating schema, data, or the PostgreSQL version. If you need to generate completely new `sis-importer-db` and `sis-db` images (which shouldn't happen too often), follow the instructions in the section [Generate new sis-importer-db and sis-db completely from scratch](#generate-new-sis-importer-db-and-sis-db-completely-from-scratch).
 
+## Scripts
+
+`manage_test_dbs.sh` is the front end for the test database tooling. Run it without arguments for an interactive menu, or give it an action and one or more databases (or `all`) to run it directly:
+
+```bash
+./manage_test_dbs.sh dump sis-db      # dump sis-db from its running container
+./manage_test_dbs.sh upload sis-db    # upload the current dump to s3
+./manage_test_dbs.sh release all      # dump, upload and push images for every database
+```
+
+Actions are `download`, `dump`, `upload` (upload + prune), `push`, `prune`, `release` (dump + upload + prune + push) and `rebuild` (download + push). Each one is a thin call to a script in this folder, which can also be used on its own:
+
+| Script                     | What it does                                                                                                           |
+| -------------------------- | ---------------------------------------------------------------------------------------------------------------------- |
+| `dump_local_db.sh`         | `pg_dump`s the databases from the running containers into `.databasedumps/`                                            |
+| `download_dump_from_s3.sh` | Downloads the newest dump of each database from s3 (`-b <branch>` to pick dumps from a single branch)                  |
+| `upload_dump_to_s3.sh`     | Uploads the dumps in `.databasedumps/test` to s3 as `<database>-<git_branch>-<datetime>.sql`                           |
+| `prune_s3_dumps.sh`        | Deletes dumps older than 30 days from s3, always keeping the newest `master` dump of each database (`-d <days>`, `-n`) |
+| `build_and_push_image.sh`  | Builds database images from the current dumps and pushes them to the Toska registry                                    |
+| `create_s3_bucket.sh`      | Creates the s3 bucket if it doesn't exist yet, run automatically before an upload                                      |
+| `lib/common.sh`            | Shared configuration (dump directories, s3 bucket, registry, database names)                                           |
+
+Dumps end up in `.databasedumps/test/<database>.sql`, which is where `db.Dockerfile` reads them from when an image is built. The s3 commands read their credentials from the `ACCESS_KEY` and `SECRET_KEY` environment variables (e.g. in CI). If neither is set, they fall back to `~/.s3cfg`, which can be found on the toska/dokumentaatio repo.
+
 ## Quick guide to updating a test database
 
 1. Ensure you have the latest test-db. Just pulling images might not replace your existing volume, so you may need to remove it manually. This command might work: `docker volume rm oodikone-sis-db-data` (replace `oodikone-sis-db-data` with the correct volume name). The new image, including data, should be pulled the next time you launch Oodikone normally.

anonyymioodi/build_and_push_image.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Build database images from the dumps in .databasedumps/test and push them to
+# the Toska registry.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0") <database>... | all
+
+Builds an image of each given database and pushes it to $REGISTRY.
+The dump is read from .databasedumps/test/<database>.sql, so run
+dump_local_db.sh or download_dump_from_s3.sh first.
+
+Databases: ${DATABASES[*]}
+EOF
+}
+
+if [[ "${1-}" == "-h" || "${1-}" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+if [[ $# -eq 0 ]]; then
+  usage
+  die "No database given"
+fi
+
+build_and_push_image() {
+  local database="$1"
+  local image="$REGISTRY/$database:latest"
+  local dump_file="$TEST_DUMP_DIR/$database.sql"
+
+  [[ -f "$dump_file" ]] || die "No dump found at $dump_file, run dump_local_db.sh or download_dump_from_s3.sh first"
+
+  infomsg "Building $image"
+  docker build "$PROJECT_ROOT" -f "$PROJECT_ROOT/db.Dockerfile" -t "$image" --build-arg DB_NAME="$database"
+
+  infomsg "Pushing $image"
+  docker push "$image"
+  successmsg "Pushed $image"
+}
+
+resolve_databases "$@"
+
+for database in "${selected_databases[@]}"; do
+  build_and_push_image "$database"
+done

anonyymioodi/create_s3_bucket.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Create the s3 bucket used for the test database dumps, if it doesn't exist yet.
+# Doing nothing when the bucket is already there, so this is safe to run every time.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0")
+
+Creates the bucket $S3_BUCKET if it doesn't exist yet, and does nothing if it does.
+EOF
+}
+
+if [[ "${1-}" == "-h" || "${1-}" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+require_s3_config
+
+# s3cmd info exits with an error when the bucket doesn't exist
+if s3 info "$S3_BUCKET" > /dev/null 2>&1; then
+  infomsg "Bucket $S3_BUCKET already exists"
+  exit 0
+fi
+
+warningmsg "Bucket $S3_BUCKET doesn't exist, creating it"
+s3 mb "$S3_BUCKET"
+successmsg "Created bucket $S3_BUCKET"

anonyymioodi/db_util.sh

@@ -1,214 +0,0 @@
-#!/usr/bin/env bash
-
-# === Config ===
-
-# Quit if failing
-set -euo pipefail
-
-# Print messages and logs that are not script output to stderr
-msg() {
-    echo >&2 -e "${1-}"
-}
-
-# Some special types of messages with colours
-successmsg() {
-  msg "${GREEN}$1${NOFORMAT}"
-}
-errormsg() {
-  msg "${RED}$1${NOFORMAT}"
-}
-infomsg() {
-  msg "${BLUE}$1${NOFORMAT}"
-}
-warningmsg() {
-  msg "${ORANGE}$1${NOFORMAT}"
-}
-
-# Setup colors for messages if running interactive shell
-if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
-  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
-else
-  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
-fi
-
-
-# === #
-
-PROJECT_ROOT="$(git rev-parse --show-toplevel)"
-
-get_project_relative_path () {
-  local absolute_path="$1"
-  local root="$2"
-  realpath -s --relative-to="${root:-$PROJECT_ROOT}" "$absolute_path"
-}
-
-LOCAL_DUMP_DIR="$(get_project_relative_path ".databasedumps/local_containers" "$PWD")" # Dumps from running docker containers
-S3_DUMP_DIR="$(get_project_relative_path ".databasedumps/s3" "$PWD")" # Dumps from s3
-TEST_DUMP_DIR="$(get_project_relative_path ".databasedumps/test" "$PWD")" # Location where .sql files are pulled while building the new image
-
-S3_CONFIG_FILE=~/.s3cfg
-S3_BUCKET="s3://oodikone-test"
-
-REGISTRY="registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi"
-
-## Following the naming convention in docker compose, these are names for database
-## services. The real data databases inside service will have names suffixed by "-real".
-KONE_DB_NAME="kone-db"
-SIS_DB_NAME="sis-db"
-SIS_IMPORTER_DB_NAME="sis-importer-db"
-USER_DB_NAME="user-db"
-JAMI_DB_NAME="jami-db"
-DATABASES=("$USER_DB_NAME" "$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$JAMI_DB_NAME")
-
-mkdir -p "$TEST_DUMP_DIR"
-mkdir -p "$S3_DUMP_DIR"
-mkdir -p "$LOCAL_DUMP_DIR"
-
-
-# === Handle s3 bucket ===
-# Use s3 bucket as the dump save/pull location
-download_s3_db () {
-  local database="$1"
-
-  # TODO: Decompress compressed files
-  s3cmd -c "$S3_CONFIG_FILE" get "$S3_BUCKET/$database.sql" "$S3_DUMP_DIR/$database.sql"
-  cp --remove-destination -r "$S3_DUMP_DIR" "$TEST_DUMP_DIR" # Move dumps to the place where docker build pulls .sql files from
-}
-
-upload_s3_db () {
-  local database="$1"
-
-  if [[ $(s3cmd ls s3:// | grep -q oodikone-test; echo $?) -eq 0 ]]
-    then
-      infomsg "OODIKONE-TEST bucket exists, skipping creation";
-    else
-      warningmsg "OODIKONE-TEST bucket doesn't exist, creating new bucket...";
-      s3cmd -c "$S3_CONFIG_FILE" mb "s3://oodikone-test"
-  fi
-
-  infomsg "Pushing $TEST_DUMP_DIR/$database.sql to $S3_BUCKET"
-  # TODO: Also compress files
-  s3cmd -c "$S3_CONFIG_FILE" put "$TEST_DUMP_DIR/$database.sql" "$S3_BUCKET/$database.sql"
-}
-
-
-# === Build images ===
-# Update registry with an existing dump located in .databasedumps/test/$1.sql
-build_and_push_image () {
-  local database="$1"
-
-  infomsg "Building docker image for $database\n"
-  docker build "$(get_project_relative_path "$PROJECT_ROOT" "$PWD")" -t "$REGISTRY/$database:latest" -f db.Dockerfile --build-arg DB_NAME="$database"
-  successmsg "$database docker image built"
-  infomsg "Pushing $database to $REGISTRY\n"
-  infomsg "$REGISTRY/$database"
-  docker push "$REGISTRY/$database"
-  successmsg "Image $REGISTRY/$database pushed successfully"
-}
-
-
-# === Handle dumping local dbs ===
-# To conveniently update all dbs, build all with docker-compose.db.yml and then run dump_all_local_dbs
-dump_local_db () {
-  local database="$1"
-  local container_name="oodikone-$1"
-  local database_name="$database"
-
-  if [[ $database == "jami-db" ]]; then database_name="postgres"; fi
-
-  infomsg "Trying to dump $database to $LOCAL_DUMP_DIR/$database.sql"
-  docker exec -i "$container_name" pg_dump -Fp -U postgres -d "$database_name" > "$LOCAL_DUMP_DIR/$database.sql"
-  successmsg "Dumped $database to $LOCAL_DUMP_DIR/$database.sql"
-  cp --remove-destination -r "$LOCAL_DUMP_DIR/$database.sql" "$TEST_DUMP_DIR" # Move dump to the place where docker build pulls the .sql file from
-}
-
-
-# === Helpers ===
-do_for_all () {
-  local cmd="$1"
-  shift # Removes $1 from "$@"
-
-  infomsg "Running $cmd for all databases"
-  for database in "${DATABASES[@]}"; do
-    "$cmd" "$database" "$@"
-  done
-  successmsg "$cmd ran successfully"
-}
-
-ask_for_input () {
-  local cmd="$1"
-  shift # Removes $1 from "$@"
-
-  echo
-
-  init_database_menu
-  select database in "${options[@]}"; do
-    "$cmd" "$database" "$@"
-    break
-  done
-}
-
-
-init_action_menu () {
-  PS3="Please enter your choice: "
-
-  options=(
-    "Pull dumps from s3"
-    "Dump from running docker images"
-    "Update dumps in s3"
-    "Push new images to registry"
-  )
-}
-
-init_mode_menu () {
-  PS3="Select command mode: "
-  options=(
-    "Run command for all databases"
-    "Select single database"
-  )
-}
-
-init_database_menu () {
-  PS3="Select database: "
-  options=("${DATABASES[@]}")
-}
-
-
-# === RUN ===
-while true; do
-  init_mode_menu
-  select mode in "${options[@]}"; do
-    case "$mode" in
-      "Run command for all databases")
-        cmd="do_for_all"
-        break;;
-      "Select single database")
-        cmd="ask_for_input"
-        break;;
-    esac
-  done
-
-  echo
-
-  echo
-
-  init_action_menu
-  select opt in "${options[@]}"; do
-    case "$opt" in
-      "Pull dumps from s3")
-        "$cmd" download_s3_db "$S3_DUMP_DIR"
-        break;;
-      "Dump from running docker images")
-        "$cmd" dump_local_db "$LOCAL_DUMP_DIR"
-        break;;
-      "Update dumps in s3")
-        "$cmd" upload_s3_db
-        break;;
-      "Push new images to registry")
-        "$cmd" build_and_push_image
-        break;;
-    esac
-  done
-
-  echo
-done

anonyymioodi/download_dump_from_s3.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+# Download database dumps from s3 into .databasedumps/s3, and copy them to
+# .databasedumps/test where db.Dockerfile reads them from.
+#
+# Dumps are uploaded as <db_name>-<git_branch>-<datetime>.sql (see
+# upload_dump_to_s3.sh), so the newest matching dump is the one downloaded.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0") [-b|--branch <branch>] <database>... | all
+
+Downloads the newest dump of each given database from $S3_TARGET.
+Databases: ${DATABASES[*]}
+
+Options:
+  -b, --branch <branch>  Only consider dumps uploaded from this git branch
+  -h, --help             Show this help
+EOF
+}
+
+branch=""
+arguments=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -b | --branch)
+      branch="${2-}"
+      [[ -n "$branch" ]] || {
+        usage
+        die "Missing value for $1"
+      }
+      shift 2
+      ;;
+    -h | --help)
+      usage
+      exit 0
+      ;;
+    -*)
+      usage
+      die "Unknown option: $1"
+      ;;
+    *)
+      arguments+=("$1")
+      shift
+      ;;
+  esac
+done
+
+# Newest object in s3 for the given database, or empty if there are none
+newest_dump_in_s3() {
+  local database="$1"
+  local prefix="$S3_TARGET/$database-"
+  if [[ -n "$branch" ]]; then prefix="$prefix$(sanitize_branch_name "$branch")-"; fi
+
+  # s3cmd ls prints "<date> <time> <size> <object>", so sorting by the first two
+  # columns in reverse puts the newest dump first
+  s3 ls "$prefix" | sort -rk1,2 | awk 'NR == 1 { print $4 }'
+}
+
+download_database() {
+  local database="$1"
+  local dump_file="$S3_DUMP_DIR/$database.sql"
+  local newest_dump
+  newest_dump="$(newest_dump_in_s3 "$database")"
+
+  [[ -n "$newest_dump" ]] || die "No dumps found for $database in $S3_TARGET${branch:+ (branch $branch)}"
+
+  infomsg "Downloading $newest_dump"
+  s3 get --force "$newest_dump" "$dump_file"
+  cp --remove-destination "$dump_file" "$TEST_DUMP_DIR/$database.sql"
+  successmsg "Downloaded $newest_dump to $dump_file"
+}
+
+if [[ ${#arguments[@]} -eq 0 ]]; then
+  usage
+  die "No database given"
+fi
+
+resolve_databases "${arguments[@]}"
+require_s3_config
+make_dump_dirs
+
+for database in "${selected_databases[@]}"; do
+  download_database "$database"
+done

anonyymioodi/dump_local_db.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# Dump one or more databases from the running docker containers into
+# .databasedumps/local_containers, and copy them to .databasedumps/test where
+# db.Dockerfile reads them from.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0") <database>... | all
+
+Dumps the given databases from their running containers.
+Databases: ${DATABASES[*]}
+EOF
+}
+
+if [[ "${1-}" == "-h" || "${1-}" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+if [[ $# -eq 0 ]]; then
+  usage
+  die "No database given"
+fi
+
+dump_database() {
+  local database="$1"
+  local container_name="oodikone-$database"
+  local database_name
+  database_name="$(postgres_database_name "$database")"
+  local dump_file="$LOCAL_DUMP_DIR/$database.sql"
+
+  infomsg "Dumping $database from container $container_name"
+  # Dump to a temporary file first, so that a failed dump doesn't overwrite the previous one
+  docker exec -i "$container_name" pg_dump -Fp -U postgres -d "$database_name" > "$dump_file.tmp"
+  mv "$dump_file.tmp" "$dump_file"
+  cp --remove-destination "$dump_file" "$TEST_DUMP_DIR/$database.sql"
+  successmsg "Dumped $database to $dump_file"
+}
+
+resolve_databases "$@"
+make_dump_dirs
+
+for database in "${selected_databases[@]}"; do
+  dump_database "$database"
+done

anonyymioodi/lib/common.sh

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+
+# Shared configuration and helpers for the anonyymioodi database scripts.
+# Meant to be sourced by the other scripts in this folder, not run on its own.
+
+# The configuration below is used by the sourcing scripts, not by this file
+# shellcheck disable=SC2034
+
+PROJECT_ROOT="$(git rev-parse --show-toplevel)"
+source "$PROJECT_ROOT"/scripts/utils.sh
+
+# Dumps taken from running docker containers
+LOCAL_DUMP_DIR="$PROJECT_ROOT/.databasedumps/local_containers"
+# Dumps downloaded from s3
+S3_DUMP_DIR="$PROJECT_ROOT/.databasedumps/s3"
+# Where db.Dockerfile copies the .sql files from when an image is built
+TEST_DUMP_DIR="$PROJECT_ROOT/.databasedumps/test"
+
+# Used as a fallback when ACCESS_KEY and SECRET_KEY aren't set in the environment
+S3_CONFIG_FILE=~/.s3cfg
+S3_HOST="s3.datacloud.helsinki.fi"
+S3_BUCKET="s3://oodikone"
+# Dumps live under a prefix of their own inside the bucket
+S3_TARGET="$S3_BUCKET/test-dbs"
+# Dumps older than this are pruned from s3, except for the newest dump of each database
+S3_DUMP_RETENTION_DAYS=30
+
+REGISTRY="registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi"
+
+# Databases, named after their docker compose services
+DATABASES=("user-db" "kone-db" "sis-db" "sis-importer-db" "jami-db")
+
+# Set by resolve_databases, read by the scripts sourcing this file
+selected_databases=()
+
+# The name of the postgres database inside the container. It matches the service
+# name for every database except jami-db.
+postgres_database_name() {
+  local database="$1"
+  if [[ "$database" == "jami-db" ]]; then
+    echo "postgres"
+  else
+    echo "$database"
+  fi
+}
+
+# Validates the given database names and expands "all" to every database.
+# The result is set to the global array $selected_databases, because dying
+# inside a subshell wouldn't stop the calling script.
+resolve_databases() {
+  selected_databases=()
+
+  if [[ $# -eq 0 ]]; then
+    die "No database given, expected one or more of: ${DATABASES[*]} all"
+  fi
+
+  for argument in "$@"; do
+    if [[ "$argument" == "all" ]]; then
+      selected_databases=("${DATABASES[@]}")
+      return
+    fi
+
+    # Check if given databases are correct ones
+    local known=false
+    for database in "${DATABASES[@]}"; do
+      if [[ "$argument" == "$database" ]]; then known=true; fi
+    done
+    $known || die "Unknown database: $argument. Expected one or more of: ${DATABASES[*]} all"
+
+    selected_databases+=("$argument")
+  done
+}
+
+# Dumps in s3 are named after the branch they were uploaded from, so the branch
+# name has to be usable as a part of a file name. E.g. feature/foo -> feature-foo.
+# feature/foo would result in a new bucket "feature" with a sub-bucket "foo"
+sanitize_branch_name() {
+  echo "${1//[^A-Za-z0-9._-]/-}"
+}
+
+current_branch_name() {
+  local branch
+  branch="$(git rev-parse --abbrev-ref HEAD)"
+  if [[ "$branch" == "HEAD" ]]; then # Detached head, use the commit instead
+    branch="$(git rev-parse --short HEAD)"
+  fi
+  sanitize_branch_name "$branch"
+}
+
+has_s3_env_credentials() {
+  [[ -n "${ACCESS_KEY-}" && -n "${SECRET_KEY-}" ]]
+}
+
+# Credentials come from ACCESS_KEY and SECRET_KEY when both are set, and from
+# $S3_CONFIG_FILE when neither is
+require_s3_config() {
+  command -v s3cmd > /dev/null || die "s3cmd not found, please install it first"
+
+  if [[ -n "${ACCESS_KEY-}" || -n "${SECRET_KEY-}" ]]; then
+    has_s3_env_credentials || die "Only one of ACCESS_KEY and SECRET_KEY is set, set both or neither"
+    return
+  fi
+
+  [[ -f "$S3_CONFIG_FILE" ]] || die "Set ACCESS_KEY and SECRET_KEY, or add $S3_CONFIG_FILE. The \".s3cfg\" config file can be found on toska/dokumentaatio repo"
+}
+
+s3_env_config() {
+  cat <<EOF
+[default]
+access_key = $ACCESS_KEY
+secret_key = $SECRET_KEY
+host_base = $S3_HOST
+host_bucket = $S3_HOST
+use_https = True
+check_ssl_certificate = True
+EOF
+}
+
+# Runs s3cmd with the credentials chosen by require_s3_config. The environment
+# credentials are passed through process substitution, so that they aren't
+# written to disk or visible in the process list.
+s3() {
+  if has_s3_env_credentials; then
+    s3cmd -c <(s3_env_config) "$@"
+  else
+    s3cmd -c "$S3_CONFIG_FILE" "$@"
+  fi
+}
+
+# The dump directories are created lazily by the scripts that write into them
+make_dump_dirs() {
+  mkdir -p "$LOCAL_DUMP_DIR" "$S3_DUMP_DIR" "$TEST_DUMP_DIR"
+}

anonyymioodi/manage_test_dbs.sh

@@ -0,0 +1,155 @@
+#!/usr/bin/env bash
+
+# Front end for the test database scripts in this folder. Run it without
+# arguments for an interactive menu, or give it an action and a database to run
+# it directly, e.g. in CI.
+#
+# Each action is just a call to one of the scripts next to this one, so they can
+# also be used on their own.
+
+set -euo pipefail
+
+# Here BASH_SOURCE[0] is used instead of $0 because it works
+# consistently when ran normally and when sourced
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+ACTIONS=("download" "dump" "upload" "push" "prune" "release" "rebuild")
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0") [<action> <database>... | all]
+
+Runs without arguments for an interactive menu.
+
+Actions:
+  download  Download the newest dumps from s3
+  dump      Dump the databases from the running docker containers
+
+  upload    Upload the current dumps to s3, then prune old dumps
+  push      Build images from the current dumps and push them to the registry
+  prune     Delete dumps older than $S3_DUMP_RETENTION_DAYS days from s3, keeping the newest one
+
+  release   Dump from the running containers, then upload, prune and push (shorthand)
+  rebuild   Download the dumps from s3, then push (shorthand)
+
+Databases: ${DATABASES[*]} all
+EOF
+}
+
+# === Actions ===
+
+download_dump() {
+  "$SCRIPT_DIR"/download_dump_from_s3.sh "$1"
+}
+
+dump_database() {
+  "$SCRIPT_DIR"/dump_local_db.sh "$1"
+}
+
+upload_dump() {
+  "$SCRIPT_DIR"/upload_dump_to_s3.sh "$1"
+}
+
+prune_dumps() {
+  "$SCRIPT_DIR"/prune_s3_dumps.sh "$1"
+}
+
+push_image() {
+  "$SCRIPT_DIR"/build_and_push_image.sh "$1"
+}
+
+run_action() {
+  local action="$1"
+  local database="$2"
+
+  case "$action" in
+    download) download_dump "$database" ;;
+    dump) dump_database "$database" ;;
+    upload)
+      upload_dump "$database"
+      prune_dumps "$database"
+      ;;
+    push) push_image "$database" ;;
+    prune) prune_dumps "$database" ;;
+    release)
+      dump_database "$database"
+      upload_dump "$database"
+      prune_dumps "$database"
+      push_image "$database"
+      ;;
+    rebuild)
+      download_dump "$database"
+      push_image "$database"
+      ;;
+    *)
+      usage
+      die "Unknown action: $action"
+      ;;
+  esac
+}
+
+run_action_for_all() {
+  local action="$1"
+  shift
+
+  # Check if command was a proper command
+  local known_action=false
+  for known in "${ACTIONS[@]}"; do
+    if [[ "$action" == "$known" ]]; then known_action=true; fi
+  done
+  if ! $known_action; then
+    usage
+    die "Unknown action: $action"
+  fi
+
+  resolve_databases "$@"
+  for database in "${selected_databases[@]}"; do
+    run_action "$action" "$database"
+  done
+  successmsg "Action $action done for: ${selected_databases[*]}"
+}
+
+# === Interactive menu ===
+
+ask_and_run() {
+  local action database
+
+  PS3="Select action: "
+  select action in "${ACTIONS[@]}" "quit"; do
+    [[ -n "${action-}" ]] || continue
+    if [[ "$action" == "quit" ]]; then exit 0; fi
+    break
+  done
+
+  PS3="Select database: "
+  select database in "${DATABASES[@]}" "all"; do
+    [[ -n "${database-}" ]] || continue
+    break
+  done
+
+  run_action_for_all "$action" "$database"
+}
+
+# === Run ===
+
+if [[ "${1-}" == "-h" || "${1-}" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+if [[ $# -eq 0 ]]; then
+  while true; do
+    ask_and_run
+    echo
+  done
+fi
+
+if [[ $# -lt 2 ]]; then
+  usage
+  die "Both an action and a database are required"
+fi
+
+run_action_for_all "$@"

anonyymioodi/prune_s3_dumps.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+# Delete old database dumps from s3. Dumps older than the retention period are
+# removed, but the newest master dump of each database is always kept, so that a
+# database which hasn't changed in a while can still be downloaded. Dumps from
+# other branches aren't kept, so that old branches don't accumulate dumps.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+# The newest dump of each database from this branch is never pruned
+MASTER_BRANCH="master"
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0") [-n|--dry-run] [-d|--days <days>] <database>... | all
+
+Deletes the dumps in $S3_TARGET older than the given number of days
+(default $S3_DUMP_RETENTION_DAYS). The newest $MASTER_BRANCH dump of each database is
+always kept, or the newest dump of any branch if the database has no $MASTER_BRANCH dumps.
+
+Databases: ${DATABASES[*]}
+
+Options:
+  -n, --dry-run      Print the dumps that would be deleted instead of deleting them
+  -d, --days <days>  Delete dumps older than this many days
+  -h, --help         Show this help
+EOF
+}
+
+dry_run=false
+days="$S3_DUMP_RETENTION_DAYS"
+arguments=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -n | --dry-run)
+      dry_run=true
+      shift
+      ;;
+    -d | --days)
+      days="${2-}"
+      [[ "$days" =~ ^[0-9]+$ ]] || {
+        usage
+        die "Expected a number of days for $1, got: $days"
+      }
+      shift 2
+      ;;
+    -h | --help)
+      usage
+      exit 0
+      ;;
+    -*)
+      usage
+      die "Unknown option: $1"
+      ;;
+    *)
+      arguments+=("$1")
+      shift
+      ;;
+  esac
+done
+
+if [[ ${#arguments[@]} -eq 0 ]]; then
+  usage
+  die "No database given"
+fi
+
+prune_database() {
+  local database="$1"
+  local deleted=0
+
+  # s3cmd ls prints "<date> <time> <size> <object>", so sorting by the first two
+  # columns in reverse puts the newest dump first
+  local dumps
+  dumps="$(s3 ls "$S3_TARGET/$database-" | sort -rk1,2)"
+
+  # Find the newest master dump or in the case of it missing, the newest dump in general
+  local kept
+  kept="$(awk -v pattern="/$database-$MASTER_BRANCH-[0-9]+-[0-9]+[.]sql$" '$4 ~ pattern { print $4; exit }' <<< "$dumps")"
+  [[ -n "$kept" ]] || kept="$(awk 'NR == 1 { print $4 }' <<< "$dumps")"
+
+  # Remove any dumps older than the dump retention date (or -d)
+  # except for master for which at least one dump is saved
+  while read -r date time _ dump; do
+    [[ -n "$dump" && "$dump" != "$kept" && "$date $time" < "$cutoff" ]] || continue
+    if [[ "$dry_run" == true ]]; then
+      infomsg "Dry run, would delete $dump"
+    else
+      s3 del "$dump"
+    fi
+    deleted=$((deleted + 1))
+  done <<< "$dumps"
+
+  if [[ "$dry_run" == true ]]; then
+    successmsg "Would have pruned $deleted dump(s) of $database older than $days days"
+  else
+    successmsg "Pruned $deleted dump(s) of $database older than $days days"
+  fi
+}
+
+resolve_databases "${arguments[@]}"
+require_s3_config
+
+cutoff="$(date -u -d "-$days days" '+%Y-%m-%d %H:%M')"
+
+for database in "${selected_databases[@]}"; do
+  prune_database "$database"
+done

anonyymioodi/upload_dump_to_s3.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+# Upload database dumps from .databasedumps/test to s3, naming the objects
+# <database>-<git_branch>-<datetime>.sql so that dumps don't overwrite each other.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0") [-n|--dry-run] <database>... | all
+
+Uploads the dumps in .databasedumps/test to $S3_TARGET
+as <database>-<git_branch>-<datetime>.sql. Run dump_local_db.sh first to
+create the dumps.
+
+Databases: ${DATABASES[*]}
+
+Options:
+  -n, --dry-run  Print the s3cmd command instead of running it
+  -h, --help     Show this help
+EOF
+}
+
+dry_run=false
+arguments=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -n | --dry-run)
+      dry_run=true
+      shift
+      ;;
+    -h | --help)
+      usage
+      exit 0
+      ;;
+    -*)
+      usage
+      die "Unknown option: $1"
+      ;;
+    *)
+      arguments+=("$1")
+      shift
+      ;;
+  esac
+done
+
+if [[ ${#arguments[@]} -eq 0 ]]; then
+  usage
+  die "No database given"
+fi
+
+upload_database() {
+  local database="$1"
+  local dump_file="$TEST_DUMP_DIR/$database.sql"
+
+  [[ -f "$dump_file" ]] || die "No dump found at $dump_file, run dump_local_db.sh first"
+  [[ -s "$dump_file" ]] || die "Dump is empty: $dump_file"
+
+  local datetime
+  datetime="$(date +%Y%m%d-%H%M%S)"
+  local target="$S3_TARGET/$database-$branch-$datetime.sql"
+
+  if [[ "$dry_run" == true ]]; then
+    infomsg "Dry run, would run: s3cmd put $dump_file $target"
+    return
+  fi
+
+  infomsg "Pushing $dump_file to $target"
+  s3 put "$dump_file" "$target"
+  successmsg "Uploaded $dump_file to $target"
+}
+
+resolve_databases "${arguments[@]}"
+require_s3_config
+
+if [[ "$dry_run" == false ]]; then
+  "$SCRIPT_DIR"/create_s3_bucket.sh
+fi
+
+branch="$(current_branch_name)"
+
+for database in "${selected_databases[@]}"; do
+  upload_database "$database"
+done

docker-compose.ci.yml

@@ -11,6 +11,19 @@ x-healthcheck-defaults: &healthcheck-defaults
   start_period: 30s # How long startup period is
 
 services:
+  adminer:
+    <<: *defaults
+    container_name: oodikone-adminer
+    environment:
+      ADMINER_DESIGN: pepa-linha
+    image: adminer:5.5.1
+    ports:
+      - 5050:8080
+    profiles:
+      - adminer
+    volumes:
+      - ./adminer-plugins:/var/www/html/plugins-enabled
+
   backend:
     <<: *defaults
     container_name: oodikone-backend

docker-compose.yml

@@ -6,6 +6,12 @@ x-defaults: &defaults
 x-build_defaults: &build_defaults
   dockerfile: dev.Dockerfile
 
+x-user_defaults: &user_defaults
+  # Run containers as user:group defined in .env
+  # If run as $(id -u):$(id -g) will reduce permission errors when mounting
+  # Defaults to registry.access.redhat.com/ubi10/nodejs-24-minimal's default user:group
+  user: '${DOCKER_UID:-1001}:${DOCKER_GID:-0}'
+
 x-healthcheck-defaults: &healthcheck-defaults
   interval: 30s # Normal healthcehck interval
   timeout: 5s
@@ -19,14 +25,14 @@ services:
     container_name: oodikone-adminer
     environment:
       ADMINER_DESIGN: pepa-linha
-    image: adminer:4.8.1
+    image: adminer:5.5.1
     ports:
       - 5050:8080
     volumes:
       - ./adminer-plugins:/var/www/html/plugins-enabled
 
   backend:
-    <<: *defaults
+    <<: [*defaults, *user_defaults]
     container_name: oodikone-backend
     build:
       dockerfile: backend/dev.Dockerfile
@@ -67,7 +73,7 @@ services:
       <<: *healthcheck-defaults
 
   frontend:
-    <<: *defaults
+    <<: [*defaults, *user_defaults]
     container_name: oodikone-frontend
     build:
       dockerfile: frontend/dev.Dockerfile
@@ -186,7 +192,7 @@ services:
       <<: *healthcheck-defaults
 
   sis-updater-worker:
-    <<: *defaults
+    <<: [*defaults, *user_defaults]
     container_name: oodikone-updater-worker
     build:
       <<: *build_defaults
@@ -213,7 +219,7 @@ services:
       - sis-updater-worker-nodemod:/opt/app-root/src/node_modules
 
   sis-updater-scheduler:
-    <<: *defaults
+    <<: [*defaults, *user_defaults]
     container_name: oodikone-updater-scheduler
     build:
       <<: *build_defaults

e2e/Course_statistics.spec.ts

@@ -64,6 +64,16 @@ const clickAway = async (page: Page) => {
   await page.locator('body').click({ position: { x: 0, y: 0 } })
 }
 
+const TKT20003 = { code: 'TKT20003', groupId: 'hy-CU-118024775' }
+const TKT20001 = { code: 'TKT20001', groupId: 'hy-CU-118024656' }
+const TKT10002 = { code: 'TKT10002', groupId: 'hy-CU-118023867' }
+const KK_RUKIRJ = { code: 'KK-RUKIRJ', groupId: 'hy-CU-117995649' }
+const course50131 = { code: '50131', groupId: 'hy-CU-2014787' }
+const course200012 = { code: '200012', groupId: 'hy-CU-53928369' }
+const BSCS1001 = { code: 'BSCS1001', groupId: 'hy-CU-128548227' }
+const BSCS1003 = { code: 'BSCS1003', groupId: 'hy-CU-128550261' }
+const TKT10004 = { code: 'TKT10004', groupId: 'hy-CU-118023990' }
+
 test.describe('Basic user', () => {
   test.beforeEach(async ({ page }) => {
     await init(page, '/coursestatistics')
@@ -71,38 +81,35 @@ test.describe('Basic user', () => {
   })
 
   test('Search should work on consecutive searches', async ({ page }) => {
-    const courseCode1 = 'TKT20003'
-    await searchByCourseCode(page, courseCode1)
-    await page.getByTestId(`course-${courseCode1}`).click()
+    await searchByCourseCode(page, TKT20003.code)
+    await page.getByTestId(`course-${TKT20003.code}`).click()
 
-    await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([courseCode1]))
+    await expect(page).toHaveURL(url => url.searchParams.get('courses') === TKT20003.groupId)
 
     await expect(page.getByRole('button', { name: 'Show population' })).toBeEnabled()
     await page.getByRole('button', { name: 'Show population' }).click()
 
-    await expect(page.getByRole('heading', { name: courseCode1 })).toBeVisible()
+    await expect(page.getByRole('heading', { name: TKT20003.code })).toBeVisible()
 
     await page.getByTestId('nav-bar-button-courseStatistics').click()
 
-    const courseCode2 = 'TKT20001'
-    await searchByCourseCode(page, courseCode2)
-    await page.getByTestId(`course-${courseCode2}`).click()
+    await searchByCourseCode(page, TKT20001.code)
+    await page.getByTestId(`course-${TKT20001.code}`).click()
 
-    await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([courseCode2]))
+    await expect(page).toHaveURL(url => url.searchParams.get('courses') === TKT20001.groupId)
 
     await expect(page.getByRole('button', { name: 'Show population' })).toBeEnabled()
     await page.getByRole('button', { name: 'Show population' }).click()
 
-    await expect(page.getByText(courseCode2, { exact: true })).toBeVisible()
+    await expect(page.getByText(TKT20001.code, { exact: true })).toBeVisible()
   })
 
   test.describe('Course table can show non-standard grades', () => {
     test('Course table can show HT-TT grade scales', async ({ page }) => {
-      const coursecode = 'KK-RUKIRJ'
-      await searchByCourseCode(page, coursecode)
-      await page.getByTestId(`course-${coursecode}`).click()
+      await searchByCourseCode(page, KK_RUKIRJ.code)
+      await page.getByTestId(`course-${KK_RUKIRJ.code}`).click()
 
-      await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([coursecode]))
+      await expect(page).toHaveURL(url => url.searchParams.get('courses') === KK_RUKIRJ.groupId)
 
       await selectFromYear(page, '2019-2020')
       await expect(page.getByRole('button', { name: 'Show population' })).toBeEnabled()
@@ -117,11 +124,10 @@ test.describe('Basic user', () => {
     })
 
     test("Course table can show old master's thesis grade scales", async ({ page }) => {
-      const coursecode = '50131'
-      await searchByCourseCode(page, coursecode)
-      await page.getByTestId(`course-${coursecode}`).click()
+      await searchByCourseCode(page, course50131.code)
+      await page.getByTestId(`course-${course50131.code}`).click()
 
-      await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([coursecode]))
+      await expect(page).toHaveURL(url => url.searchParams.get('courses') === course50131.groupId)
 
       await expect(page.getByText('Pro gradu -tutkielma tietojenkäsittelytieteessä')).toBeVisible() // 50131
 
@@ -130,7 +136,7 @@ test.describe('Basic user', () => {
 
       await expect(page.getByText('Pro gradu -tutkielma tietojenkäsittelytieteessä')).toBeVisible()
       await expect(page.getByText('Class of 2007-2020, 10 students')).toBeVisible()
-      await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+      await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
       await expect(page.getByText('Showing 10 out of 10 students')).toBeVisible()
 
       const gradesTableContents = [
@@ -143,11 +149,10 @@ test.describe('Basic user', () => {
     })
 
     test('Shows correct statistics for courses with scale passed-failed', async ({ page }) => {
-      const coursecode = '200012'
-      await searchByCourseCode(page, coursecode)
-      await page.getByTestId(`course-${coursecode}`).click()
+      await searchByCourseCode(page, course200012.code)
+      await page.getByTestId(`course-${course200012.code}`).click()
 
-      await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([coursecode]))
+      await expect(page).toHaveURL(url => url.searchParams.get('courses') === course200012.groupId)
 
       await expect(page.getByText('ON-310')).toBeVisible() // Tieteellisen kirjoittamisen seminaarin alkuopetus: Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot
       await expect(page.getByText('200012')).toBeVisible() // Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot
@@ -157,7 +162,7 @@ test.describe('Basic user', () => {
 
       await expect(page.getByText('Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot')).toBeVisible()
       await expect(page.getByText('Class of 2011-2018, 4 students')).toBeVisible()
-      await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+      await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
       await expect(page.getByText('Showing 4 out of 4 students')).toBeVisible()
 
       const gradesTableContents = [
@@ -170,11 +175,10 @@ test.describe('Basic user', () => {
 
   test.describe('Course mappings work', () => {
     test('Searching single course having substitution mappings shows course statistics', async ({ page }) => {
-      const coursecode = 'TKT20001'
-      await searchByCourseCode(page, coursecode)
-      await page.getByTestId(`course-${coursecode}`).click()
+      await searchByCourseCode(page, TKT20001.code)
+      await page.getByTestId(`course-${TKT20001.code}`).click()
 
-      await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([coursecode]))
+      await expect(page).toHaveURL(url => url.searchParams.get('courses') === TKT20001.groupId)
 
       await expect(page.getByText('Tietorakenteet ja algoritmit', { exact: true })).toBeVisible() // The old 10credit variant
       await expect(page.getByText('TKT20001', { exact: true })).toBeVisible()
@@ -196,14 +200,15 @@ test.describe('Basic user', () => {
       await expect(page.getByTestId('select-multiple-courses-toggle')).toBeChecked()
 
       await searchByCourseCode(page, 'TKT')
-      await page.getByTestId(`course-TKT20001`).click()
-      await page.getByTestId(`course-TKT10002`).click()
+      await page.getByTestId(`course-${TKT20001.code}`).click()
+      await page.getByTestId(`course-${TKT10002.code}`).click()
 
       await expect(page.getByText('Fetch statistics')).toBeEnabled()
       await page.getByText('Fetch statistics').click()
 
       await expect(page).toHaveURL(
-        url => url.searchParams.get('courseCodes') === JSON.stringify(['TKT10002', 'TKT20001'])
+        url =>
+          JSON.stringify(url.searchParams.getAll('courses')) === JSON.stringify([TKT10002.groupId, TKT20001.groupId])
       )
 
       await openSummaryTab(page)
@@ -227,20 +232,20 @@ test.describe('Basic user', () => {
     await expect(page.getByTestId('select-multiple-courses-toggle')).toBeChecked()
 
     await searchByCourseCode(page, 'BSCS')
-    await page.getByTestId(`course-BSCS1003`).click()
-    await page.getByTestId(`course-BSCS1001`).click()
+    await page.getByTestId(`course-${BSCS1003.code}`).click()
+    await page.getByTestId(`course-${BSCS1001.code}`).click()
 
     await expect(page.getByText('Fetch statistics')).toBeEnabled()
     await page.getByText('Fetch statistics').click()
 
     await expect(page).toHaveURL(
-      url => url.searchParams.get('courseCodes') === JSON.stringify(['BSCS1001', 'BSCS1003'])
+      url => JSON.stringify(url.searchParams.getAll('courses')) === JSON.stringify([BSCS1001.groupId, BSCS1003.groupId])
     )
 
     await page.getByTestId('course-population-for-2023-2024').click()
     await expect(page.getByRole('heading', { name: 'Introduction to Programming' })).toBeVisible()
     await expect(page.getByText('Class of 2023-2024')).toBeVisible()
-    await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+    await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
 
     await page.goBack()
 
@@ -248,12 +253,15 @@ test.describe('Basic user', () => {
     await page.getByTestId('CourseSelectorOptionBSCS1003').click()
 
     // Check the link has updated correctly
-    await expect(page.getByTestId('course-population-for-2022-2023')).toHaveAttribute('href', /BSCS1003/)
+    await expect(page.getByTestId('course-population-for-2022-2023')).toHaveAttribute(
+      'href',
+      new RegExp(BSCS1003.groupId)
+    )
     await expect(page.getByTestId('CourseSelector').getByText('Data Structures and Algorithms')).toBeVisible()
 
     await page.getByTestId('course-population-for-2022-2023').click()
     await expect(page.getByText('Class of 2022-2023')).toBeVisible()
-    await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+    await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
   })
 
   test('Searching course by name displays right courses', async ({ page }) => {
@@ -279,11 +287,13 @@ test.describe('Basic user', () => {
     ).toBeVisible()
   })
 
+  // FIXME
   test('Searching course by name displays right courses, 10 credit courses', async ({ page }) => {
     await expect(page.getByText('Search for courses')).toBeVisible()
     await searchByCourseName(page, 'tietorakenteet ja algoritmit')
 
-    await expect(page.getByText('Tietorakenteet ja algoritmit', { exact: true })).toBeVisible()
+    // There should be two identically named Tietorakenteet ja algoritmit courses.
+    await expect(page.getByText('Tietorakenteet ja algoritmit', { exact: true })).toHaveCount(2)
     await page.getByTestId(`course-TKT20001`).click()
     await expect(page.getByText('Search for courses')).not.toBeVisible()
 
@@ -378,7 +388,7 @@ test.describe('Basic user', () => {
 
       await expect(page.getByText('Käyttöjärjestelmät')).toBeVisible()
       await expect(page.getByText('Class of 2020-2021, 19 students')).toBeVisible()
-      await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+      await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
 
       await expect(page.getByText('TKT20003')).toBeVisible()
 
@@ -388,9 +398,9 @@ test.describe('Basic user', () => {
     })
 
     test('Population of course shows grades for each student', async ({ page }) => {
-      await searchByCourseCode(page, 'TKT20001')
-      await page.getByTestId(`course-TKT20001`).click()
-      await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify(['TKT20001']))
+      await searchByCourseCode(page, TKT20001.code)
+      await page.getByTestId(`course-${TKT20001.code}`).click()
+      await expect(page).toHaveURL(url => url.searchParams.get('courses') === TKT20001.groupId)
 
       // This step should *not* be needed to make this test work (but it is)
       await expect(page.getByRole('heading', { name: 'Selected course' })).toBeVisible()
@@ -404,7 +414,7 @@ test.describe('Basic user', () => {
 
       await expect(page.getByText('Tietorakenteet ja algoritmit')).toBeVisible()
       await expect(page.getByText('Class of 2019-2020, 33 students')).toBeVisible()
-      await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+      await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
 
       // Check grade field in Students tab
       await page.getByText('Students (33)').click()
@@ -422,6 +432,9 @@ test.describe('Basic user', () => {
       await searchByCourseCode(page, 'TKT20001')
       await page.getByTestId(`course-TKT20001`).click()
 
+      // This step should *not* be needed to make this test work (but it is)
+      await expect(page.getByRole('heading', { name: 'Selected course' })).toBeVisible()
+
       await expect(page.getByText('TKT20001', { exact: true })).toBeVisible() //  Tietorakenteet ja algoritmit
       await expect(page.getByText('AYTKT20001', { exact: true })).toBeVisible() // Avoin yo: Tietorakenteet ja algoritmit
       await expect(page.getByText('BSCS1003')).toBeVisible() // Data Structures and Algorithms
@@ -431,7 +444,7 @@ test.describe('Basic user', () => {
 
       await expect(page.getByText('Tietorakenteet ja algoritmit')).toBeVisible()
       await expect(page.getByText('Class of 2019-2020, 33 students')).toBeVisible()
-      await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+      await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
 
       await page.getByText('Students (33)').click()
       await expect(page.locator('tbody > tr').filter({ hasText: 'Hidden' })).toHaveCount(9)
@@ -448,7 +461,7 @@ test.describe('Basic user', () => {
 
     await expect(page.getByText('Käyttöjärjestelmät')).toBeVisible()
     await expect(page.getByText('Class of 2021-2022')).toBeVisible()
-    await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+    await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
 
     // Check count column of Language distribution
     await page.getByText('Language distribution').click()
@@ -631,14 +644,14 @@ test.describe('Basic user', () => {
         test('Show grades off, Separate by semesters off', async ({ page }) => {
           const tableContents = [
             // [Time, Total attempts, Passed, Failed, Pass rate, Enrollments]
-            ['Total', 146, 140, 6, '95.89 %', 73],
+            ['Total', 156, 140, 16, '89.74 %', 73],
             ['2023-2024', 7, 1, 0, '14.29 %', 7],
             ['2022-2023', 43, 27, 0, '62.79 %', 43],
             ['2021-2022', 23, 15, 0, '65.22 %', 23],
             ['2020-2021', 23, 23, 0, '100.00 %', null],
-            ['2019-2020', 28, 28, 0, '100.00 %', null],
-            ['2018-2019', 28, 26, 2, '92.86 %', null],
-            ['2017-2018', 23, 19, 4, '82.61 %', null],
+            ['2019-2020', 29, 28, 1, '96.55 %', null],
+            ['2018-2019', 32, 26, 6, '81.25 %', null],
+            ['2017-2018', 28, 19, 9, '67.86 %', null],
             ['2016-2017', 1, 1, 0, '100.00 %', null],
           ]
           await checkTableContents(page, tableContents)
@@ -671,14 +684,14 @@ test.describe('Basic user', () => {
         test('Show grades on, Separate by semesters off', async ({ page }) => {
           const tableContents = [
             // [Time, Total attempts, 0, 1, 2, 3, 4, 5, Other passed]
-            ['Total', 146, 6, 3, 8, 5, 24, 99, 1],
+            ['Total', 156, 16, 3, 8, 5, 24, 99, 1],
             ['2023-2024', 7, 0, 0, 0, 0, 0, 1, 0],
             ['2022-2023', 43, 0, 0, 0, 0, 6, 21, 0],
             ['2021-2022', 23, 0, 0, 0, 0, 2, 13, 0],
             ['2020-2021', 23, 0, 0, 2, 0, 2, 18, 1],
-            ['2019-2020', 28, 0, 1, 4, 1, 5, 17, 0],
-            ['2018-2019', 28, 2, 2, 1, 2, 3, 18, 0],
-            ['2017-2018', 23, 4, 0, 1, 2, 5, 11, 0],
+            ['2019-2020', 29, 1, 1, 4, 1, 5, 17, 0],
+            ['2018-2019', 32, 6, 2, 1, 2, 3, 18, 0],
+            ['2017-2018', 28, 9, 0, 1, 2, 5, 11, 0],
             ['2016-2017', 1, 0, 0, 0, 0, 1, 0, 0],
           ]
           await toggleShowGrades(page)
@@ -799,7 +812,7 @@ test.describe('Basic user', () => {
             ['Kevät 2015', 3, 3, 0, null, '100.00 %', '0.00 %'],
             ['Syksy 2014', 3, 3, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2014', 1, 1, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2013', 0, 0, 0, 0, '–', '–'],
+            ['Syksy 2013', 0, 0, 0, null, '–', '–'],
             ['Kevät 2013', 0, 0, 0, 0, '–', '–'],
             ['Syksy 2012', 4, 4, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2012', 1, 1, 0, null, '100.00 %', '0.00 %'],
@@ -891,7 +904,7 @@ test.describe('Basic user', () => {
             ['Kevät 2015', 3, 0, 0, 0, 0, 0, 0, 3, null, '100.00 %', '0.00 %'],
             ['Syksy 2014', 3, 0, 0, 1, 0, 0, 2, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2014', 1, 0, 0, 0, 0, 0, 1, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2013', 0, 0, 0, 0, 0, 0, 0, 0, 0, '–', '–'],
+            ['Syksy 2013', 0, 0, 0, 0, 0, 0, 0, 0, null, '–', '–'],
             ['Kevät 2013', 0, 0, 0, 0, 0, 0, 0, 0, 0, '–', '–'],
             ['Syksy 2012', 4, 0, 0, 0, 0, 3, 1, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2012', 1, 0, 0, 0, 0, 0, 1, 0, null, '100.00 %', '0.00 %'],
@@ -935,18 +948,18 @@ test.describe('Basic user', () => {
         test('Show grades off, Separate by semesters off', async ({ page }) => {
           const tableContents = [
             // [Time, Total attempts, Passed, Failed, Pass rate, Enrollments]
-            ['Total', 245, 236, 9, '96.33 %', 90],
+            ['Total', 266, 241, 25, '90.60 %', 90],
             ['2023-2024', 8, 2, 0, '25.00 %', 8],
             ['2022-2023', 44, 28, 0, '63.64 %', 44],
             ['2021-2022', 38, 27, 0, '71.05 %', 38],
-            ['2020-2021', 33, 33, 0, '100.00 %', null],
-            ['2019-2020', 57, 56, 1, '98.25 %', null],
-            ['2018-2019', 40, 38, 2, '95.00 %', null],
-            ['2017-2018', 28, 24, 4, '85.71 %', null],
+            ['2020-2021', 34, 34, 0, '100.00 %', null],
+            ['2019-2020', 60, 57, 3, '95.00 %', null],
+            ['2018-2019', 46, 39, 7, '84.78 %', null],
+            ['2017-2018', 35, 26, 9, '74.29 %', null],
             ['2016-2017', 7, 6, 1, '85.71 %', null],
-            ['2015-2016', 3, 3, 0, '100.00 %', null],
-            ['2014-2015', 6, 6, 0, '100.00 %', null],
-            ['2013-2014', 1, 1, 0, '100.00 %', null],
+            ['2015-2016', 5, 3, 2, '60.00 %', null],
+            ['2014-2015', 7, 6, 1, '85.71 %', null],
+            ['2013-2014', 2, 1, 1, '50.00 %', null],
             ['2012-2013', 4, 4, 0, '100.00 %', null],
             ['2011-2012', 1, 1, 0, '100.00 %', null],
             ['2010-2011', 1, 1, 0, '100.00 %', null],
@@ -989,7 +1002,7 @@ test.describe('Basic user', () => {
             ['Kevät 2015', 3, 3, 0, null, '100.00 %', '0.00 %'],
             ['Syksy 2014', 3, 3, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2014', 1, 1, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2013', 0, 0, 0, 0, '–', '–'],
+            ['Syksy 2013', 0, 0, 0, null, '–', '–'],
             ['Kevät 2013', 0, 0, 0, 0, '–', '–'],
             ['Syksy 2012', 4, 4, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2012', 1, 1, 0, null, '100.00 %', '0.00 %'],
@@ -1026,18 +1039,18 @@ test.describe('Basic user', () => {
         test('Show grades on, Separate by semesters off', async ({ page }) => {
           const tableContents = [
             // [Time, Total attempts, 0, 1, 2, 3, 4, 5, Other passed]
-            ['Total', 245, 9, 4, 13, 9, 37, 168, 5],
+            ['Total', 266, 25, 6, 15, 9, 37, 169, 5],
             ['2023-2024', 8, 0, 0, 0, 0, 0, 2, 0],
             ['2022-2023', 44, 0, 0, 0, 0, 6, 22, 0],
             ['2021-2022', 38, 0, 0, 1, 0, 2, 24, 0],
-            ['2020-2021', 33, 0, 0, 2, 1, 2, 27, 1],
-            ['2019-2020', 57, 1, 1, 5, 3, 9, 38, 0],
-            ['2018-2019', 40, 2, 3, 2, 2, 6, 25, 0],
-            ['2017-2018', 28, 4, 0, 1, 2, 6, 15, 0],
+            ['2020-2021', 34, 0, 0, 2, 1, 2, 28, 1],
+            ['2019-2020', 60, 3, 2, 5, 3, 9, 38, 0],
+            ['2018-2019', 46, 7, 3, 3, 2, 6, 25, 0],
+            ['2017-2018', 35, 9, 1, 2, 2, 6, 15, 0],
             ['2016-2017', 7, 1, 0, 0, 1, 1, 4, 0],
-            ['2015-2016', 3, 0, 0, 1, 0, 0, 1, 1],
-            ['2014-2015', 6, 0, 0, 1, 0, 0, 2, 3],
-            ['2013-2014', 1, 0, 0, 0, 0, 0, 1, 0],
+            ['2015-2016', 5, 2, 0, 1, 0, 0, 1, 1],
+            ['2014-2015', 7, 1, 0, 1, 0, 0, 2, 3],
+            ['2013-2014', 2, 1, 0, 0, 0, 0, 1, 0],
             ['2012-2013', 4, 0, 0, 0, 0, 3, 1, 0],
             ['2011-2012', 1, 0, 0, 0, 0, 0, 1, 0],
             ['2010-2011', 1, 0, 0, 0, 0, 1, 0, 0],
@@ -1081,7 +1094,7 @@ test.describe('Basic user', () => {
             ['Kevät 2015', 3, 0, 0, 0, 0, 0, 0, 3, null, '100.00 %', '0.00 %'],
             ['Syksy 2014', 3, 0, 0, 1, 0, 0, 2, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2014', 1, 0, 0, 0, 0, 0, 1, 0, null, '100.00 %', '0.00 %'],
-            ['Syksy 2013', 0, 0, 0, 0, 0, 0, 0, 0, 0, '–', '–'],
+            ['Syksy 2013', 0, 0, 0, 0, 0, 0, 0, 0, null, '–', '–'],
             ['Kevät 2013', 0, 0, 0, 0, 0, 0, 0, 0, 0, '–', '–'],
             ['Syksy 2012', 4, 0, 0, 0, 0, 3, 1, 0, null, '100.00 %', '0.00 %'],
             ['Kevät 2012', 1, 0, 0, 0, 0, 0, 1, 0, null, '100.00 %', '0.00 %'],
@@ -1182,7 +1195,7 @@ test.describe('Basic user', () => {
     })
 
     test('Has right to see all the students, because course provider is TKT', async ({ page }) => {
-      await page.goto('coursestatistics?courseCodes=%5B%22TKT10004%22%5D&separate=false')
+      await page.goto(`coursestatistics?courses=${encodeURIComponent(TKT10004.groupId)}&separate=false`)
       await page.getByTestId('course-population-for-2021-2022').click()
       await expect(page.getByText('Students (28)')).toBeVisible()
     })
@@ -1198,11 +1211,10 @@ test.describe('Only course statistics', () => {
   test('Some features of Course Statistics are hidden for courseStatistics-users without other rights', async ({
     page,
   }) => {
-    const coursecode = 'TKT10002'
-    await searchByCourseCode(page, coursecode)
-    await page.getByTestId(`course-${coursecode}`).click()
+    await searchByCourseCode(page, TKT10002.code)
+    await page.getByTestId(`course-${TKT10002.code}`).click()
 
-    await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([coursecode]))
+    await expect(page).toHaveURL(url => url.searchParams.get('courses') === TKT10002.groupId)
 
     await expect(page.getByText('Filter statistics by degree programmes')).not.toBeVisible()
     await expect(page.getByText('Show population')).not.toBeVisible()
@@ -1213,17 +1225,17 @@ test.describe('Only course statistics', () => {
 
     const attemptsTableContents = [
       // [Time, Total attempts, Passed, Failed, Pass rate, Enrollments]
-      ['Total *', 226, 218, 8, '96.46 %', 82],
+      ['Total *', 244, 223, 21, '91.39 %', 82],
       emptyYear('2023-2024'),
       ['2022-2023', 44, 28, 0, '63.64 %', 44],
       ['2021-2022', 38, 27, 0, '71.05 %', 38],
-      ['2020-2021', 33, 33, 0, '100.00 %', null],
-      ['2019-2020', 57, 56, 1, '98.25 %', null],
-      ['2018-2019', 40, 38, 2, '95.00 %', null],
-      ['2017-2018', 28, 24, 4, '85.71 %', null],
+      ['2020-2021', 34, 34, 0, '100.00 %', null],
+      ['2019-2020', 60, 57, 3, '95.00 %', null],
+      ['2018-2019', 46, 39, 7, '84.78 %', null],
+      ['2017-2018', 35, 26, 9, '74.29 %', null],
       ['2016-2017', 7, 6, 1, '85.71 %', null],
       emptyYear('2015-2016'),
-      ['2014-2015', 6, 6, 0, '100.00 %', null],
+      ['2014-2015', 7, 6, 1, '85.71 %', null],
       emptyYear('2013-2014'),
       emptyYear('2012-2013'),
       emptyYear('2011-2012'),

e2e/Degree_programme_overview.spec.ts

@@ -664,9 +664,7 @@ test.describe('Degree programme overview', () => {
       test('info box', async ({ page }) => {
         await expect(page.getByTestId('credit-criteria-section')).toBeVisible()
         await page.getByTestId('credit-criteria-info-box-button').click()
-        await expect(
-          page.getByTestId('credit-criteria-info-box-content').getByText('Here you can change')
-        ).toBeVisible()
+        await expect(page.getByTestId('credit-criteria-info-box-content').getByText('Tässä asetettu')).toBeVisible()
       })
 
       test.skip('changing the credit criteria works', () => {

e2e/Filters.spec.ts

@@ -190,7 +190,8 @@ test.describe('Custom Population Statistics', () => {
     await page.getByRole('combobox').fill('MAT11005 - Integraalilaskenta')
     await page.getByRole('option', { name: 'MAT11005 - Integraalilaskenta' }).click()
     await checkFilteredStudentCount(page, 7)
-    await page.getByTestId('courseFilter-MAT11005-selector').click()
+    // MAT11005 = hy-CU-117375607
+    await page.getByTestId('courseFilter-hy-CU-117375607-selector').click()
     await page.getByRole('option', { name: 'Passed' }).click()
     await checkFilteredStudentCount(page, 6)
   })

e2e/Language_center.spec.ts

@@ -4,7 +4,8 @@ import { init } from './support/commands'
 const chooseSemester = async (page: Page, semester: string, fromOrTo: 'from' | 'to') => {
   await page.getByTestId(`semester-${fromOrTo}`).click()
   await page.getByTestId(`select-opt-${semester}`).click()
-  await expect(page.getByTestId(`select-opt-${semester}`)).not.toHaveClass(/visible/)
+  // The dropdown unmounts its options on close, so wait for it to be gone before reading the table
+  await expect(page.getByRole('listbox')).toHaveCount(0)
 }
 
 const checkNumbers = async (page: Page, numbers: number[], numberOfColumns: number, mode: string) => {
@@ -36,18 +37,20 @@ test.describe('When language center is opened', () => {
         await expect(page.getByText('AYKK-RUKIRJ')).toBeVisible()
       })
 
-      test('Faculties tab shows numbers', async ({ page }) => {
-        await checkNumbers(page, [2076, 36, 9, 34, 759, 15, 40, 6, 51, 1, 24, 1059, 42, 0], 15, 'faculties')
+      // FIXME: This changes per updater run
+      // Example case: 	550789 - KK-JAOK15
+      test.skip('Faculties tab shows numbers', async ({ page }) => {
+        await checkNumbers(page, [2074, 36, 9, 33, 758, 15, 40, 6, 51, 1, 24, 1059, 42, 0], 15, 'faculties')
       })
 
-      test('Faculties tab "exceeding" button works', async ({ page }) => {
+      test.skip('Faculties tab "exceeding" button works', async ({ page }) => {
         await page.getByTestId('difference-button').click()
         await checkNumbers(page, [66, 3, 0, 4, 18, 0, 1, 1, 4, 0, 0, 31, 3, 1], 15, 'faculties')
       })
 
-      test('Faculties tab semester selector changes numbers', async ({ page }) => {
+      test.skip('Faculties tab semester selector changes numbers', async ({ page }) => {
         await chooseSemester(page, 'Syksy 2020', 'from')
-        await checkNumbers(page, [1184, 28, 6, 28, 499, 4, 10, 2, 36, 0, 9, 535, 27, 0], 15, 'faculties')
+        await checkNumbers(page, [1182, 28, 6, 27, 498, 4, 10, 2, 36, 0, 9, 535, 27, 0], 15, 'faculties')
       })
     })
 
@@ -58,8 +61,8 @@ test.describe('When language center is opened', () => {
         await chooseSemester(page, 'Kevät 2024', 'to')
       })
 
-      test('Semester tab shows numbers', async ({ page }) => {
-        await checkNumbers(page, [2076, 69, 26, 298, 58, 343, 98, 438, 138, 310, 90, 123, 74, 10, 1], 16, 'semesters')
+      test.skip('Semester tab shows numbers', async ({ page }) => {
+        await checkNumbers(page, [2074, 69, 26, 298, 58, 343, 98, 437, 138, 310, 90, 123, 73, 10, 1], 16, 'semesters')
       })
     })
   })

e2e/Numbers_match.spec.ts

@@ -4,6 +4,7 @@ import { init } from './support/commands'
 type CourseData = [
   courseOrModule: string,
   code: string,
+  groupId: string,
   name: string,
   total: number,
   passed: number,
@@ -13,20 +14,22 @@ type CourseData = [
 ]
 
 const courseData: CourseData[] = [
-  ['Course', 'MAT11001', 'Johdatus yliopistomatematiikkaan', 272, 253, 1 + 18, 1, 18],
-  ['Course', 'MAT11003', 'Raja-arvot', 270, 249, 1 + 20, 1, 20],
-  ['Course', 'MAT11004', 'Differentiaalilaskenta', 262, 248, 1 + 13, 1, 13],
-  ['Course', 'MAT21002', 'Sarjat', 256, 250, 0 + 6, 0, 6],
-  ['Course', 'MAT11005', 'Integraalilaskenta', 251, 234, 0 + 17, 0, 17],
-  ['Course', 'MAT11002', 'Lineaarialgebra ja matriisilaskenta I', 247, 242, 1 + 4, 1, 4],
-  ['Course', 'MAT12003', 'Todennäköisyyslaskenta I', 234, 227, 3 + 4, 3, 4],
-  ['Course', 'MAT21001', 'Lineaarialgebra ja matriisilaskenta II', 230, 216, 4 + 10, 4, 10],
-  ['Course', 'MAT21003', 'Vektorianalyysi I', 228, 202, 3 + 23, 3, 23],
-  ['Course', 'MAT20005', 'Akateemiset taidot', 201, 178, 0 + 23, 0, 23],
-  ['Module', 'MAT110', 'Matematiikka, perusopinnot', 195, 195, 0 + 0, 0, 0],
-  ['Course', 'MAT21014', 'Johdatus logiikkaan I', 188, 170, 0 + 18, 0, 18],
+  ['Course', 'MAT11001', 'hy-CU-117375151', 'Johdatus yliopistomatematiikkaan', 272, 253, 1 + 18, 1, 18],
+  ['Course', 'MAT11003', 'hy-CU-117375413', 'Raja-arvot', 270, 249, 1 + 20, 1, 20],
+  ['Course', 'MAT11004', 'hy-CU-117375487', 'Differentiaalilaskenta', 262, 248, 1 + 13, 1, 13],
+  ['Course', 'MAT21002', 'hy-CU-117375793', 'Sarjat', 256, 250, 0 + 6, 0, 6],
+  ['Course', 'MAT11005', 'hy-CU-117375607', 'Integraalilaskenta', 251, 234, 0 + 17, 0, 17],
+  ['Course', 'MAT11002', 'hy-CU-117375394', 'Lineaarialgebra ja matriisilaskenta I', 247, 242, 1 + 4, 1, 4],
+  ['Course', 'MAT12003', 'hy-CU-117376940', 'Todennäköisyyslaskenta I', 234, 227, 3 + 4, 3, 4],
+  ['Course', 'MAT21001', 'hy-CU-117375754', 'Lineaarialgebra ja matriisilaskenta II', 230, 216, 4 + 10, 4, 10],
+  ['Course', 'MAT21003', 'hy-CU-117375829', 'Vektorianalyysi I', 228, 202, 3 + 23, 3, 23],
+  ['Course', 'MAT20005', 'hy-CU-120230575', 'Akateemiset taidot', 201, 178, 0 + 23, 0, 23],
+  ['Module', 'MAT110', 'hy-SM-117412902', 'Matematiikka, perusopinnot', 195, 195, 0 + 0, 0, 0],
+  ['Course', 'MAT21014', 'hy-CU-117377242', 'Johdatus logiikkaan I', 188, 170, 0 + 18, 0, 18],
 ]
 
+const MAT21003_GROUP_ID = 'hy-CU-117375829'
+
 const yearlyData = [
   [2017, 1, 0, 1, 0, 1 + 0],
   [2018, 31, 27, 4, 0, 4 + 0],
@@ -45,8 +48,8 @@ const getPercentage = (value: any, total: any) => {
   return `${((value / total) * 100).toFixed(2)} %`
 }
 
-const selectCourseStatus = async (page: Page, courseCode: string, status: string) => {
-  await page.getByTestId(`courseFilter-${courseCode}-selector`).click()
+const selectCourseStatus = async (page: Page, courseGroupId: string, status: string) => {
+  await page.getByTestId(`courseFilter-${courseGroupId}-selector`).click()
   await page.getByRole('listbox').getByRole('option', { name: status }).click()
 }
 
@@ -71,18 +74,32 @@ test.describe('Numbers should match between', () => {
       const rows = page.locator('tbody tr')
       for (const [index, course] of courseData.entries()) {
         const row = rows.nth(index)
+        const [courseOrModule, code, , name, total, passed, notCompleted] = course
         // Not optimal, but not checking all locator("td") matches directly with toHaveText is hard.
-        for (const [fieldIndex, field] of course.slice(0, 6).entries()) {
+        for (const [fieldIndex, field] of [courseOrModule, code, name, total, passed, notCompleted].entries()) {
           await expect(row.locator('td').nth(fieldIndex)).toHaveText(String(field))
         }
       }
     })
 
     test('in Course statistics', async ({ page }) => {
-      for (const [_courseOrModule, code, name, total, passed, _notCompleted, failed, enrolledNoGrade] of courseData) {
-        await init(page, `/coursestatistics?courseCodes=%5B%22${code}%22%5D&combineSubstitutions=false`, 'basic')
+      for (const [
+        _courseOrModule,
+        _code,
+        groupId,
+        name,
+        total,
+        passed,
+        _notCompleted,
+        failed,
+        enrolledNoGrade,
+      ] of courseData) {
+        await init(page, `/coursestatistics?courses=${groupId}&substitutions=false`, 'basic')
         await expect(page).toHaveURL(
-          url => url.pathname === '/coursestatistics' && url.searchParams.get('courseCodes') === JSON.stringify([code])
+          url =>
+            url.pathname === '/coursestatistics' &&
+            url.searchParams.get('courses') === groupId &&
+            url.searchParams.get('substitutions') === 'false'
         )
         await expect(page.getByText('Course statistics')).toBeVisible()
         await expect(page.getByText(name)).toBeVisible()
@@ -105,12 +122,28 @@ test.describe('Numbers should match between', () => {
     })
 
     test('in Course population', async ({ page }) => {
-      for (const [_courseOrModule, code, name, total, passed, _notCompleted, failed, enrolledNoGrade] of courseData) {
+      for (const [
+        _courseOrModule,
+        code,
+        groupId,
+        name,
+        total,
+        passed,
+        _notCompleted,
+        failed,
+        enrolledNoGrade,
+      ] of courseData) {
         await init(
           page,
-          `/coursepopulation?from=${2017 - 1949}&to=${2023 - 1949}&coursecodes=%5B%22${code}%22%5D&includeSubstitutions=false`,
+          `/coursepopulation?from=${2017 - 1949}&to=${2023 - 1949}&courses=${groupId}&substitutions=false`,
           'basic'
         )
+        await expect(page).toHaveURL(
+          url =>
+            url.pathname === '/coursepopulation' &&
+            url.searchParams.get('courses') === groupId &&
+            url.searchParams.get('substitutions') === 'false'
+        )
         await expect(page.getByText(name)).toBeVisible()
 
         await page.getByTestId('courseFilter-filter-card').click()
@@ -126,7 +159,7 @@ test.describe('Numbers should match between', () => {
           ['Failed', failed],
           ['Enrolled, No Grade', enrolledNoGrade],
         ] as const) {
-          await selectCourseStatus(page, code, status)
+          await selectCourseStatus(page, groupId, status)
           await expect(page.getByText(`Students (${expected})`)).toBeVisible()
         }
       }
@@ -152,7 +185,13 @@ test.describe('Numbers should match between', () => {
 
     test('in Course statistics', async ({ page }) => {
       for (const [year, total, passed, failed, enrolledNoGrade] of yearlyData) {
-        await init(page, '/coursestatistics?courseCodes=%5B%22MAT21003%22%5D&combineSubstitutions=false', 'basic')
+        await init(page, `/coursestatistics?courses=${MAT21003_GROUP_ID}&substitutions=false`, 'basic')
+        await expect(page).toHaveURL(
+          url =>
+            url.pathname === '/coursestatistics' &&
+            url.searchParams.get('courses') === MAT21003_GROUP_ID &&
+            url.searchParams.get('substitutions') === 'false'
+        )
         const yearString = `${year}-${year + 1}`
 
         await selectYear(page, 'ToYearSelector', `ToYearSelectorOption${yearString}`)
@@ -180,9 +219,15 @@ test.describe('Numbers should match between', () => {
       for (const [year, _total, passed, failed, enrolledNoGrade] of yearlyData) {
         await init(
           page,
-          `/coursepopulation?from=${year - 1949}&to=${year - 1949}&coursecodes=%5B%22MAT21003%22%5D&includeSubstitutions=false`,
+          `/coursepopulation?from=${year - 1949}&to=${year - 1949}&courses=${MAT21003_GROUP_ID}&substitutions=false`,
           'basic'
         )
+        await expect(page).toHaveURL(
+          url =>
+            url.pathname === '/coursepopulation' &&
+            url.searchParams.get('courses') === MAT21003_GROUP_ID &&
+            url.searchParams.get('substitutions') === 'false'
+        )
         await expect(page.getByText('Vektorianalyysi I')).toBeVisible()
 
         await page.getByTestId('courseFilter-filter-card').click()
@@ -198,7 +243,7 @@ test.describe('Numbers should match between', () => {
           ['Failed', failed],
           ['Enrolled, No Grade', enrolledNoGrade],
         ] as const) {
-          await selectCourseStatus(page, 'MAT21003', status)
+          await selectCourseStatus(page, MAT21003_GROUP_ID, status)
           await expect(page.getByText(`Students (${expected})`)).toBeVisible()
         }
       }

ok.sh

@@ -0,0 +1,399 @@
+#!/usr/bin/env bash
+
+# Oodikone CLI tool (revision 2.0)
+# Bash template provided by:
+# https://betterdev.blog/minimal-safe-bash-script-template/
+
+set -Eueo pipefail
+trap cleanup SIGINT SIGTERM ERR EXIT
+
+# Constants
+# Resolve relative to the script's own location (not the caller's cwd), so
+# `ok.sh` behaves the same whether it's run as `./ok.sh` from the repo or
+# invoked with an absolute/symlinked path from anywhere (e.g. the `ok`
+# shell function installed via `./ok.sh install`).
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel)"
+
+cd "$PROJECT_ROOT"
+
+# Include scripts from other sources
+source "$PROJECT_ROOT"/scripts/functions_for_setup.sh
+
+usage() {
+  cat << EOF # remove the space between << and EOF, this is due to web plugin issue
+Usage: $(basename "${BASH_SOURCE[0]}") [command]
+
+Run without a command to open the interactive menu.
+
+Available commands:
+
+  up, u                 Start Oodikone with real data
+  rs, rl                 Restart Oodikone with real data (down, then up)
+  anon, a               Start Oodikone with anonymous data
+  down, d               Stop Oodikone
+
+  logs, l               Follow oodikone service logs
+  logs:updater, lu      Follow sis-updater service logs
+  lint, li              Lint the project or the path given as parameter
+  format, fmt, f        Format the project or the path given as parameter
+
+  test:backend, tb      Run backend unit tests
+  test:frontend, tf     Run frontend unit tests
+  test:updater, tu      Run updater (worker) unit tests
+  test:services, ts     Run backend service tests
+  test:api, ta          Run backend api tests
+  test:component, tc    Run frontend component tests tests
+  test:e2e, te          Run all e2e tests (add --ui to run in headed mode)
+
+  importer, i           Open a psql/pgcli shell to the importer database
+  updater               Open a psql/pgcli shell to the updater (sis) database
+  install               Add the 'ok' shell function to your shell rc file
+  -h, --help            Print this help and exit
+EOF
+  exit
+}
+
+cleanup() {
+  trap - SIGINT SIGTERM ERR EXIT
+  msg ""
+  errormsg "(╯°□°)╯︵ ┻━┻"
+}
+
+setup_colors() {
+  if [[ -t 2 ]] && [[ -z "${NO_COLOR-}" ]] && [[ "${TERM-}" != "dumb" ]]; then
+    NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+  else
+    NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+  fi
+}
+
+die() {
+  local msg=$1
+  local code=${2-1} # default exit status 1
+  msg "$msg"
+  exit "$code"
+}
+
+get_pgclient() {
+  if command -v pgcli >/dev/null 2>&1; then
+    echo "pgcli"
+  else
+    warningmsg "Couldn't verify pgcli installation, using psql as fallback" >&2
+    echo "psql"
+  fi
+}
+
+show_welcome() {
+  local cashmoneyyellow
+  cashmoneyyellow=$(tput setaf 221)
+  local normal
+  normal=$(tput sgr0)
+  if [ "$(tput cols)" -gt "76" ]; then
+    while IFS="" read -r p || [ -n "$p" ]; do
+      printf '%40s\n' "${cashmoneyyellow}$p${normal}"
+    done < "$PROJECT_ROOT"/scripts/assets/logo.txt
+  fi
+  msg "Welcome to Oodikone CLI!"
+  msg ""
+}
+
+main_menu_opts() {
+  msg "1) Start Oodikone    2) Databases    3) Data refresh    4) Setup    q) Quit\n"
+}
+
+database_menu_opts() {
+  msg "\nView local databases ($datasource data)"
+  msg "suffix: $dbsuffix"
+  msg "1) Oodikone"
+  msg "2) Importer"
+  msg "3) Kone"
+  msg "4) Toggle data source (test/real)"
+  msg "5) Go back"
+  msg ""
+}
+
+database_menu() {
+  # If a single mandatory service is down, an assumption is made that they all are.
+  if ! docker compose ps 2>/dev/null | grep -q 'oodikone-backend'; then
+    msg ""
+    warningmsg "Can't access databases, oodikone is not running!\nPlease run option 1 first with preferred data source."
+    return
+  fi
+
+
+  declare dbsuffix="-real"
+  declare datasource="real"
+
+  toggle_environment() {
+    if [[ $dbsuffix = "" ]];
+      then
+        dbsuffix="-real"
+        datasource="real"
+      else
+        dbsuffix=""
+        datasource="test"
+    fi
+  }
+
+  local pgclient
+  pgclient="$(get_pgclient)"
+  msg "${GREEN}\nUsing $pgclient for database connections${NOFORMAT}"
+  database_menu_opts $datasource
+
+  local userinput
+  while true; do
+    read -r -s -n 1 -p "Please select an option: " userinput
+
+    case ${userinput} in
+      1|o|O)
+        msg "Oodikone-db"
+        $pgclient -h localhost -p 12345 -U postgres -d "sis-db$dbsuffix" -w
+        database_menu_opts
+        ;;
+      2|i|I)
+        msg "Importer-db"
+        $pgclient -h localhost -p 12346 -U postgres -d "sis-importer-db$dbsuffix" -w
+        database_menu_opts
+        ;;
+      3|k|K)
+        msg "Kone-db"
+        $pgclient -h localhost -p 4321 -U postgres -d "kone-db$dbsuffix" -w
+        database_menu_opts
+        ;;
+      4|t|T)
+        msg "Toggling environment..."
+        database_menu_opts
+        ;;
+      5|q|Q)
+        msg ""
+        break
+        ;;
+      *)
+        true
+        ;;
+    esac
+  done
+}
+
+main_loop() {
+  local userinput
+
+  while true; do
+    main_menu_opts
+    read -r -s -n 1 -p "Please select an option: " userinput
+
+    case ${userinput} in
+      1)
+        msg "\nTODO: todo\n";;
+      2)
+        database_menu;;
+      3)
+        msg "\nTODO: todo\n";;
+      4)
+        msg "\nTODO: todo\n";;
+      5|q|Q)
+        break 2;;
+      *)
+        errormsg "${RED}Invalid option:${NOFORMAT} $userinput";;
+    esac
+  done
+}
+
+# === Direct (non-interactive) commands ===
+run_up() {
+  infomsg "Starting Oodikone with real data..."
+  npm run both:real "$@"
+}
+
+run_restart() {
+  infomsg "Restarting Oodikone with real data..."
+  npm run docker:down
+  npm run both:real
+}
+
+run_anon() {
+  infomsg "Starting Oodikone with anonymous data..."
+  npm run both "$@"
+}
+
+run_down() {
+  infomsg "Stopping Oodikone..."
+  npm run docker:down "$@"
+}
+
+run_logs() {
+  npm run logs "$@"
+}
+
+run_lint() {
+  npm run lint -- "$@"
+}
+
+run_format() {
+  npm run fmt -- "$@"
+}
+
+run_unit_tests() {
+  npm run test:unit --prefix "$@"
+}
+
+run_component_tests() {
+  npm run test:component --prefix services/frontend "$@"
+}
+
+run_service_tests() {
+  npm run test:services --prefix services/backend "$@"
+}
+
+run_api_tests() {
+  npm run test:api --prefix services/backend "$@"
+}
+
+run_e2e_tests() {
+  npm run playwright -- "$@"
+}
+
+run_logs_updater() {
+  docker compose --file docker-compose.yml logs sis-updater-worker sis-updater-scheduler -f
+}
+
+
+run_importer_db() {
+  local pgclient
+  pgclient="$(get_pgclient)"
+  msg "Importer-db"
+  $pgclient -h localhost -p 12346 -U postgres -d sis-importer-db-real -w
+}
+
+run_updater_db() {
+  local pgclient
+  pgclient="$(get_pgclient)"
+  msg "Updater-db"
+  $pgclient -h localhost -p 12345 -U postgres -d sis-db-real -w
+}
+
+# Add (or refresh) an `ok` shell function in the user's shell rc file, which
+# simply forwards all arguments to this script. This is the one-time setup
+# step referenced above the shell function no longer needs its own copy of
+# oodikone-specific logic.
+cmd_install() {
+  local shell_name
+  shell_name="$(basename "${SHELL:-bash}")"
+
+  local shell_rc
+  case "$shell_name" in
+    zsh) shell_rc="$HOME/.zshrc" ;;
+    *) shell_rc="$HOME/.bashrc" ;;
+  esac
+
+  local ok_path="$PROJECT_ROOT/ok.sh"
+  local marker_start="# >>> oodikone ok.sh >>>"
+  local marker_end="# <<< oodikone ok.sh <<<"
+
+  if [ -f "$shell_rc" ] && grep -qF "$marker_start" "$shell_rc"; then
+    warningmsg "ok.sh integration already present in $shell_rc, skipping."
+    infomsg "To reinstall (e.g. after moving the repo), remove the block between\n'$marker_start' and '$marker_end' in $shell_rc and re-run 'ok.sh install'."
+    return 0
+  fi
+
+  {
+    echo ""
+    echo "$marker_start"
+    echo "ok() { \"$ok_path\" \"\$@\"; }"
+    echo "$marker_end"
+  } >> "$shell_rc"
+
+  successmsg "Added the 'ok' function to $shell_rc"
+  infomsg "Run 'source $shell_rc' or restart your shell, then try 'ok up'."
+}
+
+# Main
+setup_colors
+
+case "${1-}" in
+  up|u)
+    shift
+    run_up "$@"
+    ;;
+  rs|rl)
+    shift
+    run_restart "$@"
+    ;;
+  anon|a)
+    shift
+    run_anon "$@"
+    ;;
+  down|d)
+    shift
+    run_down "$@"
+    ;;
+  logs|l)
+    shift
+    run_logs "$@"
+    ;;
+  lint|li)
+    shift
+    run_lint "$@"
+    ;;
+  format|fmt|f)
+    shift
+    run_format "$@"
+    ;;
+  test:backend|tb)
+    shift
+    run_unit_tests services/backend "$@"
+    ;;
+  test:frontend|tf)
+    shift
+    run_unit_tests services/frontend "$@"
+    ;;
+  test:updater|tu)
+    shift
+    run_unit_tests updater/sis-updater-worker "$@"
+    ;;
+  test:services|ts)
+    shift
+    run_service_tests "$@"
+    ;;
+  test:api|ta)
+    shift
+    run_api_tests "$@"
+    ;;
+  test:component|tc)
+    shift
+    run_component_tests "$@"
+    ;;
+  test:e2e|te)
+    shift
+    run_e2e_tests "$@"
+    ;;
+  logs:updater|lu)
+    shift
+    run_logs_updater "$@"
+    ;;
+  importer|i)
+    shift
+    run_importer_db
+    ;;
+  updater)
+    shift
+    run_updater_db
+    ;;
+  install)
+    shift
+    cmd_install
+    ;;
+  -h|--help)
+    usage
+    ;;
+  "")
+    show_welcome
+    get_s3_config
+    main_loop
+    ;;
+  *)
+    errormsg "Unknown command: $1"
+    usage
+    ;;
+esac

services/.dockerignore

@@ -114,3 +114,4 @@ cypress/
 **/*Dockerfile
 **/build
 **/dist
+**/.npm

services/backend/src/config/programmeCodes.ts

@@ -1,4 +1,4 @@
-// Map programme codes to "letter indentifier"
+// Map programme codes to "letter indentifiers" i.e. abbreviations
 
 export const programmeCodes = {
   KH10_001: 'TUK',
@@ -68,6 +68,8 @@ export const programmeCodes = {
   MH50_012: 'GEOG',
   MH50_013: 'USP',
   MH50_014: 'EnCHiL',
+  MH50_015: 'ATM-MP-CE',
+  MH50_015_ATMCE: 'ATM-MP-CE',
   KH55_001: 'FARM',
   MH55_001: 'PROV',
   MH55_002: 'MPHARM',
@@ -101,13 +103,13 @@ export const programmeCodes = {
   KH74_001: 'KSV',
   KH80_001: 'MAATK',
   KH80_002: 'METSAK',
-  KH80_003: 'ETK',
+  KH80_003: 'ERK',
   KH80_004: 'YET',
   MH80_001: 'AGRI',
   MH80_002: 'AGERE',
   MH80_003: 'FOR',
   MH80_004: 'FOOD',
-  MH80_005: 'HNFB',
+  MH80_005: 'NUTR',
   MH80_006: 'EKM',
   MH80_007: 'MMB',
   MH80_008: 'emPlant',
@@ -157,9 +159,9 @@ export const programmeCodes = {
   DP007: 'DPDR',
   DP008: 'SciDoc',
   DP009: 'LLD',
-  DP0010: 'TRES',
-  DP0011: 'RENEW',
-  DP0012: 'PopBe',
-  DP0013: 'HEAL',
-  DP0014: 'SOC',
+  DP010: 'TRES',
+  DP011: 'RENEW',
+  DP012: 'PopBe',
+  DP013: 'HEAL',
+  DP014: 'SOC',
 } as const

services/backend/src/database/migrations_kone/20260929_00_add_course_group_id_to_excluded_courses.js

@@ -0,0 +1,14 @@
+const { STRING } = require('sequelize')
+
+// Migration step 1 of 2
+module.exports = {
+  up: async ({ context: queryInterface }) => {
+    await queryInterface.addColumn('excluded_courses', 'course_group_id', {
+      type: STRING,
+    })
+  },
+
+  down: async ({ context: queryInterface }) => {
+    await queryInterface.removeColumn('excluded_courses', 'course_group_id')
+  },
+}

services/backend/src/database/migrations_kone/20260929_01_add_course_group_ids_to_progress_criteria.js

@@ -0,0 +1,20 @@
+const { ARRAY, STRING } = require('sequelize')
+
+const YEARS = ['one', 'two', 'three', 'four', 'five', 'six']
+
+// Migration step 1 of 2
+module.exports = {
+  up: async ({ context: queryInterface }) => {
+    for (const year of YEARS) {
+      await queryInterface.addColumn('progress_criteria', `course_group_ids_year_${year}`, {
+        type: ARRAY(STRING),
+      })
+    }
+  },
+
+  down: async ({ context: queryInterface }) => {
+    for (const year of YEARS) {
+      await queryInterface.removeColumn('progress_criteria', `course_group_ids_year_${year}`)
+    }
+  },
+}

services/backend/src/database/migrations_kone/20260929_02_drop_course_codes_from_kone_tables.js

@@ -0,0 +1,94 @@
+/* eslint-disable no-console */
+const { ARRAY, STRING } = require('sequelize')
+
+const YEARS = ['one', 'two', 'three', 'four', 'five', 'six']
+const UNIQUE_CONSTRAINT = 'excluded_courses_programme_version_group_key'
+
+// Migration step 2 of 2
+// Rows the backfill could not map to a course group id are removed, as they no longer reference a course.
+// NOTE: down only restores the schema, the dropped course codes cannot be restored.
+module.exports = {
+  up: async ({ context: queryInterface }) => {
+    await queryInterface.sequelize.transaction(async transaction => {
+      const [, unmapped] = await queryInterface.sequelize.query(
+        'DELETE FROM excluded_courses WHERE course_group_id IS NULL',
+        { transaction }
+      )
+      console.log(`Deleted ${unmapped.rowCount} excluded courses without a course group id`)
+
+      const [, duplicates] = await queryInterface.sequelize.query(
+        `DELETE FROM excluded_courses a
+          USING excluded_courses b
+          WHERE a.id > b.id
+            AND a.programme_code = b.programme_code
+            AND a.curriculum_version IS NOT DISTINCT FROM b.curriculum_version
+            AND a.course_group_id = b.course_group_id`,
+        { transaction }
+      )
+      console.log(`Deleted ${duplicates.rowCount} duplicate excluded courses`)
+
+      await queryInterface.removeColumn('excluded_courses', 'course_code', { transaction })
+      await queryInterface.changeColumn(
+        'excluded_courses',
+        'course_group_id',
+        { type: STRING, allowNull: false },
+        { transaction }
+      )
+      await queryInterface.addConstraint('excluded_courses', {
+        fields: ['programme_code', 'curriculum_version', 'course_group_id'],
+        type: 'unique',
+        name: UNIQUE_CONSTRAINT,
+        transaction,
+      })
+
+      for (const year of YEARS) {
+        const [[{ count }]] = await queryInterface.sequelize.query(
+          `SELECT COUNT(*) AS count FROM progress_criteria
+            WHERE course_group_ids_year_${year} IS NULL AND cardinality(courses_year_${year}) > 0`,
+          { transaction }
+        )
+        console.log(`Emptied year ${year} of ${count} progress criteria without course group ids`)
+
+        await queryInterface.sequelize.query(
+          `UPDATE progress_criteria SET course_group_ids_year_${year} = '{}' WHERE course_group_ids_year_${year} IS NULL`,
+          { transaction }
+        )
+        await queryInterface.removeColumn('progress_criteria', `courses_year_${year}`, { transaction })
+        await queryInterface.changeColumn(
+          'progress_criteria',
+          `course_group_ids_year_${year}`,
+          { type: ARRAY(STRING), allowNull: false, defaultValue: [] },
+          { transaction }
+        )
+      }
+    })
+  },
+
+  down: async ({ context: queryInterface }) => {
+    await queryInterface.sequelize.transaction(async transaction => {
+      await queryInterface.removeConstraint('excluded_courses', UNIQUE_CONSTRAINT, { transaction })
+      await queryInterface.changeColumn(
+        'excluded_courses',
+        'course_group_id',
+        { type: STRING, allowNull: true },
+        { transaction }
+      )
+      await queryInterface.addColumn('excluded_courses', 'course_code', { type: STRING }, { transaction })
+
+      for (const year of YEARS) {
+        await queryInterface.changeColumn(
+          'progress_criteria',
+          `course_group_ids_year_${year}`,
+          { type: ARRAY(STRING), allowNull: true, defaultValue: null },
+          { transaction }
+        )
+        await queryInterface.addColumn(
+          'progress_criteria',
+          `courses_year_${year}`,
+          { type: ARRAY(STRING) },
+          { transaction }
+        )
+      }
+    })
+  },
+}

services/backend/src/models/course.ts

@@ -21,15 +21,19 @@ import { EnrollmentModel } from './enrollment'
 import { OrganizationModel } from './organization'
 
 @Table({
-  underscored: false,
+  underscored: true,
   modelName: 'course',
   tableName: 'course',
 })
 export class CourseModel extends Model implements Course {
+  /* COLUMNS */
   @PrimaryKey
   @Column(DataType.STRING)
   declare id: Course['id']
 
+  @Column(DataType.STRING)
+  declare groupId: Course['id']
+
   @ForeignKey(() => CourseModel)
   @Column(DataType.STRING)
   declare code: Course['code']
@@ -38,29 +42,17 @@ export class CourseModel extends Model implements Course {
   @Column(DataType.STRING)
   declare coursetypecode: Course['coursetypecode']
 
-  @BelongsTo(() => CourseTypeModel, { foreignKey: 'coursetypecode', targetKey: 'coursetypecode' })
-  declare courseType: Course['courseType']
-
-  @HasMany(() => CreditModel, { foreignKey: 'course_id', sourceKey: 'id' })
-  declare credits: Course['credits']
-
-  @HasMany(() => EnrollmentModel, { foreignKey: 'course_id' })
-  declare enrollments: Course['enrollments']
-
-  @BelongsToMany(() => OrganizationModel, () => CourseProviderModel, 'coursecode')
-  declare organizations: Course['organizations']
-
   @Column(DataType.BOOLEAN)
-  declare is_study_module: Course['is_study_module']
+  declare isStudyModule: Course['isStudyModule']
 
   @Column(DataType.JSONB)
   declare name: Course['name']
 
   @Column(DataType.DATE)
-  declare max_attainment_date: Course['max_attainment_date']
+  declare maxAttainmentDate: Course['maxAttainmentDate']
 
   @Column(DataType.DATE)
-  declare min_attainment_date: Course['min_attainment_date']
+  declare minAttainmentDate: Course['minAttainmentDate']
 
   @CreatedAt
   @Column({ field: 'created_at', type: DataType.DATE })
@@ -71,14 +63,31 @@ export class CourseModel extends Model implements Course {
   declare updatedAt: Course['updatedAt']
 
   @Column(DataType.JSONB)
-  declare substitutions: Course['substitutions']
+  declare validityPeriod: Course['validityPeriod']
 
   @Column(DataType.JSONB)
-  declare substitution_groups: Course['substitution_groups']
+  declare substitutionGroups: Course['substitutionGroups']
+
+  @Column(DataType.BOOLEAN)
+  declare isPrimary: Course['isPrimary']
 
   @Column(DataType.STRING)
-  declare course_unit_type: Course['course_unit_type']
+  declare courseUnitType: Course['courseUnitType']
 
-  @Column({ field: 'main_course_code', type: DataType.STRING })
-  declare mainCourseCode: Course['mainCourseCode']
+  /* RELATIONS */
+  @BelongsTo(() => CourseTypeModel, { foreignKey: 'coursetypecode', targetKey: 'coursetypecode' })
+  declare courseType: Course['courseType']
+
+  @HasMany(() => CreditModel, { foreignKey: 'course_id', sourceKey: 'id' })
+  declare credits: Course['credits']
+
+  @HasMany(() => EnrollmentModel, { foreignKey: 'course_id' })
+  declare enrollments: Course['enrollments']
+
+  @BelongsToMany(() => OrganizationModel, {
+    through: () => CourseProviderModel,
+    foreignKey: 'coursecode',
+    sourceKey: 'groupId',
+  })
+  declare organizations: Course['organizations']
 }

services/backend/src/models/courseProvider.ts

@@ -13,6 +13,7 @@ import { OrganizationModel } from './organization'
 export class CourseProviderModel extends Model implements CourseProvider {
   @ForeignKey(() => CourseModel)
   @Column(DataType.STRING)
+  // TODO: RENAME: this is courseGroupId, not code
   declare coursecode: CourseProvider['coursecode']
 
   @Column(DataType.JSONB)
@@ -20,6 +21,7 @@ export class CourseProviderModel extends Model implements CourseProvider {
 
   @ForeignKey(() => OrganizationModel)
   @Column(DataType.STRING)
+  // TODO: RENAME: this is organizationId, not code
   declare organizationcode: CourseProvider['organizationcode']
 
   @CreatedAt

services/backend/src/models/kone/excludedCourse.ts

@@ -17,7 +17,7 @@ export class ExcludedCourseModel extends Model implements ExcludedCourse {
   declare programme_code: ExcludedCourse['programme_code']
 
   @Column(DataType.STRING)
-  declare course_code: ExcludedCourse['course_code']
+  declare course_group_id: ExcludedCourse['course_group_id']
 
   @Column(DataType.STRING)
   declare curriculum_version: ExcludedCourse['curriculum_version']

services/backend/src/models/kone/progressCriteria.ts

@@ -17,22 +17,22 @@ export class ProgressCriteriaModel extends Model implements ProgressCriteria {
   declare curriculumVersion: ProgressCriteria['curriculumVersion']
 
   @Column(DataType.ARRAY(DataType.STRING))
-  declare coursesYearOne: ProgressCriteria['coursesYearOne']
+  declare courseGroupIdsYearOne: ProgressCriteria['courseGroupIdsYearOne']
 
   @Column(DataType.ARRAY(DataType.STRING))
-  declare coursesYearTwo: ProgressCriteria['coursesYearTwo']
+  declare courseGroupIdsYearTwo: ProgressCriteria['courseGroupIdsYearTwo']
 
   @Column(DataType.ARRAY(DataType.STRING))
-  declare coursesYearThree: ProgressCriteria['coursesYearThree']
+  declare courseGroupIdsYearThree: ProgressCriteria['courseGroupIdsYearThree']
 
   @Column(DataType.ARRAY(DataType.STRING))
-  declare coursesYearFour: ProgressCriteria['coursesYearFour']
+  declare courseGroupIdsYearFour: ProgressCriteria['courseGroupIdsYearFour']
 
   @Column(DataType.ARRAY(DataType.STRING))
-  declare coursesYearFive: ProgressCriteria['coursesYearFive']
+  declare courseGroupIdsYearFive: ProgressCriteria['courseGroupIdsYearFive']
 
   @Column(DataType.ARRAY(DataType.STRING))
-  declare coursesYearSix: ProgressCriteria['coursesYearSix']
+  declare courseGroupIdsYearSix: ProgressCriteria['courseGroupIdsYearSix']
 
   @Column(DataType.INTEGER)
   declare creditsYearOne: ProgressCriteria['creditsYearOne']

services/backend/src/models/organization.ts

@@ -45,7 +45,11 @@ export class OrganizationModel extends Model implements Organization {
   @HasMany(() => SISStudyRightModel, { foreignKey: 'facultyCode', sourceKey: 'code' })
   declare SISStudyRights: Organization['SISStudyRights']
 
-  @BelongsToMany(() => CourseModel, () => CourseProviderModel, 'organizationcode')
+  @BelongsToMany(() => CourseModel, {
+    through: () => CourseProviderModel,
+    foreignKey: 'organizationcode',
+    targetKey: 'groupId',
+  })
   declare courses: Organization['courses']
 
   @CreatedAt

services/backend/src/routes/courseExclusions.ts

@@ -7,7 +7,7 @@ import { hasFullAccessToStudentData } from '../util'
 const router = Router()
 
 router.post<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async (req, res) => {
-  const { programmeCode, courseCodes, curriculumVersion } = req.body
+  const { programmeCode, courseGroupIds, curriculumVersion } = req.body
   const { roles, programmeRights } = req.user
 
   const hasFullAccess = hasFullAccessToStudentData(roles)
@@ -15,7 +15,7 @@ router.post<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async (r
 
   if (!hasFullAccess && !hasAccessToProgramme) return res.status(403).end()
 
-  const result = await addExcludedCourses(courseCodes, curriculumVersion, programmeCode)
+  const result = await addExcludedCourses(courseGroupIds, curriculumVersion, programmeCode)
   if (!result) {
     res.status(400).end()
     return
@@ -24,7 +24,7 @@ router.post<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async (r
 })
 
 router.delete<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async (req, res) => {
-  const { programmeCode, courseCodes, curriculumVersion } = req.body
+  const { programmeCode, courseGroupIds, curriculumVersion } = req.body
   const { roles, programmeRights } = req.user
 
   const hasFullAccess = hasFullAccessToStudentData(roles)
@@ -32,7 +32,7 @@ router.delete<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async
 
   if (!hasFullAccess && !hasAccessToProgramme) return res.status(403).end()
 
-  const result = await removeExcludedCourses(courseCodes, curriculumVersion, programmeCode)
+  const result = await removeExcludedCourses(courseGroupIds, curriculumVersion, programmeCode)
   if (!result) {
     res.status(400).end()
     return

services/backend/src/routes/courses.ts

@@ -8,8 +8,9 @@ import {
   CoursesMultiQuery,
   CourseYearlyStatsReqBody,
   CourseYearlyStatsQuery,
+  CourseDetails,
+  CourseDetailsQuery,
 } from '@oodikone/shared/routes/courses'
-import type { CourseWithSubsDetails } from '@oodikone/shared/types/course'
 import { getCourseDetails, getCourseYearlyStats } from '../services/courses'
 import { getCoursesByNameAndOrCode } from '../services/courses/courseFinders'
 import {
@@ -18,49 +19,31 @@ import {
   hasFullAccessToStudentData,
   validateParamLength,
 } from '../util'
+import { CourseYearlyStats } from '@oodikone/shared/types/courseYearlyStats'
 
 const router = Router()
 
 router.get<never, CanError<CoursesMultiResBody>, CoursesMultiReqBody, CoursesMultiQuery>(
   '/coursesmulti',
   async (req, res) => {
-    const { name, code, includeSpecial } = req.query
+    const { name, code } = req.query
     if (!(validateParamLength(name, 5) || validateParamLength(code, 2))) {
       return res.status(400).json({ error: 'Query parameter name or code is invalid' })
     }
 
-    const courses = await getCoursesByNameAndOrCode(name, code, includeSpecial === 'true')
-
-    const mergedCourses = new Map<string, CourseWithSubsDetails>()
-
-    for (const course of courses) {
-      if (!(course.max_attainment_date && course.min_attainment_date)) continue
-
-      if (!mergedCourses.has(course.code)) mergedCourses.set(course.code, structuredClone(course))
-      const mergedCourse = mergedCourses.get(course.code)!
-
-      if (mergedCourse.max_attainment_date < course.max_attainment_date) {
-        mergedCourse.max_attainment_date = course.max_attainment_date
-      }
-      if (mergedCourse.min_attainment_date > course.min_attainment_date) {
-        mergedCourse.min_attainment_date = course.min_attainment_date
-      }
-    }
-
-    res.json({ courses: Array.from(mergedCourses.values()) })
+    const courses = await getCoursesByNameAndOrCode(name, code)
+    res.json({ courses }).end()
   }
 )
 
-export type CourseYearlyStatsResBody = Awaited<ReturnType<typeof getCourseYearlyStats>>
+export type CourseYearlyStatsResBody = CourseYearlyStats[]
 
 router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody, CourseYearlyStatsQuery>(
   '/courseyearlystats',
   async (req, res) => {
-    const { codes, combineSubstitutions, separate, fromYearCode, toYearCode } = req.query
+    const { courses, substitutions, separate, fromYearCode, toYearCode } = req.query
 
-    const courseCodes = handleQueryArrays(codes)
-
-    if (!courseCodes?.length) {
+    if (!courses?.length) {
       return res.status(422).send({ error: 'Missing required query parameters' })
     }
 
@@ -79,14 +62,13 @@ router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody,
     const anonymize = !userHasFullAccessToStudentData && fullStudyProgrammeRights.length === 0
     const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
 
-    const useCombineSubstitutions = combineSubstitutions !== 'false'
     const useSeparate = separate === 'true'
 
     const results = await getCourseYearlyStats(
-      courseCodes,
+      handleQueryArrays(courses),
       useSeparate,
       anonymizationSalt,
-      useCombineSubstitutions,
+      substitutions === 'true',
       fromYearCode,
       toYearCode
     )
@@ -94,12 +76,12 @@ router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody,
   }
 )
 
-router.get<never, CanError<unknown>, never, { codes: string | string[] }>('/coursedetails', async (req, res) => {
-  const { codes: coursecodes } = req.query
-  const codes = handleQueryArrays(coursecodes)
+router.get<never, CanError<CourseDetails>, never, CourseDetailsQuery>('/coursedetails', async (req, res) => {
+  const { courses } = req.query
+  const courseIds = handleQueryArrays(courses)
 
-  if (!codes?.length) return res.status(500).end()
-  const details = await getCourseDetails(codes)
+  if (!courseIds.length) return res.status(422).send({ error: 'Missing required parameters' })
+  const details = await getCourseDetails(courseIds)
 
   return res.json(details)
 })

services/backend/src/routes/population.ts

@@ -14,17 +14,11 @@ import {
   type CustomPopulationByProgrammesQuery,
   type CustomPopulationByProgrammesResBody,
   type PopulationstatisticsStudyprogrammesResBody,
-  PopulationstatisticsMaxYearsToCreatePopulationFormQuery,
-  PopulationstatisticsMaxYearsToCreatePopulationFormResBody,
 } from '@oodikone/shared/routes/populations'
 import { Unification } from '@oodikone/shared/types'
-import { keyBy, mapToProviders } from '@oodikone/shared/util'
+import { keyBy, mapToProviders, MAX_POPULATION_SIZE } from '@oodikone/shared/util'
 import { rootOrgId } from '../config'
-import {
-  getCourseProvidersForCourses,
-  maxYearsToCreatePopulationFrom,
-  searchAndCombineSubstitutionGroupsToCodes,
-} from '../services/courses'
+import { getCourseProvidersForCourses, getRelevantCourseIdMap } from '../services/courses'
 import { encrypt } from '../services/encrypt'
 import { getDegreeProgrammesOfOrganization } from '../services/faculty/faculty'
 import { getStudentTagMap } from '../services/populations/getStudentData'
@@ -33,6 +27,8 @@ import { statisticsOf } from '../services/populations/statisticsOf'
 import { getStudentNumbersWithStudyRights } from '../services/populations/studentNumbersWithStudyRights'
 import { findByCourseAndSemesters, getStartAndEndDates } from '../services/students'
 import { getFullStudyProgrammeRights, handleQueryArrays, hasFullAccessToStudentData } from '../util'
+import { parseUnification } from '@oodikone/shared/types/unification'
+import logger from '../util/logger'
 
 const router = Router()
 
@@ -91,6 +87,7 @@ const obfuscateStuff = ({
     }),
     coursestatistics: {
       courses: coursestatistics.courses,
+      idToGroupIdMap: coursestatistics.idToGroupIdMap,
       enrollments: coursestatistics.enrollments.map(enrollment => ({
         ...enrollment,
         studentnumber: encryptionMap.get(enrollment.studentnumber) ?? enrollment.studentnumber,
@@ -182,42 +179,49 @@ router.get<
     programmeRights: userProgrammeRights,
     studentsUserCanAccess: allStudentsUserCanAccess,
   } = req.user
-  const { coursecodes: coursecodeJSON, from, to, separate, unifyCourses, ...restParams } = req.query
-  const includeSubstitutions = restParams.includeSubstitutions === 'true'
+  const { courses, from, to, separate, unifyCourses, substitutions } = req.query
 
-  if (!coursecodeJSON || !from || !to) {
-    return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
+  const coursesArray = handleQueryArrays(courses) as string[]
+
+  if (!coursesArray.length || !from || !to) {
+    return res.status(400).json({ error: 'Missing required parameters.' })
   }
 
+  const includeSubstitutions = substitutions === 'true'
   const isSeparate = separate === 'true'
-  const coursecodes = JSON.parse(coursecodeJSON)
-  const codes = includeSubstitutions ? await searchAndCombineSubstitutionGroupsToCodes(coursecodes) : coursecodes
+  const unification = parseUnification(unifyCourses) ?? Unification.UNIFY
 
-  const toFromDiff = Math.abs(Number(to) - Number(from) + 1)
-  const requestedYearsToCreatePopulationFrom = Math.ceil(isSeparate ? toFromDiff / 2 : toFromDiff) // 2 semesters = 1 year
+  const idToGroupIdMap = await getRelevantCourseIdMap(coursesArray, includeSubstitutions)
+  const relevantCourseIds = Object.keys(idToGroupIdMap)
 
-  const maxYearsForPopulation = await maxYearsToCreatePopulationFrom(codes, Unification.REGULAR)
-  if (requestedYearsToCreatePopulationFrom > maxYearsForPopulation) {
-    return res.status(400).json({ error: `Max years to create population from is ${maxYearsForPopulation}` })
+  const studentNumbers = await findByCourseAndSemesters(
+    relevantCourseIds,
+    Number(from),
+    Number(to),
+    isSeparate,
+    unification
+  )
+
+  if (studentNumbers.size > MAX_POPULATION_SIZE) {
+    logger.warn(`Attempted to load course population beyond allowed limits. ${JSON.stringify(req.query)}`)
+    return res.status(400).json({ error: 'Population size too large.' }).end()
   }
 
-  const studentNumbers = await findByCourseAndSemesters(codes, Number(from), Number(to), isSeparate, unifyCourses)
-
-  const courseProviders: string[] = await getCourseProvidersForCourses(coursecodes)
+  const courseProviders = await getCourseProvidersForCourses(relevantCourseIds)
   const fullStudyProgrammeRights = getFullStudyProgrammeRights(userProgrammeRights)
   const rightsMappedToProviders = mapToProviders(fullStudyProgrammeRights)
 
   const studentsUserCanAccess = courseProviders.some(provider => rightsMappedToProviders.includes(provider))
-    ? new Set(studentNumbers)
+    ? studentNumbers
     : new Set(allStudentsUserCanAccess)
 
-  const { startDate: startDate, endDate: endDate } = await getStartAndEndDates(Number(from), Number(to), isSeparate)
+  const { startDate, endDate } = await getStartAndEndDates(Number(from), Number(to), isSeparate)
 
   const studyRights = []
-  const tagMap = await getStudentTagMap(studyRights, studentNumbers, userId)
+  const tagMap = await getStudentTagMap(studyRights, [...studentNumbers], userId)
   const filterCreditsAndEnrollmentsByDate = true
   const result = await statisticsOf(
-    studentNumbers,
+    [...studentNumbers],
     studyRights,
     tagMap,
     filterCreditsAndEnrollmentsByDate,
@@ -232,7 +236,7 @@ router.get<
     studentsUserCanAccess,
   })
 
-  return res.json({ ...processed, mainCourseCodes: coursecodes, allCourseCodes: codes })
+  return res.json({ ...processed, idToGroupIdMap }).end()
 })
 
 // Used in custom population and single study guidance groups
@@ -355,21 +359,4 @@ router.get<never, PopulationstatisticsStudyprogrammesResBody>(
   }
 )
 
-router.get<
-  never,
-  PopulationstatisticsMaxYearsToCreatePopulationFormResBody,
-  never,
-  PopulationstatisticsMaxYearsToCreatePopulationFormQuery
->('/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
-  const courseCodes = JSON.parse(req.query.courseCodes) as string[]
-  const maxYearsToCreatePopulationFromBoth = await maxYearsToCreatePopulationFrom(courseCodes, Unification.UNIFY)
-  const maxYearsToCreatePopulationFromUni = await maxYearsToCreatePopulationFrom(courseCodes, Unification.REGULAR)
-  const maxYearsToCreatePopulationFromOpen = await maxYearsToCreatePopulationFrom(courseCodes, Unification.OPEN)
-  return res.json({
-    unifyCourses: maxYearsToCreatePopulationFromBoth,
-    uniCourses: maxYearsToCreatePopulationFromUni,
-    openCourses: maxYearsToCreatePopulationFromOpen,
-  })
-})
-
 export default router

services/backend/src/routes/studyProgrammeCriteria.ts

@@ -26,17 +26,17 @@ router.get('/', async (req: StudyProgrammeCriteriaRequest, res: Response) => {
 interface CriteriaCoursesRequest extends Request {
   body: {
     code: string
-    courses: string[]
+    courseGroupIds: string[]
     year: number
   }
 }
 
 router.post('/courses', async (req: CriteriaCoursesRequest, res: Response) => {
-  const { code, courses, year } = req.body
-  if (!code || !courses || !year) {
+  const { code, courseGroupIds, year } = req.body
+  if (!code || !courseGroupIds || !year) {
     return res.status(400).end()
   }
-  const studyProgrammeCriteria = await saveYearlyCourseCriteria(code, courses, year)
+  const studyProgrammeCriteria = await saveYearlyCourseCriteria(code, courseGroupIds, year)
   if (!studyProgrammeCriteria) {
     return res.status(404).end()
   }

services/backend/src/services/completedCoursesSearch.ts

@@ -23,39 +23,58 @@ type StudentWithCourses = Omit<StudentWithStudyplanNested, 'studyplans'> & {
 }
 
 const getCourses = async (courseCodes: string[]) => {
-  const courses: CompletedCoursesCourse[] = (
-    await CourseModel.findAll({
-      attributes: ['code', 'name', 'substitution_groups'],
-      where: where(fn('LOWER', col('code')), {
-        [Op.in]: courseCodes.map(code => code.toLowerCase()),
-      }),
-    })
-  ).map(course => course.toJSON())
+  if (courseCodes.length === 0) return []
+
+  const matchedCourses: Pick<CourseModel, 'groupId'>[] = await CourseModel.findAll({
+    attributes: ['groupId'],
+    where: where(fn('LOWER', col('code')), {
+      [Op.in]: courseCodes.map(code => code.toLowerCase()),
+    }),
+    raw: true,
+  })
+  const groupIds = [...new Set(matchedCourses.map(({ groupId }) => groupId))]
+  if (groupIds.length === 0) return []
+
+  const courses: CompletedCoursesCourse[] = await CourseModel.findAll({
+    attributes: ['code', 'name', 'groupId', 'substitutionGroups'],
+    where: { groupId: { [Op.in]: groupIds }, isPrimary: true },
+    raw: true,
+  })
   return courses
 }
 
+// Every course-position a student must hold a credit/enrollment for, in order to satisfy a substitution
+// combination: the main course's own groupId, plus each substitution group's member groupIds.
+const getGroupIdCombinationsFor = (course: CompletedCoursesCourse): string[][] => [
+  [course.groupId],
+  ...(course.substitutionGroups ?? []),
+]
+
 const getPassedCredits = async (
   courses: CompletedCoursesCourse[],
-  fullCourseCodes: string[],
-  studentNumbers: string[]
+  courseIds: string[],
+  studentNumbers: string[],
+  idToGroupId: Record<string, string>
 ) => {
-  const credits: Pick<CreditModel, 'course_code' | 'student_studentnumber' | 'credittypecode' | 'attainment_date'>[] =
-    await CreditModel.findAll({
-      raw: true,
-      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'attainment_date'],
-      order: [['attainment_date', 'DESC']],
-      where: {
-        course_code: {
-          [Op.in]: fullCourseCodes,
-        },
-        student_studentnumber: {
-          [Op.in]: studentNumbers,
-        },
-        credittypecode: {
-          [Op.not]: CreditTypeCode.FAILED,
-        },
+  const credits: Pick<
+    CreditModel,
+    'course_code' | 'course_id' | 'student_studentnumber' | 'credittypecode' | 'attainment_date'
+  >[] = await CreditModel.findAll({
+    raw: true,
+    attributes: ['course_code', 'course_id', 'student_studentnumber', 'credittypecode', 'attainment_date'],
+    order: [['attainment_date', 'DESC']],
+    where: {
+      course_id: {
+        [Op.in]: courseIds,
       },
-    })
+      student_studentnumber: {
+        [Op.in]: studentNumbers,
+      },
+      credittypecode: {
+        [Op.not]: CreditTypeCode.FAILED,
+      },
+    },
+  })
 
   const creditsByStudentNumber = Object.groupBy(credits, ({ student_studentnumber }) => student_studentnumber)
 
@@ -69,26 +88,28 @@ const getPassedCredits = async (
 
   Object.entries(creditsByStudentNumber).map(([studentNumber, credits]) => {
     // We know that credits will exist, because of the way we created the previous object
-    const studentCreditCourseCodes = credits!.map(({ course_code }) => course_code)
+    const studentCreditGroupIds = credits!.map(credit => idToGroupId[credit.course_id])
     for (const course of courses) {
-      // Also handle the main course code by adding it as a group
-      for (const group of [[course.code]].concat(course.substitution_groups)) {
-        if (group.every(code => studentCreditCourseCodes.includes(code))) {
-          // We just checked that (group) course code exists in credits so .find(...)! is ok
-          const groupCredits = group.map(code => credits!.find(credit => credit.course_code === code)!)
+      // The first combination is always the main course itself; the rest are substitution groups
+      getGroupIdCombinationsFor(course).forEach((positions, index) => {
+        const isMainCourse = index === 0
+        if (positions.every(groupId => studentCreditGroupIds.includes(groupId))) {
+          // We just checked that each position has a matching credit so .find(...)! is ok
+          const groupCredits = positions.map(
+            groupId => credits!.find(credit => idToGroupId[credit.course_id] === groupId)!
+          )
           if (groupCredits.length) {
             const groupCreditCodes = groupCredits.map(({ course_code }) => course_code)
             formattedCredits.push({
               courseCode: course.code,
-              substitution:
-                groupCreditCodes.includes(course.code) && groupCreditCodes.length === 1 ? null : groupCreditCodes,
+              substitution: isMainCourse ? null : groupCreditCodes,
               studentNumber,
               creditType: groupCredits.length > 1 ? CreditTypeCode.PASSED : groupCredits.at(0)!.credittypecode,
               date: groupCredits?.at(0)?.attainment_date ?? now(), // Credits are sorted by date in desc. order
             })
           }
         }
-      }
+      })
     }
   })
 
@@ -97,26 +118,28 @@ const getPassedCredits = async (
 
 const getEnrollments = async (
   courses: CompletedCoursesCourse[],
-  fullCourseCodes: string[],
-  studentNumbers: string[]
+  courseIds: string[],
+  studentNumbers: string[],
+  idToGroupId: Record<string, string>
 ) => {
-  const enrollments: Array<Pick<EnrollmentModel, 'course_code' | 'enrollment_date_time' | 'studentnumber'>> =
-    await EnrollmentModel.findAll({
-      attributes: ['course_code', 'enrollment_date_time', 'studentnumber'],
-      order: [['enrollment_date_time', 'DESC']],
-      where: {
-        course_code: {
-          [Op.in]: fullCourseCodes,
-        },
-        studentnumber: {
-          [Op.in]: studentNumbers,
-        },
-        state: {
-          [Op.eq]: EnrollmentState.ENROLLED,
-        },
-        enrollment_date_time: { [Op.gte]: enrollmentTimeDateThreshold },
+  const enrollments: Array<
+    Pick<EnrollmentModel, 'course_code' | 'course_id' | 'enrollment_date_time' | 'studentnumber'>
+  > = await EnrollmentModel.findAll({
+    attributes: ['course_code', 'course_id', 'enrollment_date_time', 'studentnumber'],
+    order: [['enrollment_date_time', 'DESC']],
+    where: {
+      course_id: {
+        [Op.in]: courseIds,
       },
-    })
+      studentnumber: {
+        [Op.in]: studentNumbers,
+      },
+      state: {
+        [Op.eq]: EnrollmentState.ENROLLED,
+      },
+      enrollment_date_time: { [Op.gte]: enrollmentTimeDateThreshold },
+    },
+  })
 
   const enrollmentsByStudents = Object.groupBy(enrollments, ({ studentnumber }) => studentnumber)
 
@@ -129,27 +152,27 @@ const getEnrollments = async (
 
   Object.entries(enrollmentsByStudents).map(([studentNumber, enrollments]) => {
     // We know that enrollments will exist, because of the way we created the previous object
-    const studentEnrollmentCourseCodes = enrollments!.map(({ course_code }) => course_code)
+    const studentEnrollmentGroupIds = enrollments!.map(enrollment => idToGroupId[enrollment.course_id])
     for (const course of courses) {
-      // Also handle the main course code by adding it as a group
-      for (const group of [[course.code]].concat(course.substitution_groups)) {
-        if (group.every(code => studentEnrollmentCourseCodes.includes(code))) {
-          // We just checked that (group) course code exists in enrollments so .find(...)! is ok
-          const groupEnrollments = group.map(code => enrollments!.find(enrollment => enrollment.course_code === code)!)
+      // The first combination is always the main course itself; the rest are substitution groups
+      getGroupIdCombinationsFor(course).forEach((positions, index) => {
+        const isMainCourse = index === 0
+        if (positions.every(groupId => studentEnrollmentGroupIds.includes(groupId))) {
+          // We just checked that each position has a matching enrollment so .find(...)! is ok
+          const groupEnrollments = positions.map(
+            groupId => enrollments!.find(enrollment => idToGroupId[enrollment.course_id] === groupId)!
+          )
           if (groupEnrollments.length) {
             const groupEnrollmentCodes = groupEnrollments.map(({ course_code }) => course_code)
             formattedEnrollments.push({
               courseCode: course.code,
-              substitution:
-                groupEnrollmentCodes.includes(course.code) && groupEnrollmentCodes.length === 1
-                  ? null
-                  : groupEnrollmentCodes,
+              substitution: isMainCourse ? null : groupEnrollmentCodes,
               studentNumber,
               date: groupEnrollments?.at(0)?.enrollment_date_time ?? now(), // Enrollments are sorted by date in desc. order
             })
           }
         }
-      }
+      })
     }
   })
 
@@ -173,9 +196,26 @@ const getStudents = async (studentNumbers: string[]): Promise<StudentWithCourses
     ],
   })
 
-  return students.map(student => {
-    const { studyplans, ...rest }: StudentWithStudyplanNested = student.toJSON()
-    const coursesInStudyPlan = studyplans.flatMap(studyplan => studyplan.included_courses)
+  const plainStudents = students.map((student): StudentWithStudyplanNested => student.toJSON())
+
+  // included_courses holds course ids, with a rare fallback to a raw code for "custom" (non-catalogued)
+  // entries. Resolve ids to their current code for display - anything missing from the map is already a code.
+  const includedCourseIds = [
+    ...new Set(
+      plainStudents.flatMap(student => student.studyplans.flatMap(({ included_courses }) => included_courses))
+    ),
+  ]
+  const courses: Pick<CourseModel, 'id' | 'code'>[] = await CourseModel.findAll({
+    attributes: ['id', 'code'],
+    where: { id: { [Op.in]: includedCourseIds } },
+    raw: true,
+  })
+  const idToCode = Object.fromEntries(courses.map(({ id, code }) => [id, code]))
+
+  return plainStudents.map(({ studyplans, ...rest }) => {
+    const coursesInStudyPlan = studyplans.flatMap(studyplan =>
+      studyplan.included_courses.map(idOrCode => idToCode[idOrCode] ?? idOrCode)
+    )
     return { ...rest, coursesInStudyPlan }
   })
 }
@@ -185,22 +225,21 @@ export const getCompletedCourses = async (
   courseCodes: string[]
 ): Promise<{ students: Omit<CompletedCoursesStudent, 'allEnrollments'>[]; courses: CompletedCoursesCourse[] }> => {
   const courses = await getCourses(courseCodes)
-  const courseCodesSet = new Set(courseCodes)
 
-  // Get *ALL* courses including any courses in any substitution groups
-  for (const course of courses) {
-    courseCodesSet.add(course.code)
-    for (const group of course.substitution_groups) {
-      for (const code of group) {
-        courseCodesSet.add(code)
-      }
-    }
-  }
+  // Get *ALL* courses' ids, including any courses in any substitution groups
+  const substitutionGroupIds = courses.flatMap(course => (course.substitutionGroups ?? []).flat())
+  const allGroupIds = [...new Set([...courses.map(course => course.groupId), ...substitutionGroupIds])]
 
-  const fullCourseCodes = Array.from(courseCodesSet)
+  const relevantCourses: Pick<CourseModel, 'id' | 'groupId'>[] = await CourseModel.findAll({
+    attributes: ['id', 'groupId'],
+    where: { groupId: { [Op.in]: allGroupIds } },
+    raw: true,
+  })
+  const courseIds = relevantCourses.map(({ id }) => id)
+  const idToGroupId = Object.fromEntries(relevantCourses.map(({ id, groupId }) => [id, groupId]))
 
-  const credits = await getPassedCredits(courses, fullCourseCodes, studentNumbers)
-  const enrollments = await getEnrollments(courses, fullCourseCodes, studentNumbers)
+  const credits = await getPassedCredits(courses, courseIds, studentNumbers, idToGroupId)
+  const enrollments = await getEnrollments(courses, courseIds, studentNumbers, idToGroupId)
   const studentInfo = await getStudents(studentNumbers)
 
   const studentCredits: StudentCredits = {}
@@ -276,7 +315,7 @@ export const getCompletedCourses = async (
   )
 
   students.forEach(student => {
-    courseCodes.forEach(courseCode => {
+    courses.forEach(({ code: courseCode }) => {
       const [latestEnrollment] = student.allEnrollments
         .filter(enrollment => enrollment.courseCode === courseCode)
         .sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime())

services/backend/src/services/courses/courseFinders.ts

@@ -1,70 +1,116 @@
-import { Op } from 'sequelize'
+import { col, fn, Op, QueryTypes } from 'sequelize'
 
 import { CourseWithSubsDetails } from '@oodikone/shared/types/course'
 import { CourseModel } from '../../models'
+import { dbConnections } from '../../database/connection'
 
-const nameLikeTerm = (name: string) => {
-  if (!name) {
-    return undefined
-  }
-  const term = `%${name.trim()}%`
-  return {
-    name: {
-      [Op.or]: {
-        fi: {
-          [Op.iLike]: term,
+const likeTerm = (userInput: string | undefined, type: 'name' | 'code') => {
+  const searchTerm = userInput?.trim().replace(/[\\%_]/g, '')
+  if (!searchTerm) return undefined
+
+  return type === 'name'
+    ? {
+        name: {
+          [Op.or]: {
+            fi: {
+              [Op.iLike]: `%${searchTerm}%`,
+            },
+            sv: {
+              [Op.iLike]: `%${searchTerm}%`,
+            },
+            en: {
+              [Op.iLike]: `%${searchTerm}%`,
+            },
+          },
         },
-        sv: {
-          [Op.iLike]: term,
-        },
-        en: {
-          [Op.iLike]: term,
+      }
+    : {
+        code: {
+          [Op.iLike]: `${searchTerm}%`,
         },
+      }
+}
+
+export const getCoursesByNameAndOrCode = async (name: string, code: string): Promise<CourseWithSubsDetails[]> => {
+  // Get groupIds for the course units (CU) the user is querying
+  // This step allows users to search with old or upcoming course codes and names
+  // Example: searching for GEOK_2011 (old code) returns results for GEOK2011 (new code)
+  const groupIds = (
+    await CourseModel.findAll({
+      attributes: [[fn('DISTINCT', col('group_id')), 'groupId']],
+      where: {
+        ...likeTerm(name, 'name'),
+        ...likeTerm(code, 'code'),
+        groupId: { [Op.ne]: null },
       },
-    },
-  }
-}
-
-const escapeRegExp = (string: string) => string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
-
-const codeLikeTerm = (code: string) => {
-  if (!code) {
-    return undefined
-  }
-  return {
-    code: {
-      [Op.iRegexp]: `^(AY|A)?${escapeRegExp(code)}`,
-    },
-  }
-}
-
-export const getCoursesByNameAndOrCode = async (name: string, code: string, includeSpecial: boolean) => {
-  const courses = await CourseModel.findAll({
-    where: {
-      ...(includeSpecial ? {} : { mainCourseCode: { [Op.ne]: null } }),
-      ...nameLikeTerm(name),
-      ...codeLikeTerm(code),
-    },
-    raw: true,
-  })
-
-  const coursesWithSubstitutionGroupDetails = await Promise.all(
-    courses.map<Promise<CourseWithSubsDetails>>(async course => {
-      const groupDetails = await Promise.all(
-        course?.substitution_groups?.map(async group => {
-          return await CourseModel.findAll({
-            attributes: ['name', 'code'],
-            where: { code: { [Op.in]: group } },
-            raw: true,
-          })
-        })
-      )
-
-      return { ...course, substitution_groups: groupDetails }
+      raw: true,
     })
+  ).map(({ groupId }) => groupId)
+
+  if (!groupIds.length) return []
+
+  // Gets name/code from the "primary" CU, and attainment dates from all CUs with the same groupId
+  const primaryCourses = await dbConnections.sequelize.query<CourseModel>(
+    `
+      WITH filtered AS (
+        SELECT *
+        FROM course
+        WHERE group_id IN (:groupIds)
+      ),
+      grouped AS (
+        SELECT
+          group_id,
+          MIN(min_attainment_date) AS min_attainment_date,
+          MAX(max_attainment_date) AS max_attainment_date
+        FROM filtered
+        GROUP BY group_id
+      ),
+      primary_course AS (
+        SELECT code, name, group_id, substitution_groups
+        FROM filtered
+        where is_primary = true
+      )
+      SELECT
+        g.group_id AS "groupId",
+        g.min_attainment_date AS "minAttainmentDate",
+        g.max_attainment_date AS "maxAttainmentDate",
+        p.name,
+        p.code,
+        p.substitution_groups AS "substitutionGroups"
+      FROM grouped g
+      JOIN primary_course p
+        ON p.group_id = g.group_id
+    `,
+    {
+      replacements: {
+        groupIds,
+      },
+      type: QueryTypes.SELECT,
+    }
   )
 
-  return coursesWithSubstitutionGroupDetails
+  const substitutionGroupIds = [...new Set(primaryCourses.map(course => course.substitutionGroups).flat(2))]
+  const substitutionCourses = substitutionGroupIds.length
+    ? Object.groupBy(
+        await CourseModel.findAll({
+          attributes: ['name', 'groupId', 'code'],
+          where: { groupId: { [Op.in]: substitutionGroupIds }, isPrimary: true },
+          raw: true,
+        }),
+        ({ groupId }) => groupId
+      )
+    : {}
+
+  const coursesWithSubstitutionDetails = primaryCourses.map(course => ({
+    ...course,
+    // NOTE: There are some substitutions that just don't exist in importer -> don't send undefined to client
+    substitutionGroups:
+      course.substitutionGroups
+        ?.map(groupIds => groupIds.flatMap(groupId => substitutionCourses[groupId]).filter(sub => !!sub))
+        .filter(group => group.length) ?? [],
+  }))
+
+  return coursesWithSubstitutionDetails
 }
 
 export const getCoursesByCodes = (codes: string[]) => {

services/backend/src/services/courses/courseYearlyStatsCounter.ts

@@ -1,7 +1,16 @@
-import type { Name } from '@oodikone/shared/types'
-import type { Programme, FacultyYearStats, Grades, Group, Student } from '@oodikone/shared/types/courseYearlyStats'
+import { CreditTypeCode, type Name } from '@oodikone/shared/types'
+import type {
+  Programme,
+  FacultyYearStats,
+  Grades,
+  Group,
+  Student,
+  CourseStat,
+  Realisation,
+} from '@oodikone/shared/types/courseYearlyStats'
 import { getSemesterNamesByCode } from '../semesters'
 import type { OrganizationDetails } from './helpers'
+import { yearCodeToYear } from '@oodikone/shared/util'
 
 export class CourseYearlyStatsCounter {
   private groups: Record<number, Group>
@@ -55,7 +64,7 @@ export class CourseYearlyStatsCounter {
       credits: {},
     }
 
-    const year = `${1949 + Number(yearCode)}-${1950 + Number(yearCode)}`
+    const year = `${yearCodeToYear(yearCode)}-${yearCodeToYear(yearCode) + 1}`
     this.facultyStats[yearCode] ??= {
       year,
       allStudents: [],
@@ -93,7 +102,8 @@ export class CourseYearlyStatsCounter {
     }
   }
 
-  public markCreditToGroup(
+  /** Add student's credit to attempts, attempt-stats include possibly duplicate students */
+  public markAttemptToGroup(
     studentNumber: string,
     passed: boolean,
     grade: string,
@@ -104,7 +114,6 @@ export class CourseYearlyStatsCounter {
   ) {
     this.initGroup(groupCode, groupName, courseCode, yearCode)
 
-    // mark credit to attempts
     const { grades, categories } = this.groups[groupCode].attempts
 
     grades[grade] ??= []
@@ -115,8 +124,24 @@ export class CourseYearlyStatsCounter {
     } else {
       categories.failed.push(studentNumber)
     }
+  }
+
+  /** Add student's credit to student-stats, student-stats include only unique students */
+  public markStudentGradeToGroup(
+    studentNumber: string,
+    passed: boolean,
+    grade: string,
+    groupCode: number,
+    groupName: string | Name,
+    courseCode: string,
+    yearCode: number,
+    creditTypeCode: CreditTypeCode
+  ) {
+    this.initGroup(groupCode, groupName, courseCode, yearCode)
+
+    // Don't add students to student stats based on improved grades
+    if (creditTypeCode === CreditTypeCode.IMPROVED) return
 
-    // mark best effort grade
     const current = this.groups[groupCode].students.grades[studentNumber]
 
     if (current?.passed && !passed) return
@@ -198,7 +223,7 @@ export class CourseYearlyStatsCounter {
     })
   }
 
-  private async parseGroupStatistics(anonymizationSalt: string | null) {
+  private async parseGroupStatistics(anonymizationSalt: string | null): Promise<Realisation[]> {
     await this.insertEmptyRows()
     for (const [studentNumber, data] of this.students) {
       this.groups[data.code].students.studentNumbers.push(studentNumber)
@@ -213,8 +238,8 @@ export class CourseYearlyStatsCounter {
           coursecode: group.coursecode,
           attempts: {
             categories: {
-              failed: [] as string[],
-              passed: [] as string[],
+              failed: [],
+              passed: [],
             },
             grades: Object.keys(group.attempts.grades).reduce<Grades>((grades, grade) => {
               grades[grade] = []
@@ -223,8 +248,11 @@ export class CourseYearlyStatsCounter {
           },
           yearCode: group.yearCode,
           students: {
-            studentNumbers: [] as string[],
+            studentNumbers: [],
+            grades: {},
           },
+          allEnrollments: [],
+          enrollments: [],
         }
       }
 
@@ -276,7 +304,9 @@ export class CourseYearlyStatsCounter {
     return this.facultyStats
   }
 
-  public async getFinalStatistics(anonymizationSalt: string | null) {
+  public async getFinalStatistics(
+    anonymizationSalt: string | null
+  ): Promise<Pick<CourseStat, 'obfuscated' | 'statistics' | 'programmes' | 'facultyStats'>> {
     const statistics = await this.parseGroupStatistics(anonymizationSalt)
     const obfuscated = statistics.some(row => row.obfuscated)
 

services/backend/src/services/courses/creditsAndEnrollmentsOfCourse.ts

@@ -1,90 +1,93 @@
 import { Op } from 'sequelize'
 
 import { Credit, Enrollment } from '@oodikone/shared/models'
-import { CreditTypeCode, EnrollmentState, Unification } from '@oodikone/shared/types'
+import { EnrollmentState, Unification } from '@oodikone/shared/types'
 import { enrollmentTimeDateThreshold } from '@oodikone/shared/util'
 import {
-  StudentModel,
   CreditModel,
   SemesterModel,
   OrganizationModel,
   EnrollmentModel,
   SISStudyRightModel,
   SISStudyRightElementModel,
+  CourseModel,
 } from '../../models'
 import { now } from '../../util/clock'
 import { getIsOpen } from './helpers'
 
-export const getCreditsForCourses = async (codeGroups: string[][], unification: Unification, from: Date, to: Date) => {
-  const allCourseCodes = codeGroups.flatMap(group => group)
-
+export const getCreditsForCourses = async (
+  groupIdGroups: string[][],
+  allCourseIds: string[],
+  unification: Unification,
+  from: Date,
+  to: Date
+) => {
   // We need the credits grouped by student numbers so that we can check if a student has
   // completed a substitution group
-  const students = await StudentModel.findAll({
+  const attainments = await CreditModel.findAll({
     raw: true,
     nest: true,
-    attributes: ['studentnumber'],
+    attributes: [
+      'grade',
+      'course_code',
+      'course_id',
+      'credits',
+      'attainment_date',
+      'student_studentnumber',
+      'studyright_id',
+      'credittypecode',
+    ],
+    where: {
+      course_id: {
+        [Op.in]: allCourseIds,
+      },
+      is_open: getIsOpen(unification),
+      attainment_date: { [Op.between]: [from, to] },
+    },
+    order: [['attainment_date', 'ASC']],
     include: [
       {
-        model: CreditModel,
-        attributes: [
-          'grade',
-          'course_code',
-          'credits',
-          'attainment_date',
-          'student_studentnumber',
-          'studyright_id',
-          'credittypecode',
-        ],
+        model: SemesterModel,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
         where: {
-          course_code: {
-            [Op.in]: allCourseCodes,
+          startdate: {
+            [Op.lte]: now(),
           },
-          is_open: getIsOpen(unification),
-          credittypecode: { [Op.not]: CreditTypeCode.IMPROVED }, // We do not care about improved grades
-          attainment_date: { [Op.between]: [from, to] },
         },
-        order: [['attainment_date', 'ASC']],
-        include: [
-          {
-            model: SemesterModel,
-            attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
-            where: {
-              startdate: {
-                [Op.lte]: now(),
-              },
-            },
-          },
-        ],
+      },
+      {
+        model: CourseModel,
+        attributes: ['groupId'],
+        required: true,
       },
     ],
   })
 
-  const studentNumberToCredits = students.reduce<Record<string, Credit[]>>((acc, student) => {
-    acc[student.studentnumber] ??= []
-    acc[student.studentnumber] = acc[student.studentnumber].concat(student.credits)
-
+  const studentNumberToCredits = attainments.reduce<Record<string, Credit[]>>((acc, att) => {
+    acc[att.student_studentnumber] ??= []
+    acc[att.student_studentnumber].push(att)
     return acc
   }, {})
 
   const completedGroups: Credit[][] = []
   for (const studentCredits of Object.values(studentNumberToCredits)) {
-    const codesOfPassedCredits = studentCredits
+    const groupIdsOfPassedCredits = studentCredits
       .filter(credit => CreditModel.passed(credit))
-      .map(credit => credit.course_code)
-    for (const group of codeGroups) {
+      .map(credit => credit.course.groupId)
+
+    for (const group of groupIdGroups) {
       if (group.length === 1) {
         // Failed courses are only calculated for the original course and 1-to-1 substitutions
-        const credits = studentCredits.filter(credit => credit.course_code === group.at(0))
+        const credits = studentCredits.filter(credit => credit.course.groupId === group[0])
         if (credits.length) {
           completedGroups.push(credits)
         }
       }
       // For substitution groups we must only get passed groups
-      else if (group.every(code => codesOfPassedCredits.includes(code))) {
+      else if (group.every(groupId => groupIdsOfPassedCredits.includes(groupId))) {
         // The credit in question should always exist because we just checked that it does
         completedGroups.push(
-          studentCredits.filter(credit => group.includes(credit.course_code) && CreditModel.passed(credit))
+          studentCredits.filter(credit => group.includes(credit.course.groupId) && CreditModel.passed(credit))
         )
       } else {
         // "Partially completed substitution, do not count towards anything"
@@ -137,68 +140,63 @@ export const getStudentNumberToSrElementsMap = async (studentNumbers: string[])
 }
 
 export const getEnrollmentsForCourses = async (
-  codeGroups: string[][],
+  groupIdGroups: string[][],
+  allCourseIds: string[],
   unification: Unification,
   from: Date,
   to: Date
 ) => {
-  const allCourseCodes = codeGroups.flatMap(group => group)
-
-  const students = await StudentModel.findAll({
+  const enrollments = await EnrollmentModel.findAll({
     raw: true,
     nest: true,
-    attributes: ['studentnumber'],
+    attributes: ['studentnumber', 'enrollment_date_time', 'course_code', 'course_id', 'studyright_id'],
+    where: {
+      course_id: {
+        [Op.in]: allCourseIds,
+      },
+      enrollment_date_time: {
+        [Op.between]: [from, to],
+        // Date when OK changed from Oodi to Sisu data, studyright_id is null before that date
+        [Op.gte]: enrollmentTimeDateThreshold,
+      },
+      state: EnrollmentState.ENROLLED,
+      is_open: getIsOpen(unification),
+    },
     include: [
       {
-        model: EnrollmentModel,
-        attributes: ['studentnumber', 'enrollment_date_time', 'course_code', 'studyright_id'],
-        include: [
-          {
-            model: SemesterModel,
-            attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
-            where: {
-              startdate: {
-                [Op.lte]: now(),
-              },
-            },
-          },
-        ],
+        model: SemesterModel,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
         where: {
-          course_code: {
-            [Op.in]: allCourseCodes,
+          startdate: {
+            [Op.lte]: now(),
           },
-          enrollment_date_time: {
-            [Op.and]: {
-              [Op.between]: [from, to],
-              // Date when OK changed from Oodi to Sisu data, studyright_id is null before that date
-              [Op.gte]: enrollmentTimeDateThreshold,
-            },
-          },
-          state: EnrollmentState.ENROLLED,
-          is_open: getIsOpen(unification),
         },
       },
+      {
+        model: CourseModel,
+        attributes: ['groupId'],
+        required: true,
+      },
     ],
   })
 
-  const studentNumberToEnrollments = students.reduce<Record<string, Enrollment[]>>((acc, student) => {
-    acc[student.studentnumber] ??= []
-    acc[student.studentnumber] = acc[student.studentnumber].concat(student.enrollments)
-
+  const studentNumberToEnrollments = enrollments.reduce<Record<string, Enrollment[]>>((acc, enrollment) => {
+    acc[enrollment.studentnumber] ??= []
+    acc[enrollment.studentnumber].push(enrollment)
     return acc
   }, {})
 
   // Calculate all code groups that have enrollments for all of the courses in the group
   const completedEnrollments: Enrollment[][] = []
   for (const studentEnrollments of Object.values(studentNumberToEnrollments)) {
-    const codesOfEnrollments = studentEnrollments.map(enrollment => enrollment.course_code)
+    const groupIdsOfEnrollments = studentEnrollments.map(enrollment => enrollment.course.groupId)
 
-    for (const group of codeGroups) {
-      if (group.every(code => codesOfEnrollments.includes(code))) {
+    for (const group of groupIdGroups) {
+      if (group.every(groupId => groupIdsOfEnrollments.includes(groupId))) {
         completedEnrollments.push(
           studentEnrollments
-            .filter(enrollment => group.includes(enrollment.course_code))
-            .sort((a, b) => new Date(b.enrollment_date_time).getTime() - new Date(a.enrollment_date_time).getTime())
+            .filter(enrollment => group.includes(enrollment.course.groupId))
+            .sort((a, b) => b.enrollment_date_time.getTime() - a.enrollment_date_time.getTime())
         )
       } else {
         // Partially completed enrollments for a group, skipping...
@@ -206,5 +204,5 @@ export const getEnrollmentsForCourses = async (
     }
   }
 
-  return completedEnrollments
+  return completedEnrollments.filter(enrollmentGroup => enrollmentGroup.length)
 }

services/backend/src/services/courses/helpers.ts

@@ -1,6 +1,7 @@
 import { Op } from 'sequelize'
 
 import { Name, Unification } from '@oodikone/shared/types'
+import { CourseModel } from '../../models'
 
 export type FormattedProgramme = {
   code: string
@@ -23,3 +24,13 @@ export type OrganizationDetails = {
   code?: string
   name: Name
 }
+
+/** Gets all course ids for a course given course groupId(s) */
+export const getAllCourseIds = async (groupIds: string | string[]) =>
+  (
+    await CourseModel.findAll({
+      attributes: ['id'],
+      where: { groupId: { [Op.in]: Array.isArray(groupIds) ? groupIds : [groupIds] } },
+      raw: true,
+    })
+  ).map(course => course.id)

services/backend/src/services/courses/index.ts

@@ -1,13 +1,12 @@
 import crypto from 'crypto'
-import { Op, fn as dbFn, col as dbCol } from 'sequelize'
+import { Op } from 'sequelize'
 
 import { Credit, Enrollment } from '@oodikone/shared/models'
-import { Name, EnrollmentState, Unification } from '@oodikone/shared/types'
+import { Name, EnrollmentState, Unification, CreditTypeCode } from '@oodikone/shared/types'
 import { enrollmentTimeDateThreshold, getSemesterCodeAt, yearCodeToYear, yearToYearCode } from '@oodikone/shared/util'
 import { dateIsBetween } from '@oodikone/shared/util/datetime'
 import logger from '../../../src/util/logger'
 import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, SISStudyRightElementModel } from '../../models'
-import { isOpenUniCourseCode } from '../../util'
 import { now } from '../../util/clock'
 import { getSemestersAndYears, SemestersAndYears } from '../semesters'
 import { CourseYearlyStatsCounter } from './courseYearlyStatsCounter'
@@ -16,7 +15,8 @@ import {
   getEnrollmentsForCourses,
   getStudentNumberToSrElementsMap,
 } from './creditsAndEnrollmentsOfCourse'
-import { FormattedProgramme, getIsOpen } from './helpers'
+import { FormattedProgramme } from './helpers'
+import { CourseYearlyStats } from '@oodikone/shared/types/courseYearlyStats'
 
 const formatStudyRightElement = (studyRightElement: SISStudyRightElementModel): FormattedProgramme => ({
   code: studyRightElement.code,
@@ -46,14 +46,14 @@ type FormattedCredit = {
 
 // Is group in question for a single course (original or 1-to-1 substitution) or a substitution group with multiple courses
 const isSingleCourse = (group: Credit[] | Enrollment[]): boolean =>
-  group?.length === 1 || [...new Set(group?.map((course: Credit | Enrollment) => course.course_code))].length === 1
+  group?.length === 1 || [...new Set(group?.map((course: Credit | Enrollment) => course.course.groupId))].length === 1
 
-const parseCredit = (
+export const parseCredit = (
   creditGroup: Credit[],
   anonymizationSalt: string | null,
   mainCourseCode: string,
   studyRightElements: Array<SISStudyRightElementModel>
-): FormattedCredit => {
+): FormattedCredit & { creditTypeCode: CreditTypeCode } => {
   const singleCourse = isSingleCourse(creditGroup)
 
   // TODO: Calculate substitutions correctly once course_unit_realisation hits
@@ -67,6 +67,7 @@ const parseCredit = (
   const courseCode = singleCourse ? attainment.course_code : mainCourseCode
   const grade = singleCourse ? attainment.grade : 'substituted'
   const credits = singleCourse ? attainment.credits : creditGroup.reduce((acc, credit) => acc + credit.credits, 0)
+  const creditTypeCode = singleCourse ? attainment.credittypecode : CreditTypeCode.PASSED
 
   const {
     semester,
@@ -105,6 +106,7 @@ const parseCredit = (
     studentNumber: anonymizationSalt ? anonymizeStudentNumber(studentNumber, anonymizationSalt) : studentNumber,
     programme,
     credits,
+    creditTypeCode,
   }
 }
 
@@ -151,17 +153,20 @@ const parseEnrollment = (
   }
 }
 
-const getSubstitutionGroupDetails = async (codeGroups: string[][]) => {
+const getSubstitutionGroupDetails = async (
+  groupIdGroups: string[][]
+): Promise<Pick<CourseModel, 'code' | 'name' | 'groupId'>[][]> => {
   const substitutionGroupDetails = await CourseModel.findAll({
-    attributes: ['code', 'name'],
+    attributes: ['code', 'name', 'groupId'],
     where: {
-      code: { [Op.in]: codeGroups.flatMap(group => group) },
+      groupId: { [Op.in]: groupIdGroups.flatMap(group => group) },
+      isPrimary: true,
     },
     raw: true,
   })
 
-  return codeGroups.map(group =>
-    group.map(code => substitutionGroupDetails.find(subCourse => subCourse.code === code)!)
+  return groupIdGroups.map(group =>
+    group.map(groupId => substitutionGroupDetails.find(subCourse => subCourse.groupId === groupId)!)
   )
 }
 
@@ -178,186 +183,177 @@ const getSemesterAndYearByDate = (
   return { year, semester }
 }
 
-const getYearlyStatsOfNew = async (
-  course: Pick<CourseModel, 'code' | 'name' | 'substitution_groups'> | null,
-  courseCode: string,
-  separate: boolean,
-  unification: Unification,
-  anonymizationSalt: string | null,
-  combineSubstitutions: boolean,
-  studentNumberToSrElementsMap: Record<string, SISStudyRightElementModel[]>,
-  from: Date,
-  to: Date
-) => {
-  // Includes main course code and substitutions (if enabled)
-  const creditGroupCodes =
-    combineSubstitutions && course?.substitution_groups
-      ? [[courseCode]].concat(course.substitution_groups)
-      : [[courseCode]]
+export const getYearlyStatsOfNew =
+  (
+    primaryCourse: Pick<CourseModel, 'id' | 'groupId' | 'code' | 'name' | 'substitutionGroups'>,
+    separate: boolean,
+    anonymizationSalt: string | null,
+    substitutions: boolean,
+    studentNumberToSrElementsMap: Record<string, SISStudyRightElementModel[]>,
+    from: Date,
+    to: Date
+  ) =>
+  async (unification: Unification): Promise<CourseYearlyStats['openStats']> => {
+    const { groupId, name, code: courseCode } = primaryCourse
 
-  let filteredCreditGroupCodes: string[][]
-  switch (unification) {
-    // Include only course codes / group of course codes WITHOUT ANY AY prefix
-    case Unification.REGULAR:
-      filteredCreditGroupCodes = creditGroupCodes.filter(group => group.every(course => !isOpenUniCourseCode(course)))
-      break
-    // Include all courses / group of courses
-    case Unification.OPEN:
-    case Unification.UNIFY:
-    default:
-      filteredCreditGroupCodes = creditGroupCodes
-      break
-  }
+    // Includes main course code and substitutions (if enabled)
+    const creditGroupIds = substitutions ? [[groupId]].concat(primaryCourse.substitutionGroups ?? []) : [[groupId]]
 
-  const { semesters, years } = await getSemestersAndYears()
+    const allCourseIds = (
+      await CourseModel.findAll({
+        raw: true,
+        attributes: ['id'],
+        where: { groupId: { [Op.in]: creditGroupIds.flat() } },
+      })
+    ).map(({ id }) => id)
 
-  const [creditGroups, enrollmentGroups] = await Promise.all([
-    getCreditsForCourses(filteredCreditGroupCodes, unification, from, to),
-    getEnrollmentsForCourses(filteredCreditGroupCodes, unification, from, to),
-  ])
+    const { semesters, years } = await getSemestersAndYears()
 
-  const counter = new CourseYearlyStatsCounter()
+    const [creditGroups, enrollmentGroups] = await Promise.all([
+      getCreditsForCourses(creditGroupIds, allCourseIds, unification, from, to),
+      getEnrollmentsForCourses(creditGroupIds, allCourseIds, unification, from, to),
+    ])
 
-  for (const creditGroup of creditGroups) {
-    if (!creditGroup?.length) continue
+    const counter = new CourseYearlyStatsCounter()
 
-    const {
-      studentNumber,
-      grade,
-      passed,
-      semesterCode,
-      semesterName,
-      yearCode,
-      yearName,
-      attainmentDate,
-      programme,
-      courseCode: creditCourseCode,
-      credits,
-    } = parseCredit(
-      creditGroup,
-      anonymizationSalt,
-      courseCode,
-      studentNumberToSrElementsMap[creditGroup.at(0)?.student_studentnumber ?? 0] ?? []
-    )
+    // A single-course credit group can contain more than one real attempt for the same course code
+    // (e.g. failed once and passed on a retry). parseCredit collapses a group into a single
+    // attainment for student-stats (no student is counted twice), but attempt-stats (include all attempts for a course)
+    // need every attempt counted in its own year/semester.
+    // Substitution groups are left untouched
+    const splitIntoAttempts = (group: Credit[]): Credit[][] =>
+      isSingleCourse(group) && group.length > 1 ? group.map(credit => [credit]) : [group]
 
-    counter.markStudyProgramme(
-      studentNumber,
-      yearCode,
-      passed,
-      credits,
-      programme.code,
-      programme.name,
-      programme.facultyCode,
-      programme.organization
-    )
+    for (const creditGroup of creditGroups) {
+      if (!creditGroup?.length) continue
 
-    // Credits/attainments have quaranteed matching attainment_date and semesters/years
-    const groupCode = separate ? semesterCode : yearCode
-    const groupName = separate ? semesterName : yearName
-    counter.markCreditToGroup(studentNumber, passed, grade, groupCode, groupName, creditCourseCode, yearCode)
-    counter.markCreditToStudentCategories(studentNumber, attainmentDate, groupCode)
-  }
+      const studyRightElements = studentNumberToSrElementsMap[creditGroup.at(0)?.student_studentnumber ?? 0] ?? []
 
-  for (const enrollments of enrollmentGroups) {
-    if (!enrollments?.length) continue
-
-    for (const enrollment of enrollments) {
       const {
         studentNumber,
-        courseCode: enrollmentCourseCode,
-        enrollmentDateTime,
+        grade,
+        passed,
+        semesterCode,
+        semesterName,
+        yearCode,
+        yearName,
+        attainmentDate,
         programme,
-      } = parseEnrollment(
-        enrollment,
-        anonymizationSalt,
-        studentNumberToSrElementsMap[enrollment.studentnumber ?? 0] ?? []
-      )
-
-      // Enrollments can have conflicting enrollment_date_time and semestercode, so we need to manually select
-      // semestercode matching enrollment_date_time
-      const { semester, year } = getSemesterAndYearByDate(enrollment.enrollment_date_time, semesters, years)
+        courseCode: creditCourseCode,
+        credits,
+        creditTypeCode,
+      } = parseCredit(creditGroup, anonymizationSalt, courseCode, studyRightElements)
 
       counter.markStudyProgramme(
         studentNumber,
-        year.yearcode,
-        false, // passed
-        0, // credits
+        yearCode,
+        passed,
+        credits,
         programme.code,
         programme.name,
         programme.facultyCode,
         programme.organization
       )
 
-      const groupCode = separate ? semester.semestercode : year.yearcode
-      const groupName = separate ? semester.name : year.yearname
-
-      counter.markEnrollmentToGroup(
+      // Credits/attainments have quaranteed matching attainment_date and semesters/years
+      const groupCode = separate ? semesterCode : yearCode
+      const groupName = separate ? semesterName : yearName
+      counter.markStudentGradeToGroup(
         studentNumber,
-        enrollmentDateTime,
+        passed,
+        grade,
         groupCode,
         groupName,
-        enrollmentCourseCode,
-        year.yearcode
+        creditCourseCode,
+        yearCode,
+        creditTypeCode
       )
+
+      // Don't add students to student stats based on improved grades
+      if (creditTypeCode !== CreditTypeCode.IMPROVED) {
+        counter.markCreditToStudentCategories(studentNumber, attainmentDate, groupCode)
+      }
+
+      // The same group can include a failed and a passed credit in which case the parseCredit
+      // would return only the passed credit. Hence we split the failed and passed credit into
+      // two separate groups
+      for (const attemptCredits of splitIntoAttempts(creditGroup)) {
+        const attempt = parseCredit(attemptCredits, anonymizationSalt, courseCode, studyRightElements)
+        const attemptGroupCode = separate ? attempt.semesterCode : attempt.yearCode
+        const attemptGroupName = separate ? attempt.semesterName : attempt.yearName
+        counter.markAttemptToGroup(
+          attempt.studentNumber,
+          attempt.passed,
+          attempt.grade,
+          attemptGroupCode,
+          attemptGroupName,
+          attempt.courseCode,
+          attempt.yearCode
+        )
+      }
+    }
+
+    for (const enrollments of enrollmentGroups) {
+      for (const enrollment of enrollments) {
+        const studyRightElements = studentNumberToSrElementsMap[enrollment.studentnumber ?? 0] ?? []
+
+        const {
+          studentNumber,
+          courseCode: enrollmentCourseCode,
+          enrollmentDateTime,
+          programme,
+        } = parseEnrollment(enrollment, anonymizationSalt, studyRightElements)
+
+        // Enrollments can have conflicting enrollment_date_time and semestercode, so we need to manually select
+        // semestercode matching enrollment_date_time
+        const { semester, year } = getSemesterAndYearByDate(enrollment.enrollment_date_time, semesters, years)
+
+        counter.markStudyProgramme(
+          studentNumber,
+          year.yearcode,
+          false, // passed
+          0, // credits
+          programme.code,
+          programme.name,
+          programme.facultyCode,
+          programme.organization
+        )
+
+        const groupCode = separate ? semester.semestercode : year.yearcode
+        const groupName = separate ? semester.name : year.yearname
+
+        counter.markEnrollmentToGroup(
+          studentNumber,
+          enrollmentDateTime,
+          groupCode,
+          groupName,
+          enrollmentCourseCode,
+          year.yearcode
+        )
+      }
+    }
+
+    const statistics = await counter.getFinalStatistics(anonymizationSalt)
+
+    const substitutionGroups =
+      substitutions && primaryCourse.substitutionGroups?.length
+        ? await getSubstitutionGroupDetails(primaryCourse.substitutionGroups)
+        : [[{ code: courseCode, name, groupId }]]
+
+    return {
+      ...statistics,
+      courseCode,
+      groupId,
+      substitutionGroups,
+      name,
     }
   }
 
-  const statistics = await counter.getFinalStatistics(anonymizationSalt)
-
-  const substitutionGroups: Pick<CourseModel, 'code' | 'name'>[][] =
-    combineSubstitutions && course?.substitution_groups?.length
-      ? await getSubstitutionGroupDetails(course.substitution_groups)
-      : [[{ code: course!.code, name: course!.name }]]
-
-  return {
-    ...statistics,
-    coursecode: courseCode,
-    alternatives: substitutionGroups,
-    name: course?.name,
-  }
-}
-
-/**
-  Mostly-legacy function that prevents user from opening too big of a populations.
-  At the moment the threshold to prevent a population is so high it will never trigger.
-
- */
-export const maxYearsToCreatePopulationFrom = async (courseCodes: string[], unification: Unification) => {
-  const lastAttainmentDate = (await CourseModel.findOne({
-    attributes: [[dbFn('MAX', dbCol('max_attainment_date')), 'date']],
-    where: {
-      code: { [Op.in]: courseCodes },
-    },
-    raw: true,
-  })) as { date: Date } | null
-
-  if (lastAttainmentDate?.date == null) return 0
-
-  /** Amount of years before latest attainment that the attainment count is calcucated for */
-  const yearRange = 6
-
-  const newestAttainmentYear = lastAttainmentDate?.date.getFullYear()
-  const attainmentDateThreshold = new Date(newestAttainmentYear - yearRange, 0, 1)
-
-  /** Amount of attainments between latest-attainment-year - yearRange and latest attainment date */
-  const attainmentsWithinThreshold = await CreditModel.count({
-    where: {
-      course_code: { [Op.in]: courseCodes },
-      attainment_date: { [Op.gt]: attainmentDateThreshold },
-      is_open: getIsOpen(unification),
-    },
-  })
-
-  /** Limit the allowed attainments to 15_000 during the $yearRange years */
-  const maxAllowedAttainments = 15_000 * yearRange
-  return Math.round(Math.max(1, maxAllowedAttainments / attainmentsWithinThreshold))
-}
-
 export const getCourseYearlyStats = async (
-  courseCodes: string[],
+  courseGroupIds: string[],
   separate: boolean,
   anonymizationSalt: string | null,
-  combineSubstitutions: boolean,
+  substitutions: boolean,
   fromYearCode = yearToYearCode(1950).toString(),
   toYearCode: string = yearToYearCode(now().getFullYear()).toString()
 ) => {
@@ -365,26 +361,36 @@ export const getCourseYearlyStats = async (
   const from = new Date(`${yearCodeToYear(fromYearCode)}-08-01`) // FALL
   const to = new Date(`${yearCodeToYear(toYearCode) + 1}-07-31`) // SPRING next year
 
+  const relevantCourseIds = (
+    await CourseModel.findAll({
+      attributes: ['id'],
+      raw: true,
+      where: {
+        groupId: { [Op.in]: courseGroupIds },
+      },
+    })
+  ).map(({ id }) => id)
+
   const [credits, enrollments] = await Promise.all([
     CreditModel.findAll({
       attributes: ['student_studentnumber'],
+      raw: true,
       where: {
-        course_code: { [Op.in]: courseCodes },
+        course_id: { [Op.in]: relevantCourseIds },
         attainment_date: { [Op.between]: [from, to] },
       },
     }),
     EnrollmentModel.findAll({
       attributes: ['studentnumber'],
+      raw: true,
       where: {
-        course_code: {
-          [Op.in]: courseCodes,
+        course_id: {
+          [Op.in]: relevantCourseIds,
         },
         state: EnrollmentState.ENROLLED,
         enrollment_date_time: {
-          [Op.and]: {
-            [Op.between]: [from, to],
-            [Op.gte]: enrollmentTimeDateThreshold,
-          },
+          [Op.between]: [from, to],
+          [Op.gte]: enrollmentTimeDateThreshold,
         },
       },
     }),
@@ -402,71 +408,52 @@ export const getCourseYearlyStats = async (
 
   const studentNumberToSrElementsMap = await getStudentNumberToSrElementsMap([...studentNumbers])
 
-  const statsRegular = await Promise.all(
-    courseCodes.map(async courseCode => {
-      const course: Pick<CourseModel, 'code' | 'name' | 'substitution_groups'> | null = await CourseModel.findOne({
-        raw: true,
-        attributes: ['code', 'name', 'substitution_groups'],
-        where: { code: courseCode },
-      })
+  const stats = await Promise.all(
+    courseGroupIds.map(async groupId => {
+      const primaryCourse: Pick<CourseModel, 'id' | 'groupId' | 'code' | 'name' | 'substitutionGroups'> | null =
+        await CourseModel.findOne({
+          raw: true,
+          attributes: ['groupId', 'id', 'code', 'name', 'substitutionGroups'],
+          where: { groupId, isPrimary: true },
+        })
 
-      if (!course) {
-        logger.error('Course for course stats not found with code' + courseCode)
+      if (!primaryCourse) {
+        logger.error(`Primary course for course stats not found for: ${groupId}`)
         return {}
       }
 
+      const getYearlyStatsOfUnification = getYearlyStatsOfNew(
+        primaryCourse,
+        separate,
+        anonymizationSalt,
+        substitutions,
+        studentNumberToSrElementsMap,
+        from,
+        to
+      )
+
       const [openStats, regularStats, unifyStats] = await Promise.all([
-        getYearlyStatsOfNew(
-          course,
-          courseCode,
-          separate,
-          Unification.OPEN,
-          anonymizationSalt,
-          combineSubstitutions,
-          studentNumberToSrElementsMap,
-          from,
-          to
-        ),
-        getYearlyStatsOfNew(
-          course,
-          courseCode,
-          separate,
-          Unification.REGULAR,
-          anonymizationSalt,
-          combineSubstitutions,
-          studentNumberToSrElementsMap,
-          from,
-          to
-        ),
-        getYearlyStatsOfNew(
-          course,
-          courseCode,
-          separate,
-          Unification.UNIFY,
-          anonymizationSalt,
-          combineSubstitutions,
-          studentNumberToSrElementsMap,
-          from,
-          to
-        ),
+        getYearlyStatsOfUnification(Unification.OPEN),
+        getYearlyStatsOfUnification(Unification.REGULAR),
+        getYearlyStatsOfUnification(Unification.UNIFY),
       ])
 
       return { unifyStats, regularStats, openStats }
     })
   )
 
-  return statsRegular
+  return stats as CourseYearlyStats[]
 }
 
-export const getCourseProvidersForCourses = async (codes: string[]) =>
+export const getCourseProvidersForCourses = async (courseIds: string[]) =>
   (
     await OrganizationModel.findAll({
       attributes: ['code'],
       include: {
         model: CourseModel,
         where: {
-          code: {
-            [Op.in]: codes,
+          id: {
+            [Op.in]: courseIds,
           },
         },
       },
@@ -474,28 +461,47 @@ export const getCourseProvidersForCourses = async (codes: string[]) =>
     })
   ).map(({ code }) => code)
 
-export const getCourseDetails = async (codes: string[]) =>
+export const getCourseDetails = async (courseGroupIds: string[]) =>
   CourseModel.findAll({
-    attributes: ['code', 'name', 'substitution_groups'],
-    where: { code: { [Op.in]: codes } },
+    attributes: ['id', 'groupId', 'code', 'name', 'substitutionGroups', 'isStudyModule'],
+    where: { groupId: { [Op.in]: courseGroupIds }, isPrimary: true },
     raw: true,
   })
 
-// Add all substitution_groups and coursecodes together
-export const searchAndCombineSubstitutionGroupsToCodes = async (coursecodes: string[]) => {
-  const substitutionGroups = await CourseModel.findAll({
-    raw: true,
-    attributes: ['substitution_groups'],
-    where: {
-      code: { [Op.in]: coursecodes },
-    },
-  })
+/** First gets all unique groupIds from main course and substitutions (if enabled)
+ * Then returns an id-groupId map for all courses that match above groupId
+ */
+export const getRelevantCourseIdMap = async (courseGroupIds: string[], substitutions: boolean) => {
+  const combinedGroupIds = substitutions
+    ? [
+        ...new Set(
+          (
+            await CourseModel.findAll({
+              raw: true,
+              attributes: ['groupId', 'substitutionGroups'],
+              where: {
+                groupId: { [Op.in]: courseGroupIds },
+              },
+            })
+          )
+            .flatMap(({ groupId, substitutionGroups }) => [groupId, substitutionGroups])
+            .flat(2)
+        ),
+      ]
+    : courseGroupIds
 
-  return [
-    ...new Set(
-      coursecodes.concat(
-        substitutionGroups.flatMap(({ substitution_groups }) => substitution_groups.flatMap(code => code))
-      )
-    ),
-  ]
+  const idToGroupIdMap = (
+    await CourseModel.findAll({
+      raw: true,
+      attributes: ['id', 'groupId'],
+      where: {
+        groupId: { [Op.in]: combinedGroupIds },
+      },
+    })
+  ).reduce<Record<string, string>>((acc, { id, groupId }) => {
+    acc[id] = groupId
+    return acc
+  }, {})
+
+  return idToGroupIdMap
 }

services/backend/src/services/excludedCourses.ts

@@ -2,18 +2,35 @@ import { Op } from 'sequelize'
 
 import { ExcludedCourseModel } from '../models/kone'
 
-export const addExcludedCourses = async (courseCodes: string[], curriculumVersion: string, programmeCode: string) => {
-  return ExcludedCourseModel.bulkCreate(
-    courseCodes.map(courseCode => ({
+export const addExcludedCourses = async (
+  courseGroupIds: string[],
+  curriculumVersion: string,
+  programmeCode: string
+) => {
+  const existing = await ExcludedCourseModel.findAll({
+    attributes: ['course_group_id'],
+    where: {
       programme_code: programmeCode,
       curriculum_version: curriculumVersion,
-      course_code: courseCode,
-    }))
+      course_group_id: { [Op.in]: courseGroupIds },
+    },
+    raw: true,
+  })
+  const alreadyExcluded = new Set(existing.map(({ course_group_id }) => course_group_id))
+
+  return ExcludedCourseModel.bulkCreate(
+    courseGroupIds
+      .filter(courseGroupId => !alreadyExcluded.has(courseGroupId))
+      .map(courseGroupId => ({
+        programme_code: programmeCode,
+        curriculum_version: curriculumVersion,
+        course_group_id: courseGroupId,
+      }))
   )
 }
 
 export const removeExcludedCourses = async (
-  courseCodes: string[],
+  courseGroupIds: string[],
   curriculumVersion: string,
   programmeCode: string
 ) => {
@@ -21,7 +38,7 @@ export const removeExcludedCourses = async (
     where: {
       programme_code: programmeCode,
       curriculum_version: curriculumVersion,
-      course_code: { [Op.in]: courseCodes },
+      course_group_id: { [Op.in]: courseGroupIds },
     },
   })
 }

services/backend/src/services/faculty/faculty.ts

@@ -59,7 +59,7 @@ export const getDegreeProgrammesOfOrganization = async (organizationId: string,
     progId:
       programme.code in programmeCodes ? programmeCodes[programme.code as keyof typeof programmeCodes] : programme.code,
   }))
-  const programmesGroupedByCode = groupBy(orderBy(programmesWithProgIds, ['valid_from'], ['desc']), prog => prog.code)
+  const programmesGroupedByCode = groupBy(orderBy(programmesWithProgIds, ['validFrom'], ['desc']), prog => prog.code)
   const curriculumPeriods = await getCurriculumPeriods()
   const relevantProgrammes: ProgrammeModuleWithRelevantAttributes[] = []
 

services/backend/src/services/languageCenterData.ts

@@ -4,6 +4,7 @@ import { Op } from 'sequelize'
 import { EnrollmentState } from '@oodikone/shared/types'
 import { CourseModel, CreditModel, EnrollmentModel, SISStudyRightModel } from '../models'
 import { redisClient } from './redis'
+import { getPrimaryCoursesByGroupIds } from './studyProgramme'
 
 export const LANGUAGE_CENTER_REDIS_KEY = 'LANGUAGE_CENTER_DATA'
 
@@ -25,8 +26,8 @@ const getDifference = stats => {
 }
 
 const getLanguageCenterCourses = async () => {
-  const courses: Pick<CourseModel, 'code' | 'name'>[] = await CourseModel.findAll({
-    attributes: ['code', 'name'],
+  const courses: Pick<CourseModel, 'id' | 'groupId' | 'code' | 'name'>[] = await CourseModel.findAll({
+    attributes: ['id', 'groupId', 'code', 'name'],
     where: {
       [Op.or]: [{ code: { [Op.like]: 'KK%' } }, { code: { [Op.like]: 'AYKK%' } }],
     },
@@ -111,30 +112,54 @@ type Attempt = {
 export const computeLanguageCenterData = async () => {
   const courses = await getLanguageCenterCourses()
   const autumnSemester2017 = 135
+  const courseIds = courses.map(course => course.id)
+  const idToGroupId = new Map(courses.map(course => [course.id, course.groupId]))
+
+  const primaryCourses = await getPrimaryCoursesByGroupIds([...new Set(courses.map(course => course.groupId))])
+  const groupIdToPrimaryCode = new Map(primaryCourses.map(course => [course.groupId, course.code]))
+
+  const courseCodeOf = (courseId: string) => {
+    const groupId = idToGroupId.get(courseId)
+    return groupId ? groupIdToPrimaryCode.get(groupId) : undefined
+  }
 
   const credits = await CreditModel.findAll({
     raw: true,
-    attributes: ['course_code', 'student_studentnumber', 'semestercode', 'attainment_date', 'studyright_id'],
+    attributes: ['course_id', 'student_studentnumber', 'semestercode', 'attainment_date', 'studyright_id'],
     where: {
-      [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
+      course_id: { [Op.in]: courseIds },
       semestercode: { [Op.gte]: autumnSemester2017 },
       credittypecode: 4,
     },
+    order: [
+      ['attainment_date', 'DESC NULLS LAST'],
+      ['student_studentnumber', 'ASC'],
+      ['course_id', 'ASC'],
+    ], // Add a fallbacks sorting so that runs would be more deterministic
   })
 
   const enrollments = await EnrollmentModel.findAll({
     raw: true,
-    attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'studyright_id', 'state'],
+    attributes: ['studentnumber', 'semestercode', 'course_id', 'enrollment_date_time', 'studyright_id', 'state'],
     where: {
-      [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
+      course_id: { [Op.in]: courseIds },
       state: { [Op.in]: ['ENROLLED', 'REJECTED'] },
       // EnrollmentDateTimeThreshold need not be used here as we are not counting failed courses
     },
+    order: [
+      ['enrollment_date_time', 'DESC NULLS LAST'],
+      ['studentnumber', 'ASC'],
+      ['course_id', 'ASC'], // Last fallback
+    ],
   })
 
   const studyRights = await SISStudyRightModel.findAll({
     raw: true,
-    order: [['end_date', 'DESC NULLS LAST']],
+    order: [
+      ['end_date', 'DESC NULLS LAST'],
+      ['start_date', 'DESC'],
+      ['id', 'DESC'],
+    ],
   })
 
   const attemptStudyRightToFacultyMap = studyRights.reduce<
@@ -149,12 +174,14 @@ export const computeLanguageCenterData = async () => {
   const attemptsByStudents: Record<string, Attempt[]> = {}
 
   credits.forEach(credit => {
+    const courseCode = courseCodeOf(credit.course_id)
+    if (!courseCode) return
     const studentNumber = credit.student_studentnumber
     studentNumbers.add(studentNumber)
     attemptsByStudents[studentNumber] ??= []
     attemptsByStudents[studentNumber].push({
       studentNumber,
-      courseCode: credit.course_code,
+      courseCode,
       completed: true,
       date: credit.attainment_date,
       faculty: attemptStudyRightToFacultyMap[credit.studyright_id],
@@ -163,15 +190,15 @@ export const computeLanguageCenterData = async () => {
   })
 
   enrollments.forEach(enrollment => {
+    const courseCode = courseCodeOf(enrollment.course_id)
+    if (!courseCode) return
     const studentNumber = enrollment.studentnumber
     studentNumbers.add(studentNumber)
     attemptsByStudents[studentNumber] ??= []
     if (
       attemptsByStudents[studentNumber].find(
         attempt =>
-          !attempt.completed &&
-          attempt.semestercode === enrollment.semestercode &&
-          attempt.courseCode === enrollment.course_code
+          !attempt.completed && attempt.semestercode === enrollment.semestercode && attempt.courseCode === courseCode
       )
     ) {
       return
@@ -179,7 +206,7 @@ export const computeLanguageCenterData = async () => {
 
     attemptsByStudents[studentNumber].push({
       studentNumber,
-      courseCode: enrollment.course_code,
+      courseCode,
       completed: false,
       date: enrollment.enrollment_date_time,
       faculty: attemptStudyRightToFacultyMap[enrollment.studyright_id],
@@ -226,7 +253,7 @@ export const computeLanguageCenterData = async () => {
     }
   })
 
-  const unorderedTableData = createArrayOfCourses(attempts, courses)
+  const unorderedTableData = createArrayOfCourses(attempts, primaryCourses)
   const tableData = orderBy(unorderedTableData, 'code')
 
   const faculties = [...new Set(attempts.map(({ faculty }) => faculty))]

services/backend/src/services/populations/closeToGraduation.ts

@@ -24,7 +24,8 @@ const findThesisAndLatestAndEarliestAttainments = (
   studyPlan: Studyplan,
   attainments: Credit[],
   degreeProgrammeType: DegreeProgrammeType,
-  studyRightId: string
+  studyRightId: string,
+  idToGroupIdMap: Record<string, string>
 ) => {
   let thesisData: Credit | undefined
   const attainmentDates: AttainmentDates = {}
@@ -32,16 +33,19 @@ const findThesisAndLatestAndEarliestAttainments = (
     [DegreeProgrammeType.BACHELOR]: 'urn:code:course-unit-type:bachelors-thesis',
     [DegreeProgrammeType.MASTER]: 'urn:code:course-unit-type:masters-thesis',
   }
+  const includedCourseGroupIds = new Set(
+    studyPlan.included_courses.map(id => idToGroupIdMap[id]).filter((groupId): groupId is string => !!groupId)
+  )
 
   for (const attainment of attainments) {
     if (
-      attainment.course?.course_unit_type === thesisCodes[degreeProgrammeType] &&
+      attainment.course?.courseUnitType === thesisCodes[degreeProgrammeType] &&
       attainment.studyright_id === studyRightId
     ) {
       thesisData = attainment
     }
     attainmentDates.latestTotal ??= attainment.attainment_date
-    if (studyPlan.included_courses.includes(attainment.course?.code)) {
+    if (attainment.course?.groupId && includedCourseGroupIds.has(attainment.course.groupId)) {
       attainmentDates.latestHops ??= attainment.attainment_date
       if (
         !attainmentDates.earliestHops ||
@@ -55,7 +59,12 @@ const findThesisAndLatestAndEarliestAttainments = (
   return { attainmentDates, thesisData }
 }
 
-const formatStudent = (student: Student, facultyMap: Record<string, Name>, currentSemesterCode: number) => {
+const formatStudent = (
+  student: Student,
+  facultyMap: Record<string, Name>,
+  currentSemesterCode: number,
+  idToGroupIdMap: Record<string, string>
+) => {
   const {
     studentnumber: studentNumber,
     abbreviatedname: name,
@@ -101,7 +110,8 @@ const formatStudent = (student: Student, facultyMap: Record<string, Name>, curre
       studyPlan,
       student.credits,
       degreeProgrammeType,
-      studyRightId
+      studyRightId,
+      idToGroupIdMap
     )
 
     acc.push({
@@ -230,12 +240,12 @@ export const findStudentsCloseToGraduation = async (studentNumbers?: string[]) =
         },
         {
           model: CreditModel,
-          attributes: ['attainment_date', 'grade', 'studyright_id'],
+          attributes: ['attainment_date', 'grade', 'studyright_id', 'course_id'],
           where: { credittypecode: CreditTypeCode.PASSED },
           include: [
             {
               model: CourseModel,
-              attributes: ['code', 'course_unit_type'],
+              attributes: ['code', 'courseUnitType', 'groupId'],
             },
           ],
         },
@@ -264,10 +274,20 @@ export const findStudentsCloseToGraduation = async (studentNumbers?: string[]) =
     facultyMap[programme.code] = programme['organization.name']
   }
 
+  const includedCourseIds = [
+    ...new Set(students.flatMap(student => student.studyplans.flatMap(sp => sp.included_courses))),
+  ]
+  const includedCourses = await CourseModel.findAll({
+    attributes: ['id', 'groupId'],
+    where: { id: { [Op.in]: includedCourseIds } },
+    raw: true,
+  })
+  const idToGroupIdMap = Object.fromEntries(includedCourses.map(({ id, groupId }) => [id, groupId]))
+
   const { semestercode: currentSemesterCode } = await getCurrentSemester()
 
   return students
-    .flatMap(student => formatStudent(student, facultyMap, currentSemesterCode))
+    .flatMap(student => formatStudent(student, facultyMap, currentSemesterCode, idToGroupIdMap))
     .reduce(
       (acc, student) => {
         if (student.programme.degreeProgrammeType === DegreeProgrammeType.BACHELOR) {

services/backend/src/services/populations/getStudentData.ts

@@ -1,6 +1,6 @@
-import { Op } from 'sequelize'
+import { col, Op } from 'sequelize'
 
-import { Credit, Enrollment, SISStudyRight, SISStudyRightElement, Student, Studyplan } from '@oodikone/shared/models'
+import { SISStudyRight, SISStudyRightElement, Student, Studyplan } from '@oodikone/shared/models'
 import { Tag, TagStudent } from '@oodikone/shared/models/kone'
 import { EnrollmentState } from '@oodikone/shared/types'
 import { enrollmentTimeDateThreshold } from '@oodikone/shared/util'
@@ -12,8 +12,10 @@ import {
   StudyplanModel,
   SISStudyRightModel,
   SISStudyRightElementModel,
+  CourseModel,
 } from '../../models'
 import { TagModel, TagStudentModel } from '../../models/kone'
+import { StudentCredit, StudentEnrollment } from '@oodikone/shared/types/studentData'
 
 export type StudentTags = TagStudent & {
   tag: Pick<Tag, 'tag_id' | 'tagname' | 'personal_user_id'>
@@ -46,17 +48,21 @@ export const getStudentTagMap = async (studyRights: string[], studentNumbers: st
   return studentTagMap
 }
 
-export type StudentEnrollment = Pick<
-  Enrollment,
-  'course_code' | 'state' | 'enrollment_date_time' | 'semestercode' | 'studentnumber' | 'studyright_id'
->
 export const getEnrollments = (
   studentNumbers: string[],
   startDate: string,
   endDate: string
-): Promise<Array<StudentEnrollment>> =>
+): Promise<StudentEnrollment[]> =>
   EnrollmentModel.findAll({
-    attributes: ['course_code', 'state', 'enrollment_date_time', 'semestercode', 'studentnumber', 'studyright_id'],
+    attributes: [
+      'course_id',
+      'course_code',
+      'state',
+      'enrollment_date_time',
+      'semestercode',
+      'studentnumber',
+      'studyright_id',
+    ],
     where: {
       studentnumber: {
         [Op.in]: studentNumbers,
@@ -185,24 +191,7 @@ export const getStudents = (studentNumbers: string[]): Promise<Array<StudentData
     },
   }).then(data => data.map(x => x.get({ plain: true })))
 
-export type StudentCredit = Pick<
-  Credit,
-  | 'grade'
-  | 'credits'
-  | 'credittypecode'
-  | 'attainment_date'
-  | 'isStudyModule'
-  | 'student_studentnumber'
-  | 'course_code'
-  | 'language'
-  | 'studyright_id'
->
-
-export const getCredits = async (
-  studentNumbers: string[],
-  startDate: string,
-  endDate: string
-): Promise<Array<StudentCredit>> =>
+export const getCredits = async (studentNumbers: string[], startDate: string, endDate: string) =>
   CreditModel.findAll({
     raw: true,
     attributes: [
@@ -212,15 +201,23 @@ export const getCredits = async (
       'attainment_date',
       'isStudyModule',
       'student_studentnumber',
+      'course_id',
       'course_code',
       'language',
       'studyright_id',
+      [col('course.group_id'), 'courseGroupId'],
     ],
+    include: {
+      model: CourseModel,
+      attributes: [],
+      as: 'course',
+      required: true,
+    },
     where: {
       student_studentnumber: { [Op.in]: studentNumbers },
       attainment_date: { [Op.between]: [startDate, endDate] },
     },
-  })
+  }) as unknown as Promise<StudentCredit[]>
 
 export const getStudyRightElementsForStudyRight = async (studentNumbers: string[], code: string) =>
   SISStudyRightElementModel.findAll({

services/backend/src/services/populations/shared.ts

@@ -77,11 +77,11 @@ export const getOptionsForStudents = (
   return optionMap
 }
 
-export const getCourses = (courses: string[]): Promise<Array<CourseStats>> =>
+export const getCourseGroupIds = (courseIds: string[]): Promise<Array<Pick<CourseStats, 'id' | 'groupId'>>> =>
   CourseModel.findAll({
-    attributes: ['code', 'name', 'substitution_groups', 'is_study_module'],
+    attributes: ['id', 'groupId'],
     where: {
-      code: { [Op.in]: courses },
+      id: { [Op.in]: courseIds },
     },
     raw: true,
   })

services/backend/src/services/populations/statisticsOf.ts

@@ -1,17 +1,17 @@
 import { getDegreeProgrammeType } from '../../util'
 import { now } from '../../util/clock'
 import { getCriteria } from '../studyProgramme/studyProgrammeCriteria'
+import { getCourseDetails } from '../courses'
 import { formatStudentForAPI } from './formatStatisticsForApi'
 import {
-  type StudentEnrollment,
-  type StudentCredit,
   type StudentTags,
   getStudents,
   getEnrollments,
   getCredits,
   getStudyRightElementsForStudyRight,
 } from './getStudentData'
-import { getCourses, getOptionsForStudents } from './shared'
+import { getCourseGroupIds, getOptionsForStudents } from './shared'
+import { StudentCredit, StudentEnrollment } from '@oodikone/shared/types/studentData'
 
 export type OptimizedStatisticsQuery = {
   userId: string
@@ -63,10 +63,13 @@ export const statisticsOf = async (
       getStudyRightElementsForStudyRight(studentNumbers, code),
     ])
 
-  const courseCodes = new Set<string>()
-  for (const { course_code } of credits) courseCodes.add(course_code)
-  for (const { course_code } of enrollments) courseCodes.add(course_code)
-  const courses = await getCourses(Array.from(courseCodes))
+  const courseIds = new Set<string>()
+  for (const { course_id } of credits) courseIds.add(course_id)
+  for (const { course_id } of enrollments) courseIds.add(course_id)
+  const idToGroupIdRows = await getCourseGroupIds([...courseIds])
+  const idToGroupIdMap = Object.fromEntries(idToGroupIdRows.map(({ id, groupId }) => [id, groupId]))
+  const groupIds = [...new Set(idToGroupIdRows.map(({ groupId }) => groupId))]
+  const courses = await getCourseDetails(groupIds)
 
   const optionData = getOptionsForStudents(studyRightElementsForStudyRight, degreeProgrammeType)
   const formattedStudents = students.map(student => {
@@ -79,6 +82,6 @@ export const statisticsOf = async (
   return {
     students: formattedStudents,
     criteria,
-    coursestatistics: { courses, enrollments, credits },
+    coursestatistics: { courses, idToGroupIdMap, enrollments, credits },
   }
 }

services/backend/src/services/programmeModules.ts

@@ -47,7 +47,7 @@ export const getCurriculumOptions = async (code: string) => {
   }
 }
 
-type ModuleWithChildren = Pick<ProgrammeModule, 'id' | 'type' | 'code' | 'name'> & {
+type ModuleWithChildren = Pick<ProgrammeModule, 'id' | 'group_id' | 'type' | 'code' | 'name'> & {
   // TODO: This is supposed to be in the ProgrammeModule
   degree_programme_type: DegreeProgrammeType | null
 
@@ -90,6 +90,7 @@ const recursivelyGetModuleAndChildren = async (code: string, curriculum_period_i
         GROUP BY pm.id, c.name, c.code, c.order, c.id
       ) SELECT
         id,
+        group_id,
         type,
         module_order,
         parent_code,
@@ -141,7 +142,7 @@ const labelProgrammes = (modules: ModuleWithChildren[], excludedCourses: Exclude
       label: `${module.parent_code}\n${module.parent_name?.fi}`,
       orderNumber: module.module_order,
     }
-    const foundCourse = excludedCourses.find(course => course.course_code === module.code)
+    const foundCourse = excludedCourses.find(course => course.course_group_id === module.group_id)
     const visible = { visibility: !foundCourse, id: foundCourse?.id ?? null }
 
     return { ...module, label, visible }
@@ -180,6 +181,7 @@ const getCoursesAndModulesForProgramme = async (code: string, periodIds: string)
 
   return {
     courses: labelProgrammes(modifiedCourses, excludedCourses).map(mod => ({
+      group_id: mod.group_id,
       code: mod.code,
       name: mod.name,
       degree_programme_type: mod.degree_programme_type,

services/backend/src/services/providerCredits.ts

@@ -3,7 +3,7 @@ import { EnrollmentType, ExtentCode } from '@oodikone/shared/types'
 import { CreditStats } from '@oodikone/shared/types/studyProgramme'
 import { mapToProviders } from '@oodikone/shared/util'
 import { getCreditStats, setCreditStats } from './analyticsService'
-import { getCourseCodesOfProvider } from './providers'
+import { getCourseIdsOfProvider } from './providers'
 import { getCreditsForProvider, getTransferredCredits } from './studyProgramme/creditGetters'
 import { getStudentHetuStateMap } from './studyProgramme/studentGetters'
 import { defineYear } from './studyProgramme/studyProgrammeHelpers'
@@ -107,8 +107,8 @@ const getBasicDegreeStudyRight = (
 export const computeCreditsProduced = async (providerCode: string, isAcademicYear: boolean): Promise<CreditStats> => {
   const since = new Date('2017-01-01')
   const rapoFormattedProviderCode = mapToProviders([providerCode])[0]
-  const courses = await getCourseCodesOfProvider(rapoFormattedProviderCode)
-  const credits = await getCreditsForProvider(rapoFormattedProviderCode, courses, since)
+  const courseIds = await getCourseIdsOfProvider(rapoFormattedProviderCode)
+  const credits = await getCreditsForProvider(rapoFormattedProviderCode, courseIds, since)
 
   // This part also adds oikis Vaasa which is provided by a different organization.
   // Unknown if there are other similar cases!
@@ -119,8 +119,8 @@ export const computeCreditsProduced = async (providerCode: string, isAcademicYea
   const vaasaProvider =
     rapoFormattedProviderCode in vaasaCodes ? vaasaCodes[rapoFormattedProviderCode as keyof typeof vaasaCodes] : null
   if (vaasaProvider) {
-    const courses = await getCourseCodesOfProvider(vaasaProvider)
-    const vaasaCredits = await getCreditsForProvider(vaasaProvider, courses, since)
+    const vaasaCourseIds = await getCourseIdsOfProvider(vaasaProvider)
+    const vaasaCredits = await getCreditsForProvider(vaasaProvider, vaasaCourseIds, since)
     credits.push(...vaasaCredits)
   }
 

services/backend/src/services/providers.ts

@@ -1,4 +1,5 @@
-import { getOpenUniCourseCode } from '../../src/util'
+import { Op } from 'sequelize'
+
 import { CourseModel, OrganizationModel } from '../models'
 
 export const getAllProviders = async () => {
@@ -9,10 +10,10 @@ export const getAllProviders = async () => {
   return providers
 }
 
-export const getCourseCodesOfProvider = async (provider: string) => {
+export const getCourseIdsOfProvider = async (provider: string): Promise<string[]> => {
   const coursesByProvider = await CourseModel.findAll({
     raw: true,
-    attributes: ['id', 'code', 'substitution_groups'],
+    attributes: ['id', 'substitutionGroups'],
     include: {
       model: OrganizationModel,
       required: true,
@@ -25,14 +26,28 @@ export const getCourseCodesOfProvider = async (provider: string) => {
     },
   })
 
-  const coursesWithOpenUniSubstitutions = coursesByProvider.map(({ code, substitution_groups }) => {
-    if (!substitution_groups?.length) {
-      return [code]
-    }
-    // This is ok to flatten, because we only want all credits completed by students, not what courses are
-    // completed and how => we don't care about substitutions
-    return [code].concat(substitution_groups.filter(group => group.some(code => getOpenUniCourseCode(code))).flat())
+  const directIds = coursesByProvider.map(({ id }) => id)
+  const substitutionGroupIds = [
+    ...new Set(coursesByProvider.flatMap(({ substitutionGroups }) => substitutionGroups ?? []).flat()),
+  ]
+
+  if (substitutionGroupIds.length === 0) return directIds
+
+  const substitutionCourses = await CourseModel.findAll({
+    raw: true,
+    attributes: ['id'],
+    where: { groupId: { [Op.in]: substitutionGroupIds } },
+    include: {
+      model: OrganizationModel,
+      required: true,
+      where: {
+        code: provider,
+      },
+      through: {
+        attributes: [],
+      },
+    },
   })
 
-  return coursesWithOpenUniSubstitutions.flat()
+  return [...new Set([...directIds, ...substitutionCourses.map(({ id }) => id)])]
 }

services/backend/src/services/students.ts

@@ -1,7 +1,7 @@
 import { Op } from 'sequelize'
 
-import type { Credit, Student } from '@oodikone/shared/models'
-import { CreditTypeCode, EnrollmentState, UnifyStatus } from '@oodikone/shared/types'
+import type { Credit, Student, Studyplan } from '@oodikone/shared/models'
+import { CreditTypeCode, EnrollmentState, Unification } from '@oodikone/shared/types'
 import { FormattedStudentForSearch, StudentPageStudent } from '@oodikone/shared/types/studentData'
 import { enrollmentTimeDateThresholdAcademicYear, splitByEmptySpace } from '@oodikone/shared/util'
 import { dateMaxFromList } from '@oodikone/shared/util/datetime'
@@ -43,7 +43,7 @@ const byStudentNumber = async (studentNumber: string) => {
           include: [
             {
               model: CourseModel,
-              attributes: ['code', 'name'],
+              attributes: ['code', 'name', 'groupId'],
               required: true,
             },
           ],
@@ -110,13 +110,13 @@ const byStudentNumber = async (studentNumber: string) => {
   }
 }
 
-const getUnifyStatus = (unifyCourses: UnifyStatus): [boolean] | [true, false] => {
+const getUnifyStatus = (unifyCourses: Unification): [boolean] | [true, false] => {
   switch (unifyCourses) {
-    case 'unifyStats':
+    case Unification.UNIFY:
       return [true, false]
-    case 'openStats':
+    case Unification.OPEN:
       return [true]
-    case 'regularStats':
+    case Unification.REGULAR:
       return [false]
     default:
       return [true, false]
@@ -156,104 +156,47 @@ export const getStartAndEndDates = async (from: number, to: number, separate: bo
   Returns students who have an attainment/enrollment between to and from.
  */
 export const findByCourseAndSemesters = async (
-  codes: string[],
+  courseIds: string[],
   from: number,
   to: number,
   separate: boolean,
-  unifyCourses: UnifyStatus = 'unifyStats'
+  unifyCourses: Unification = Unification.UNIFY
 ) => {
   /* from & to are semestercodes if separate = false, or yearcodes in case separate is true. */
-
-  const { startDate: startDate, endDate: endDate } = await getStartAndEndDates(from, to, separate)
+  const { startDate, endDate } = await getStartAndEndDates(from, to, separate)
+  if (!courseIds.length || !startDate || !endDate) return new Set<string>()
 
   const unifyStatus = getUnifyStatus(unifyCourses)
 
-  const credits = (await CreditModel.findAll({
-    raw: true,
-    attributes: [
-      ['student_studentnumber', 'studentnumber'],
-      'course_code',
-      ['attainment_date', 'date'],
-      'credittypecode',
-    ],
-    where: {
-      course_code: { [Op.in]: codes },
-      is_open: unifyStatus,
-      attainment_date: { [Op.between]: [startDate, endDate] },
-      credittypecode: { [Op.not]: CreditTypeCode.IMPROVED }, // We do not care about improved grades
-    },
-    order: [
-      ['attainment_date', 'DESC'],
-      ['credittypecode', 'ASC'],
-    ],
-  })) as unknown as Array<{
-    studentnumber: CreditModel['student_studentnumber']
-    course_code: CreditModel['course_code']
-    date: CreditModel['attainment_date']
-    credittypecode: CreditModel['credittypecode']
-  }>
-  // HACK: Sequelize doesn't like renaming attributes so we have to force-cast
-
-  const enrollments = (await EnrollmentModel.findAll({
-    raw: true,
-    attributes: ['studentnumber', 'course_code', ['enrollment_date_time', 'date']],
-    where: {
-      course_code: { [Op.in]: codes },
-      is_open: unifyStatus,
-      enrollment_date_time: {
-        [Op.between]: [dateMaxFromList(startDate ?? new Date(0), enrollmentTimeDateThresholdAcademicYear), endDate],
+  const creditStudents = (
+    await CreditModel.findAll({
+      raw: true,
+      attributes: ['student_studentnumber'],
+      where: {
+        course_id: { [Op.in]: courseIds },
+        is_open: unifyStatus,
+        attainment_date: { [Op.between]: [startDate, endDate] },
+        credittypecode: { [Op.not]: CreditTypeCode.IMPROVED }, // We do not care about improved grades
       },
-      state: EnrollmentState.ENROLLED,
-    },
-    order: [['enrollment_date_time', 'DESC']],
-  })) as unknown as Array<{
-    studentnumber: EnrollmentModel['studentnumber']
-    course_code: EnrollmentModel['course_code']
-    date: EnrollmentModel['enrollment_date_time']
-  }>
-  // HACK: Sequelize doesn't like renaming attributes so we have to force-cast
+    })
+  ).map(({ student_studentnumber }) => student_studentnumber)
 
-  const studentCreditsAndEnrollments: Record<
-    string,
-    Array<{
-      studentnumber: string
-      course_code: string
-      date: Date
-      type: 'credit' | 'enrollment'
-      credittypecode?: CreditTypeCode
-    }>
-  > = {}
+  const enrollmentStudents = (
+    await EnrollmentModel.findAll({
+      raw: true,
+      attributes: ['studentnumber'],
+      where: {
+        course_id: { [Op.in]: courseIds },
+        is_open: unifyStatus,
+        enrollment_date_time: {
+          [Op.between]: [dateMaxFromList(startDate ?? new Date(0), enrollmentTimeDateThresholdAcademicYear), endDate],
+        },
+        state: EnrollmentState.ENROLLED,
+      },
+    })
+  ).map(({ studentnumber }) => studentnumber)
 
-  for (const credit of credits) {
-    studentCreditsAndEnrollments[credit.studentnumber] ??= []
-    studentCreditsAndEnrollments[credit.studentnumber].push({ ...credit, type: 'credit' })
-  }
-  for (const enrollment of enrollments) {
-    studentCreditsAndEnrollments[enrollment.studentnumber] ??= []
-    studentCreditsAndEnrollments[enrollment.studentnumber].push({ ...enrollment, type: 'enrollment' })
-  }
-
-  const passedCreditTypes = [CreditTypeCode.PASSED, CreditTypeCode.APPROVED]
-
-  const filteredStudentNumbers = new Set<string>()
-  Object.entries(studentCreditsAndEnrollments).map(([studentNumber, creditsOrEnrollments]) => {
-    const latestCredit = creditsOrEnrollments.find(
-      entry => entry.type === 'credit' && codes.includes(entry.course_code)
-    )
-    const latestEnrollment = creditsOrEnrollments.find(
-      entry => entry.type === 'enrollment' && codes.includes(entry.course_code)
-    )
-
-    if (latestCredit && passedCreditTypes.includes(latestCredit.credittypecode!)) {
-      filteredStudentNumbers.add(studentNumber)
-    } else if (latestCredit?.credittypecode === CreditTypeCode.FAILED) {
-      filteredStudentNumbers.add(studentNumber)
-    } else if (latestEnrollment) {
-      filteredStudentNumbers.add(studentNumber)
-    }
-  })
-
-  return [...filteredStudentNumbers]
+  return new Set([...creditStudents, ...enrollmentStudents])
 }
 
 const formatSharedStudentData = ({
@@ -270,12 +213,13 @@ const formatSharedStudentData = ({
   updatedAt,
   createdAt,
   sis_person_id,
-}: Student): Omit<StudentPageStudent, 'tags'> => {
+}: Student): Omit<StudentPageStudent, 'tags' | 'idToGroupIdMap'> => {
   const toCourse = ({ grade, credits, credittypecode, is_open, attainment_date, course, isStudyModule }: Credit) => {
     return {
       course: {
         code: course.code,
         name: course.name,
+        groupId: course.groupId,
       },
       date: attainment_date,
       passed: CreditModel.passed({ credittypecode }) || CreditModel.improved({ credittypecode }),
@@ -307,13 +251,25 @@ const formatSharedStudentData = ({
   }
 }
 
+const getIdToGroupIdMap = async (studyplans: Pick<Studyplan, 'included_courses' | 'includedModules'>[]) => {
+  const ids = [...new Set(studyplans.flatMap(plan => [...plan.included_courses, ...plan.includedModules]))]
+  const courses = await CourseModel.findAll({
+    attributes: ['id', 'groupId'],
+    where: { id: { [Op.in]: ids } },
+    raw: true,
+  })
+  return Object.fromEntries(courses.map(({ id, groupId }) => [id, groupId]))
+}
+
 export const withStudentNumber = async (studentNumber: string, userId: string): Promise<StudentPageStudent | null> => {
   const student = await byStudentNumber(studentNumber)
   if (student == null) {
     return null
   }
+  const idToGroupIdMap = await getIdToGroupIdMap(student.studyplans)
   return {
     ...formatSharedStudentData(student),
+    idToGroupIdMap,
     tags: student.tags
       .filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
       .map(tag => ({

services/backend/src/services/studyProgramme/creditGetters.ts

@@ -4,22 +4,27 @@ import { CreditTypeCode } from '@oodikone/shared/types'
 import { CourseModel, CourseProviderModel, CreditModel, OrganizationModel } from '../../models'
 import { formatCredit } from './format'
 
-export const getCreditsForProvider = async (provider: string, codes: string[], since: Date) => {
+export const getCreditsForProvider = async (provider: string, courseIds: string[], since: Date) => {
   const credits = await CreditModel.findAll({
+    include: {
+      model: CourseModel,
+      attributes: ['groupId'],
+    },
     where: {
-      course_code: { [Op.in]: codes },
+      course_id: { [Op.in]: courseIds },
       attainment_date: { [Op.gte]: since },
       isStudyModule: { [Op.not]: true },
       credittypecode: CreditTypeCode.PASSED,
     },
     raw: true,
+    nest: true,
   })
 
-  const courseIds = credits.map(cr => cr.course_id)
+  const courseGroupIds = credits.map(cr => cr.course.groupId)
 
   const providers = await CourseProviderModel.findAll({
     where: {
-      coursecode: { [Op.in]: courseIds },
+      coursecode: { [Op.in]: courseGroupIds },
     },
     raw: true,
   })
@@ -38,14 +43,14 @@ export const getCreditsForProvider = async (provider: string, codes: string[], s
     return obj
   }, {})
 
-  const courseIdToShareMap = providers.reduce<Record<string, CourseProviderModel[]>>((obj, provider) => {
+  const courseGroupIdToShareMap = providers.reduce<Record<string, CourseProviderModel[]>>((obj, provider) => {
     obj[provider.coursecode] ??= []
     obj[provider.coursecode].push(provider)
     return obj
   }, {})
 
-  const courseIdToShare = (courseId: string, attainmentDate: Date) => {
-    const providers = courseIdToShareMap[courseId]
+  const courseGroupIdToShare = (courseGroupId: string, attainmentDate: Date) => {
+    const providers = courseGroupIdToShareMap[courseGroupId]
     const relevantProvider = providers?.find(p => organizationIdToCodeMap[p.organizationcode] === provider)
     if (!relevantProvider?.shares) return 0
     const relevantShare = relevantProvider.shares
@@ -71,7 +76,7 @@ export const getCreditsForProvider = async (provider: string, codes: string[], s
         return bTime - aTime
       })[0]
 
-    // Odd logic, but if there are multiple providers for same course code, if no fitting dates are
+    // Odd logic, but if there are multiple providers for same course group, if no fitting dates are
     // not found for our relevant provider, we can assume the share of that date is of some other provider.
     // But if only 1 provider exists, we can assume it has share of 1.
     if (!relevantShare) {
@@ -82,7 +87,7 @@ export const getCreditsForProvider = async (provider: string, codes: string[], s
   }
 
   return credits
-    .map(cr => ({ ...cr, credits: cr.credits * courseIdToShare(cr.course_id, cr.attainment_date) }))
+    .map(cr => ({ ...cr, credits: cr.credits * courseGroupIdToShare(cr.course.groupId, cr.attainment_date) }))
     .map(formatCredit)
 }
 

services/backend/src/services/studyProgramme/index.ts

@@ -1,7 +1,7 @@
 import { memoize } from 'lodash-es'
 import { Op, QueryTypes } from 'sequelize'
 
-import { Name, EnrollmentState } from '@oodikone/shared/types'
+import { EnrollmentState } from '@oodikone/shared/types'
 import { enrollmentTimeDateThresholdAcademicYear } from '@oodikone/shared/util'
 import { dateIsBetween } from '@oodikone/shared/util/datetime'
 import { dbConnections } from '../../database/connection'
@@ -9,9 +9,9 @@ import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, Programme
 import logger from '../../util/logger'
 
 export const getAllProgrammeCourses = async (providerCode: string) => {
-  const courses: Pick<CourseModel, 'id' | 'code' | 'name' | 'substitution_groups'>[] = await CourseModel.findAll({
+  const courses: Pick<CourseModel, 'id' | 'groupId' | 'code' | 'name'>[] = await CourseModel.findAll({
     raw: true,
-    attributes: ['id', 'code', 'name', 'substitution_groups'],
+    attributes: ['id', 'groupId', 'code', 'name'],
     include: [
       {
         model: OrganizationModel,
@@ -29,193 +29,210 @@ export const getAllProgrammeCourses = async (providerCode: string) => {
   return courses
 }
 
+export const getCoursesByGroupIds = async (groupIds: string[]) => {
+  if (groupIds.length === 0) return []
+  return (
+    await CourseModel.findAll({
+      raw: true,
+      attributes: ['id'],
+      where: { groupId: { [Op.in]: groupIds } },
+    })
+  ).map(course => course.id)
+}
+
+export const getPrimaryCoursesByGroupIds = async (
+  groupIds: string[]
+): Promise<Pick<CourseModel, 'code' | 'name' | 'groupId'>[]> => {
+  if (groupIds.length === 0) return []
+  return CourseModel.findAll({
+    raw: true,
+    attributes: ['code', 'name', 'groupId'],
+    where: { groupId: { [Op.in]: groupIds }, isPrimary: true },
+  })
+}
+
+type NotCompletedCourseRow = {
+  groupId: string
+  isStudyModule: boolean
+  allNotPassed: number
+}
+
 export const getNotCompletedForProgrammeCourses = async (
-  from: Date,
-  to: Date,
-  programmeCourses: string[],
+  programmeCourseIds: string[],
+  yearRanges: Array<{ year: number; from: Date; to: Date }>,
   start: Date,
   end: Date
-) => {
+): Promise<Map<number, NotCompletedCourseRow[]>> => {
+  const emptyResult = new Map<number, NotCompletedCourseRow[]>(yearRanges.map(({ year }) => [year, []]))
+
   try {
-    const enrollments: Array<
-      Pick<EnrollmentModel, 'studentnumber' | 'course_code' | 'enrollment_date_time' | 'course'>
-    > = await EnrollmentModel.findAll({
-      raw: true,
-      nest: true,
-      attributes: ['studentnumber', 'course_code', 'enrollment_date_time'],
-      include: {
-        model: CourseModel,
-        attributes: ['is_study_module'],
-      },
-      where: {
-        course_code: {
-          [Op.in]: programmeCourses,
+    const [enrollments, credits]: [
+      Pick<EnrollmentModel, 'studentnumber' | 'enrollment_date_time' | 'course'>[],
+      Pick<CreditModel, 'student_studentnumber' | 'credittypecode' | 'isStudyModule' | 'attainment_date' | 'course'>[],
+    ] = await Promise.all([
+      EnrollmentModel.findAll({
+        raw: true,
+        nest: true,
+        attributes: ['studentnumber', 'enrollment_date_time'],
+        include: {
+          model: CourseModel,
+          attributes: ['isStudyModule', 'groupId'],
         },
-        enrollment_date_time: {
-          [Op.and]: {
+        where: {
+          course_id: {
+            [Op.in]: programmeCourseIds,
+          },
+          enrollment_date_time: {
+            [Op.and]: {
+              [Op.between]: [start, end],
+              [Op.gte]: enrollmentTimeDateThresholdAcademicYear, // This has to be Academic year to match with Course statistics which doesn't display enrollments for ...-2021
+            },
+          },
+          state: EnrollmentState.ENROLLED,
+        },
+        order: [['enrollment_date_time', 'DESC']],
+      }),
+      CreditModel.findAll({
+        raw: true,
+        nest: true,
+        attributes: ['student_studentnumber', 'credittypecode', 'isStudyModule', 'attainment_date'],
+        include: {
+          model: CourseModel,
+          attributes: ['groupId'],
+        },
+        where: {
+          course_id: {
+            [Op.in]: programmeCourseIds,
+          },
+          attainment_date: {
             [Op.between]: [start, end],
-            [Op.gte]: enrollmentTimeDateThresholdAcademicYear, // This has to be Academic year to match with Course statistics which doesn't display enrollments for ...-2021
           },
         },
-        state: EnrollmentState.ENROLLED,
-      },
-      order: [['enrollment_date_time', 'DESC']],
-    })
+        order: [
+          ['attainment_date', 'DESC'],
+          ['credittypecode', 'ASC'],
+        ], // Passed credits for a student have to be handled first (only PASSED, and FAILED matter)
+      }),
+    ])
 
-    // TODO: This is being called for each year, could be optimized to call only once
-    const credits: Array<
-      Pick<
-        CreditModel,
-        'course_code' | 'student_studentnumber' | 'credittypecode' | 'isStudyModule' | 'attainment_date'
-      >
-    > = await CreditModel.findAll({
-      raw: true,
-      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'isStudyModule', 'attainment_date'],
-      where: {
-        course_code: {
-          [Op.in]: programmeCourses,
-        },
-        attainment_date: {
-          [Op.between]: [start, end],
-        },
-      },
-      order: [
-        ['attainment_date', 'DESC'],
-        ['credittypecode', 'ASC'],
-      ], // Passed credits for a student have to be handled first (only PASSED, and FAILED matter)
-    })
+    const creditKey = (studentNumber: string, courseGroupId: string) => `${studentNumber}::${courseGroupId}`
 
-    const enrolledStudentsByCourseCode = new Map<string, Set<string>>()
+    const everPassedByStudentCourse = new Set<string>()
+    const latestFailedDateByStudentCourse = new Map<string, Date>()
 
-    // Pick only this (between to, from) year's credits
-    const filteredCredits = credits.filter(credit => dateIsBetween(credit.attainment_date, from, to))
+    for (const credit of credits) {
+      const groupId = credit.course.groupId
 
-    const filteredEnrollments = enrollments
-      // Only include the first enrollment for a course for a student.
-      // This assumes that enrollments are sorted in DESC order by enrollment_date_time
-      .filter(({ course_code: courseCode, studentnumber: studentNumber }) => {
-        const courseEnrollments = enrolledStudentsByCourseCode.get(courseCode)
-        const hasEnrolled = courseEnrollments?.has(studentNumber) ?? false
-
-        if (!courseEnrollments) enrolledStudentsByCourseCode.set(courseCode, new Set<string>())
-        if (!hasEnrolled) enrolledStudentsByCourseCode.get(courseCode)!.add(studentNumber)
-
-        return !hasEnrolled
-      })
-      .filter(enrollment => dateIsBetween(enrollment.enrollment_date_time, from, to))
-
-    const courseCodeToName = (
-      await CourseModel.findAll({
-        raw: true,
-        attributes: ['code', 'name'],
-        where: {
-          code: programmeCourses,
-        },
-      })
-    ).reduce<Map<string, Name>>((acc, val) => {
-      acc.set(val.code, val.name)
-      return acc
-    }, new Map())
-
-    const creditCourses = filteredCredits.map(credit => ({
-      code: credit.course_code,
-      studentNumber: credit.student_studentnumber,
-      creditTypeCode: credit.credittypecode,
-      courseName: courseCodeToName.get(credit.course_code)!,
-      isStudyModule: credit.isStudyModule,
-      attainmentDate: credit.attainment_date,
-    }))
-
-    const passedByCourseCodes = new Map<string, Set<string>>()
-    const failedByCourseCodes = new Map<string, Set<string>>()
-    const notCompletedByCourseCodes = new Map<string, Set<string>>()
-    const courses = new Map<string, { name: Name; isStudyModule: boolean }>()
-
-    const studentHasPassedCourse = (studentNumber: string, courseCode: string): boolean =>
-      (passedByCourseCodes.get(courseCode)?.has(studentNumber) ?? false) ||
-      !!credits.find(
-        credit =>
-          credit.student_studentnumber === studentNumber &&
-          credit.course_code === courseCode &&
-          CreditModel.passed(credit)
-      )
-
-    /** Student has already been counted as failed or there is a newer failed credit */
-    const studentHasFailedCourse = (
-      studentNumber: string,
-      courseCode: string,
-      attainmentDate: Date = new Date(0)
-    ): boolean =>
-      (failedByCourseCodes.get(courseCode)?.has(studentNumber) ?? false) ||
-      !!credits.find(
-        credit =>
-          credit.student_studentnumber === studentNumber &&
-          credit.course_code === courseCode &&
-          CreditModel.failed(credit) &&
-          credit.attainment_date > attainmentDate
-      )
-
-    const studentHasCredit = (studentNumber: string, courseCode: string): boolean =>
-      studentHasPassedCourse(studentNumber, courseCode) || studentHasFailedCourse(studentNumber, courseCode)
-
-    for (const {
-      code: courseCode,
-      courseName,
-      creditTypeCode,
-      isStudyModule,
-      studentNumber,
-      attainmentDate,
-    } of creditCourses) {
-      if (!courses.has(courseCode)) {
-        courses.set(courseCode, {
-          isStudyModule,
-          name: courseName,
-        })
-        passedByCourseCodes.set(courseCode, new Set())
-        failedByCourseCodes.set(courseCode, new Set())
-        notCompletedByCourseCodes.set(courseCode, new Set())
+      if (CreditModel.passed(credit)) {
+        everPassedByStudentCourse.add(creditKey(credit.student_studentnumber, groupId))
       }
 
-      if (CreditModel.passed({ credittypecode: creditTypeCode })) {
-        passedByCourseCodes.get(courseCode)!.add(studentNumber)
-      }
-      if (
-        CreditModel.failed({ credittypecode: creditTypeCode }) &&
-        !studentHasPassedCourse(studentNumber, courseCode) &&
-        !studentHasFailedCourse(studentNumber, courseCode, attainmentDate)
-      ) {
-        failedByCourseCodes.get(courseCode)!.add(studentNumber)
+      if (CreditModel.failed(credit)) {
+        const mapKey = creditKey(credit.student_studentnumber, groupId)
+        const latestFailedDate = latestFailedDateByStudentCourse.get(mapKey)
+        if (!latestFailedDate || credit.attainment_date > latestFailedDate) {
+          latestFailedDateByStudentCourse.set(mapKey, credit.attainment_date)
+        }
       }
     }
 
-    // Add course details to courseMap if there are only enrollments
-    for (const { studentnumber: studentNumber, course_code: courseCode, course } of filteredEnrollments) {
-      if (!courses.has(courseCode)) {
-        courses.set(courseCode, {
-          isStudyModule: course.is_study_module,
-          name: courseCodeToName.get(courseCode)!,
-        })
-        passedByCourseCodes.set(courseCode, new Set())
-        failedByCourseCodes.set(courseCode, new Set())
-        notCompletedByCourseCodes.set(courseCode, new Set())
+    const enrolledStudentsByCourseGroupId = new Map<string, Set<string>>()
+
+    // Only include the latest enrollment for a course per student per given time range.
+    // This assumes that enrollments are sorted in DESC order by enrollment_date_time
+    const dedupedEnrollments = enrollments.filter(({ course, studentnumber: studentNumber }) => {
+      const groupId = course.groupId
+      const courseEnrollments = enrolledStudentsByCourseGroupId.get(groupId)
+      const hasEnrolled = !!courseEnrollments?.has(studentNumber)
+
+      if (!courseEnrollments) enrolledStudentsByCourseGroupId.set(groupId, new Set<string>())
+      if (!hasEnrolled) enrolledStudentsByCourseGroupId.get(groupId)!.add(studentNumber)
+
+      return !hasEnrolled
+    })
+
+    const result = new Map<number, NotCompletedCourseRow[]>()
+
+    for (const { year, from, to } of yearRanges) {
+      const filteredCredits = credits.filter(credit => dateIsBetween(credit.attainment_date, from, to))
+      const filteredEnrollments = dedupedEnrollments.filter(enrollment =>
+        dateIsBetween(enrollment.enrollment_date_time, from, to)
+      )
+
+      // Keys are groupIds
+      const failedByCourse = new Map<string, Set<string>>()
+      const notCompletedByCourse = new Map<string, Set<string>>()
+      const courses = new Map<string, { isStudyModule: boolean }>()
+
+      const studentHasPassedCourse = (studentNumber: string, courseGroupId: string): boolean =>
+        everPassedByStudentCourse.has(creditKey(studentNumber, courseGroupId))
+
+      // Whether there is a newer failed credit
+      const studentHasFailedCourse = (
+        studentNumber: string,
+        courseGroupId: string,
+        attainmentDate: Date = new Date(0)
+      ): boolean => {
+        const latestFailedDate = latestFailedDateByStudentCourse.get(creditKey(studentNumber, courseGroupId))
+        return !!latestFailedDate && latestFailedDate > attainmentDate
       }
 
-      if (!studentHasCredit(studentNumber, courseCode)) {
-        notCompletedByCourseCodes.get(courseCode)!.add(studentNumber)
-      }
-    }
+      const studentHasCredit = (studentNumber: string, courseGroupId: string): boolean =>
+        studentHasPassedCourse(studentNumber, courseGroupId) || studentHasFailedCourse(studentNumber, courseGroupId)
 
-    return [...courses.entries()].map(([code, { isStudyModule, name }]) => {
-      return {
-        code,
-        name,
+      const ensureCourseExists = (groupId: string, isStudyModule: boolean) => {
+        if (!courses.has(groupId)) {
+          courses.set(groupId, {
+            isStudyModule,
+          })
+          failedByCourse.set(groupId, new Set())
+          notCompletedByCourse.set(groupId, new Set())
+        }
+      }
+
+      for (const {
+        credittypecode: creditTypeCode,
         isStudyModule,
-        allNotPassed: (notCompletedByCourseCodes.get(code)?.size ?? 0) + (failedByCourseCodes.get(code)?.size ?? 0),
+        student_studentnumber: studentNumber,
+        attainment_date: attainmentDate,
+        course,
+      } of filteredCredits) {
+        const groupId = course.groupId
+        ensureCourseExists(groupId, isStudyModule)
+
+        if (
+          CreditModel.failed({ credittypecode: creditTypeCode }) &&
+          !studentHasPassedCourse(studentNumber, groupId) &&
+          !studentHasFailedCourse(studentNumber, groupId, attainmentDate)
+        ) {
+          failedByCourse.get(groupId)!.add(studentNumber)
+        }
       }
-    })
+
+      // Add course details to courseMap if there are only enrollments
+      for (const { studentnumber: studentNumber, course } of filteredEnrollments) {
+        const groupId = course.groupId
+        ensureCourseExists(groupId, course.isStudyModule)
+
+        if (!studentHasCredit(studentNumber, groupId)) {
+          notCompletedByCourse.get(groupId)!.add(studentNumber)
+        }
+      }
+
+      result.set(
+        year,
+        [...courses.entries()].map(([groupId, { isStudyModule }]) => ({
+          groupId,
+          isStudyModule,
+          allNotPassed: (notCompletedByCourse.get(groupId)?.size ?? 0) + (failedByCourse.get(groupId)?.size ?? 0),
+        }))
+      )
+    }
+
+    return result
   } catch (error) {
     logger.error(`getNotCompletedForProgrammeCourses failed ${error}`)
-    return []
+    return emptyResult
   }
 }
 

services/backend/src/services/studyProgramme/studentGetters.ts

@@ -34,6 +34,7 @@ const openUniversityExtentCodes = [ExtentCode.OPEN_UNIVERSITY_STUDIES]
 export type ProgrammeCourseCreditRow = {
   courseCode: string
   courseName: Name
+  groupId: string
   studentNumber: string
   attainmentDate: string
   credits: number
@@ -46,6 +47,7 @@ export type ProgrammeCourseCreditRow = {
 export type TransferCourseCreditRow = {
   courseCode: string
   courseName: Name
+  groupId: string
   studentNumber: string
   attainmentDate: string
   credits: number
@@ -57,7 +59,7 @@ const { sequelize } = dbConnections
 /**
 NOTE:
  1. There are some attainments that are linked to a valid study right, but the attainmentdate
-    is outside the study right's validity period. (maybe human error when the credits were registered)
+    is outside the study right's validity period.
  2. There are some attainments with no attached study right at all.
  3. Possibly some other edge cases.
 
@@ -67,39 +69,34 @@ NOTE:
    degree student -> has any degree (bsc / msc) study right in the university
    openuni -> only if no other study rights
 **/
-export const getProgrammeCourseAggregates = async (params: {
-  courseCodes: string[]
-  from: Date
+export const getProgrammeCourseAggregates = async (
+  courseIds: string[],
+  from: Date,
   to: Date
-}): Promise<ProgrammeCourseCreditRow[]> => {
-  const { courseCodes, from, to } = params
-
-  if (courseCodes.length === 0) {
-    return []
-  }
+): Promise<ProgrammeCourseCreditRow[]> => {
+  if (!courseIds.length) return []
 
   try {
     return await sequelize.query<ProgrammeCourseCreditRow>(
       `
         WITH filtered_credits AS (
           SELECT
-            cr.course_code AS "courseCode",
-            co.name AS "courseName",
+            co.group_id AS "groupId",
             cr.student_studentnumber AS "studentNumber",
             cr.attainment_date AS "attainmentDate",
             cr.credits AS credits,
             cr."isStudyModule" AS "isStudyModule"
           FROM
             credit cr
-            INNER JOIN course co ON co.code = cr.course_code
+            INNER JOIN course co ON co.id = cr.course_id
           WHERE
-            cr.course_code IN (:courseCodes)
+            cr.course_id IN (:courseIds)
+            AND co.group_id is NOT NULL
             AND cr.attainment_date BETWEEN :from AND :to
             AND cr.credittypecode IN (:creditTypes)
         )
         SELECT
-          fc."courseCode",
-          fc."courseName",
+          fc."groupId",
           fc."studentNumber",
           fc."attainmentDate",
           fc.credits,
@@ -173,7 +170,7 @@ export const getProgrammeCourseAggregates = async (params: {
       {
         type: QueryTypes.SELECT,
         replacements: {
-          courseCodes,
+          courseIds,
           from,
           to,
           creditTypes: [CreditTypeCode.PASSED, CreditTypeCode.APPROVED],
@@ -192,39 +189,35 @@ export const getProgrammeCourseAggregates = async (params: {
   }
 }
 
-export const getTransferCourseAggregates = async (params: {
-  courseCodes: string[]
-  from: Date
+export const getTransferCourseAggregates = async (
+  courseIds: string[],
+  from: Date,
   to: Date
-}): Promise<TransferCourseCreditRow[]> => {
-  const { courseCodes, from, to } = params
-
-  if (courseCodes.length === 0) {
-    return []
-  }
+): Promise<TransferCourseCreditRow[]> => {
+  if (!courseIds.length) return []
 
   try {
     return await sequelize.query<TransferCourseCreditRow>(
       `
         SELECT
-          cr.course_code AS "courseCode",
-          co.name AS "courseName",
+          co.group_id AS "groupId",
           cr.student_studentnumber AS "studentNumber",
           cr.attainment_date AS "attainmentDate",
           cr.credits AS credits,
           cr."isStudyModule" AS "isStudyModule"
         FROM
           credit cr
-          INNER JOIN course co ON co.code = cr.course_code
+          INNER JOIN course co ON co.id = cr.course_id
         WHERE
-          cr.course_code IN (:courseCodes)
+          cr.course_id IN (:courseIds)
+          AND co.group_id IS NOT NULL
           AND cr.attainment_date BETWEEN :from AND :to
           AND cr.credittypecode = :creditType
       `,
       {
         type: QueryTypes.SELECT,
         replacements: {
-          courseCodes,
+          courseIds,
           from,
           to,
           creditType: CreditTypeCode.APPROVED,

services/backend/src/services/studyProgramme/studyProgrammeCourses.ts

@@ -1,10 +1,17 @@
-import { Name, StudyProgrammeCourse, YearType } from '@oodikone/shared/types'
+import { StudyProgrammeCourse, YearType } from '@oodikone/shared/types'
 import { mapToProviders, range, yearCodeToYear } from '@oodikone/shared/util'
 import { createEmptyStats, YearStats } from '@oodikone/shared/util/studyProgramme'
 import { now } from '../../util/clock'
 
-import { getAllProgrammeCourses, getCurrentStudyYearStartDate, getNotCompletedForProgrammeCourses } from '.'
+import {
+  getAllProgrammeCourses,
+  getCoursesByGroupIds,
+  getCurrentStudyYearStartDate,
+  getNotCompletedForProgrammeCourses,
+  getPrimaryCoursesByGroupIds,
+} from '.'
 import { getStudentHetuStateMap, getProgrammeCourseAggregates, getTransferCourseAggregates } from './studentGetters'
+import logger from '../../util/logger'
 
 const START_YEAR = 2017
 const JULY = 6
@@ -18,22 +25,12 @@ const getFrom = (yearType: YearType, year: number) =>
 const getTo = (yearType: YearType, year: number) =>
   yearType === 'ACADEMIC_YEAR' ? new Date(year + 1, JULY, 31, 23, 59, 59) : new Date(year, 11, 31, 23, 59, 59)
 
-const getAllStudyProgrammeCourses = async (studyProgramme: string) => {
+const getAllStudyProgrammeCourseGroupIds = async (studyProgramme: string): Promise<string[]> => {
   const providerCode = mapToProviders([studyProgramme])[0]
   if (!providerCode) return []
 
   const normalCourses = await getAllProgrammeCourses(providerCode)
-  const courseCodes = new Set<string>()
-
-  for (const course of normalCourses) {
-    courseCodes.add(course.code)
-    // We want all AY course codes variants, even from partially completed susbtitution groups
-    for (const code of course?.substitution_groups?.flat() ?? []) {
-      if (code === 'AY' + course.code) courseCodes.add(code)
-    }
-  }
-
-  return [...courseCodes]
+  return [...new Set(normalCourses.map(course => course.groupId))]
 }
 
 const getYearKey = (date: Date, isAcademicYear: boolean) => {
@@ -58,8 +55,7 @@ type YearAccumulator = {
 }
 
 type CourseAccumulator = {
-  code: string
-  name: Name
+  groupId: string
   isStudyModule: boolean
   years: Map<number, YearAccumulator>
 }
@@ -80,23 +76,18 @@ const createYearAccumulator = (isStudyModule: boolean): YearAccumulator => ({
 
 const ensureCourse = (
   courseMap: Map<string, CourseAccumulator>,
-  code: string,
-  name: Name,
+  groupId: string,
   isStudyModule: boolean
 ): CourseAccumulator => {
-  if (!courseMap.has(code)) {
-    courseMap.set(code, {
-      code,
-      name,
+  if (!courseMap.has(groupId)) {
+    courseMap.set(groupId, {
+      groupId,
       isStudyModule,
       years: new Map(),
     })
   }
 
-  const course = courseMap.get(code)!
-  course.isStudyModule = course.isStudyModule || isStudyModule
-  course.name = course.name ?? name
-  return course
+  return courseMap.get(groupId)!
 }
 
 const ensureYear = (course: CourseAccumulator, year: number, isStudyModule: boolean) => {
@@ -154,30 +145,35 @@ export const getStudyProgrammeCoursesForStudyTrack = async (
     return []
   }
 
-  const mainProgrammeCourses = await getAllStudyProgrammeCourses(studyProgramme)
-  const secondProgrammeCourses = combinedProgramme ? await getAllStudyProgrammeCourses(combinedProgramme) : []
-  const programmeCourses = [...new Set([...mainProgrammeCourses, ...secondProgrammeCourses])]
+  const [mainProgrammeGroupIds, secondProgrammeGroupIds] = await Promise.all([
+    getAllStudyProgrammeCourseGroupIds(studyProgramme),
+    combinedProgramme ? getAllStudyProgrammeCourseGroupIds(combinedProgramme) : [],
+  ])
 
-  if (programmeCourses.length === 0) {
-    return []
-  }
+  const programmeGroupIds = [...new Set([...mainProgrammeGroupIds, ...secondProgrammeGroupIds])]
+
+  if (!programmeGroupIds.length) return []
+
+  const programmeCourseIds = [...new Set(await getCoursesByGroupIds(programmeGroupIds))]
 
   const firstYear = yearRange[0]
   const lastYear = yearRange[yearRange.length - 1]
   const from = getFrom(academicYear, firstYear)
   const to = getTo(academicYear, lastYear)
 
+  const notCompletedByYearPromise = getNotCompletedForProgrammeCourses(
+    programmeCourseIds,
+    yearRange.map(year => ({ year, from: getFrom(academicYear, year), to: getTo(academicYear, year) })),
+    from,
+    to
+  )
+  const primaryByGroupIdPromise = getPrimaryCoursesByGroupIds(programmeGroupIds).then(
+    rows => new Map(rows.map(({ groupId, code, name }) => [groupId, { code, name }]))
+  )
+
   const [programmeCredits, transferCredits] = await Promise.all([
-    getProgrammeCourseAggregates({
-      courseCodes: programmeCourses,
-      from,
-      to,
-    }),
-    getTransferCourseAggregates({
-      courseCodes: programmeCourses,
-      from,
-      to,
-    }),
+    getProgrammeCourseAggregates(programmeCourseIds, from, to),
+    getTransferCourseAggregates(programmeCourseIds, from, to),
   ])
 
   const openUniStudentHetuMap = await getStudentHetuStateMap(
@@ -194,7 +190,8 @@ export const getStudyProgrammeCoursesForStudyTrack = async (
     if (!yearRange.includes(year)) continue
     maxYear = Math.max(maxYear, year)
 
-    const course = ensureCourse(courseMap, row.courseCode, row.courseName, row.isStudyModule)
+    const course = ensureCourse(courseMap, row.groupId, row.isStudyModule)
+
     const yearAccumulator = ensureYear(course, year, row.isStudyModule)
     const { stats } = yearAccumulator
 
@@ -242,53 +239,56 @@ export const getStudyProgrammeCoursesForStudyTrack = async (
     if (!yearRange.includes(year)) continue
     maxYear = Math.max(maxYear, year)
 
-    const course = ensureCourse(courseMap, row.courseCode, row.courseName, row.isStudyModule)
+    const course = ensureCourse(courseMap, row.groupId, row.isStudyModule)
     const yearAccumulator = ensureYear(course, year, row.isStudyModule)
     yearAccumulator.transferStudents.add(row.studentNumber)
     yearAccumulator.stats.transferStudentsCredits += row.credits
   }
 
-  const notCompletedResults = await Promise.all(
-    yearRange.map(year => {
-      const yearFrom = getFrom(academicYear, year)
-      const yearTo = getTo(academicYear, year)
-      return getNotCompletedForProgrammeCourses(yearFrom, yearTo, programmeCourses, from, to)
-    })
-  )
+  const notCompletedByYear = await notCompletedByYearPromise
 
-  notCompletedResults.forEach((results, index) => {
-    const year = yearRange[index]
-    if (!results) return
+  for (const year of yearRange) {
+    const results = notCompletedByYear.get(year)
+    if (!results) continue
     if (results.length > 0) {
       maxYear = Math.max(maxYear, year)
     }
     for (const result of results) {
-      const course = ensureCourse(courseMap, result.code, result.name, result.isStudyModule ?? false)
+      const course = ensureCourse(courseMap, result.groupId, result.isStudyModule ?? false)
       const yearAccumulator = ensureYear(course, year, result.isStudyModule ?? false)
       yearAccumulator.stats.allNotPassed += result.allNotPassed
     }
-  })
+  }
 
   if (maxYear === 0) {
     maxYear = lastYear
   }
 
+  const primaryByGroupId = await primaryByGroupIdPromise
+
   const coursesRecord: Record<string, StudyProgrammeCourse> = {}
 
-  for (const [code, course] of courseMap.entries()) {
+  // Code and name for each CU group is fetched from the "primary" instance
+  for (const [groupId, course] of courseMap.entries()) {
     const years: Record<number, YearStats> = {}
     for (const [year, yearAccumulator] of course.years.entries()) {
       if (year > maxYear) continue
       years[year] = finalizeYearStats(yearAccumulator)
     }
 
-    if (Object.keys(years).length === 0) {
+    if (!Object.keys(years).length) continue
+
+    const primary = primaryByGroupId.get(groupId)
+
+    if (!primary?.code || !primary?.name) {
+      logger.warn(`Warning: Programme courses: No primary course for groupId: ${groupId ?? '(groupId is undefined)'}`)
       continue
     }
 
-    coursesRecord[code] = {
-      code,
-      name: course.name,
+    coursesRecord[groupId] = {
+      code: primary.code,
+      groupId,
+      name: primary.name,
       isStudyModule: course.isStudyModule,
       years,
     }

services/backend/src/services/studyProgramme/studyProgrammeCoursesBySemester.ts

@@ -5,7 +5,7 @@ import { CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
 import { mapToProviders } from '@oodikone/shared/util'
 import { CreditModel, EnrollmentModel } from '../../models'
 import { createArrayOfCourses } from '../languageCenterData'
-import { getAllProgrammeCourses } from '.'
+import { getAllProgrammeCourses, getPrimaryCoursesByGroupIds } from '.'
 
 type Attempt = {
   studentNumber: string
@@ -19,12 +19,21 @@ type Attempt = {
 export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgramme: string) => {
   const courses = await getAllProgrammeCourses(mapToProviders([studyProgramme])[0])
   const autumnSemester2017 = 135
-  const courseCodes = courses.map(course => course.code)
+  const courseIds = courses.map(course => course.id)
+  const idToGroupId = new Map(courses.map(course => [course.id, course.groupId]))
+
+  const primaryCourses = await getPrimaryCoursesByGroupIds([...new Set(courses.map(course => course.groupId))])
+  const groupIdToPrimaryCode = new Map(primaryCourses.map(course => [course.groupId, course.code]))
+
+  const courseCodeOf = (courseId: string) => {
+    const groupId = idToGroupId.get(courseId)
+    return groupId ? groupIdToPrimaryCode.get(groupId) : undefined
+  }
 
   const credits = await CreditModel.findAll({
-    attributes: ['course_code', 'student_studentnumber', 'semestercode', 'attainment_date'],
+    attributes: ['course_id', 'student_studentnumber', 'semestercode', 'attainment_date'],
     where: {
-      course_code: { [Op.in]: courseCodes },
+      course_id: { [Op.in]: courseIds },
       semestercode: { [Op.gte]: autumnSemester2017 },
       credittypecode: CreditTypeCode.PASSED,
     },
@@ -32,9 +41,9 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
   })
 
   const enrollments = await EnrollmentModel.findAll({
-    attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'state'],
+    attributes: ['studentnumber', 'semestercode', 'course_id', 'enrollment_date_time', 'state'],
     where: {
-      course_code: { [Op.in]: courseCodes },
+      course_id: { [Op.in]: courseIds },
       semestercode: { [Op.gte]: autumnSemester2017 }, // It is ok to use enrollments before OK sisu-migration, ColorizedTableData doesn't need it
       state: { [Op.in]: [EnrollmentState.ENROLLED, EnrollmentState.REJECTED] },
     },
@@ -45,12 +54,14 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
   const attemptsByStudents = {} as Record<string, Attempt[]>
 
   credits.forEach(credit => {
+    const courseCode = courseCodeOf(credit.course_id)
+    if (!courseCode) return
     const studentNumber = credit.student_studentnumber
     studentList.add(studentNumber)
     attemptsByStudents[studentNumber] ??= []
     attemptsByStudents[studentNumber].push({
       studentNumber,
-      courseCode: credit.course_code,
+      courseCode,
       completed: true,
       date: credit.attainment_date,
       semestercode: credit.semestercode,
@@ -58,22 +69,22 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
   })
 
   enrollments.forEach(enrollment => {
+    const courseCode = courseCodeOf(enrollment.course_id)
+    if (!courseCode) return
     const studentNumber = enrollment.studentnumber
     attemptsByStudents[studentNumber] ??= []
     studentList.add(studentNumber)
     if (
       attemptsByStudents[studentNumber].find(
         attempt =>
-          !attempt.completed &&
-          attempt.semestercode === enrollment.semestercode &&
-          attempt.courseCode === enrollment.course_code
+          !attempt.completed && attempt.semestercode === enrollment.semestercode && attempt.courseCode === courseCode
       )
     ) {
       return
     }
     attemptsByStudents[studentNumber].push({
       studentNumber,
-      courseCode: enrollment.course_code,
+      courseCode,
       completed: false,
       date: enrollment.enrollment_date_time,
       semestercode: enrollment.semestercode,
@@ -84,7 +95,7 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
   const attemptsArray = [] as Attempt[]
   studentList.forEach(studentNumber => attemptsArray.push(...attemptsByStudents[studentNumber]))
 
-  const unorderedTableData = createArrayOfCourses(attemptsArray, courses)
+  const unorderedTableData = createArrayOfCourses(attemptsArray, primaryCourses)
   const tableData = orderBy(unorderedTableData, 'code')
 
   return { tableData }

services/backend/src/services/studyProgramme/studyProgrammeCriteria.ts

@@ -1,3 +1,5 @@
+import { Op } from 'sequelize'
+
 import { ProgressCriteria } from '@oodikone/shared/types'
 import { CourseModel } from '../../models'
 import { ProgressCriteriaModel } from '../../models/kone'
@@ -11,31 +13,42 @@ const getCriteriaByStudyProgramme = async (code: string): Promise<CriteriaWithou
     where: { code },
   })
 
-/** Construct course_code => substitution_groups object */
-const getSubstitutions = async (codes: string[]) => {
-  const courses: Array<Pick<CourseModel, 'code' | 'substitution_groups'>> = await CourseModel.findAll({
-    attributes: ['code', 'substitution_groups'],
-    where: { code: codes },
+/** Construct course_group_id => substitutionGroups object */
+const getSubstitutions = async (courseGroupIds: string[]) => {
+  const courses: Array<Pick<CourseModel, 'groupId' | 'substitutionGroups' | 'isPrimary'>> = await CourseModel.findAll({
+    attributes: ['groupId', 'substitutionGroups', 'isPrimary'],
+    where: { groupId: { [Op.in]: courseGroupIds } },
     raw: true,
   })
 
-  // Sort substitution_groups by length, because (usually) the shortest substitution is the "correct" one
+  // A group can have several rows (one per course code), prefer the primary one
+  const substitutionsByGroupId = new Map<string, string[][]>()
+  for (const { groupId, substitutionGroups, isPrimary } of courses) {
+    if (isPrimary || !substitutionsByGroupId.has(groupId)) {
+      substitutionsByGroupId.set(groupId, substitutionGroups)
+    }
+  }
+
+  // Sort substitutionGroups by length, because (usually) the shortest substitution is the "correct" one
   return Object.fromEntries(
-    courses.map(({ code, substitution_groups }) => [code, substitution_groups.sort((a, b) => b.length - a.length)])
+    [...substitutionsByGroupId].map(([groupId, substitutionGroups]) => [
+      groupId,
+      [...substitutionGroups].sort((a, b) => b.length - a.length),
+    ])
   )
 }
 
 const formatCriteria = async (criteria: CriteriaWithoutCurriculumVersion | null) => {
-  const yearOne = criteria?.coursesYearOne ?? []
-  const yearTwo = criteria?.coursesYearTwo ?? []
-  const yearThree = criteria?.coursesYearThree ?? []
-  const yearFour = criteria?.coursesYearFour ?? []
-  const yearFive = criteria?.coursesYearFive ?? []
-  const yearSix = criteria?.coursesYearSix ?? []
-  const courseCodes = [...yearOne, ...yearTwo, ...yearThree, ...yearFour, ...yearFive, ...yearSix]
+  const yearOne = criteria?.courseGroupIdsYearOne ?? []
+  const yearTwo = criteria?.courseGroupIdsYearTwo ?? []
+  const yearThree = criteria?.courseGroupIdsYearThree ?? []
+  const yearFour = criteria?.courseGroupIdsYearFour ?? []
+  const yearFive = criteria?.courseGroupIdsYearFive ?? []
+  const yearSix = criteria?.courseGroupIdsYearSix ?? []
+  const courseGroupIds = [...yearOne, ...yearTwo, ...yearThree, ...yearFour, ...yearFive, ...yearSix]
 
   const formattedCriteria: ProgressCriteria = {
-    allCourseGroups: await getSubstitutions(courseCodes),
+    allCourseGroups: await getSubstitutions(courseGroupIds),
     courses: { yearOne, yearTwo, yearThree, yearFour, yearFive, yearSix },
     credits: {
       yearOne: criteria?.creditsYearOne ?? 0,
@@ -56,12 +69,12 @@ const createCriteria = async (
 ) => {
   const newProgrammeCriteria = {
     code: studyProgramme,
-    coursesYearOne: courses.year1,
-    coursesYearTwo: courses.year2,
-    coursesYearThree: courses.year3,
-    coursesYearFour: courses.year4,
-    coursesYearFive: courses.year5,
-    coursesYearSix: courses.year6,
+    courseGroupIdsYearOne: courses.year1,
+    courseGroupIdsYearTwo: courses.year2,
+    courseGroupIdsYearThree: courses.year3,
+    courseGroupIdsYearFour: courses.year4,
+    courseGroupIdsYearFive: courses.year5,
+    courseGroupIdsYearSix: courses.year6,
     creditsYearOne: credits.year1,
     creditsYearTwo: credits.year2,
     creditsYearThree: credits.year3,
@@ -109,35 +122,35 @@ export const saveYearlyCreditCriteria = async (studyProgramme: string, credits:
   }
 }
 
-export const saveYearlyCourseCriteria = async (studyProgramme: string, courses: string[], year: number) => {
+export const saveYearlyCourseCriteria = async (studyProgramme: string, courseGroupIds: string[], year: number) => {
   const studyProgrammeToUpdate = await getCriteriaByStudyProgramme(studyProgramme)
   if (!studyProgrammeToUpdate) {
     const courseObj: Record<string, string[]> = { year1: [], year2: [], year3: [], year4: [], year5: [], year6: [] }
     const creditObj = { year1: 0, year2: 0, year3: 0, year4: 0, year5: 0, year6: 0 }
     if (year === 1) {
-      courseObj.year1 = courses
+      courseObj.year1 = courseGroupIds
     } else if (year === 2) {
-      courseObj.year2 = courses
+      courseObj.year2 = courseGroupIds
     } else if (year === 3) {
-      courseObj.year3 = courses
+      courseObj.year3 = courseGroupIds
     } else if (year === 4) {
-      courseObj.year4 = courses
+      courseObj.year4 = courseGroupIds
     } else if (year === 5) {
-      courseObj.year5 = courses
+      courseObj.year5 = courseGroupIds
     } else {
-      courseObj.year6 = courses
+      courseObj.year6 = courseGroupIds
     }
     return await createCriteria(studyProgramme, courseObj, creditObj)
   }
 
   try {
     const years = {
-      1: 'coursesYearOne',
-      2: 'coursesYearTwo',
-      3: 'coursesYearThree',
-      4: 'coursesYearFour',
-      5: 'coursesYearFive',
-      6: 'coursesYearSix',
+      1: 'courseGroupIdsYearOne',
+      2: 'courseGroupIdsYearTwo',
+      3: 'courseGroupIdsYearThree',
+      4: 'courseGroupIdsYearFour',
+      5: 'courseGroupIdsYearFive',
+      6: 'courseGroupIdsYearSix',
     } as const
     if (!(year in years)) {
       throw new Error(`Invalid year: ${year}`)
@@ -145,7 +158,7 @@ export const saveYearlyCourseCriteria = async (studyProgramme: string, courses:
 
     const yearToUpdate = years[year as keyof typeof years]
 
-    const updatedCriteria = await studyProgrammeToUpdate.update({ [yearToUpdate]: courses })
+    const updatedCriteria = await studyProgrammeToUpdate.update({ [yearToUpdate]: courseGroupIds })
     return await formatCriteria(updatedCriteria)
   } catch (error) {
     logger.error(`Updating yearly credit criteria failed: ${error}`)

services/backend/src/services/studyProgramme/studyProgrammeHelpers.ts

@@ -312,7 +312,9 @@ export const getGoal = async (programme?: string) => {
   return ((minimumCredits / 60) * 12) / 6 // 60 Credits per year divided to semesters
 }
 
-export const isRelevantProgramme = (code: string) => /^(KH|MH)\d{2}_\d{3}$/.test(code) || /^T\d{6}$/.test(code)
+/** Returns true for all programme codes that are determined to be newer than 2017 */
+export const isRelevantProgramme = (code: string) =>
+  /^(KH|MH)\d{2}_\d{3}$/.test(code) || /^T\d{6}$/.test(code) || /^DP\d{3}$/.test(code)
 
 export const getStudyRightElementsWithPhase = (studyRight: Pick<SISStudyRight, 'studyRightElements'>, phase: Phase) => {
   return orderBy(

services/backend/src/services/studyProgramme/studyRightFinders.ts

@@ -43,7 +43,7 @@ export const getStudyRightsInProgramme = async (
         include: [
           {
             model: CreditModel,
-            attributes: ['attainment_date', 'credits', 'course_code'],
+            attributes: ['attainment_date', 'credits', 'course_code', 'course_id'],
             required: false,
             where: {
               isStudyModule: false,

services/backend/src/services/studyProgramme/studyTrackStats.ts

@@ -247,7 +247,7 @@ const getMainStatsByTrackAndYear = async (
 
     const latestAttainmentWithinYearAndInHops = dayjs(
       studyRight.student.credits
-        .filter(credit => studyRight.studyPlans.some(sp => sp.included_courses.includes(credit.course_code)))
+        .filter(credit => studyRight.studyPlans.some(sp => sp.included_courses.includes(credit.course_id)))
         .reduce((latest, { attainment_date }) => {
           return attainment_date > latest ? attainment_date : latest
         }, new Date('1900-01-01'))

services/backend/src/services/teachers/index.ts

@@ -34,7 +34,7 @@ const findTeacherCredits = async (teacherId: string) => {
       include: [
         {
           model: CourseModel,
-          attributes: ['name', 'code'],
+          attributes: ['name', 'code', 'groupId'],
           required: true,
         },
         {
@@ -105,9 +105,9 @@ const markCreditForYear = (years: Record<number, any>, credit: Credit) => {
 
 const markCreditForCourse = (courses: Record<string, any>, credit: Credit) => {
   const { passed, failed, credits, course, semester, transferred } = parseCreditInfo(credit)
-  const { code, name } = course
+  const { code, name, groupId } = course
   const { semestercode } = semester
-  const { stats, semesters = {}, ...rest } = courses[code] ?? { id: code, name }
+  const { stats, semesters = {}, ...rest } = courses[code] ?? { id: code, groupId, name }
   return {
     ...courses,
     [code]: {
@@ -151,7 +151,7 @@ const getActiveTeachers = async (providers: string[], startSemester: number, end
     JOIN credit_teachers ON teacher.id = credit_teachers.teacher_id
     JOIN credit ON credit.id = credit_teachers.credit_id
     JOIN course ON credit.course_id = course.id
-    JOIN course_providers ON course.id = course_providers.coursecode
+    JOIN course_providers ON course.group_id = course_providers.coursecode
     JOIN organization ON organization.id = course_providers.organizationcode
     WHERE credit.semestercode BETWEEN :startSemester AND :endSemester
       AND organization.code IN (:providers)

services/backend/src/util/index.ts

@@ -51,8 +51,6 @@ export const hasFullAccessToStudentData = (roles?: Role[]): boolean => {
 
 // Only open uni courses
 export const isOpenUniCourseCode = (code: string): boolean => /^AY.+?(?:en|fi|sv)?$/.test(code)
-// Open uni AND normal courses
-export const getOpenUniCourseCode = (code: string): string[] | null => /^AY?(.+?)(?:en|fi|sv)?$/.exec(code)
 
 /**
  * Returns the keys of the given object as an array of strings, typed as the keys of the object.

services/backend/tests/api/courses/courseYearlyStats/attemptStatistics/mat11001.test.ts

@@ -0,0 +1,22 @@
+import { Express } from 'express'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { initTests } from '../../../../utils'
+import { getCourseYearlyStats, CourseYearlyStats } from '../helpers'
+
+void describe('Course yearly attempt statistics - MAT11001 (hy-CU-117375151, no substitutions)', () => {
+  let app: Express
+  let body: CourseYearlyStats
+  beforeAll(async () => {
+    app = await initTests()
+    body = await getCourseYearlyStats(app, 'courses=hy-CU-117375151&substitutions=false')
+  })
+
+  it('should include every attempt of a student who has only failed the course', () => {
+    const stats = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
+    assert(stats && 'enrollments' in stats, 'Missing field enrollment in statsitics')
+
+    assert.strictEqual(stats.attempts.categories.failed.length, 1, "Didn't include student's failed attempt")
+    assert.deepStrictEqual(stats.attempts.categories.failed, ['542874'], "Didn't include student's failed attempt")
+  })
+})

services/backend/tests/api/courses/courseYearlyStats/attemptStatistics/mat11002.test.ts

@@ -0,0 +1,44 @@
+import { Express } from 'express'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { initTests } from '../../../../utils'
+import { getCourseYearlyStats, CourseYearlyStats } from '../helpers'
+
+void describe('Course yearly attempt statistics - MAT11002 (hy-CU-117375394, no substitutions)', () => {
+  let app: Express
+  let body: CourseYearlyStats
+  beforeAll(async () => {
+    app = await initTests()
+    body = await getCourseYearlyStats(app, 'courses=hy-CU-117375394&substitutions=false')
+  })
+
+  // FIXME: What is this even testing??
+  it.skip('should not include attempts for student with only AY credit (501716)', () => {
+    const stats = body.unifyStats?.statistics.find(year => year.name === '2017-2018')
+    assert(stats && 'enrollments' in stats, 'Missing field enrollment in statsitics')
+    assert.strictEqual(stats.attempts.categories.failed.length, 1)
+    assert.deepStrictEqual(stats.attempts.categories.failed, ['501716'])
+  })
+
+  it('should include a failed and a passed attempt for a student with both (501716)', () => {
+    const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
+    assert(year && 'enrollments' in year, 'Stats missing completely')
+    assert.include(year.attempts.categories.failed, '501716', 'Failed attempts missing the student in question')
+    assert.include(year.attempts.categories.passed, '501716', "Passed attempts didn't include the passed attempt")
+    assert.include(year.attempts.grades['1'] ?? [], '501716', 'Grades should include the attempt in the correct grade')
+  })
+
+  it('should include an attempt with only approved grade (543385)', () => {
+    const year = body.unifyStats?.statistics.find(year => year.name === '2022-2023')
+    assert(year && 'enrollments' in year, 'Stats missing completely')
+    assert.notInclude(year.attempts.categories.failed, '543385')
+    assert.include(year.attempts.categories.passed, '543385', "Passed attempts didn't include the approved attempt")
+    assert.include(year.attempts.grades['5'] ?? [], '543385', 'Grades should include the attempt in the correct grade')
+  })
+
+  it('should include an attempt with only improved grade (509770)', () => {
+    const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
+    assert(year && 'enrollments' in year, 'Missing stats for 2018-2019')
+    assert.include(year.attempts.categories.passed, '509770', 'Passed attempts should include in passed attempts')
+  })
+})

services/backend/tests/api/courses/courseYearlyStats/attemptStatistics/mat21003.test.ts

@@ -0,0 +1,87 @@
+import { Express } from 'express'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { yearToYearCode } from '@oodikone/shared/util'
+import { initTests } from '../../../../utils'
+import { getCourseYearlyStats, CourseYearlyStats } from '../helpers'
+
+const MAT21003 = 'hy-CU-117375829'
+
+void describe('Course yearly attempt statistics - MAT21003 (hy-CU-117375394, no substitutions)', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
+
+  describe('duplicate failed attempts across a large timespan are counted separately', () => {
+    let body: CourseYearlyStats
+    beforeAll(async () => {
+      body = await getCourseYearlyStats(app, `courses=${MAT21003}&substitutions=false`)
+    })
+
+    it('2017-2018 should include a single failed attempt (539036)', () => {
+      const year = body.unifyStats?.statistics.find(year => year.name === '2017-2018')
+      assert(year && 'enrollments' in year, 'Missing stats for 2017-2018')
+      assert.deepStrictEqual(
+        year.attempts.categories.failed,
+        ['539036'],
+        'Failed attempts should not include any students'
+      )
+    })
+
+    it('2018-2019 should include a failed attempt (539036)', () => {
+      const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
+      assert(year && 'enrollments' in year, 'Missing stats for 2018-2019')
+      assert.include(year.attempts.categories.failed, '539036', 'Failed attempts should include the failed attempt')
+    })
+  })
+
+  describe('year selection for smaller timeframes', () => {
+    it.each([
+      ['2017-2018', 2017, 2017],
+      ['2020-2023', 2020, 2022],
+      ['2016-2024', 2016, 2023],
+    ])('should include correct years for %s', async (name, from, to) => {
+      const body = await getCourseYearlyStats(
+        app,
+        `courses=hy-CU-117375394&substitutions=false&fromYearCode=${yearToYearCode(from)}&toYearCode=${yearToYearCode(to)}`
+      )
+
+      const years: string[] = []
+      for (let i = from; i <= to; i++) {
+        years.push(`${i}-${i + 1}`)
+      }
+      assert.deepStrictEqual(
+        body.unifyStats.statistics.map(({ name }) => name),
+        years,
+        `${name} included incorrect years`
+      )
+    })
+  })
+
+  describe('duplicate attempts are counted again once an attempt falls out of the timespan', () => {
+    it('2017-2018 should include a failed attempt (539036)', async () => {
+      const body = await getCourseYearlyStats(
+        app,
+        `courses=${MAT21003}&substitutions=false&fromYearCode=${yearToYearCode(2017)}&toYearCode=${yearToYearCode(2017)}`
+      )
+      const year = body.unifyStats.statistics.find(year => year.name === '2017-2018')
+      assert(year && 'enrollments' in year, 'Missing stats for 2017-2018')
+      assert.deepStrictEqual(
+        year.attempts.categories.failed,
+        ['539036'],
+        'Failed attempts should have included a student'
+      )
+    })
+
+    it('2018-2019 should include a failed attempt (539036)', async () => {
+      const body = await getCourseYearlyStats(
+        app,
+        `courses=${MAT21003}&substitutions=false&fromYearCode=${yearToYearCode(2018)}&toYearCode=${yearToYearCode(2018)}`
+      )
+      const year = body.unifyStats.statistics.find(year => year.name === '2018-2019')
+      assert(year && 'enrollments' in year, 'Missing stats for 2018-2019')
+      assert.include(year.attempts.categories.failed, '539036', 'Failed attempts included the incorrect students')
+    })
+  })
+})

services/backend/tests/api/courses/courseYearlyStats/attemptStatistics/tkt10002.test.ts

@@ -0,0 +1,149 @@
+import { Express } from 'express'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { yearToYearCode } from '@oodikone/shared/util'
+import { initTests } from '../../../../utils'
+import { getCourseYearlyStats, CourseYearlyStats } from '../helpers'
+
+void describe('Course yearly attempt statistics - TKT10002 (hy-CU-118023867, no substitutions)', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
+
+  describe('full range (2016-2023)', () => {
+    let body: CourseYearlyStats
+    beforeAll(async () => {
+      body = await getCourseYearlyStats(
+        app,
+        `courses=hy-CU-118023867&substitutions=false&fromYearCode=${yearToYearCode(2016)}&toYearCode=${yearToYearCode(2023)}`
+      )
+    })
+
+    it('has one entry per statistic year', () => {
+      assert.strictEqual(body.unifyStats.statistics.length, 8, 'unifyStats years')
+      assert.strictEqual(body.regularStats.statistics.length, 8, 'regularStats years')
+      assert.strictEqual(body.openStats.statistics.length, 4, 'openStats years')
+    })
+
+    it.each([
+      ['2016-2017', { total: 1, passed: 1, failed: 0 }, { 5: 0, 4: 1, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 }],
+      [
+        '2017-2018',
+        { total: 28, passed: 19, failed: 9 },
+        { 5: 11, 4: 5, 3: 2, 2: 1, 1: 0, 0: 0, 'Hyl.': 9, 'Hyv.': 0 },
+      ],
+      [
+        '2018-2019',
+        { total: 32, passed: 26, failed: 6 },
+        { 5: 18, 4: 3, 3: 2, 2: 1, 1: 2, 0: 5, 'Hyl.': 1, 'Hyv.': 0 },
+      ],
+      [
+        '2019-2020',
+        { total: 29, passed: 28, failed: 1 },
+        { 5: 17, 4: 5, 3: 1, 2: 4, 1: 1, 0: 0, 'Hyl.': 1, 'Hyv.': 0 },
+      ],
+      [
+        '2020-2021',
+        { total: 23, passed: 23, failed: 0 },
+        { 5: 18, 4: 2, 3: 0, 2: 2, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 1 },
+      ],
+      [
+        '2021-2022',
+        { total: 15, passed: 15, failed: 0 },
+        { 5: 13, 4: 2, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
+      ],
+      [
+        '2022-2023',
+        { total: 27, passed: 27, failed: 0 },
+        { 5: 21, 4: 6, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
+      ],
+      ['2023-2024', { total: 1, passed: 1, failed: 0 }, { 5: 1, 4: 0, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 }],
+    ])('for attempt statistic content during (%s)', (year, categories, grades) => {
+      const stats = body.unifyStats.statistics.find(yearStats => yearStats.name === year)!
+      assert('enrollments' in stats, 'Missing field enrollment in statsitics')
+
+      assert.strictEqual(
+        stats.attempts.categories.failed.length,
+        categories.failed,
+        `Incorrect amount of failed attempts for ${year}`
+      )
+      assert.strictEqual(
+        stats.attempts.categories.passed.length,
+        categories.passed,
+        `Incorrect amount of passed attempts for ${year}`
+      )
+      assert.strictEqual(
+        stats.attempts.categories.failed.length + stats.attempts.categories.passed.length,
+        categories.total,
+        `Incorrect amount of total attempts for ${year}`
+      )
+
+      Object.entries(grades).forEach(([grade, count]) =>
+        assert.strictEqual(
+          stats.attempts.grades[grade]?.length ?? 0,
+          count,
+          `Incorrect amount of attempts with grade ${grade}`
+        )
+      )
+    })
+
+    it('should include every attempt of a student who has failed a course in 2017 and later completed it', () => {
+      const year2017 = body.unifyStats.statistics.find(year => year.name === '2017-2018')
+      const year2018 = body.unifyStats.statistics.find(year => year.name === '2018-2019')
+      const year2020 = body.unifyStats.statistics.find(year => year.name === '2020-2021')
+      const year2022 = body.unifyStats.statistics.find(year => year.name === '2022-2023')
+      assert(year2017 && 'enrollments' in year2017, 'Missing stats for 2017-2018')
+      assert(year2018 && 'enrollments' in year2018, 'Missing stats for 2018-2019')
+      assert(year2020 && 'enrollments' in year2020, 'Missing stats for 2020-2021')
+      assert(year2022 && 'enrollments' in year2022, 'Missing stats for 2021-2022')
+
+      assert.include(year2017.attempts.categories.failed, '457686')
+      assert.include(year2018.attempts.categories.passed, '457686')
+
+      assert.include(year2017.attempts.categories.failed, '455478')
+      assert.include(year2020.attempts.categories.passed, '455478')
+      assert.strictEqual(year2017.attempts.categories.failed.filter(sNum => sNum === '455478').length, 2) // Failed twice
+
+      assert.include(year2017.attempts.categories.failed, '547994')
+      assert.include(year2022.attempts.categories.passed, '547994')
+    })
+  })
+
+  describe('single academic year ranges', () => {
+    it.each([
+      // Same as above
+      ['2016-2017', { total: 1, passed: 1, failed: 0 }],
+      ['2017-2018', { total: 28, passed: 19, failed: 9 }],
+      ['2018-2019', { total: 32, passed: 26, failed: 6 }],
+      ['2019-2020', { total: 29, passed: 28, failed: 1 }],
+      ['2020-2021', { total: 23, passed: 23, failed: 0 }],
+      ['2021-2022', { total: 15, passed: 15, failed: 0 }],
+      ['2022-2023', { total: 27, passed: 27, failed: 0 }],
+      ['2023-2024', { total: 1, passed: 1, failed: 0 }],
+    ])('should include correct attempt stats for one academic year (%s)', async (year, categories) => {
+      const body = await getCourseYearlyStats(
+        app,
+        `courses=hy-CU-118023867&substitutions=false&fromYearCode=${yearToYearCode(year.split('-').at(0))}&toYearCode=${yearToYearCode(parseInt(year.split('-').at(1)!)) - 1}`
+      )
+      const stats = body.unifyStats.statistics.find(yearStats => yearStats.name === year)!
+      assert('enrollments' in stats, 'Missing field enrollment in statsitics')
+
+      assert.strictEqual(
+        stats.attempts.categories.failed.length,
+        categories.failed,
+        `Incorrect amount of failed attempts for ${year}`
+      )
+      assert.strictEqual(
+        stats.attempts.categories.passed.length,
+        categories.passed,
+        `Incorrect amount of passed attempts for ${year}`
+      )
+      assert.strictEqual(
+        stats.attempts.categories.failed.length + stats.attempts.categories.passed.length,
+        categories.total,
+        `Incorrect amount of total attempts for ${year}`
+      )
+    })
+  })
+})

services/backend/tests/api/courses/courseYearlyStats/helpers.ts

@@ -0,0 +1,40 @@
+import { Express } from 'express'
+import request from 'supertest'
+import { assert } from 'vitest'
+
+import { Unarray } from '@oodikone/shared/types'
+import { Grades } from '@oodikone/shared/types/courseYearlyStats'
+import { CourseYearlyStatsResBody } from '@/routes/courses'
+import { ResponseWithBody } from '../../../utils'
+
+export type CourseYearlyStats = Unarray<CourseYearlyStatsResBody>
+
+/** GET /courseyearlystats as a basic user and return the course's stats. */
+export const getCourseYearlyStats = async (app: Express, query: string): Promise<CourseYearlyStats> => {
+  const res = (await request(app)
+    .get(`/courseyearlystats?${query}`)
+    .set('shib-session-id', 'test')
+    .set('uid', 'basic')
+    .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
+
+  assert.strictEqual(res.status, 200)
+  assert.strictEqual(res.body.length, 1, 'Query to return anything')
+  const body = res.body.at(0)!
+  assert(
+    'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
+    'All keys of courseyearlystats not defined'
+  )
+  return body
+}
+
+/** Calculates passed and failed from students.grades instead of attempts.categories
+ * because students includes only deduplicated credits and attempts include all credits
+ * student has done
+ */
+export const calculatePassedAndFailed = (studentGrades: Grades) => {
+  const studentCategories: { passed: string[]; failed: string[] } = { passed: [], failed: [] }
+  Object.entries(studentGrades).forEach(([grade, studentNumbers]) => {
+    studentCategories[grade === '0' ? 'failed' : 'passed'].push(...studentNumbers)
+  })
+  return studentCategories
+}

services/backend/tests/api/courses/courseYearlyStats/studentStatistics/general.test.ts

@@ -0,0 +1,27 @@
+import { Express } from 'express'
+import request from 'supertest'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { initTests } from '../../../../utils'
+
+void describe('Course yearly statistics - validation', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
+
+  it('should return nothing with missing parameters', async () => {
+    const res = await request(app)
+      .get('/courseyearlystats')
+      .set('shib-session-id', 'test')
+      .set('uid', 'basic')
+      .set('hygroupcn', 'grp-oodikone-basic-users')
+
+    assert.strictEqual(res.status, 422)
+    assert.strictEqual(res.body?.error, 'Missing required query parameters')
+  })
+
+  it.todo('should return correct amount of students for a course with substitutions')
+  it.todo('should return correct amount of students for multiple coursecodes without substitutions')
+  it.todo('should return correct amount of students for multiple coursecodes with substitutions')
+})

services/backend/tests/api/courses/courseYearlyStats/studentStatistics/mat11001.test.ts

@@ -0,0 +1,34 @@
+import { Express } from 'express'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { initTests } from '../../../../utils'
+import { calculatePassedAndFailed, getCourseYearlyStats, CourseYearlyStats } from '../helpers'
+
+void describe('Course yearly statistics - MAT11001 (hy-CU-117375151, no substitutions)', () => {
+  let app: Express
+  let body: CourseYearlyStats
+  beforeAll(async () => {
+    app = await initTests()
+    body = await getCourseYearlyStats(app, 'courses=hy-CU-117375151&substitutions=false')
+  })
+
+  it('should include a student who has only failed the course', () => {
+    const stats = body.unifyStats.statistics.find(year => year.name === '2018-2019')
+    assert(stats && 'enrollments' in stats, 'Missing field enrollment in statsitics')
+
+    const studentCategories = calculatePassedAndFailed(stats.students.grades)
+
+    assert(
+      studentCategories.failed.includes('542874'),
+      "Stats didn't include student with only failed course attainment (failed)"
+    )
+    assert(
+      !studentCategories.passed.includes('542874'),
+      'Stats did incorrectly include student with only failed course attainment (passed)'
+    )
+    assert(
+      !stats.enrollments.map(({ studentNumber }) => studentNumber).includes('542874'),
+      "Stats didn't include student with only failed course attainment (enrollments)"
+    )
+  })
+})

services/backend/tests/api/courses/courseYearlyStats/studentStatistics/mat11002.test.ts

@@ -0,0 +1,120 @@
+import { Express } from 'express'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { initTests } from '../../../../utils'
+import { calculatePassedAndFailed, getCourseYearlyStats, CourseYearlyStats } from '../helpers'
+
+const MAT11002 = 'hy-CU-117375394'
+
+void describe('Course yearly statistics - MAT11002 (hy-CU-117375394, no substitutions)', () => {
+  let app: Express
+  let body: CourseYearlyStats
+  beforeAll(async () => {
+    app = await initTests()
+    body = await getCourseYearlyStats(app, `courses=${MAT11002}&combineSubstitutions=false`)
+  })
+
+  it('should not include students for AY code', () => {
+    const stats = body.unifyStats.statistics.find(year => year.name === '2017-2018')
+    assert(stats && 'enrollments' in stats, 'Missing field enrollment in statsitics')
+    const studentCategories = calculatePassedAndFailed(stats.students.grades)
+    assert(!studentCategories.failed.includes('534980'))
+  })
+
+  it('should not include student with failed grade after passed grade', () => {
+    const year = body.unifyStats.statistics.find(year => year.name === '2018-2019')
+    assert(year && 'enrollments' in year, 'Stats missing completely')
+    const studentCategories = calculatePassedAndFailed(year.students.grades)
+    assert(!studentCategories.failed.includes('501716'), 'Failed students included incorrectly the student in question')
+    assert(studentCategories.passed.includes('501716'), "Passed students didn't include student with a passed grade")
+    assert(year.students.grades['1'].includes('501716'), 'Grades should include the student in the correct grade')
+  })
+
+  // FIXME: Doesn't test anything acually. Student doesn't have a passed AY grade
+  it.skip('should include student with failed grade and passed AY grade', () => {
+    const year = body.unifyStats.statistics.find(year => year.name === '2020-2021')
+    assert(year && 'enrollments' in year, 'Stats missing completely')
+    const studentCategories = calculatePassedAndFailed(year.students.grades)
+    assert(
+      studentCategories.failed.includes('0011812135'),
+      "Failed students didn't incorrectly include student in question"
+    )
+    assert(
+      !studentCategories.passed.includes('0011812135'),
+      'Passed students incorrectly included student without a passing grade'
+    )
+    assert(year.students.grades['0'].includes('0011812135'), 'Grades should include the student in the correct grade')
+  })
+
+  it('should include student with only approved grade', () => {
+    const year = body.unifyStats.statistics.find(year => year.name === '2022-2023')
+    assert(year && 'enrollments' in year, 'Stats missing completely')
+    const studentCategories = calculatePassedAndFailed(year.students.grades)
+    assert(!studentCategories.failed.includes('543385'), 'Failed students included incorrectly the student in question')
+    assert(studentCategories.passed.includes('543385'), "Passed students didn't include student with a passed grade")
+    assert(year.students.grades['5'].includes('543385'), 'Grades should include the student in the correct grade')
+  })
+
+  it('should not count a student with only improved grades as passed (509770)', () => {
+    const year = body.unifyStats.statistics.find(year => year.name === '2018-2019')
+    assert(year && 'enrollments' in year, 'Missing stats for 2018-2019')
+
+    const studentCategories = calculatePassedAndFailed(year.students.grades)
+    assert.notInclude(studentCategories.passed, '509770')
+    assert.strictEqual(studentCategories.passed.length, 57)
+
+    assert.notInclude(year.students.studentNumbers, '509770')
+  })
+
+  describe('enrollments', () => {
+    it('2021-2022', () => {
+      const year = body.unifyStats?.statistics.find(year => year.name === '2021-2022')
+      assert(year && 'enrollments' in year, 'Missing field enrollment in statsitics')
+      assert.strictEqual(year.enrollments.length, 27, 'Incorrect amount of distinct enrolled students 2021')
+      assert.strictEqual(year.allEnrollments.length, 29, 'Incorrect amount of total enrolled students 2021')
+    })
+
+    it('2022-2023', () => {
+      const year = body.unifyStats?.statistics.find(year => year.name === '2022-2023')
+      assert(year && 'enrollments' in year, 'Missing field enrollment in statsitics')
+      assert.strictEqual(year.enrollments.length, 23, 'Incorrect amount of distinct enrolled students 2022')
+      assert.strictEqual(year.allEnrollments.length, 26, 'Incorrect amount of total enrolled students 2022')
+    })
+
+    it('2023-2024', () => {
+      const year = body.unifyStats?.statistics.find(year => year.name === '2023-2024')
+      assert(year && 'enrollments' in year, 'Missing field enrollment in statsitics')
+      assert.strictEqual(year.enrollments.length, 1, 'Incorrect amount of distinct enrolled students 2023')
+      assert.strictEqual(year.allEnrollments.length, 1, 'Incorrect amount of total enrolled students 2023')
+    })
+
+    it('total', () => {
+      assert(
+        body.unifyStats?.statistics.every(year => 'enrollments' in year),
+        'Missing field enrollment in statsitics'
+      )
+      assert.strictEqual(
+        body.unifyStats?.statistics.reduce((acc, yearStats) => acc + yearStats.enrollments.length, 0),
+        27 + 23 + 1
+      )
+      assert.strictEqual(
+        body.unifyStats?.statistics.reduce((acc, yearStats) => acc + yearStats.allEnrollments.length, 0),
+        29 + 26 + 1
+      )
+      assert.strictEqual(
+        body.unifyStats?.statistics.reduce(
+          (acc, yearStats) => acc + calculatePassedAndFailed(yearStats.students.grades).failed.length,
+          0
+        ),
+        1
+      )
+      assert.strictEqual(
+        body.unifyStats?.statistics.reduce(
+          (acc, yearStats) => acc + calculatePassedAndFailed(yearStats.students.grades).passed.length,
+          0
+        ),
+        249
+      )
+    })
+  })
+})

services/backend/tests/api/courses/courseYearlyStats/studentStatistics/mat21003.test.ts

@@ -0,0 +1,92 @@
+import { Express } from 'express'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { yearToYearCode } from '@oodikone/shared/util'
+import { initTests } from '../../../../utils'
+import { calculatePassedAndFailed, getCourseYearlyStats, CourseYearlyStats } from '../helpers'
+
+const MAT21003 = 'hy-CU-117375829'
+
+void describe('Course yearly statistics - MAT21003 (no substitutions)', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
+
+  describe('duplicate failed grades across a large timespan are not double-counted', () => {
+    let body: CourseYearlyStats
+    beforeAll(async () => {
+      body = await getCourseYearlyStats(app, `courses=${MAT21003}&substitutions=false`)
+    })
+
+    it('2017-2018 should not include a failed grade', () => {
+      const year = body.unifyStats.statistics.find(year => year.name === '2017-2018')
+      assert(year && 'enrollments' in year, 'Missing stats for 2017-2018')
+      const studentCategories = calculatePassedAndFailed(year.students.grades)
+      assert.strictEqual(studentCategories.failed.length, 0, 'Failed stats should not include any students')
+      assert.deepStrictEqual(studentCategories.failed, [], 'Failed stats should not include any students')
+    })
+
+    it('2018-2019 should include a failed grade', () => {
+      const year = body.unifyStats.statistics.find(year => year.name === '2018-2019')
+      assert(year && 'enrollments' in year, 'Missing stats for 2018-2019')
+      const studentCategories = calculatePassedAndFailed(year.students.grades)
+      assert.strictEqual(studentCategories.failed.length, 1, 'Failed stats should include only one student')
+      assert.deepStrictEqual(studentCategories.failed, ['539036'], 'Failed stats included the incorrect student')
+    })
+  })
+
+  describe('year selection for smaller timeframes', () => {
+    it.each([
+      ['2017-2018', 2017, 2017],
+      ['2020-2023', 2020, 2022],
+      ['2016-2024', 2016, 2023],
+    ])('should include correct years for %s', async (name, from, to) => {
+      const body = await getCourseYearlyStats(
+        app,
+        `courses=${MAT21003}&substitutions=false&fromYearCode=${yearToYearCode(from)}&toYearCode=${yearToYearCode(to)}`
+      )
+
+      const years: string[] = []
+      for (let i = from; i <= to; i++) {
+        years.push(`${i}-${i + 1}`)
+      }
+      assert.deepStrictEqual(
+        body.unifyStats.statistics.map(({ name }) => name),
+        years,
+        `${name} included incorrect years`
+      )
+    })
+  })
+
+  describe('duplicate grades are counted again once an attainment falls out of the timespan', () => {
+    it('2017-2018 should include a failed grade', async () => {
+      const body = await getCourseYearlyStats(
+        app,
+        `courses=${MAT21003}&substitutions=false&fromYearCode=${yearToYearCode(2017)}&toYearCode=${yearToYearCode(2017)}`
+      )
+      const year = body.unifyStats.statistics.find(year => year.name === '2017-2018')
+      assert(year && 'enrollments' in year, 'Missing stats for 2017-2018')
+      const studentCategories = calculatePassedAndFailed(year.students.grades)
+
+      assert.deepStrictEqual(studentCategories.failed, ['539036'], 'Failed stats should have include a students')
+      assert.strictEqual(studentCategories.failed.length, 1, 'Failed stats should have included a failed student')
+    })
+
+    it('2018-2019 should include a failed grade', async () => {
+      const body = await getCourseYearlyStats(
+        app,
+        `courses=${MAT21003}&substitutions=false&fromYearCode=${yearToYearCode(2018)}&toYearCode=${yearToYearCode(2018)}`
+      )
+      const year = body.unifyStats.statistics.find(year => year.name === '2018-2019')
+      assert(year && 'enrollments' in year, 'Missing stats for 2018-2019')
+      const studentCategories = calculatePassedAndFailed(year.students.grades)
+      assert.deepStrictEqual(
+        studentCategories.failed,
+        ['539036', '540698', '542927', '544688'],
+        'Failed stats included the incorrect students'
+      )
+      assert.strictEqual(studentCategories.failed.length, 4, 'Failed stats should include only one student')
+    })
+  })
+})

services/backend/tests/api/courses/courseYearlyStats/studentStatistics/tkt10002.test.ts

@@ -0,0 +1,287 @@
+import { Express } from 'express'
+import { describe, it, beforeAll, assert } from 'vitest'
+
+import { yearToYearCode } from '@oodikone/shared/util'
+import { initTests } from '../../../../utils'
+import { calculatePassedAndFailed, getCourseYearlyStats, CourseYearlyStats } from '../helpers'
+
+const TKT10002 = 'hy-CU-118023867'
+
+void describe('Course yearly statistics - TKT10002 (no substitutions)', () => {
+  let app: Express
+  beforeAll(async () => {
+    app = await initTests()
+  })
+
+  describe('full range (2016-2023)', () => {
+    let body: CourseYearlyStats
+    beforeAll(async () => {
+      body = await getCourseYearlyStats(
+        app,
+        `courses=${TKT10002}&substitutions=false&fromYearCode=${yearToYearCode(2016)}&toYearCode=${yearToYearCode(2023)}`
+      )
+    })
+
+    it('has one entry per statistic year', () => {
+      assert.strictEqual(body.unifyStats.statistics.length, 8, 'unifyStats years')
+      assert.strictEqual(body.regularStats.statistics.length, 8, 'regularStats years')
+      assert.strictEqual(body.openStats.statistics.length, 4, 'openStats years')
+
+      // Faculties should have the same amount of years as normal statistics (above)
+      assert.strictEqual(Object.keys(body.unifyStats.facultyStats).length, 8, 'unifyStats years')
+      assert.strictEqual(Object.keys(body.regularStats.facultyStats).length, 8, 'regularStats years')
+      assert.strictEqual(Object.keys(body.openStats.facultyStats).length, 4, 'openStats years')
+    })
+
+    // TODO: Rewrite /courseyearlystats so that total = passed + failed + enrolledNoGrade
+    it.each([
+      [
+        '2016-2017',
+        { total: 1, passed: 1, failed: 0, enrolledNoGrade: 0 },
+        { 5: 0, 4: 1, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyv.': 0 },
+      ],
+      [
+        '2017-2018',
+        { total: 23, passed: 19, failed: 4, enrolledNoGrade: 0 },
+        // NOTE: If looking at credits, all Hyl. will be added as grade '0'
+        // eg. here 0 students got grade 0 and 4 students got grade Hyl.
+        { 5: 11, 4: 5, 3: 2, 2: 1, 1: 0, 0: 4, 'Hyv.': 0 },
+      ],
+      [
+        '2018-2019',
+        { total: 28, passed: 26, failed: 2, enrolledNoGrade: 0 },
+        { 5: 18, 4: 3, 3: 2, 2: 1, 1: 2, 0: 2, 'Hyv.': 0 },
+      ],
+      [
+        '2019-2020',
+        { total: 28, passed: 28, failed: 0, enrolledNoGrade: 0 },
+        { 5: 17, 4: 5, 3: 1, 2: 4, 1: 1, 0: 0, 'Hyv.': 0 },
+      ],
+      [
+        '2020-2021',
+        { total: 23, passed: 23, failed: 0, enrolledNoGrade: 0 },
+        { 5: 18, 4: 2, 3: 0, 2: 2, 1: 0, 0: 0, 'Hyv.': 1 },
+      ],
+      [
+        '2021-2022',
+        { total: 15, passed: 15, failed: 0, enrolledNoGrade: 20 },
+        { 5: 13, 4: 2, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyv.': 0 },
+      ],
+      [
+        '2022-2023',
+        { total: 27, passed: 27, failed: 0, enrolledNoGrade: 34 },
+        { 5: 21, 4: 6, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyv.': 0 },
+      ],
+      [
+        '2023-2024',
+        { total: 1, passed: 1, failed: 0, enrolledNoGrade: 7 },
+        { 5: 1, 4: 0, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyv.': 0 },
+      ],
+    ])('for statistic content during ($0)', (year, categories, grades) => {
+      const stats = body.unifyStats.statistics.find(yearStats => yearStats.name === year)!
+      assert('enrollments' in stats, 'Missing field enrollment in statsitics')
+
+      const studentCategories = calculatePassedAndFailed(stats.students.grades)
+
+      if (categories) {
+        assert.strictEqual(
+          studentCategories.failed.length,
+          categories.failed,
+          `Incorrect amount of failed students for ${year}`
+        )
+        assert.strictEqual(
+          studentCategories.passed.length,
+          categories.passed,
+          `Incorrect amount of passed students for ${year}`
+        )
+        assert.strictEqual(
+          stats.students.studentNumbers.length,
+          categories.total,
+          `Incorrect amount of total students for ${year}`
+        )
+        assert.strictEqual(
+          stats.enrollments.length,
+          categories.enrolledNoGrade,
+          `Incorrect amount of enrolled students for ${year}`
+        )
+      }
+      if (grades) {
+        Object.entries(grades).forEach(([grade, count]) =>
+          assert.strictEqual(
+            stats.students.grades[grade]?.length ?? 0,
+            count,
+            `Incorrect amount of students with grade ${grade}`
+          )
+        )
+      }
+    })
+
+    it('should include students who have failed a course and then completed it only once', () => {
+      const stats2017 = body.unifyStats.statistics.find(year => year.name === '2017-2018')!
+      const { studentNumbers } = stats2017.students
+      assert.notIncludeMembers(
+        studentNumbers,
+        ['457686' /* SPRING 2019*/, '455478' /* FALL 2020 */, '547994' /* FALL 2022*/],
+        "Students that have completed course later should not be included in the previous year's stats"
+      )
+
+      const year2017 = body.unifyStats.statistics.find(year => year.name === '2017-2018')
+      const year2018 = body.unifyStats.statistics.find(year => year.name === '2018-2019')
+      const year2020 = body.unifyStats.statistics.find(year => year.name === '2020-2021')
+      const year2022 = body.unifyStats.statistics.find(year => year.name === '2022-2023')
+      assert(year2017 && 'enrollments' in year2017, 'Missing stats for 2017-2018')
+      assert(year2018 && 'enrollments' in year2018, 'Missing stats for 2018-2019')
+      assert(year2020 && 'enrollments' in year2020, 'Missing stats for 2020-2021')
+      assert(year2022 && 'enrollments' in year2022, 'Missing stats for 2022-2023')
+
+      const studentCategories2017 = calculatePassedAndFailed(year2017.students.grades)
+      const studentCategories2018 = calculatePassedAndFailed(year2018.students.grades)
+      const studentCategories2020 = calculatePassedAndFailed(year2020.students.grades)
+      const studentCategories2022 = calculatePassedAndFailed(year2022.students.grades)
+
+      assert(
+        !studentCategories2017.passed.includes('457686') && !studentCategories2017.failed.includes('457686'),
+        "Student was incorrectly included to the failed course code's year stats (457686)"
+      )
+      assert(
+        studentCategories2018.passed.includes('457686') && year2018.students.grades['5'].includes('457686'),
+        "Student was incorrectly excluded from the passed course's completion year stats (457686)"
+      )
+
+      assert(
+        !studentCategories2017.passed.includes('455478') && !studentCategories2017.failed.includes('455478'),
+        "Student was incorrectly included to the failed course code's year stats (455478)"
+      )
+      assert(
+        studentCategories2020.passed.includes('455478') && year2020.students.grades['2'].includes('455478'),
+        "Student was incorrectly excluded from the passed course's completion year stats (455478)"
+      )
+
+      assert(
+        !studentCategories2017.passed.includes('547994') && !studentCategories2017.failed.includes('547994'),
+        "Student was incorrectly included to the failed course code's year stats (547994)"
+      )
+      assert(
+        studentCategories2022.passed.includes('547994') && year2022.students.grades['5'].includes('547994'),
+        "Student was incorrectly excluded from the passed course's completion year stats (547994)"
+      )
+    })
+
+    it("should mark student's enrollment time correctly with mismatched semestercode and enrollment_date_time", () => {
+      const stats2021 = body.unifyStats.statistics.find(year => year.name === '2021-2022')!
+      const stats2022 = body.unifyStats.statistics.find(year => year.name === '2022-2023')!
+      assert('enrollments' in stats2021, 'Missing field enrollment in statsitics (2021)')
+      assert('enrollments' in stats2022, 'Missing field enrollment in statsitics (2022)')
+      // Student with enrollment_date_time outside of semestert start and end dates
+      assert(
+        !stats2021.enrollments.map(enrollment => enrollment.studentNumber).includes('455129') &&
+          stats2022.enrollments.map(enrollment => enrollment.studentNumber).includes('455129'),
+        'Student with incorrect semester (by updater) but a correct enrollment_date was not found in the correct year (by enrollment_date)'
+      )
+    })
+  })
+
+  describe('single academic year ranges', () => {
+    // TODO: Rewrite /courseyearlystats so that total = passed + failed + enrolledNoGrade
+    it.each([
+      [
+        '2016-2017',
+        { total: 1, passed: 1, failed: 0, enrolledNoGrade: 0, extra: {} },
+        { 5: 0, 4: 1, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
+      ],
+      [
+        '2017-2018',
+        {
+          total: 26,
+          passed: 19,
+          failed: 7,
+          enrolledNoGrade: 0,
+          extra: { failed: ['455478', '457686', '474032', '497183', '509165', '547994', '550840'] },
+        },
+        // students.grades buckets every failing grade under '0', so 'Hyl.' is folded into it here.
+        { 5: 11, 4: 5, 3: 2, 2: 1, 1: 0, 0: 7, 'Hyl.': 0, 'Hyv.': 0 },
+      ],
+      [
+        '2018-2019',
+        { total: 30, passed: 26, failed: 4, enrolledNoGrade: 0, extra: {} },
+        { 5: 18, 4: 3, 3: 2, 2: 1, 1: 2, 0: 4, 'Hyl.': 0, 'Hyv.': 0 },
+      ],
+      [
+        '2019-2020',
+        { total: 28, passed: 28, failed: 0, enrolledNoGrade: 0, extra: {} },
+        { 5: 17, 4: 5, 3: 1, 2: 4, 1: 1, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
+      ],
+      [
+        '2020-2021',
+        { total: 23, passed: 23, failed: 0, enrolledNoGrade: 0, extra: {} },
+        { 5: 18, 4: 2, 3: 0, 2: 2, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 1 },
+      ],
+      [
+        '2021-2022',
+        { total: 15, passed: 15, failed: 0, enrolledNoGrade: 20, extra: {} },
+        { 5: 13, 4: 2, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
+      ],
+      [
+        '2022-2023',
+        { total: 27, passed: 27, failed: 0, enrolledNoGrade: 34, extra: {} },
+        { 5: 21, 4: 6, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
+      ],
+      [
+        '2023-2024',
+        { total: 1, passed: 1, failed: 0, enrolledNoGrade: 7, extra: {} },
+        { 5: 1, 4: 0, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
+      ],
+    ])('should include correct stats for one academic year ($0)', async (year, categories, grades) => {
+      const body = await getCourseYearlyStats(
+        app,
+        `courses=${TKT10002}&substitutions=false&fromYearCode=${yearToYearCode(year.split('-').at(0))}&toYearCode=${yearToYearCode(parseInt(year.split('-').at(1)!)) - 1}`
+      )
+      const stats = body.unifyStats.statistics.find(yearStats => yearStats.name === year)!
+      assert('enrollments' in stats, 'Missing field enrollment in statsitics')
+
+      const studentCategories = calculatePassedAndFailed(stats.students.grades)
+
+      if (Object.keys(categories.extra).length) {
+        Object.keys(categories.extra).forEach(category => {
+          assert.deepStrictEqual(
+            studentCategories[category as 'passed' | 'failed'].sort(),
+            categories.extra[category].sort(),
+            `"${category}" included incorrect students`
+          )
+        })
+      }
+
+      if (categories) {
+        assert.strictEqual(
+          studentCategories.failed.length,
+          categories.failed,
+          `Incorrect amount of failed students for ${year}`
+        )
+        assert.strictEqual(
+          studentCategories.passed.length,
+          categories.passed,
+          `Incorrect amount of passed students for ${year}`
+        )
+        assert.strictEqual(
+          stats.students.studentNumbers.length,
+          categories.total,
+          `Incorrect amount of total students for ${year}`
+        )
+        assert.strictEqual(
+          stats.enrollments.length,
+          categories.enrolledNoGrade,
+          `Incorrect amount of enrolled students for ${year}`
+        )
+      }
+      if (grades) {
+        Object.entries(grades).forEach(([grade, count]) =>
+          assert.strictEqual(
+            stats.students.grades[grade]?.length ?? 0,
+            count,
+            `Incorrect amount of students with grade ${grade}`
+          )
+        )
+      }
+    })
+  })
+})

services/backend/tests/api/courses/courseyearlystats.test.ts

@@ -1,665 +0,0 @@
-import { Express } from 'express'
-import request from 'supertest'
-import { describe, it, beforeAll, assert } from 'vitest'
-
-import { Unarray } from '@oodikone/shared/types'
-import { yearToYearCode } from '@oodikone/shared/util'
-import { CourseYearlyStatsResBody } from '../../../src/routes/courses'
-import { initTests, ResponseWithBody } from '../../utils'
-
-void describe('Course yearly statistics (2016-2023)', () => {
-  let app: Express
-  beforeAll(async () => {
-    app = await initTests()
-  })
-
-  it('should return nothing with missing parameters', async () => {
-    const res = await request(app)
-      .get('/courseyearlystats')
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')
-
-    assert.strictEqual(res.status, 422)
-    assert.strictEqual(res.body?.error, 'Missing required query parameters')
-  })
-
-  describe('should return correct amount of values for a course without substitutions (TKT10002)', () => {
-    let body: Unarray<CourseYearlyStatsResBody>
-    beforeAll(async () => {
-      const res = (await request(app)
-        .get(
-          `/courseyearlystats?codes=TKT10002&combineSubstitutions=false&fromYearCode=${yearToYearCode(2016)}&toYearCode=${yearToYearCode(2023)}`
-        )
-        .set('shib-session-id', 'test')
-        .set('uid', 'basic')
-        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
-
-      assert.strictEqual(res.status, 200)
-      assert.strictEqual(res.body.length, 1, 'Query to return anything')
-      body = res.body.at(0)!
-      assert(
-        'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
-        'All keys of courseyearlystats not defined'
-      )
-    })
-
-    it('for statistic years', () => {
-      assert.strictEqual(body.unifyStats!.statistics.length, 8, 'unifyStats years')
-      assert.strictEqual(body.regularStats!.statistics.length, 8, 'regularStats years')
-      assert.strictEqual(body.openStats!.statistics.length, 4, 'openStats years')
-
-      // Faculties should have the same amount of years as normal statistics (above)
-      assert.strictEqual(Object.keys(body.unifyStats!.facultyStats).length, 8, 'unifyStats years')
-      assert.strictEqual(Object.keys(body.regularStats!.facultyStats).length, 8, 'regularStats years')
-      assert.strictEqual(Object.keys(body.openStats!.facultyStats).length, 4, 'openStats years')
-    })
-
-    // TODO: Rewrite /courseyearlystats so that total = passed + failed + enrolledNoGrade
-    it.each([
-      [
-        '2016-2017',
-        { total: 1, passed: 1, failed: 0, enrolledNoGrade: 0 },
-        { 5: 0, 4: 1, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
-      ],
-      [
-        '2017-2018',
-        { total: 23, passed: 19, failed: 4, enrolledNoGrade: 0 },
-        { 5: 11, 4: 5, 3: 2, 2: 1, 1: 0, 0: 0, 'Hyl.': 4, 'Hyv.': 0 },
-      ],
-      [
-        '2018-2019',
-        { total: 28, passed: 26, failed: 2, enrolledNoGrade: 0 },
-        { 5: 18, 4: 3, 3: 2, 2: 1, 1: 2, 0: 2, 'Hyl.': 0, 'Hyv.': 0 },
-      ],
-      [
-        '2019-2020',
-        { total: 28, passed: 28, failed: 0, enrolledNoGrade: 0 },
-        { 5: 17, 4: 5, 3: 1, 2: 4, 1: 1, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
-      ],
-      [
-        '2020-2021',
-        { total: 23, passed: 23, failed: 0, enrolledNoGrade: 0 },
-        { 5: 18, 4: 2, 3: 0, 2: 2, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 1 },
-      ],
-      [
-        '2021-2022',
-        { total: 15, passed: 15, failed: 0, enrolledNoGrade: 20 },
-        { 5: 13, 4: 2, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
-      ],
-      [
-        '2022-2023',
-        { total: 27, passed: 27, failed: 0, enrolledNoGrade: 34 },
-        { 5: 21, 4: 6, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
-      ],
-      [
-        '2023-2024',
-        { total: 1, passed: 1, failed: 0, enrolledNoGrade: 7 },
-        { 5: 1, 4: 0, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
-      ],
-    ])('for statistic content during ($0)', (year, categories, grades) => {
-      const stats = body.unifyStats!.statistics.find(yearStats => yearStats.name === year)!
-      assert('enrollments' in stats, 'Missing field enrollment in statsitics')
-
-      if (categories) {
-        assert.strictEqual(
-          stats.attempts.categories.failed.length,
-          categories.failed,
-          `Incorrect amount of failed students for ${year}`
-        )
-        assert.strictEqual(
-          stats.attempts.categories.passed.length,
-          categories.passed,
-          `Incorrect amount of passed students for ${year}`
-        )
-        assert.strictEqual(
-          stats.students.studentNumbers.length,
-          categories.total,
-          `Incorrect amount of total students for ${year}`
-        )
-        assert.strictEqual(
-          stats.enrollments.length,
-          categories.enrolledNoGrade,
-          `Incorrect amount of enrolled students for ${year}`
-        )
-      }
-      if (grades) {
-        Object.entries(grades).forEach(([grade, count]) =>
-          assert.strictEqual(
-            stats.attempts.grades[grade]?.length ?? 0,
-            count,
-            `Incorrect amount of students with grade ${grade}`
-          )
-        )
-      }
-    })
-
-    describe('should work correctly in specific cases (TKT10002)', () => {
-      let body: Unarray<CourseYearlyStatsResBody>
-      beforeAll(async () => {
-        const res = (await request(app)
-          .get(
-            `/courseyearlystats?codes=TKT10002&combineSubstitutions=false&fromYearCode=${yearToYearCode(2016)}&toYearCode=${yearToYearCode(2023)}`
-          )
-          .set('shib-session-id', 'test')
-          .set('uid', 'basic')
-          .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
-
-        assert.strictEqual(res.status, 200)
-        assert.strictEqual(res.body.length, 1, 'Query to return anything')
-        body = res.body.at(0)!
-        assert(
-          'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
-          'All keys of courseyearlystats not defined'
-        )
-      })
-
-      it('should include students who have failed a course and then completed it only once', () => {
-        const stats2017 = body.unifyStats!.statistics.find(year => year.name === '2017-2018')!
-        const { studentNumbers } = stats2017.students
-
-        assert(
-          !(
-            (
-              '457686' in studentNumbers || // SPRING 2019
-              '455478' in studentNumbers || // FALL 2020
-              '547994' in studentNumbers
-            ) // FALL 2022
-          ),
-          "Students that have completed course later should not be included in the previous year's stats"
-        )
-
-        assert(
-          !body
-            .unifyStats!.statistics.find(year => year.name === '2017-2018')
-            ?.attempts.categories.passed.includes('457686') &&
-            !body
-              .unifyStats!.statistics.find(year => year.name === '2017-2018')
-              ?.attempts.categories.failed.includes('457686'),
-          "Student was incorrectly included to the failed course code's year stats (457686)"
-        )
-        assert(
-          body
-            .unifyStats!.statistics.find(year => year.name === '2018-2019')
-            ?.attempts.categories.passed.includes('457686') &&
-            body
-              .unifyStats!.statistics.find(year => year.name === '2018-2019')
-              ?.attempts.grades['5'].includes('457686'),
-          "Student was incorrectly excluded from the passed course's completion year stats (457686)"
-        )
-
-        assert(
-          !body
-            .unifyStats!.statistics.find(year => year.name === '2017-2018')
-            ?.attempts.categories.passed.includes('455478') &&
-            !body
-              .unifyStats!.statistics.find(year => year.name === '2017-2018')
-              ?.attempts.categories.failed.includes('455478'),
-          "Student was incorrectly included to the failed course code's year stats (455478)"
-        )
-        assert(
-          body
-            .unifyStats!.statistics.find(year => year.name === '2020-2021')
-            ?.attempts.categories.passed.includes('455478') &&
-            body
-              .unifyStats!.statistics.find(year => year.name === '2020-2021')
-              ?.attempts.grades['2'].includes('455478'),
-          "Student was incorrectly excluded from the passed course's completion year stats (455478)"
-        )
-
-        assert(
-          !body
-            .unifyStats!.statistics.find(year => year.name === '2017-2018')
-            ?.attempts.categories.passed.includes('547994') &&
-            !body
-              .unifyStats!.statistics.find(year => year.name === '2017-2018')
-              ?.attempts.categories.failed.includes('547994'),
-          "Student was incorrectly included to the failed course code's year stats (547994)"
-        )
-        assert(
-          body
-            .unifyStats!.statistics.find(year => year.name === '2022-2023')
-            ?.attempts.categories.passed.includes('547994') &&
-            body
-              .unifyStats!.statistics.find(year => year.name === '2022-2023')
-              ?.attempts.grades['5'].includes('547994'),
-          "Student was incorrectly excluded from the passed course's completion year stats (547994)"
-        )
-      })
-
-      it("should mark student's enrollment time correctly with mismatched semestercode and enrollment_date_time", () => {
-        const stats2021 = body.unifyStats!.statistics.find(year => year.name === '2021-2022')!
-        const stats2022 = body.unifyStats!.statistics.find(year => year.name === '2022-2023')!
-        assert('enrollments' in stats2021, 'Missing field enrollment in statsitics (2021)')
-        assert('enrollments' in stats2022, 'Missing field enrollment in statsitics (2022)')
-        // Student with enrollment_date_time outside of semestert start and end dates
-        assert(
-          !stats2021.enrollments.map(enrollment => enrollment.studentNumber).includes('455129') &&
-            stats2022.enrollments.map(enrollment => enrollment.studentNumber).includes('455129'),
-          'Student with incorrect semester (by updater) but a correct enrollment_date was not found in the correct year (by enrollment_date)'
-        )
-      })
-    })
-
-    describe('should work correctly in specific cases (MAT11001)', () => {
-      let body: Unarray<CourseYearlyStatsResBody>
-      beforeAll(async () => {
-        const res = (await request(app)
-          .get('/courseyearlystats?codes=MAT11001&combineSubstitutions=false')
-          .set('shib-session-id', 'test')
-          .set('uid', 'basic')
-          .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
-
-        assert.strictEqual(res.status, 200)
-        assert.strictEqual(res.body.length, 1)
-        body = res.body.at(0)!
-        assert(
-          'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
-          'All keys of courseyearlystats not defined'
-        )
-      })
-
-      it('where a student has failed a course', () => {
-        const stats = body.unifyStats?.statistics!.find(year => year.name === '2018-2019')
-        assert(stats && 'enrollments' in stats, 'Missing field enrollment in statsitics')
-
-        assert(
-          stats.attempts.categories.failed.includes('542874'),
-          "Stats didn't include student with only failed course attainment (failed)"
-        )
-        assert(
-          !stats.attempts.categories.passed.includes('542874'),
-          'Stats did incorrectly include student with only failed course attainment (passed)'
-        )
-        assert(
-          !stats.enrollments.map(({ studentNumber }) => studentNumber).includes('542874'),
-          "Stats didn't include student with only failed course attainment (enrollments)"
-        )
-      })
-    })
-
-    describe('should work correctly in specific cases (MAT11002)', () => {
-      let body: Unarray<CourseYearlyStatsResBody>
-      beforeAll(async () => {
-        const res = (await request(app)
-          .get('/courseyearlystats?codes=MAT11002&combineSubstitutions=false')
-          .set('shib-session-id', 'test')
-          .set('uid', 'basic')
-          .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
-
-        assert.strictEqual(res.status, 200)
-        assert.strictEqual(res.body.length, 1)
-        body = res.body.at(0)!
-        assert(
-          'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
-          'All keys of courseyearlystats not defined'
-        )
-      })
-
-      it('should not include students for AY code', () => {
-        const stats = body.unifyStats?.statistics!.find(year => year.name === '2017-2018')
-        assert(stats && 'enrollments' in stats, 'Missing field enrollment in statsitics')
-        assert(!('534980' in stats.attempts.categories.failed))
-      })
-
-      it('should not include student with failed grade after passed grade', () => {
-        const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
-        assert(year, 'Stats missing completely')
-        assert(
-          !year.attempts.categories.failed.includes('501716'),
-          'Failed students included incorrectly the student in question'
-        )
-        assert(
-          year.attempts.categories.passed.includes('501716'),
-          "Passed students didn't include student with a passed grade"
-        )
-        assert(year.attempts.grades['1'].includes('501716'), 'Grades should include the student in the correct grade')
-      })
-
-      it('should include student with failed grade and passed AY grade', () => {
-        const year = body.unifyStats?.statistics.find(year => year.name === '2020-2021')
-        assert(year, 'Stats missing completely')
-        assert(
-          year.attempts.categories.failed.includes('0011812135'),
-          "Failed students didn't incorrectly include student in question"
-        )
-        assert(
-          !year.attempts.categories.passed.includes('0011812135'),
-          'Passed students incorrectly included student without a passing grade'
-        )
-        assert(
-          year.attempts.grades['Hyl.'].includes('0011812135'),
-          'Grades should include the student in the correct grade'
-        )
-      })
-
-      it('should include student with only approved grade', () => {
-        const year = body.unifyStats?.statistics.find(year => year.name === '2022-2023')
-        assert(year, 'Stats missing completely')
-        assert(
-          !year.attempts.categories.failed.includes('543385'),
-          'Failed students included incorrectly the student in question'
-        )
-        assert(
-          year.attempts.categories.passed.includes('543385'),
-          "Passed students didn't include student with a passed grade"
-        )
-        assert(year.attempts.grades['5'].includes('543385'), 'Grades should include the student in the correct grade')
-      })
-
-      describe('should calculate enrollments correctly', () => {
-        it('- 2021-2022', () => {
-          const year = body.unifyStats?.statistics.find(year => year.name === '2021-2022')
-          assert(year && 'enrollments' in year, 'Missing field enrollment in statsitics')
-          assert.strictEqual(year.enrollments.length, 27, 'Incorrect amount of distinct enrolled students 2021')
-          assert.strictEqual(year.allEnrollments.length, 29, 'Incorrect amount of total enrolled students 2021')
-        })
-        it('- 2022-2023', () => {
-          const year = body.unifyStats?.statistics.find(year => year.name === '2022-2023')
-          assert(year && 'enrollments' in year, 'Missing field enrollment in statsitics')
-          assert.strictEqual(year.enrollments.length, 23, 'Incorrect amount of distinct enrolled students 2022')
-          assert.strictEqual(year.allEnrollments.length, 26, 'Incorrect amount of total enrolled students 2022')
-        })
-        it('- 2023-2024', () => {
-          const year = body.unifyStats?.statistics.find(year => year.name === '2023-2024')
-          assert(year && 'enrollments' in year, 'Missing field enrollment in statsitics')
-          assert.strictEqual(year.enrollments.length, 1, 'Incorrect amount of distinct enrolled students 2023')
-          assert.strictEqual(year.allEnrollments.length, 1, 'Incorrect amount of total enrolled students 2023')
-        })
-
-        it('- total', () => {
-          assert(
-            body.unifyStats?.statistics.every(year => 'enrollments' in year),
-            'Missing field enrollment in statsitics'
-          )
-          assert.strictEqual(
-            body.unifyStats?.statistics.reduce((acc, yearStats) => acc + yearStats.enrollments.length, 0),
-            27 + 23 + 1
-          )
-          assert.strictEqual(
-            body.unifyStats?.statistics.reduce((acc, yearStats) => acc + yearStats.allEnrollments.length, 0),
-            29 + 26 + 1
-          )
-          assert.strictEqual(
-            body.unifyStats?.statistics.reduce(
-              (acc, yearStats) => acc + yearStats.attempts.categories.failed.length,
-              0
-            ),
-            1
-          )
-          assert.strictEqual(
-            body.unifyStats?.statistics.reduce(
-              (acc, yearStats) => acc + yearStats.attempts.categories.passed.length,
-              0
-            ),
-            249
-          )
-        })
-      })
-    })
-
-    describe('should work correctly in specific cases (MAT21003)', () => {
-      describe('should not count duplicate failed grades to different years when choosing a large timespan', () => {
-        let body: Unarray<CourseYearlyStatsResBody>
-        beforeAll(async () => {
-          const res = (await request(app)
-            .get('/courseyearlystats?codes=MAT21003&combineSubstitutions=false')
-            .set('shib-session-id', 'test')
-            .set('uid', 'basic')
-            .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
-
-          assert.strictEqual(res.status, 200)
-          assert.strictEqual(res.body.length, 1)
-          body = res.body.at(0)!
-          assert(
-            'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
-            'All keys of courseyearlystats not defined'
-          )
-        })
-
-        it('- 2017-2018 should not include failed grade', () => {
-          const year = body.unifyStats?.statistics.find(year => year.name === '2017-2018')
-          assert.strictEqual(year?.attempts.categories.failed.length, 0, 'Failed stats should not include any students')
-          assert.deepStrictEqual(year?.attempts.categories.failed, [], 'Failed stats should not include any students')
-        })
-        it('- 2018-2019 should include a failed grade', () => {
-          const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
-          assert.strictEqual(year?.attempts.categories.failed.length, 1, 'Failed stats should include only one student')
-          assert.deepStrictEqual(
-            year?.attempts.categories.failed,
-            ['539036'],
-            'Failed stats included the incorrect student'
-          )
-        })
-      })
-
-      describe('should not count students with only improved grades as passed', () => {
-        let body: Unarray<CourseYearlyStatsResBody>
-        beforeAll(async () => {
-          const res = (await request(app)
-            .get('/courseyearlystats?codes=MAT11002&combineSubstitutions=false')
-            .set('shib-session-id', 'test')
-            .set('uid', 'basic')
-            .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
-
-          assert.strictEqual(res.status, 200)
-          assert.strictEqual(res.body.length, 1)
-          body = res.body.at(0)!
-          assert(
-            'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
-            'All keys of courseyearlystats not defined'
-          )
-        })
-
-        it('2018-2019 MAT11002 should not include student with only improved grade (509770)', () => {
-          const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
-          assert.notInclude(year?.attempts.categories.passed, ['509770'])
-          assert.strictEqual(year?.attempts.categories.passed.length, 57)
-        })
-      })
-    })
-  })
-
-  it.todo('should return correct amount of students for a course with substitutions')
-  it.todo('should return correct amount of students for multiple coursecodes without substitutions')
-  it.todo('should return correct amount of students for multiple coursecodes with substitutions')
-})
-
-describe('Course yearly statistics (smaller timeframes)', () => {
-  let app: Express
-  beforeAll(async () => {
-    app = await initTests()
-  })
-
-  it.each([
-    ['2017-2018', 2017, 2017],
-    ['2020-2023', 2020, 2022],
-    ['2016-2024', 2016, 2023],
-  ])('should include correct years for smaller timeframe (MAT21003 $0)', async (name, from, to) => {
-    const res = (await request(app)
-      .get(
-        `/courseyearlystats?codes=MAT21003&combineSubstitutions=false&fromYearCode=${yearToYearCode(from)}&toYearCode=${yearToYearCode(to)}`
-      )
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
-
-    assert.strictEqual(res.status, 200)
-    assert.strictEqual(res.body.length, 1)
-    const body = res.body.at(0)!
-
-    const years: string[] = []
-    for (let i = from; i <= to; i++) {
-      years.push(`${i}-${i + 1}`)
-    }
-    assert.deepStrictEqual(
-      body.unifyStats!.statistics.map(({ name }) => name),
-      years,
-      `${name} included incorrect years`
-    )
-  })
-
-  // TODO: Rewrite /courseyearlystats so that total = passed + failed + enrolledNoGrade
-  it.each([
-    [
-      '2016-2017',
-      { total: 1, passed: 1, failed: 0, enrolledNoGrade: 0, extra: {} },
-      { 5: 0, 4: 1, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
-    ],
-    [
-      '2017-2018',
-      {
-        total: 26,
-        passed: 19,
-        failed: 7,
-        enrolledNoGrade: 0,
-        extra: { failed: ['455478', '457686', '474032', '497183', '509165', '547994', '550840'] },
-      },
-      { 5: 11, 4: 5, 3: 2, 2: 1, 1: 0, 0: 0, 'Hyl.': 7, 'Hyv.': 0 },
-    ],
-    [
-      '2018-2019',
-      { total: 30, passed: 26, failed: 4, enrolledNoGrade: 0, extra: {} },
-      { 5: 18, 4: 3, 3: 2, 2: 1, 1: 2, 0: 3, 'Hyl.': 1, 'Hyv.': 0 },
-    ],
-    [
-      '2019-2020',
-      { total: 28, passed: 28, failed: 0, enrolledNoGrade: 0, extra: {} },
-      { 5: 17, 4: 5, 3: 1, 2: 4, 1: 1, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
-    ],
-    [
-      '2020-2021',
-      { total: 23, passed: 23, failed: 0, enrolledNoGrade: 0, extra: {} },
-      { 5: 18, 4: 2, 3: 0, 2: 2, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 1 },
-    ],
-    [
-      '2021-2022',
-      { total: 15, passed: 15, failed: 0, enrolledNoGrade: 20, extra: {} },
-      { 5: 13, 4: 2, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
-    ],
-    [
-      '2022-2023',
-      { total: 27, passed: 27, failed: 0, enrolledNoGrade: 34, extra: {} },
-      { 5: 21, 4: 6, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
-    ],
-    [
-      '2023-2024',
-      { total: 1, passed: 1, failed: 0, enrolledNoGrade: 7, extra: {} },
-      { 5: 1, 4: 0, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 },
-    ],
-  ])('should include correct stats for one academic year (TKT10002, $0)', async (year, categories, grades) => {
-    const res = (await request(app)
-      .get(
-        `/courseyearlystats?codes=TKT10002&combineSubstitutions=false&fromYearCode=${yearToYearCode(year.split('-').at(0))}&toYearCode=${yearToYearCode(parseInt(year.split('-').at(1)!)) - 1}`
-      )
-      .set('shib-session-id', 'test')
-      .set('uid', 'basic')
-      .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
-
-    assert.strictEqual(res.status, 200)
-    assert.strictEqual(res.body.length, 1)
-    const body = res.body.at(0)!
-    assert(
-      'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
-      'All keys of courseyearlystats not defined'
-    )
-    const stats = body.unifyStats!.statistics.find(yearStats => yearStats.name === year)!
-    assert('enrollments' in stats, 'Missing field enrollment in statsitics')
-
-    if (Object.keys(categories.extra).length) {
-      Object.keys(categories.extra).forEach(category => {
-        assert.deepStrictEqual(
-          stats.attempts.categories[category].sort(),
-          categories.extra[category].sort(),
-          `"${category}" included incorrect students`
-        )
-      })
-    }
-
-    if (categories) {
-      assert.strictEqual(
-        stats.attempts.categories.failed.length,
-        categories.failed,
-        `Incorrect amount of failed students for ${year}`
-      )
-      assert.strictEqual(
-        stats.attempts.categories.passed.length,
-        categories.passed,
-        `Incorrect amount of passed students for ${year}`
-      )
-      assert.strictEqual(
-        stats.students.studentNumbers.length,
-        categories.total,
-        `Incorrect amount of total students for ${year}`
-      )
-      assert.strictEqual(
-        stats.enrollments.length,
-        categories.enrolledNoGrade,
-        `Incorrect amount of enrolled students for ${year}`
-      )
-    }
-    if (grades) {
-      Object.entries(grades).forEach(([grade, count]) =>
-        assert.strictEqual(
-          stats.attempts.grades[grade]?.length ?? 0,
-          count,
-          `Incorrect amount of students with grade ${grade}`
-        )
-      )
-    }
-  })
-
-  describe('should count duplicate grades if an attainment falls out of the timespan (MAT21003)', () => {
-    it('- 2017-2018 should include a failed grade', async () => {
-      const res = (await request(app)
-        .get(
-          `/courseyearlystats?codes=MAT21003&combineSubstitutions=false&fromYearCode=${yearToYearCode(2017)}&toYearCode=${yearToYearCode(2017)}`
-        )
-        .set('shib-session-id', 'test')
-        .set('uid', 'basic')
-        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
-
-      assert.strictEqual(res.status, 200)
-      assert.strictEqual(res.body.length, 1)
-      const body = res.body.at(0)!
-      assert(
-        'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
-        'All keys of courseyearlystats not defined'
-      )
-      const year = body.unifyStats!.statistics.find(year => year.name === '2017-2018')!
-
-      assert.deepStrictEqual(year.attempts.categories.failed, ['539036'], 'Failed stats should have include a students')
-      assert.strictEqual(
-        year.attempts.categories.failed.length,
-        1,
-        'Failed stats should have included a failed student'
-      )
-    })
-
-    it('- 2018-2019 should include a failed grade', async () => {
-      const res = (await request(app)
-        .get(
-          `/courseyearlystats?codes=MAT21003&combineSubstitutions=false&fromYearCode=${yearToYearCode(2018)}&toYearCode=${yearToYearCode(2018)}`
-        )
-        .set('shib-session-id', 'test')
-        .set('uid', 'basic')
-        .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<CourseYearlyStatsResBody>
-
-      assert.strictEqual(res.status, 200)
-      assert.strictEqual(res.body.length, 1)
-      const body = res.body.at(0)!
-      assert(
-        'unifyStats' in body && 'regularStats' in body && 'openStats' in body,
-        'All keys of courseyearlystats not defined'
-      )
-
-      const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
-      assert.deepStrictEqual(
-        year?.attempts.categories.failed,
-        ['539036', '540698', '542927', '544688'],
-        'Failed stats included the incorrect students'
-      )
-      assert.strictEqual(year?.attempts.categories.failed.length, 4, 'Failed stats should include only one student')
-    })
-  })
-})

services/backend/tests/api/population/populationStatistics.test.ts

@@ -5,7 +5,7 @@ import { describe, it, beforeAll, assert } from 'vitest'
 
 import { CanError } from '@oodikone/shared/routes'
 import { PopulationstatisticsResBody } from '@oodikone/shared/routes/populations'
-import { dbConnections } from '../../../src/database/connection'
+import { dbConnections } from '@/database/connection'
 import { initTests, ResponseWithBody } from '../../utils'
 
 const populationUrl = (

services/backend/tests/api/population/populationStatisticsByCourse.test.ts

@@ -5,10 +5,13 @@ import { describe, it, beforeAll, assert } from 'vitest'
 import { initTests } from '../../utils'
 import { yearToYearCode } from '@oodikone/shared/util'
 import { PopulationstatisticsbycourseResBody } from '@oodikone/shared/routes/populations'
-import { CreditModel } from '../../../src/models'
+import { CreditModel } from '@/models'
 import { EnrollmentState } from '@oodikone/shared/types'
 
-void describe('Population statistics by course', () => {
+const MAT21003 = 'hy-CU-117375829'
+const MAT21005 = 'hy-CU-117376344'
+
+void describe('Population statistics by course MAT21003 + MAT21005 (hy-CU-117375829 + hy-CU-117376344)', () => {
   let app: Express
   beforeAll(async () => {
     app = await initTests()
@@ -22,7 +25,7 @@ void describe('Population statistics by course', () => {
       .set('hygroupcn', 'grp-oodikone-basic-users')
 
     assert.strictEqual(res.status, 400)
-    assert.deepStrictEqual(res.body.error, 'The body should have a yearcode and coursecode defined')
+    assert.deepStrictEqual(res.body.error, 'Missing required parameters.')
   })
 
   // Vektorianalyysi I, MAT21003
@@ -40,9 +43,7 @@ void describe('Population statistics by course', () => {
     [2023, 21, 1, 0, 36],
   ])('should return correct amount of students for single year ($0)', async (year, total, passed, failed, enrolled) => {
     const res = await request(app)
-      .get(
-        `/populationstatisticsbycourse?coursecodes=["MAT21003"]&from=${yearToYearCode(year)}&to=${yearToYearCode(year)}`
-      )
+      .get(`/populationstatisticsbycourse?courses=${MAT21003}&from=${yearToYearCode(year)}&to=${yearToYearCode(year)}`)
       .set('shib-session-id', 'test')
       .set('uid', 'basic')
       .set('hygroupcn', 'grp-oodikone-basic-users')
@@ -52,20 +53,23 @@ void describe('Population statistics by course', () => {
 
     assert.strictEqual(body.students.length, total, 'Incorrect amount of students')
     assert.strictEqual(
-      body.coursestatistics.credits.filter(credit => credit.course_code === 'MAT21003' && CreditModel.passed(credit))
-        .length,
+      body.coursestatistics.credits.filter(
+        credit => body.idToGroupIdMap[credit.course_id] === MAT21003 && CreditModel.passed(credit)
+      ).length,
       passed,
       'Incorrect amount of passed credits'
     )
     assert.strictEqual(
-      body.coursestatistics.credits.filter(credit => credit.course_code === 'MAT21003' && CreditModel.failed(credit))
-        .length,
+      body.coursestatistics.credits.filter(
+        credit => body.idToGroupIdMap[credit.course_id] === MAT21003 && CreditModel.failed(credit)
+      ).length,
       failed,
       'Incorrect amount of failed credits'
     )
     assert.strictEqual(
       body.coursestatistics.enrollments.filter(
-        enrollment => enrollment.course_code === 'MAT21003' && enrollment.state === EnrollmentState.ENROLLED
+        enrollment =>
+          body.idToGroupIdMap[enrollment.course_id] === MAT21003 && enrollment.state === EnrollmentState.ENROLLED
       ).length,
       enrolled,
       'Incorrect amount of enrollments'
@@ -74,7 +78,7 @@ void describe('Population statistics by course', () => {
 
   it('should return correct amount of students for a course', async () => {
     const res = await request(app)
-      .get('/populationstatisticsbycourse?coursecodes=["MAT21005"]&from=70&to=70')
+      .get(`/populationstatisticsbycourse?courses=${MAT21005}&from=70&to=70`)
       .set('shib-session-id', 'test')
       .set('uid', 'basic')
       .set('hygroupcn', 'grp-oodikone-basic-users')

services/backend/tests/api/studyProgramme/studyProgrammeCourseStats.test.ts

@@ -97,13 +97,12 @@ void describe("Study programme's course stats (all years)", () => {
       .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<StudyProgrammeCourse[]>
 
     assert.strictEqual(res.status, 200)
-    assert.strictEqual(res.body.length, 102, 'Response included incorrect amount of courses') // Not checked number
+    assert.strictEqual(res.body.length, 88, 'Response included incorrect amount of courses') // Not checked number
 
     const courseCodes = res.body.map(({ code }) => code)
-    assert(
-      ['MAT11003', 'MAT11001', 'MAT11004', 'AYMAT11003', 'AYMAT11001', 'AYMAT11004'].every(code =>
-        courseCodes.includes(code)
-      ),
+    assert.includeMembers(
+      courseCodes,
+      ['MAT11003', 'MAT11001', 'MAT11004' /* , 'AYMAT11003', 'AYMAT11001', 'AYMAT11004' */], // AY codes were previously required
       'Required courses or their open uni variants missing'
     )
 

services/backend/tests/api/testTimeHeader.test.ts

@@ -2,8 +2,8 @@ import express from 'express'
 import request from 'supertest'
 import { describe, it, assert } from 'vitest'
 
-import { testTimeMiddleware } from '../../src/middleware/testTime'
-import { now } from '../../src/util/clock'
+import { testTimeMiddleware } from '@/middleware/testTime'
+import { now } from '@/util/clock'
 
 describe('test time middleware', () => {
   it('uses the date from the x-test-now header', async () => {

services/backend/tests/services/courses/index.test.ts

@@ -1,164 +1,154 @@
-import { describe, it, assert, beforeAll, vi } from 'vitest'
-import { Unification } from '@oodikone/shared/types'
-import { initializeDatabaseConnection } from '../../../src/database/connection'
-import {
-  maxYearsToCreatePopulationFrom,
-  searchAndCombineSubstitutionGroupsToCodes,
-} from '../../../src/services/courses'
-import { findByCourseAndSemesters } from '../../../src/services/students'
+import { describe, it, assert, beforeAll } from 'vitest'
+import { initializeDatabaseConnection } from '@/database/connection'
+import { findByCourseAndSemesters } from '@/services/students'
 import { yearToYearCode } from '@oodikone/shared/util'
+import { Unification } from '@oodikone/shared/types'
+import { getRelevantCourseIdMap } from '@/services/courses'
+
+const CSM14204 = 'hy-CU-119366777'
+const MAT21018 = 'hy-CU-117377764'
 
 void describe.concurrent('Search and complete substitution groups to codes', () => {
   beforeAll(async () => {
     await initializeDatabaseConnection()
   })
 
-  it('should return correct course codes for MAT11001 (only single-length groups)', async () => {
-    const res = await searchAndCombineSubstitutionGroupsToCodes(['MAT11001'])
-    assert.deepStrictEqual(res, ['MAT11001', '57033', 'AYMAT11001', 'A57033'])
-  })
+  type TestList = [string, number, number][]
 
-  it('should return correct course codes for MAT21016 (also multicourse groups)', async () => {
-    const res = await searchAndCombineSubstitutionGroupsToCodes(['MAT21016'])
-    assert.deepStrictEqual(res, ['MAT21016', '57282', 'MFK-M310', 'MAT21023'])
-  })
-  it('should return correct course codes for MAT11001 + MAT21016', async () => {
-    const res = await searchAndCombineSubstitutionGroupsToCodes(['MAT11001', 'MAT21016'])
-    assert.deepStrictEqual(res, [
-      'MAT11001',
-      'MAT21016',
-      '57033',
-      'AYMAT11001',
-      'A57033',
-      '57282',
-      'MFK-M310',
-      'MAT21023',
-    ])
-  })
-  it.todo('should return correct course codes for X + Y (duplicate courses in substitution groups)')
-})
+  const semesterToSemesterCode = (semesterYear: number, springOrFall: 'spring' | 'fall') => {
+    return (semesterYear - 1950) * 2 + (springOrFall === 'spring' ? 0 : 1)
+  }
 
-void describe.skip('Max years to generate population from', () => {
-  beforeAll(async () => {
-    await initializeDatabaseConnection()
-  })
+  void describe.concurrent('Find by course and semester', () => {
+    it.each([
+      ['semester', semesterToSemesterCode(2019, 'fall'), semesterToSemesterCode(2023, 'spring')],
+      ['year', yearToYearCode(2017), yearToYearCode(2023)],
+    ] as TestList)(
+      'should return correct student numbers for a short course (CSM14204, 2019-2023) by $0',
+      async (separate, to, from) => {
+        const courseIds = await getRelevantCourseIdMap([CSM14204], false)
+        const res = await findByCourseAndSemesters(
+          Object.keys(courseIds),
+          to,
+          from,
+          separate === 'semester',
+          Unification.UNIFY
+        )
 
-  it('should return no years without a course code', async () => {
-    const res = await maxYearsToCreatePopulationFrom([], Unification.UNIFY)
-    assert.strictEqual(res, 0, 'Got something else than 0 years')
-  })
+        assert.deepStrictEqual([...res].sort(), ['458079', '478837', '483126'])
+      }
+    )
 
-  it('should return correct years for MAT11001', async () => {
-    const res = await maxYearsToCreatePopulationFrom(['MAT11001'], Unification.UNIFY)
-    assert.strictEqual(res, 6)
-  })
+    it.each([
+      ['semester', semesterToSemesterCode(2020, 'fall'), semesterToSemesterCode(2023, 'spring')],
+      ['year', yearToYearCode(2020), yearToYearCode(2023)],
+    ] as TestList)('should filter out students outside of 2020-2023 by $0', async (separate, to, from) => {
+      const courseIds = await getRelevantCourseIdMap([CSM14204], false)
+      const res = await findByCourseAndSemesters(
+        Object.keys(courseIds),
+        to,
+        from,
+        separate === 'semester',
+        Unification.UNIFY
+      )
+      assert.deepStrictEqual([...res].sort(), ['478837', '483126'])
+    })
 
-  it.todo('should should return the same max years as before', () => {
-    const date = new Date(2020)
-    vi.useFakeTimers()
-    vi.setSystemTime(date)
+    it.each([
+      ['semester', semesterToSemesterCode(2017, 'fall'), semesterToSemesterCode(2021, 'spring')],
+      ['year', yearToYearCode(2017), yearToYearCode(2020)],
+    ] as TestList)(
+      'should return correct student numbers for a longer course (MAT21018, 2017-2021) by $0',
+      async (separate, to, from) => {
+        const courseIds = await getRelevantCourseIdMap([MAT21018], false)
+        const res = await findByCourseAndSemesters(
+          Object.keys(courseIds),
+          to,
+          from,
+          separate === 'semester',
+          Unification.UNIFY
+        )
+        const passed = ['457686', '495976', '484997', '491970', '461485', '501442', '508370']
+        const failed = [
+          '484541',
+          '487566',
+          '493344',
+          '520906',
+          '541350',
+          '544750',
+          '550840',
+          '495398',
+          '511089',
+          '538399',
+        ]
+        assert.deepStrictEqual([...res].sort(), passed.concat(failed).sort())
+      }
+    )
 
-    vi.useRealTimers()
-  })
-})
+    it.each([
+      ['semester', semesterToSemesterCode(2021, 'fall'), semesterToSemesterCode(2023, 'spring')],
+      ['year', yearToYearCode(2021), yearToYearCode(2022)],
+    ] as TestList)(
+      'should return correct student numbers for a longer course (MAT21018, 2021-2023) by $0',
+      async (separate, to, from) => {
+        const courseIds = await getRelevantCourseIdMap([MAT21018], false)
+        const res = await findByCourseAndSemesters(
+          Object.keys(courseIds),
+          to,
+          from,
+          separate === 'semester',
+          Unification.UNIFY
+        )
+        const passed = [
+          '509745',
+          '504315',
+          '520805',
+          '509881',
+          '511089',
+          '519527',
+          '522321',
+          '527445',
+          '538399',
+          '534980',
+          '474789',
+          '478837',
+          '482406',
+          '486809',
+          '547552',
+          '493345',
+          '495398',
+          '498558',
+        ]
+        const enrolledNoGrade = ['529866', '488481', '479440', '518062']
 
-type TestList = [string, number, number][]
+        assert.deepStrictEqual([...res].sort(), passed.concat(enrolledNoGrade).sort())
+        assert.strictEqual(res.size, passed.length + enrolledNoGrade.length)
+      }
+    )
 
-const semesterToSemesterCode = (semesterYear: number, springOrFall: 'spring' | 'fall') => {
-  return (semesterYear - 1950) * 2 + (springOrFall === 'spring' ? 0 : 1)
-}
+    it.todo('should return correct student numbers for multiple courses (CSM14204+MAT21018)')
 
-void describe.concurrent('Find by course and semester', () => {
-  it.each([
-    ['semester', semesterToSemesterCode(2019, 'fall'), semesterToSemesterCode(2023, 'spring')],
-    ['year', yearToYearCode(2017), yearToYearCode(2023)],
-  ] as TestList)(
-    'should return correct student numbers for a short course (CSM14204, 2019-2023) by $0',
-    async (separate, to, from) => {
-      const res = (await findByCourseAndSemesters(['CSM14204'], to, from, separate === 'semester', 'unifyStats')).sort()
-      assert.deepStrictEqual(res, ['458079', '478837', '483126'])
-    }
-  )
+    describe('should work with specific cases', () => {
+      it('Student with enrollment in -21 and passed grade in -23 should be included in both stats when querying one year at a time (522321)', async () => {
+        const courseIds = await getRelevantCourseIdMap([MAT21018], false)
+        const res21 = await findByCourseAndSemesters(
+          Object.keys(courseIds),
+          yearToYearCode(2021),
+          yearToYearCode(2021),
+          false,
+          Unification.UNIFY
+        )
+        const res23 = await findByCourseAndSemesters(
+          Object.keys(courseIds),
+          yearToYearCode(2022),
+          yearToYearCode(2022),
+          false,
+          Unification.UNIFY
+        )
 
-  it.each([
-    ['semester', semesterToSemesterCode(2020, 'fall'), semesterToSemesterCode(2023, 'spring')],
-    ['year', yearToYearCode(2020), yearToYearCode(2023)],
-  ] as TestList)('should filter out students outside of 2020-2023 by $0', async (separate, to, from) => {
-    const res = (await findByCourseAndSemesters(['CSM14204'], to, from, separate === 'semester', 'unifyStats')).sort()
-    assert.deepStrictEqual(res, ['478837', '483126'])
-  })
-
-  it.each([
-    ['semester', semesterToSemesterCode(2017, 'fall'), semesterToSemesterCode(2021, 'spring')],
-    ['year', yearToYearCode(2017), yearToYearCode(2020)],
-  ] as TestList)(
-    'should return correct student numbers for a longer course (MAT21018, 2017-2021) by $0',
-    async (separate, to, from) => {
-      const res = (await findByCourseAndSemesters(['MAT21018'], to, from, separate === 'semester', 'unifyStats')).sort()
-      const passed = ['457686', '495976', '484997', '491970', '461485', '501442', '508370']
-      const failed = [
-        '484541',
-        '487566',
-        '493344',
-        '520906',
-        '541350',
-        '544750',
-        '550840',
-        '495398',
-        '511089',
-        '538399',
-      ]
-      assert.deepStrictEqual(res, passed.concat(failed).sort())
-    }
-  )
-
-  it.each([
-    ['semester', semesterToSemesterCode(2021, 'fall'), semesterToSemesterCode(2023, 'spring')],
-    ['year', yearToYearCode(2021), yearToYearCode(2022)],
-  ] as TestList)(
-    'should return correct student numbers for a longer course (MAT21018, 2021-2023) by $0',
-    async (separate, to, from) => {
-      const res = (await findByCourseAndSemesters(['MAT21018'], to, from, separate === 'semester', 'unifyStats')).sort()
-      const passed = [
-        '509745',
-        '504315',
-        '520805',
-        '509881',
-        '511089',
-        '519527',
-        '522321',
-        '527445',
-        '538399',
-        '534980',
-        '474789',
-        '478837',
-        '482406',
-        '486809',
-        '547552',
-        '493345',
-        '495398',
-        '498558',
-      ]
-      const enrolledNoGrade = ['529866', '488481', '479440', '518062']
-
-      assert.deepStrictEqual(res, passed.concat(enrolledNoGrade).sort())
-      assert.strictEqual(res.length, passed.length + enrolledNoGrade.length)
-    }
-  )
-
-  it.todo('should return correct student numbers for multiple courses (CSM14204+MAT21018)')
-
-  describe('should work with specific cases', () => {
-    it('Student with enrollment in -21 and passed grade in -23 should be included in both stats when querying one year at a time (522321)', async () => {
-      const res21 = (
-        await findByCourseAndSemesters(['MAT21018'], yearToYearCode(2021), yearToYearCode(2021), false, 'unifyStats')
-      ).sort()
-      const res23 = (
-        await findByCourseAndSemesters(['MAT21018'], yearToYearCode(2022), yearToYearCode(2022), false, 'unifyStats')
-      ).sort()
-
-      assert.include(res21, '522321', '522321 incorrectly missing in -21 stats')
-      assert.include(res23, '522321', '522321 incorrectly missing in -23 stats')
+        assert.include([...res21], '522321', '522321 incorrectly missing in -21 stats')
+        assert.include([...res23], '522321', '522321 incorrectly missing in -23 stats')
+      })
     })
   })
 })

services/backend/tests/unit/services/courses/getCreditsForCourses.test.ts

@@ -0,0 +1,192 @@
+import { describe, it, assert, vi, afterEach } from 'vitest'
+import { Op } from 'sequelize'
+
+import { CreditTypeCode, Unification } from '@oodikone/shared/types'
+import { Credit } from '@oodikone/shared/models'
+import { createCredit as credit } from '@oodikone/shared/test/utils'
+import { CreditModel } from '@/models'
+import { getCreditsForCourses } from '@/services/courses/creditsAndEnrollmentsOfCourse'
+
+const MAT11002 = 'hy-CU-117375394'
+const MAT21001 = 'hy-CU-117375754'
+const MAT21002 = 'hy-CU-117375793'
+const OTHER101 = 'hy-CU-000000000'
+
+const mockCredits = (credits: Array<Record<string, unknown>>) =>
+  vi.spyOn(CreditModel, 'findAll').mockResolvedValue(credits as any)
+
+const from = new Date(2000, 0, 1)
+const to = new Date(2030, 0, 1)
+
+// Tests share a spy on the same CreditModel.findAll, so they must not run concurrently
+// (the suite config default is sequence.concurrent: true).
+void describe('Get credits for courses (unit, mocked db)', { concurrent: false }, () => {
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('should return no groups when there are no groupId groups to match', async () => {
+    const findAll = mockCredits([credit()])
+
+    const res = await getCreditsForCourses([], [], Unification.UNIFY, from, to)
+
+    assert.deepStrictEqual(res, [])
+    assert.strictEqual(findAll.mock.calls.length, 1)
+  })
+
+  it('should query CreditModel with course ids, date range and openness filters', async () => {
+    const findAll = mockCredits([])
+
+    await getCreditsForCourses([[MAT11002], [MAT21001, MAT21002]], ['course-1', 'course-2'], Unification.OPEN, from, to)
+
+    const call = findAll.mock.calls[0][0] as any
+
+    assert.deepStrictEqual(call.where.course_id, { [Op.in]: ['course-1', 'course-2'] })
+    assert.deepStrictEqual(call.where.attainment_date, { [Op.between]: [from, to] })
+    assert.deepStrictEqual(call.where.is_open, { [Op.eq]: true })
+  })
+
+  it('should include both passed and failed credits for a single-code group as one group', async () => {
+    mockCredits([credit({ credittypecode: CreditTypeCode.FAILED }), credit({ credittypecode: CreditTypeCode.PASSED })])
+
+    const res = await getCreditsForCourses([[MAT11002]], ['course-1'], Unification.UNIFY, from, to)
+
+    assert.strictEqual(res.length, 1)
+    assert.strictEqual(res.flat().length, 2)
+  })
+
+  it('should create a separate group per student for a single-code group', async () => {
+    mockCredits([credit({ student_studentnumber: '111' }), credit({ student_studentnumber: '222' })])
+
+    const res = await getCreditsForCourses([[MAT11002]], ['course-1'], Unification.UNIFY, from, to)
+
+    assert.strictEqual(res.length, 2)
+    assert.deepStrictEqual(res.map(group => group.map(c => c.student_studentnumber)).sort(), [['111'], ['222']])
+  })
+
+  it('should not create a group for a single course code the student has no credit for', async () => {
+    mockCredits([credit({ course_code: 'OTHER101', course: { groupId: OTHER101 } as Credit['course'] })])
+
+    const res = await getCreditsForCourses([[MAT11002]], ['course-1'], Unification.UNIFY, from, to)
+
+    assert.deepStrictEqual(res, [])
+  })
+
+  it('should only include the passed credits of a substitution group when every code was passed', async () => {
+    mockCredits([
+      credit({
+        course_code: 'MAT11002',
+        course: { groupId: MAT11002 } as Credit['course'],
+        credittypecode: CreditTypeCode.PASSED,
+      }),
+      credit({
+        course_code: 'MAT21001',
+        course: { groupId: MAT21001 } as Credit['course'],
+        credittypecode: CreditTypeCode.APPROVED,
+      }),
+    ])
+
+    const res = await getCreditsForCourses(
+      [[MAT11002, MAT21001]],
+      ['course-1', 'course-2'],
+      Unification.UNIFY,
+      from,
+      to
+    )
+
+    assert.strictEqual(res.length, 1)
+    assert.deepStrictEqual(res[0].map(c => c.course_code).sort(), ['MAT11002', 'MAT21001'])
+  })
+
+  it('should exclude a partially completed substitution group entirely', async () => {
+    mockCredits([
+      credit({
+        course_code: 'MAT11002',
+        course: { groupId: MAT11002 } as Credit['course'],
+        credittypecode: CreditTypeCode.PASSED,
+      }),
+      credit({
+        course_code: 'MAT21001',
+        course: { groupId: MAT21001 } as Credit['course'],
+        credittypecode: CreditTypeCode.FAILED,
+      }),
+    ])
+
+    const res = await getCreditsForCourses(
+      [[MAT11002, MAT21001]],
+      ['course-1', 'course-2'],
+      Unification.UNIFY,
+      from,
+      to
+    )
+
+    assert.deepStrictEqual(res, [])
+  })
+
+  it('should exclude an earlier failed attempt from a substitution group even when it later passed', async () => {
+    mockCredits([
+      credit({
+        course_code: 'MAT11002',
+        course: { groupId: MAT11002 } as Credit['course'],
+        credittypecode: CreditTypeCode.FAILED,
+      }),
+      credit({
+        course_code: 'MAT11002',
+        course: { groupId: MAT11002 } as Credit['course'],
+        credittypecode: CreditTypeCode.PASSED,
+      }),
+      credit({
+        course_code: 'MAT21001',
+        course: { groupId: MAT21001 } as Credit['course'],
+        credittypecode: CreditTypeCode.PASSED,
+      }),
+    ])
+
+    const res = await getCreditsForCourses(
+      [[MAT11002, MAT21001]],
+      ['course-1', 'course-2'],
+      Unification.UNIFY,
+      from,
+      to
+    )
+
+    assert.strictEqual(res.length, 1)
+    assert.strictEqual(res.flat().length, 2)
+    assert(
+      res[0].every(c => c.credittypecode !== CreditTypeCode.FAILED),
+      'failed attempt should not be included in the completed substitution group'
+    )
+  })
+
+  it('should handle independent groups and students together', async () => {
+    mockCredits([
+      credit({
+        student_studentnumber: '111',
+        course_code: 'MAT11002',
+        course: { groupId: MAT11002 } as Credit['course'],
+      }),
+      credit({
+        student_studentnumber: '222',
+        course_code: 'MAT21001',
+        course: { groupId: MAT21001 } as Credit['course'],
+        credittypecode: CreditTypeCode.PASSED,
+      }),
+      credit({
+        student_studentnumber: '222',
+        course_code: 'MAT21002',
+        course: { groupId: MAT21002 } as Credit['course'],
+        credittypecode: CreditTypeCode.PASSED,
+      }),
+    ])
+
+    const res = await getCreditsForCourses(
+      [[MAT11002], [MAT21001, MAT21002]],
+      ['course-1', 'course-2', 'course-3'],
+      Unification.UNIFY,
+      from,
+      to
+    )
+
+    assert.strictEqual(res.length, 2)
+  })
+})

services/backend/tests/unit/services/courses/parseCredit.test.ts

@@ -0,0 +1,61 @@
+import { describe, it, assert } from 'vitest'
+
+import { CreditTypeCode } from '@oodikone/shared/types'
+import { Credit } from '@oodikone/shared/models'
+import { createCredit as credit, createStudyRightElementModel } from '@oodikone/shared/test/utils'
+import { parseCredit } from '@/services/courses'
+import { SISStudyRightElementModel } from '@/models'
+
+const studyRightElement = (...args: Parameters<typeof createStudyRightElementModel>) =>
+  createStudyRightElementModel(...args) as SISStudyRightElementModel
+
+void describe('parseCredit', () => {
+  it('should parse a single passed credit', () => {
+    const res = parseCredit([credit()], null, 'MAT11002', [studyRightElement()])
+
+    assert.strictEqual(res.courseCode, 'MAT11002')
+    assert.strictEqual(res.grade, '5')
+    assert.strictEqual(res.credits, 5)
+    assert.strictEqual(res.passed, true)
+    assert.strictEqual(res.studentNumber, '111')
+    assert.strictEqual(res.programme.code, 'KH50_005')
+  })
+
+  it('should mark a failed single credit as not passed', () => {
+    const res = parseCredit([credit({ credittypecode: CreditTypeCode.FAILED })], null, 'MAT11002', [
+      studyRightElement(),
+    ])
+
+    assert.strictEqual(res.passed, false)
+  })
+
+  it('should combine a substitution group into one credit with summed credits', () => {
+    const res = parseCredit(
+      [
+        credit({ course_code: 'MAT11002', course: { groupId: 'hy-CU-117375394' } as Credit['course'], credits: 5 }),
+        credit({ course_code: 'MAT21001', course: { groupId: 'hy-CU-117375754' } as Credit['course'], credits: 5 }),
+      ],
+      null,
+      'MAT11002',
+      [studyRightElement()]
+    )
+
+    assert.strictEqual(res.courseCode, 'MAT11002')
+    assert.strictEqual(res.grade, 'substituted')
+    assert.strictEqual(res.credits, 10)
+    assert.strictEqual(res.passed, true)
+  })
+
+  it('should anonymize the student number when a salt is given', () => {
+    const res = parseCredit([credit()], 'some-salt', 'MAT11002', [studyRightElement()])
+
+    assert.notStrictEqual(res.studentNumber, '111')
+    assert.match(res.studentNumber, /^[a-f0-9]{64}$/)
+  })
+
+  it('should fall back to programme OTHER when no matching study right element is found', () => {
+    const res = parseCredit([credit()], null, 'MAT11002', [])
+
+    assert.deepStrictEqual(res.programme.code, 'OTHER')
+  })
+})

services/backend/tests/unit/services/populations/shared/parseDateRangeFromParams.test.ts

@@ -1,5 +1,5 @@
 import dayjs from 'dayjs'
-import { parseDateRangeFromParams } from '../../../../../src/services/populations/shared'
+import { parseDateRangeFromParams } from '@/services/populations/shared'
 import { describe, it, assert } from 'vitest'
 
 void describe('parseDateRangeFromParams', () => {

services/backend/tests/unit/services/studyProgramme/studyProgrammeHelpers.test.ts

@@ -1,9 +1,5 @@
 import { describe, it, assert, vi, beforeAll } from 'vitest'
-import {
-  getPercentage,
-  getYearsArray,
-  getYearsObject,
-} from '../../../../src/services/studyProgramme/studyProgrammeHelpers'
+import { getPercentage, getYearsArray, getYearsObject } from '@/services/studyProgramme/studyProgrammeHelpers'
 
 void describe('Get years object', () => {
   it('should return nothing with empty years', () => {

services/backend/tests/unit/services/studyProgramme/studyTrackStats/combineStats.test.ts

@@ -1,6 +1,6 @@
 import { describe, it, assert } from 'vitest'
-import { combineStats } from '../../../../../src/services/studyProgramme/studyTrackStats'
-import { getPercentage } from '../../../../../src/services/studyProgramme/studyProgrammeHelpers'
+import { combineStats } from '@/services/studyProgramme/studyTrackStats'
+import { getPercentage } from '@/services/studyProgramme/studyProgrammeHelpers'
 
 const constructStats = (mainStatsByYear = {}, mainStatsByTrack = {}, otherCountriesCount = {}) => {
   return {

services/backend/tests/unit/utils/index/hasFullAccessToStudentData.test.ts

@@ -1,4 +1,4 @@
-import { hasFullAccessToStudentData } from '../../../../src/util'
+import { hasFullAccessToStudentData } from '@/util'
 import { describe, it, assert } from 'vitest'
 
 void describe('hasFullAccessToStudentData', () => {

services/backend/tests/utils.ts

@@ -1,10 +1,10 @@
 import express from 'express'
 import { Response } from 'supertest'
 
-import { baseUrl, silentTesting } from '../src/config'
-import { initializeDatabaseConnection, dbConnections } from '../src/database/connection'
-import routes from '../src/routes'
-import logger from '../src/util/logger'
+import { baseUrl, silentTesting } from '@/config'
+import { initializeDatabaseConnection, dbConnections } from '@/database/connection'
+import routes from '@/routes'
+import logger from '@/util/logger'
 
 /** Override Supertest's Response body with our own type */
 export type ResponseWithBody<T> = Omit<Response, 'body'> & { body: T }

services/backend/tsconfig.json

@@ -9,6 +9,7 @@
     "emitDecoratorMetadata": true,
     "outDir": "dist",
     "paths": {
+      "@/*": ["./src/*"],
       "*": ["./*"]
     },
     "esModuleInterop": true,

services/backend/vitest.config.ts

@@ -1,6 +1,12 @@
+import path from 'path'
 import { defineConfig } from 'vitest/config'
 
 export default defineConfig({
+  resolve: {
+    alias: {
+      '@': path.resolve(__dirname, './src'),
+    },
+  },
   test: {
     reporters: ['tree'],
     exclude: ['./dist', 'node_modules'],

services/frontend/dev.Dockerfile

@@ -1,6 +1,9 @@
 FROM registry.access.redhat.com/ubi10/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
+# Default install location is node_modules/.vite which is not writable
+#  by user 1001 when binding the local files to the container
+ENV VITE_CACHE_DIR=/tmp/vite
 
 WORKDIR /opt/app-root/shared
 COPY ./shared .

services/frontend/src/common/InfoToolTips/studyProgramme.ts

@@ -190,16 +190,31 @@ However this is optional and you can create a new tag without selecting year.
   },
 
   degreeCoursesTab: {
-    creditCriteria: `
-Here you can change visibility of degree courses as and set course and credits criteria, for each year their
-own. Credits criteria is computed as follows: for the first academic year the credits are taken into account
-if they are completed during the first 12 months. For the second year, we take into account the completions
-during the first 24 months, for the third year the first 36 months.
+    creditCriteria: {
+      en: `The criteria set here will affect the Class Statistics → Students tab → Progress tab view.
 
-The progress of the students by these criteria will be shown in class statistics view.
+Credit criteria is computed as follows: for the first academic year, the credits are counted
+if they are completed during the first 12 months. For the second year, completions
+within the first 24 months. Third year, the first 36 months, and so on.
 `,
-  },
+      fi: `Tässä asetettu "criteria" vaikuttaa Class Statistics → Students tab → Progress tab näkymään.
 
+Laskenta toimii seuraavasti: ensimmäisen akateemisen vuoden opintopisteet lasketaan, jos ne ovat suoritettu ensimmäisen 12 kuukauden ajalla. Toisen vuoden opintopisteet, jos ne ovat suoritettu ensimmäisen 24 kuukauden aikana. Kolmas, ensimmäiset 36kk, ja niin edelleen.
+`,
+    },
+    programmeCriteriaAndVisibility: {
+      en: `Click the arrow next to the module to view courses belonging to the module in the curriculum. Setting a criterion label for a course will affect which academic year the course is placed under in Class Statistics → Students → Progress tab.
+
+Courses' visibilities can also be toggled here; hiding a course here will additionally hide it in the Courses of Class view and Students → Courses tab.
+
+Note: If hiding a module, all courses belonging to the module will also be hidden.`,
+      fi: `Klikkaa nuolta opintokokonaisuuden nimen vieressä näyttääksesi opetussuunitelmassa opintokokonaisuuden alle sijoitetut kurssit. Kursseille voi asettaa "Criterion label":eita, jotka vaikuttavat minkä akateemisen vuoden alla kurssi näytetään Class Statistics → Students → Progress tab näkymässä.
+
+Kurssien näkyvyyttä voidaan myös muuttaa tällä sivulla. Kurssin piilottaminen piilottaa sen myös Courses of Class ja Students → Courses tab näkymissä.
+
+Huom: Opintokokonaisuuden piilottaminen piilottaa myös kaikki sen alla olevat kurssit.`,
+    },
+  },
   // Not used
   notCompleted: `
 Sisältää opiskelijat, jotka ovat ilmoittautuneet kurssille, mutta eivät ole suorittaneet sitä ja opiskelijat, jotka ovat saaneet hylätyn arvosanan.

services/frontend/src/common/index.ts

@@ -106,28 +106,29 @@ export const getTextInWithOpen = (
   return courseName
 }
 
-export const getUnifyTextIn = (unifyCourses: string) => {
+export const getUnifyTextIn = (unifyCourses?: string) => {
   switch (unifyCourses) {
     case 'regularStats':
-      return 'Normal'
+      return 'normal'
     case 'openStats':
-      return 'Open'
+      return 'open'
     case 'unifyStats':
-      return 'Open and normal'
+      return 'open and normal'
     default:
       return ''
   }
 }
 
-// Gives students course completion date
-export const getStudentToTargetCourseDateMap = (
-  students: Pick<FormattedStudent, 'studentNumber' | 'courses'>[],
-  codes: string[]
-) => {
-  const uniqueCourses = new Set(codes)
+export const getStudentToCourseCompletionDateTimeMap = (
+  students: Pick<FormattedStudent, 'studentNumber' | 'courses'>[] | undefined,
+  courseIds: string[] | undefined
+): Record<string, Date | null> => {
+  if (!students?.length || !courseIds?.length) return {}
+
+  const uniqueCourseIds = new Set(courseIds)
   return students.reduce((acc, student) => {
     const targetCourse = student.courses
-      .filter(course => uniqueCourses.has(course.course_code))
+      .filter(course => uniqueCourseIds.has(course.course_id))
       .sort((a, b) => +new Date(b.date) - +new Date(a.date))[0]
     acc[student.studentNumber] = targetCourse ? targetCourse.date : null
     return acc
@@ -393,7 +394,7 @@ export const languageAbbreviations = {
 
 export const showAsUserKey = 'showAsUser'
 
-export const getEnrollmentTypeTextForExcel = (type: number, statutoryAbsence?: boolean) => {
+export const getEnrollmentTypeTextForExcel = (type: EnrollmentType | undefined, statutoryAbsence: boolean | undefined) => {
   if (type === 1) return 'Present'
   if (type === 2 && statutoryAbsence) return 'Absent (statutory)'
   if (type === 2) return 'Absent'

services/frontend/src/components/AccessDenied/index.tsx

@@ -6,7 +6,7 @@ import { ErrorBackground } from '@/components/ErrorBoundary/ErrorBackground'
 export const AccessDenied = () => {
   const [easterEggVisible, setEasterEggVisible] = useState(false)
   useEffect(() => {
-    setTimeout(() => setEasterEggVisible(true), Math.floor(Math.random() * 25000 + 5000))
+    setTimeout(() => setEasterEggVisible(true), Math.floor(Math.random() * (20 * 60 * 1000) + 30 * 1000))
   }, [])
 
   const header = 'Welcome to Oodikone!'

services/frontend/src/components/ColorizedCoursesTable/FacultiesTab/index.tsx

@@ -52,9 +52,7 @@ export const FacultiesTab = () => {
       const facultiesTotal = { ...emptyFields }
       selectedSemesters.forEach(semestercode => {
         data.faculties.forEach(faculty => {
-          if (!course.bySemesters.cellStats[faculty]) {
-            course.bySemesters.cellStats[faculty] = { ...emptyFields }
-          }
+          course.bySemesters.cellStats[faculty] ??= { ...emptyFields }
           const stats = course.bySemesters[semestercode]?.[faculty]
           if (!stats) return
           course.bySemesters.cellStats[faculty].completions += stats.completions

services/frontend/src/components/ColorizedCoursesTable/SemestersTab/logic.jsx

@@ -34,11 +34,11 @@ export const useColumns = (getTextIn, semesters, numberMode, colorMode, allTotal
               sx={{
                 'td:has(> &)':
                   colorMode !== 'course'
-                    ? getColor(row.original[numberMode], semesters.length, colorMode, numberMode, null, allTotal)
+                    ? getColor(row.original.bySemesters, semesters.length, colorMode, numberMode, null, allTotal)
                     : {},
               }}
             >
-              {row.original[numberMode] ?? 0}
+              {row.original.bySemesters[numberMode] ?? 0}
             </Box>
           )
         },
@@ -56,6 +56,8 @@ export const useColumns = (getTextIn, semesters, numberMode, colorMode, allTotal
                 ),
               0
             ),
+        sortingFn: (rowA, rowB) =>
+          (rowA.original.bySemesters[numberMode] ?? 0) - (rowB.original.bySemesters[numberMode] ?? 0),
       }),
       ...semesters.map(({ name, semestercode }) =>
         columnHelper.accessor(() => undefined, {