[E2E] Fix test infobox wording

thanks claude opus 5.5 high

.claude/settings.json

@@ -5,7 +5,8 @@
       "Read(./.databasedumps/**)",
       "Read(**/*.sqz)",
       "Read(**/*.sql)",
-      "Bash(docker exec *-db *)"
+      "Bash(docker exec *-db *)",
+      "Bash(*psql *)"
     ]
   }
 }

.env.template

@@ -3,4 +3,8 @@
 # `node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"`
 IMPORTER_DB_API_TOKEN=
 PATE_TOKEN=
-CRYPT_KEY=
+CRYPT_KEY=
+# Changes the user and group docker runs as. Use "id -u" and "id -g" to run containers as current user+group.
+# Fixes errors with permissions when mounting.
+DOCKER_UID=
+DOCKER_GID=

.github/actions/test/updater-unit/action.yaml

@@ -0,0 +1,18 @@
+name: 'Run updater unit tests'
+description: 'Runs unit tests in sis-updater-worker'
+
+runs:
+  using: composite
+  steps:
+    - name: Install node-24
+      uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - name: Install dependencies
+      shell: bash
+      run: npm ci --prefix ./updater/sis-updater-worker
+
+    - name: Run updater unit tests
+      shell: bash
+      run: npm run test:unit --prefix ./updater/sis-updater-worker

.github/workflows/production.yaml

@@ -133,7 +133,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: 'Send notification to Slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.6.1
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -155,7 +155,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: 'Send notification to Slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.6.1
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure

.github/workflows/pull_request.yaml

@@ -26,3 +26,5 @@ jobs:
     secrets:
       username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
       password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      s3_access_key: ${{ secrets.S3_ACCESS_KEY }}
+      s3_secret_key: ${{ secrets.S3_SECRET_KEY }}

.github/workflows/staging.yaml

@@ -23,6 +23,8 @@ jobs:
     secrets:
       username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
       password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      s3_access_key: ${{ secrets.S3_ACCESS_KEY }}
+      s3_secret_key: ${{ secrets.S3_SECRET_KEY }}
 
   release_backend:
     name: 'Build and release backend'
@@ -110,7 +112,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: 'Send notification to Slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.6.1
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -131,7 +133,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: 'Send notification to Slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.6.1
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure

.github/workflows/test.yaml

@@ -15,6 +15,12 @@ on:
       password:
         required: true
         description: 'Toska registry password'
+      s3_access_key:
+        required: true
+        description: 'S3 access key for the test database dumps'
+      s3_secret_key:
+        required: true
+        description: 'S3 secret key for the test database dumps'
 
 jobs:
   # === Static tests / linting ===
@@ -95,6 +101,7 @@ jobs:
       - build_frontend
       - lint_packages
       - test_unit
+      - test_updater_unit
 
     steps:
       - uses: actions/checkout@v7
@@ -106,11 +113,14 @@ jobs:
           password: ${{ secrets.password }}
 
       - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
+        env:
+          ACCESS_KEY: ${{ secrets.s3_access_key }}
+          SECRET_KEY: ${{ secrets.s3_secret_key }}
         run: |
           docker compose --file docker-compose.ci.yml up --detach --quiet-pull sis-updater-worker
           sleep 30 # wait for sis-updater-worker to start and run migrations
-          # TODO: Disable this until fixed
-          # ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
+          sudo apt update -y && sudo apt install s3cmd -y
+          ./anonyymioodi/manage_test_dbs.sh release sis-db
 
       - name: Teardown containers started by compose
         if: always()
@@ -130,6 +140,7 @@ jobs:
       - build_frontend
       - lint_packages
       - test_unit
+      - test_updater_unit
 
     steps:
       - uses: actions/checkout@v7
@@ -142,11 +153,14 @@ jobs:
           password: ${{ secrets.password }}
 
       - name: 'Let backend run migrations, then push new version of image to toska hub'
+        env:
+          ACCESS_KEY: ${{ secrets.s3_access_key }}
+          SECRET_KEY: ${{ secrets.s3_secret_key }}
         run: |
           docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
           sleep 30 # wait for backend to start and run migrations
-          # TODO: Disable this until fixed
-          # ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
+          sudo apt update -y && sudo apt install s3cmd -y
+          ./anonyymioodi/manage_test_dbs.sh release kone-db
 
       - name: Teardown containers started by compose
         if: always()
@@ -166,6 +180,7 @@ jobs:
       - build_frontend
       - lint_packages
       - test_unit
+      - test_updater_unit
 
     steps:
       - uses: actions/checkout@v7
@@ -178,11 +193,14 @@ jobs:
           password: ${{ secrets.password }}
 
       - name: 'Let backend run migrations, then push new version of image to toska hub'
+        env:
+          ACCESS_KEY: ${{ secrets.s3_access_key }}
+          SECRET_KEY: ${{ secrets.s3_secret_key }}
         run: |
           docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
           sleep 30 # wait for backend to start and run migrations
-          # TODO: Disable this until fixed
-          # ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
+          sudo apt update -y && sudo apt install s3cmd -y
+          ./anonyymioodi/manage_test_dbs.sh release user-db
 
       - name: Teardown containers started by compose
         if: always()
@@ -249,10 +267,15 @@ jobs:
           EXIT_AFTER_IMMEDIATES: yes
 
       # TODO: Push the updated sis-db to toska-hub when updater is deterministic with test-data
-      # - name: Push newly build db to toska hub
-      #   if: |
-      #   success() && inputs.skip_db_update != 'true'
-      #   run: ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
+      - name: Push newly build db to toska hub
+        if: |
+          success() && inputs.skip_db_update != 'true'
+        env:
+          ACCESS_KEY: ${{ secrets.s3_access_key }}
+          SECRET_KEY: ${{ secrets.s3_secret_key }}
+        run: |
+          sudo apt update -y && sudo apt install s3cmd -y
+          ./anonyymioodi/manage_test_dbs.sh release sis-db
 
       - name: Dump logs on failure
         if: failure()
@@ -315,6 +338,19 @@ jobs:
       - name: Run unit tests (frontend and backend)
         uses: ./.github/actions/test/unit
 
+  # === Updater unit testing ===
+  test_updater_unit:
+    name: 'Run updater unit tests'
+    runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
+    timeout-minutes: 15
+    if: |
+      !cancelled() && !failure()
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Run updater unit tests
+        uses: ./.github/actions/test/updater-unit
+
   # === Frontend component testing ===
   test_components:
     name: 'Test frontend components'
@@ -346,35 +382,35 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: 'Test population statistics'
+          - name: 'Population statistics'
             spec: e2e/Population_statistics.spec.ts
-          - name: 'Test course statistics'
+          - name: 'Course statistics'
             spec: e2e/Course_statistics.spec.ts
-          - name: 'Test degree programme overview'
+          - name: 'Degree programme overview'
             spec: e2e/Degree_programme_overview.spec.ts
-          - name: 'Test Faculty statistics'
+          - name: 'faculty statistics'
             spec: e2e/Faculty_statistics.spec.ts
-          - name: 'Test teachers'
+          - name: 'Teachers'
             spec: e2e/Teachers.spec.ts
-          - name: 'Test users'
+          - name: 'Users'
             spec: e2e/Users.spec.ts
-          - name: 'Test access rights'
+          - name: 'Access rights'
             spec: e2e/Routes.spec.ts
-          - name: 'Test university'
+          - name: 'University'
             spec: e2e/University.spec.ts
-          - name: 'Test language center view'
+          - name: 'Language center view'
             spec: e2e/Language_center.spec.ts
-          - name: 'Test study guidance groups'
+          - name: 'Study guidance groups'
             spec: e2e/Study_guidance_groups.spec.ts
-          - name: 'Test custom population'
+          - name: 'Custom population'
             spec: e2e/Custom_population.spec.ts
-          - name: 'Test filters'
+          - name: 'Filters'
             spec: e2e/Filters.spec.ts
-          - name: 'Test students'
+          - name: 'Students'
             spec: e2e/Students.spec.ts
-          - name: 'Test completed courses'
+          - name: 'Completed courses'
             spec: e2e/Completed_courses.spec.ts
-          - name: 'Test numbers match'
+          - name: 'Numbers match'
             spec: e2e/Numbers_match.spec.ts
     name: ${{ matrix.name }}
     steps:

.github/workflows/updater.yaml

@@ -23,6 +23,15 @@ jobs:
       - name: 'Run linter'
         run: npm run lint -- updater
 
+  test_updater_unit:
+    name: 'Run updater unit tests'
+    runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v7
+      - name: Run updater unit tests
+        uses: ./.github/actions/test/updater-unit
+
   build_worker:
     name: 'Build Updater worker'
     runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
@@ -79,6 +88,7 @@ jobs:
     timeout-minutes: 15
     needs:
       - lint_scheduler_and_worker
+      - test_updater_unit
       - build_worker
       - build_scheduler
 

anonyymioodi/README.md

@@ -4,6 +4,30 @@ This folder contains a guide and scripts to modify or create anonymous database
 
 The quick guide should be enough for most purposes: updating schema, data, or the PostgreSQL version. If you need to generate completely new `sis-importer-db` and `sis-db` images (which shouldn't happen too often), follow the instructions in the section [Generate new sis-importer-db and sis-db completely from scratch](#generate-new-sis-importer-db-and-sis-db-completely-from-scratch).
 
+## Scripts
+
+`manage_test_dbs.sh` is the front end for the test database tooling. Run it without arguments for an interactive menu, or give it an action and one or more databases (or `all`) to run it directly:
+
+```bash
+./manage_test_dbs.sh dump sis-db      # dump sis-db from its running container
+./manage_test_dbs.sh upload sis-db    # upload the current dump to s3
+./manage_test_dbs.sh release all      # dump, upload and push images for every database
+```
+
+Actions are `download`, `dump`, `upload` (upload + prune), `push`, `prune`, `release` (dump + upload + prune + push) and `rebuild` (download + push). Each one is a thin call to a script in this folder, which can also be used on its own:
+
+| Script                     | What it does                                                                                                           |
+| -------------------------- | ---------------------------------------------------------------------------------------------------------------------- |
+| `dump_local_db.sh`         | `pg_dump`s the databases from the running containers into `.databasedumps/`                                            |
+| `download_dump_from_s3.sh` | Downloads the newest dump of each database from s3 (`-b <branch>` to pick dumps from a single branch)                  |
+| `upload_dump_to_s3.sh`     | Uploads the dumps in `.databasedumps/test` to s3 as `<database>-<git_branch>-<datetime>.sql`                           |
+| `prune_s3_dumps.sh`        | Deletes dumps older than 30 days from s3, always keeping the newest `master` dump of each database (`-d <days>`, `-n`) |
+| `build_and_push_image.sh`  | Builds database images from the current dumps and pushes them to the Toska registry                                    |
+| `create_s3_bucket.sh`      | Creates the s3 bucket if it doesn't exist yet, run automatically before an upload                                      |
+| `lib/common.sh`            | Shared configuration (dump directories, s3 bucket, registry, database names)                                           |
+
+Dumps end up in `.databasedumps/test/<database>.sql`, which is where `db.Dockerfile` reads them from when an image is built. The s3 commands read their credentials from the `ACCESS_KEY` and `SECRET_KEY` environment variables (e.g. in CI). If neither is set, they fall back to `~/.s3cfg`, which can be found on the toska/dokumentaatio repo.
+
 ## Quick guide to updating a test database
 
 1. Ensure you have the latest test-db. Just pulling images might not replace your existing volume, so you may need to remove it manually. This command might work: `docker volume rm oodikone-sis-db-data` (replace `oodikone-sis-db-data` with the correct volume name). The new image, including data, should be pulled the next time you launch Oodikone normally.

anonyymioodi/build_and_push_image.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Build database images from the dumps in .databasedumps/test and push them to
+# the Toska registry.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0") <database>... | all
+
+Builds an image of each given database and pushes it to $REGISTRY.
+The dump is read from .databasedumps/test/<database>.sql, so run
+dump_local_db.sh or download_dump_from_s3.sh first.
+
+Databases: ${DATABASES[*]}
+EOF
+}
+
+if [[ "${1-}" == "-h" || "${1-}" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+if [[ $# -eq 0 ]]; then
+  usage
+  die "No database given"
+fi
+
+build_and_push_image() {
+  local database="$1"
+  local image="$REGISTRY/$database:latest"
+  local dump_file="$TEST_DUMP_DIR/$database.sql"
+
+  [[ -f "$dump_file" ]] || die "No dump found at $dump_file, run dump_local_db.sh or download_dump_from_s3.sh first"
+
+  infomsg "Building $image"
+  docker build "$PROJECT_ROOT" -f "$PROJECT_ROOT/db.Dockerfile" -t "$image" --build-arg DB_NAME="$database"
+
+  infomsg "Pushing $image"
+  docker push "$image"
+  successmsg "Pushed $image"
+}
+
+resolve_databases "$@"
+
+for database in "${selected_databases[@]}"; do
+  build_and_push_image "$database"
+done

anonyymioodi/create_s3_bucket.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Create the s3 bucket used for the test database dumps, if it doesn't exist yet.
+# Doing nothing when the bucket is already there, so this is safe to run every time.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0")
+
+Creates the bucket $S3_BUCKET if it doesn't exist yet, and does nothing if it does.
+EOF
+}
+
+if [[ "${1-}" == "-h" || "${1-}" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+require_s3_config
+
+# s3cmd info exits with an error when the bucket doesn't exist
+if s3 info "$S3_BUCKET" > /dev/null 2>&1; then
+  infomsg "Bucket $S3_BUCKET already exists"
+  exit 0
+fi
+
+warningmsg "Bucket $S3_BUCKET doesn't exist, creating it"
+s3 mb "$S3_BUCKET"
+successmsg "Created bucket $S3_BUCKET"

anonyymioodi/db_util.sh

@@ -1,214 +0,0 @@
-#!/usr/bin/env bash
-
-# === Config ===
-
-# Quit if failing
-set -euo pipefail
-
-# Print messages and logs that are not script output to stderr
-msg() {
-    echo >&2 -e "${1-}"
-}
-
-# Some special types of messages with colours
-successmsg() {
-  msg "${GREEN}$1${NOFORMAT}"
-}
-errormsg() {
-  msg "${RED}$1${NOFORMAT}"
-}
-infomsg() {
-  msg "${BLUE}$1${NOFORMAT}"
-}
-warningmsg() {
-  msg "${ORANGE}$1${NOFORMAT}"
-}
-
-# Setup colors for messages if running interactive shell
-if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
-  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
-else
-  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
-fi
-
-
-# === #
-
-PROJECT_ROOT="$(git rev-parse --show-toplevel)"
-
-get_project_relative_path () {
-  local absolute_path="$1"
-  local root="$2"
-  realpath -s --relative-to="${root:-$PROJECT_ROOT}" "$absolute_path"
-}
-
-LOCAL_DUMP_DIR="$(get_project_relative_path ".databasedumps/local_containers" "$PWD")" # Dumps from running docker containers
-S3_DUMP_DIR="$(get_project_relative_path ".databasedumps/s3" "$PWD")" # Dumps from s3
-TEST_DUMP_DIR="$(get_project_relative_path ".databasedumps/test" "$PWD")" # Location where .sql files are pulled while building the new image
-
-S3_CONFIG_FILE=~/.s3cfg
-S3_BUCKET="s3://oodikone-test"
-
-REGISTRY="registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi"
-
-## Following the naming convention in docker compose, these are names for database
-## services. The real data databases inside service will have names suffixed by "-real".
-KONE_DB_NAME="kone-db"
-SIS_DB_NAME="sis-db"
-SIS_IMPORTER_DB_NAME="sis-importer-db"
-USER_DB_NAME="user-db"
-JAMI_DB_NAME="jami-db"
-DATABASES=("$USER_DB_NAME" "$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$JAMI_DB_NAME")
-
-mkdir -p "$TEST_DUMP_DIR"
-mkdir -p "$S3_DUMP_DIR"
-mkdir -p "$LOCAL_DUMP_DIR"
-
-
-# === Handle s3 bucket ===
-# Use s3 bucket as the dump save/pull location
-download_s3_db () {
-  local database="$1"
-
-  # TODO: Decompress compressed files
-  s3cmd -c "$S3_CONFIG_FILE" get "$S3_BUCKET/$database.sql" "$S3_DUMP_DIR/$database.sql"
-  cp --remove-destination -r "$S3_DUMP_DIR" "$TEST_DUMP_DIR" # Move dumps to the place where docker build pulls .sql files from
-}
-
-upload_s3_db () {
-  local database="$1"
-
-  if [[ $(s3cmd ls s3:// | grep -q oodikone-test; echo $?) -eq 0 ]]
-    then
-      infomsg "OODIKONE-TEST bucket exists, skipping creation";
-    else
-      warningmsg "OODIKONE-TEST bucket doesn't exist, creating new bucket...";
-      s3cmd -c "$S3_CONFIG_FILE" mb "s3://oodikone-test"
-  fi
-
-  infomsg "Pushing $TEST_DUMP_DIR/$database.sql to $S3_BUCKET"
-  # TODO: Also compress files
-  s3cmd -c "$S3_CONFIG_FILE" put "$TEST_DUMP_DIR/$database.sql" "$S3_BUCKET/$database.sql"
-}
-
-
-# === Build images ===
-# Update registry with an existing dump located in .databasedumps/test/$1.sql
-build_and_push_image () {
-  local database="$1"
-
-  infomsg "Building docker image for $database\n"
-  docker build "$(get_project_relative_path "$PROJECT_ROOT" "$PWD")" -t "$REGISTRY/$database:latest" -f db.Dockerfile --build-arg DB_NAME="$database"
-  successmsg "$database docker image built"
-  infomsg "Pushing $database to $REGISTRY\n"
-  infomsg "$REGISTRY/$database"
-  docker push "$REGISTRY/$database"
-  successmsg "Image $REGISTRY/$database pushed successfully"
-}
-
-
-# === Handle dumping local dbs ===
-# To conveniently update all dbs, build all with docker-compose.db.yml and then run dump_all_local_dbs
-dump_local_db () {
-  local database="$1"
-  local container_name="oodikone-$1"
-  local database_name="$database"
-
-  if [[ $database == "jami-db" ]]; then database_name="postgres"; fi
-
-  infomsg "Trying to dump $database to $LOCAL_DUMP_DIR/$database.sql"
-  docker exec -i "$container_name" pg_dump -Fp -U postgres -d "$database_name" > "$LOCAL_DUMP_DIR/$database.sql"
-  successmsg "Dumped $database to $LOCAL_DUMP_DIR/$database.sql"
-  cp --remove-destination -r "$LOCAL_DUMP_DIR/$database.sql" "$TEST_DUMP_DIR" # Move dump to the place where docker build pulls the .sql file from
-}
-
-
-# === Helpers ===
-do_for_all () {
-  local cmd="$1"
-  shift # Removes $1 from "$@"
-
-  infomsg "Running $cmd for all databases"
-  for database in "${DATABASES[@]}"; do
-    "$cmd" "$database" "$@"
-  done
-  successmsg "$cmd ran successfully"
-}
-
-ask_for_input () {
-  local cmd="$1"
-  shift # Removes $1 from "$@"
-
-  echo
-
-  init_database_menu
-  select database in "${options[@]}"; do
-    "$cmd" "$database" "$@"
-    break
-  done
-}
-
-
-init_action_menu () {
-  PS3="Please enter your choice: "
-
-  options=(
-    "Pull dumps from s3"
-    "Dump from running docker images"
-    "Update dumps in s3"
-    "Push new images to registry"
-  )
-}
-
-init_mode_menu () {
-  PS3="Select command mode: "
-  options=(
-    "Run command for all databases"
-    "Select single database"
-  )
-}
-
-init_database_menu () {
-  PS3="Select database: "
-  options=("${DATABASES[@]}")
-}
-
-
-# === RUN ===
-while true; do
-  init_mode_menu
-  select mode in "${options[@]}"; do
-    case "$mode" in
-      "Run command for all databases")
-        cmd="do_for_all"
-        break;;
-      "Select single database")
-        cmd="ask_for_input"
-        break;;
-    esac
-  done
-
-  echo
-
-  echo
-
-  init_action_menu
-  select opt in "${options[@]}"; do
-    case "$opt" in
-      "Pull dumps from s3")
-        "$cmd" download_s3_db "$S3_DUMP_DIR"
-        break;;
-      "Dump from running docker images")
-        "$cmd" dump_local_db "$LOCAL_DUMP_DIR"
-        break;;
-      "Update dumps in s3")
-        "$cmd" upload_s3_db
-        break;;
-      "Push new images to registry")
-        "$cmd" build_and_push_image
-        break;;
-    esac
-  done
-
-  echo
-done

anonyymioodi/download_dump_from_s3.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+# Download database dumps from s3 into .databasedumps/s3, and copy them to
+# .databasedumps/test where db.Dockerfile reads them from.
+#
+# Dumps are uploaded as <db_name>-<git_branch>-<datetime>.sql (see
+# upload_dump_to_s3.sh), so the newest matching dump is the one downloaded.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0") [-b|--branch <branch>] <database>... | all
+
+Downloads the newest dump of each given database from $S3_TARGET.
+Databases: ${DATABASES[*]}
+
+Options:
+  -b, --branch <branch>  Only consider dumps uploaded from this git branch
+  -h, --help             Show this help
+EOF
+}
+
+branch=""
+arguments=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -b | --branch)
+      branch="${2-}"
+      [[ -n "$branch" ]] || {
+        usage
+        die "Missing value for $1"
+      }
+      shift 2
+      ;;
+    -h | --help)
+      usage
+      exit 0
+      ;;
+    -*)
+      usage
+      die "Unknown option: $1"
+      ;;
+    *)
+      arguments+=("$1")
+      shift
+      ;;
+  esac
+done
+
+# Newest object in s3 for the given database, or empty if there are none
+newest_dump_in_s3() {
+  local database="$1"
+  local prefix="$S3_TARGET/$database-"
+  if [[ -n "$branch" ]]; then prefix="$prefix$(sanitize_branch_name "$branch")-"; fi
+
+  # s3cmd ls prints "<date> <time> <size> <object>", so sorting by the first two
+  # columns in reverse puts the newest dump first
+  s3 ls "$prefix" | sort -rk1,2 | awk 'NR == 1 { print $4 }'
+}
+
+download_database() {
+  local database="$1"
+  local dump_file="$S3_DUMP_DIR/$database.sql"
+  local newest_dump
+  newest_dump="$(newest_dump_in_s3 "$database")"
+
+  [[ -n "$newest_dump" ]] || die "No dumps found for $database in $S3_TARGET${branch:+ (branch $branch)}"
+
+  infomsg "Downloading $newest_dump"
+  s3 get --force "$newest_dump" "$dump_file"
+  cp --remove-destination "$dump_file" "$TEST_DUMP_DIR/$database.sql"
+  successmsg "Downloaded $newest_dump to $dump_file"
+}
+
+if [[ ${#arguments[@]} -eq 0 ]]; then
+  usage
+  die "No database given"
+fi
+
+resolve_databases "${arguments[@]}"
+require_s3_config
+make_dump_dirs
+
+for database in "${selected_databases[@]}"; do
+  download_database "$database"
+done

anonyymioodi/dump_local_db.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# Dump one or more databases from the running docker containers into
+# .databasedumps/local_containers, and copy them to .databasedumps/test where
+# db.Dockerfile reads them from.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0") <database>... | all
+
+Dumps the given databases from their running containers.
+Databases: ${DATABASES[*]}
+EOF
+}
+
+if [[ "${1-}" == "-h" || "${1-}" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+if [[ $# -eq 0 ]]; then
+  usage
+  die "No database given"
+fi
+
+dump_database() {
+  local database="$1"
+  local container_name="oodikone-$database"
+  local database_name
+  database_name="$(postgres_database_name "$database")"
+  local dump_file="$LOCAL_DUMP_DIR/$database.sql"
+
+  infomsg "Dumping $database from container $container_name"
+  # Dump to a temporary file first, so that a failed dump doesn't overwrite the previous one
+  docker exec -i "$container_name" pg_dump -Fp -U postgres -d "$database_name" > "$dump_file.tmp"
+  mv "$dump_file.tmp" "$dump_file"
+  cp --remove-destination "$dump_file" "$TEST_DUMP_DIR/$database.sql"
+  successmsg "Dumped $database to $dump_file"
+}
+
+resolve_databases "$@"
+make_dump_dirs
+
+for database in "${selected_databases[@]}"; do
+  dump_database "$database"
+done

anonyymioodi/lib/common.sh

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+
+# Shared configuration and helpers for the anonyymioodi database scripts.
+# Meant to be sourced by the other scripts in this folder, not run on its own.
+
+# The configuration below is used by the sourcing scripts, not by this file
+# shellcheck disable=SC2034
+
+PROJECT_ROOT="$(git rev-parse --show-toplevel)"
+source "$PROJECT_ROOT"/scripts/utils.sh
+
+# Dumps taken from running docker containers
+LOCAL_DUMP_DIR="$PROJECT_ROOT/.databasedumps/local_containers"
+# Dumps downloaded from s3
+S3_DUMP_DIR="$PROJECT_ROOT/.databasedumps/s3"
+# Where db.Dockerfile copies the .sql files from when an image is built
+TEST_DUMP_DIR="$PROJECT_ROOT/.databasedumps/test"
+
+# Used as a fallback when ACCESS_KEY and SECRET_KEY aren't set in the environment
+S3_CONFIG_FILE=~/.s3cfg
+S3_HOST="s3.datacloud.helsinki.fi"
+S3_BUCKET="s3://oodikone"
+# Dumps live under a prefix of their own inside the bucket
+S3_TARGET="$S3_BUCKET/test-dbs"
+# Dumps older than this are pruned from s3, except for the newest dump of each database
+S3_DUMP_RETENTION_DAYS=30
+
+REGISTRY="registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi"
+
+# Databases, named after their docker compose services
+DATABASES=("user-db" "kone-db" "sis-db" "sis-importer-db" "jami-db")
+
+# Set by resolve_databases, read by the scripts sourcing this file
+selected_databases=()
+
+# The name of the postgres database inside the container. It matches the service
+# name for every database except jami-db.
+postgres_database_name() {
+  local database="$1"
+  if [[ "$database" == "jami-db" ]]; then
+    echo "postgres"
+  else
+    echo "$database"
+  fi
+}
+
+# Validates the given database names and expands "all" to every database.
+# The result is set to the global array $selected_databases, because dying
+# inside a subshell wouldn't stop the calling script.
+resolve_databases() {
+  selected_databases=()
+
+  if [[ $# -eq 0 ]]; then
+    die "No database given, expected one or more of: ${DATABASES[*]} all"
+  fi
+
+  for argument in "$@"; do
+    if [[ "$argument" == "all" ]]; then
+      selected_databases=("${DATABASES[@]}")
+      return
+    fi
+
+    # Check if given databases are correct ones
+    local known=false
+    for database in "${DATABASES[@]}"; do
+      if [[ "$argument" == "$database" ]]; then known=true; fi
+    done
+    $known || die "Unknown database: $argument. Expected one or more of: ${DATABASES[*]} all"
+
+    selected_databases+=("$argument")
+  done
+}
+
+# Dumps in s3 are named after the branch they were uploaded from, so the branch
+# name has to be usable as a part of a file name. E.g. feature/foo -> feature-foo.
+# feature/foo would result in a new bucket "feature" with a sub-bucket "foo"
+sanitize_branch_name() {
+  echo "${1//[^A-Za-z0-9._-]/-}"
+}
+
+current_branch_name() {
+  local branch
+  branch="$(git rev-parse --abbrev-ref HEAD)"
+  if [[ "$branch" == "HEAD" ]]; then # Detached head, use the commit instead
+    branch="$(git rev-parse --short HEAD)"
+  fi
+  sanitize_branch_name "$branch"
+}
+
+has_s3_env_credentials() {
+  [[ -n "${ACCESS_KEY-}" && -n "${SECRET_KEY-}" ]]
+}
+
+# Credentials come from ACCESS_KEY and SECRET_KEY when both are set, and from
+# $S3_CONFIG_FILE when neither is
+require_s3_config() {
+  command -v s3cmd > /dev/null || die "s3cmd not found, please install it first"
+
+  if [[ -n "${ACCESS_KEY-}" || -n "${SECRET_KEY-}" ]]; then
+    has_s3_env_credentials || die "Only one of ACCESS_KEY and SECRET_KEY is set, set both or neither"
+    return
+  fi
+
+  [[ -f "$S3_CONFIG_FILE" ]] || die "Set ACCESS_KEY and SECRET_KEY, or add $S3_CONFIG_FILE. The \".s3cfg\" config file can be found on toska/dokumentaatio repo"
+}
+
+s3_env_config() {
+  cat <<EOF
+[default]
+access_key = $ACCESS_KEY
+secret_key = $SECRET_KEY
+host_base = $S3_HOST
+host_bucket = $S3_HOST
+use_https = True
+check_ssl_certificate = True
+EOF
+}
+
+# Runs s3cmd with the credentials chosen by require_s3_config. The environment
+# credentials are passed through process substitution, so that they aren't
+# written to disk or visible in the process list.
+s3() {
+  if has_s3_env_credentials; then
+    s3cmd -c <(s3_env_config) "$@"
+  else
+    s3cmd -c "$S3_CONFIG_FILE" "$@"
+  fi
+}
+
+# The dump directories are created lazily by the scripts that write into them
+make_dump_dirs() {
+  mkdir -p "$LOCAL_DUMP_DIR" "$S3_DUMP_DIR" "$TEST_DUMP_DIR"
+}

anonyymioodi/manage_test_dbs.sh

@@ -0,0 +1,155 @@
+#!/usr/bin/env bash
+
+# Front end for the test database scripts in this folder. Run it without
+# arguments for an interactive menu, or give it an action and a database to run
+# it directly, e.g. in CI.
+#
+# Each action is just a call to one of the scripts next to this one, so they can
+# also be used on their own.
+
+set -euo pipefail
+
+# Here BASH_SOURCE[0] is used instead of $0 because it works
+# consistently when ran normally and when sourced
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+ACTIONS=("download" "dump" "upload" "push" "prune" "release" "rebuild")
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0") [<action> <database>... | all]
+
+Runs without arguments for an interactive menu.
+
+Actions:
+  download  Download the newest dumps from s3
+  dump      Dump the databases from the running docker containers
+
+  upload    Upload the current dumps to s3, then prune old dumps
+  push      Build images from the current dumps and push them to the registry
+  prune     Delete dumps older than $S3_DUMP_RETENTION_DAYS days from s3, keeping the newest one
+
+  release   Dump from the running containers, then upload, prune and push (shorthand)
+  rebuild   Download the dumps from s3, then push (shorthand)
+
+Databases: ${DATABASES[*]} all
+EOF
+}
+
+# === Actions ===
+
+download_dump() {
+  "$SCRIPT_DIR"/download_dump_from_s3.sh "$1"
+}
+
+dump_database() {
+  "$SCRIPT_DIR"/dump_local_db.sh "$1"
+}
+
+upload_dump() {
+  "$SCRIPT_DIR"/upload_dump_to_s3.sh "$1"
+}
+
+prune_dumps() {
+  "$SCRIPT_DIR"/prune_s3_dumps.sh "$1"
+}
+
+push_image() {
+  "$SCRIPT_DIR"/build_and_push_image.sh "$1"
+}
+
+run_action() {
+  local action="$1"
+  local database="$2"
+
+  case "$action" in
+    download) download_dump "$database" ;;
+    dump) dump_database "$database" ;;
+    upload)
+      upload_dump "$database"
+      prune_dumps "$database"
+      ;;
+    push) push_image "$database" ;;
+    prune) prune_dumps "$database" ;;
+    release)
+      dump_database "$database"
+      upload_dump "$database"
+      prune_dumps "$database"
+      push_image "$database"
+      ;;
+    rebuild)
+      download_dump "$database"
+      push_image "$database"
+      ;;
+    *)
+      usage
+      die "Unknown action: $action"
+      ;;
+  esac
+}
+
+run_action_for_all() {
+  local action="$1"
+  shift
+
+  # Check if command was a proper command
+  local known_action=false
+  for known in "${ACTIONS[@]}"; do
+    if [[ "$action" == "$known" ]]; then known_action=true; fi
+  done
+  if ! $known_action; then
+    usage
+    die "Unknown action: $action"
+  fi
+
+  resolve_databases "$@"
+  for database in "${selected_databases[@]}"; do
+    run_action "$action" "$database"
+  done
+  successmsg "Action $action done for: ${selected_databases[*]}"
+}
+
+# === Interactive menu ===
+
+ask_and_run() {
+  local action database
+
+  PS3="Select action: "
+  select action in "${ACTIONS[@]}" "quit"; do
+    [[ -n "${action-}" ]] || continue
+    if [[ "$action" == "quit" ]]; then exit 0; fi
+    break
+  done
+
+  PS3="Select database: "
+  select database in "${DATABASES[@]}" "all"; do
+    [[ -n "${database-}" ]] || continue
+    break
+  done
+
+  run_action_for_all "$action" "$database"
+}
+
+# === Run ===
+
+if [[ "${1-}" == "-h" || "${1-}" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+if [[ $# -eq 0 ]]; then
+  while true; do
+    ask_and_run
+    echo
+  done
+fi
+
+if [[ $# -lt 2 ]]; then
+  usage
+  die "Both an action and a database are required"
+fi
+
+run_action_for_all "$@"

anonyymioodi/prune_s3_dumps.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+# Delete old database dumps from s3. Dumps older than the retention period are
+# removed, but the newest master dump of each database is always kept, so that a
+# database which hasn't changed in a while can still be downloaded. Dumps from
+# other branches aren't kept, so that old branches don't accumulate dumps.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+# The newest dump of each database from this branch is never pruned
+MASTER_BRANCH="master"
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0") [-n|--dry-run] [-d|--days <days>] <database>... | all
+
+Deletes the dumps in $S3_TARGET older than the given number of days
+(default $S3_DUMP_RETENTION_DAYS). The newest $MASTER_BRANCH dump of each database is
+always kept, or the newest dump of any branch if the database has no $MASTER_BRANCH dumps.
+
+Databases: ${DATABASES[*]}
+
+Options:
+  -n, --dry-run      Print the dumps that would be deleted instead of deleting them
+  -d, --days <days>  Delete dumps older than this many days
+  -h, --help         Show this help
+EOF
+}
+
+dry_run=false
+days="$S3_DUMP_RETENTION_DAYS"
+arguments=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -n | --dry-run)
+      dry_run=true
+      shift
+      ;;
+    -d | --days)
+      days="${2-}"
+      [[ "$days" =~ ^[0-9]+$ ]] || {
+        usage
+        die "Expected a number of days for $1, got: $days"
+      }
+      shift 2
+      ;;
+    -h | --help)
+      usage
+      exit 0
+      ;;
+    -*)
+      usage
+      die "Unknown option: $1"
+      ;;
+    *)
+      arguments+=("$1")
+      shift
+      ;;
+  esac
+done
+
+if [[ ${#arguments[@]} -eq 0 ]]; then
+  usage
+  die "No database given"
+fi
+
+prune_database() {
+  local database="$1"
+  local deleted=0
+
+  # s3cmd ls prints "<date> <time> <size> <object>", so sorting by the first two
+  # columns in reverse puts the newest dump first
+  local dumps
+  dumps="$(s3 ls "$S3_TARGET/$database-" | sort -rk1,2)"
+
+  # Find the newest master dump or in the case of it missing, the newest dump in general
+  local kept
+  kept="$(awk -v pattern="/$database-$MASTER_BRANCH-[0-9]+-[0-9]+[.]sql$" '$4 ~ pattern { print $4; exit }' <<< "$dumps")"
+  [[ -n "$kept" ]] || kept="$(awk 'NR == 1 { print $4 }' <<< "$dumps")"
+
+  # Remove any dumps older than the dump retention date (or -d)
+  # except for master for which at least one dump is saved
+  while read -r date time _ dump; do
+    [[ -n "$dump" && "$dump" != "$kept" && "$date $time" < "$cutoff" ]] || continue
+    if [[ "$dry_run" == true ]]; then
+      infomsg "Dry run, would delete $dump"
+    else
+      s3 del "$dump"
+    fi
+    deleted=$((deleted + 1))
+  done <<< "$dumps"
+
+  if [[ "$dry_run" == true ]]; then
+    successmsg "Would have pruned $deleted dump(s) of $database older than $days days"
+  else
+    successmsg "Pruned $deleted dump(s) of $database older than $days days"
+  fi
+}
+
+resolve_databases "${arguments[@]}"
+require_s3_config
+
+cutoff="$(date -u -d "-$days days" '+%Y-%m-%d %H:%M')"
+
+for database in "${selected_databases[@]}"; do
+  prune_database "$database"
+done

anonyymioodi/upload_dump_to_s3.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+# Upload database dumps from .databasedumps/test to s3, naming the objects
+# <database>-<git_branch>-<datetime>.sql so that dumps don't overwrite each other.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0") [-n|--dry-run] <database>... | all
+
+Uploads the dumps in .databasedumps/test to $S3_TARGET
+as <database>-<git_branch>-<datetime>.sql. Run dump_local_db.sh first to
+create the dumps.
+
+Databases: ${DATABASES[*]}
+
+Options:
+  -n, --dry-run  Print the s3cmd command instead of running it
+  -h, --help     Show this help
+EOF
+}
+
+dry_run=false
+arguments=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -n | --dry-run)
+      dry_run=true
+      shift
+      ;;
+    -h | --help)
+      usage
+      exit 0
+      ;;
+    -*)
+      usage
+      die "Unknown option: $1"
+      ;;
+    *)
+      arguments+=("$1")
+      shift
+      ;;
+  esac
+done
+
+if [[ ${#arguments[@]} -eq 0 ]]; then
+  usage
+  die "No database given"
+fi
+
+upload_database() {
+  local database="$1"
+  local dump_file="$TEST_DUMP_DIR/$database.sql"
+
+  [[ -f "$dump_file" ]] || die "No dump found at $dump_file, run dump_local_db.sh first"
+  [[ -s "$dump_file" ]] || die "Dump is empty: $dump_file"
+
+  local datetime
+  datetime="$(date +%Y%m%d-%H%M%S)"
+  local target="$S3_TARGET/$database-$branch-$datetime.sql"
+
+  if [[ "$dry_run" == true ]]; then
+    infomsg "Dry run, would run: s3cmd put $dump_file $target"
+    return
+  fi
+
+  infomsg "Pushing $dump_file to $target"
+  s3 put "$dump_file" "$target"
+  successmsg "Uploaded $dump_file to $target"
+}
+
+resolve_databases "${arguments[@]}"
+require_s3_config
+
+if [[ "$dry_run" == false ]]; then
+  "$SCRIPT_DIR"/create_s3_bucket.sh
+fi
+
+branch="$(current_branch_name)"
+
+for database in "${selected_databases[@]}"; do
+  upload_database "$database"
+done

docker-compose.ci.yml

@@ -11,6 +11,19 @@ x-healthcheck-defaults: &healthcheck-defaults
   start_period: 30s # How long startup period is
 
 services:
+  adminer:
+    <<: *defaults
+    container_name: oodikone-adminer
+    environment:
+      ADMINER_DESIGN: pepa-linha
+    image: adminer:5.5.1
+    ports:
+      - 5050:8080
+    profiles:
+      - adminer
+    volumes:
+      - ./adminer-plugins:/var/www/html/plugins-enabled
+
   backend:
     <<: *defaults
     container_name: oodikone-backend

docker-compose.yml

@@ -6,6 +6,12 @@ x-defaults: &defaults
 x-build_defaults: &build_defaults
   dockerfile: dev.Dockerfile
 
+x-user_defaults: &user_defaults
+  # Run containers as user:group defined in .env
+  # If run as $(id -u):$(id -g) will reduce permission errors when mounting
+  # Defaults to registry.access.redhat.com/ubi10/nodejs-24-minimal's default user:group
+  user: '${DOCKER_UID:-1001}:${DOCKER_GID:-0}'
+
 x-healthcheck-defaults: &healthcheck-defaults
   interval: 30s # Normal healthcehck interval
   timeout: 5s
@@ -19,14 +25,14 @@ services:
     container_name: oodikone-adminer
     environment:
       ADMINER_DESIGN: pepa-linha
-    image: adminer:4.8.1
+    image: adminer:5.5.1
     ports:
       - 5050:8080
     volumes:
       - ./adminer-plugins:/var/www/html/plugins-enabled
 
   backend:
-    <<: *defaults
+    <<: [*defaults, *user_defaults]
     container_name: oodikone-backend
     build:
       dockerfile: backend/dev.Dockerfile
@@ -67,7 +73,7 @@ services:
       <<: *healthcheck-defaults
 
   frontend:
-    <<: *defaults
+    <<: [*defaults, *user_defaults]
     container_name: oodikone-frontend
     build:
       dockerfile: frontend/dev.Dockerfile
@@ -186,7 +192,7 @@ services:
       <<: *healthcheck-defaults
 
   sis-updater-worker:
-    <<: *defaults
+    <<: [*defaults, *user_defaults]
     container_name: oodikone-updater-worker
     build:
       <<: *build_defaults
@@ -213,7 +219,7 @@ services:
       - sis-updater-worker-nodemod:/opt/app-root/src/node_modules
 
   sis-updater-scheduler:
-    <<: *defaults
+    <<: [*defaults, *user_defaults]
     container_name: oodikone-updater-scheduler
     build:
       <<: *build_defaults

e2e/Course_statistics.spec.ts

@@ -64,6 +64,16 @@ const clickAway = async (page: Page) => {
   await page.locator('body').click({ position: { x: 0, y: 0 } })
 }
 
+const TKT20003 = { code: 'TKT20003', groupId: 'hy-CU-118024775' }
+const TKT20001 = { code: 'TKT20001', groupId: 'hy-CU-118024656' }
+const TKT10002 = { code: 'TKT10002', groupId: 'hy-CU-118023867' }
+const KK_RUKIRJ = { code: 'KK-RUKIRJ', groupId: 'hy-CU-117995649' }
+const course50131 = { code: '50131', groupId: 'hy-CU-2014787' }
+const course200012 = { code: '200012', groupId: 'hy-CU-53928369' }
+const BSCS1001 = { code: 'BSCS1001', groupId: 'hy-CU-128548227' }
+const BSCS1003 = { code: 'BSCS1003', groupId: 'hy-CU-128550261' }
+const TKT10004 = { code: 'TKT10004', groupId: 'hy-CU-118023990' }
+
 test.describe('Basic user', () => {
   test.beforeEach(async ({ page }) => {
     await init(page, '/coursestatistics')
@@ -71,38 +81,35 @@ test.describe('Basic user', () => {
   })
 
   test('Search should work on consecutive searches', async ({ page }) => {
-    const courseCode1 = 'TKT20003'
-    await searchByCourseCode(page, courseCode1)
-    await page.getByTestId(`course-${courseCode1}`).click()
+    await searchByCourseCode(page, TKT20003.code)
+    await page.getByTestId(`course-${TKT20003.code}`).click()
 
-    await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([courseCode1]))
+    await expect(page).toHaveURL(url => url.searchParams.get('courses') === TKT20003.groupId)
 
     await expect(page.getByRole('button', { name: 'Show population' })).toBeEnabled()
     await page.getByRole('button', { name: 'Show population' }).click()
 
-    await expect(page.getByRole('heading', { name: courseCode1 })).toBeVisible()
+    await expect(page.getByRole('heading', { name: TKT20003.code })).toBeVisible()
 
     await page.getByTestId('nav-bar-button-courseStatistics').click()
 
-    const courseCode2 = 'TKT20001'
-    await searchByCourseCode(page, courseCode2)
-    await page.getByTestId(`course-${courseCode2}`).click()
+    await searchByCourseCode(page, TKT20001.code)
+    await page.getByTestId(`course-${TKT20001.code}`).click()
 
-    await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([courseCode2]))
+    await expect(page).toHaveURL(url => url.searchParams.get('courses') === TKT20001.groupId)
 
     await expect(page.getByRole('button', { name: 'Show population' })).toBeEnabled()
     await page.getByRole('button', { name: 'Show population' }).click()
 
-    await expect(page.getByText(courseCode2, { exact: true })).toBeVisible()
+    await expect(page.getByText(TKT20001.code, { exact: true })).toBeVisible()
   })
 
   test.describe('Course table can show non-standard grades', () => {
     test('Course table can show HT-TT grade scales', async ({ page }) => {
-      const coursecode = 'KK-RUKIRJ'
-      await searchByCourseCode(page, coursecode)
-      await page.getByTestId(`course-${coursecode}`).click()
+      await searchByCourseCode(page, KK_RUKIRJ.code)
+      await page.getByTestId(`course-${KK_RUKIRJ.code}`).click()
 
-      await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([coursecode]))
+      await expect(page).toHaveURL(url => url.searchParams.get('courses') === KK_RUKIRJ.groupId)
 
       await selectFromYear(page, '2019-2020')
       await expect(page.getByRole('button', { name: 'Show population' })).toBeEnabled()
@@ -117,11 +124,10 @@ test.describe('Basic user', () => {
     })
 
     test("Course table can show old master's thesis grade scales", async ({ page }) => {
-      const coursecode = '50131'
-      await searchByCourseCode(page, coursecode)
-      await page.getByTestId(`course-${coursecode}`).click()
+      await searchByCourseCode(page, course50131.code)
+      await page.getByTestId(`course-${course50131.code}`).click()
 
-      await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([coursecode]))
+      await expect(page).toHaveURL(url => url.searchParams.get('courses') === course50131.groupId)
 
       await expect(page.getByText('Pro gradu -tutkielma tietojenkäsittelytieteessä')).toBeVisible() // 50131
 
@@ -130,7 +136,7 @@ test.describe('Basic user', () => {
 
       await expect(page.getByText('Pro gradu -tutkielma tietojenkäsittelytieteessä')).toBeVisible()
       await expect(page.getByText('Class of 2007-2020, 10 students')).toBeVisible()
-      await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+      await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
       await expect(page.getByText('Showing 10 out of 10 students')).toBeVisible()
 
       const gradesTableContents = [
@@ -143,11 +149,10 @@ test.describe('Basic user', () => {
     })
 
     test('Shows correct statistics for courses with scale passed-failed', async ({ page }) => {
-      const coursecode = '200012'
-      await searchByCourseCode(page, coursecode)
-      await page.getByTestId(`course-${coursecode}`).click()
+      await searchByCourseCode(page, course200012.code)
+      await page.getByTestId(`course-${course200012.code}`).click()
 
-      await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([coursecode]))
+      await expect(page).toHaveURL(url => url.searchParams.get('courses') === course200012.groupId)
 
       await expect(page.getByText('ON-310')).toBeVisible() // Tieteellisen kirjoittamisen seminaarin alkuopetus: Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot
       await expect(page.getByText('200012')).toBeVisible() // Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot
@@ -157,7 +162,7 @@ test.describe('Basic user', () => {
 
       await expect(page.getByText('Tieteellisen kirjallisen työn ja tiedonhankinnan perustaidot')).toBeVisible()
       await expect(page.getByText('Class of 2011-2018, 4 students')).toBeVisible()
-      await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+      await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
       await expect(page.getByText('Showing 4 out of 4 students')).toBeVisible()
 
       const gradesTableContents = [
@@ -170,11 +175,10 @@ test.describe('Basic user', () => {
 
   test.describe('Course mappings work', () => {
     test('Searching single course having substitution mappings shows course statistics', async ({ page }) => {
-      const coursecode = 'TKT20001'
-      await searchByCourseCode(page, coursecode)
-      await page.getByTestId(`course-${coursecode}`).click()
+      await searchByCourseCode(page, TKT20001.code)
+      await page.getByTestId(`course-${TKT20001.code}`).click()
 
-      await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([coursecode]))
+      await expect(page).toHaveURL(url => url.searchParams.get('courses') === TKT20001.groupId)
 
       await expect(page.getByText('Tietorakenteet ja algoritmit', { exact: true })).toBeVisible() // The old 10credit variant
       await expect(page.getByText('TKT20001', { exact: true })).toBeVisible()
@@ -196,14 +200,15 @@ test.describe('Basic user', () => {
       await expect(page.getByTestId('select-multiple-courses-toggle')).toBeChecked()
 
       await searchByCourseCode(page, 'TKT')
-      await page.getByTestId(`course-TKT20001`).click()
-      await page.getByTestId(`course-TKT10002`).click()
+      await page.getByTestId(`course-${TKT20001.code}`).click()
+      await page.getByTestId(`course-${TKT10002.code}`).click()
 
       await expect(page.getByText('Fetch statistics')).toBeEnabled()
       await page.getByText('Fetch statistics').click()
 
       await expect(page).toHaveURL(
-        url => url.searchParams.get('courseCodes') === JSON.stringify(['TKT10002', 'TKT20001'])
+        url =>
+          JSON.stringify(url.searchParams.getAll('courses')) === JSON.stringify([TKT10002.groupId, TKT20001.groupId])
       )
 
       await openSummaryTab(page)
@@ -227,20 +232,20 @@ test.describe('Basic user', () => {
     await expect(page.getByTestId('select-multiple-courses-toggle')).toBeChecked()
 
     await searchByCourseCode(page, 'BSCS')
-    await page.getByTestId(`course-BSCS1003`).click()
-    await page.getByTestId(`course-BSCS1001`).click()
+    await page.getByTestId(`course-${BSCS1003.code}`).click()
+    await page.getByTestId(`course-${BSCS1001.code}`).click()
 
     await expect(page.getByText('Fetch statistics')).toBeEnabled()
     await page.getByText('Fetch statistics').click()
 
     await expect(page).toHaveURL(
-      url => url.searchParams.get('courseCodes') === JSON.stringify(['BSCS1001', 'BSCS1003'])
+      url => JSON.stringify(url.searchParams.getAll('courses')) === JSON.stringify([BSCS1001.groupId, BSCS1003.groupId])
     )
 
     await page.getByTestId('course-population-for-2023-2024').click()
     await expect(page.getByRole('heading', { name: 'Introduction to Programming' })).toBeVisible()
     await expect(page.getByText('Class of 2023-2024')).toBeVisible()
-    await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+    await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
 
     await page.goBack()
 
@@ -248,12 +253,15 @@ test.describe('Basic user', () => {
     await page.getByTestId('CourseSelectorOptionBSCS1003').click()
 
     // Check the link has updated correctly
-    await expect(page.getByTestId('course-population-for-2022-2023')).toHaveAttribute('href', /BSCS1003/)
+    await expect(page.getByTestId('course-population-for-2022-2023')).toHaveAttribute(
+      'href',
+      new RegExp(BSCS1003.groupId)
+    )
     await expect(page.getByTestId('CourseSelector').getByText('Data Structures and Algorithms')).toBeVisible()
 
     await page.getByTestId('course-population-for-2022-2023').click()
     await expect(page.getByText('Class of 2022-2023')).toBeVisible()
-    await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+    await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
   })
 
   test('Searching course by name displays right courses', async ({ page }) => {
@@ -279,11 +287,13 @@ test.describe('Basic user', () => {
     ).toBeVisible()
   })
 
+  // FIXME
   test('Searching course by name displays right courses, 10 credit courses', async ({ page }) => {
     await expect(page.getByText('Search for courses')).toBeVisible()
     await searchByCourseName(page, 'tietorakenteet ja algoritmit')
 
-    await expect(page.getByText('Tietorakenteet ja algoritmit', { exact: true })).toBeVisible()
+    // There should be two identically named Tietorakenteet ja algoritmit courses.
+    await expect(page.getByText('Tietorakenteet ja algoritmit', { exact: true })).toHaveCount(2)
     await page.getByTestId(`course-TKT20001`).click()
     await expect(page.getByText('Search for courses')).not.toBeVisible()
 
@@ -378,7 +388,7 @@ test.describe('Basic user', () => {
 
       await expect(page.getByText('Käyttöjärjestelmät')).toBeVisible()
       await expect(page.getByText('Class of 2020-2021, 19 students')).toBeVisible()
-      await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+      await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
 
       await expect(page.getByText('TKT20003')).toBeVisible()
 
@@ -388,9 +398,9 @@ test.describe('Basic user', () => {
     })
 
     test('Population of course shows grades for each student', async ({ page }) => {
-      await searchByCourseCode(page, 'TKT20001')
-      await page.getByTestId(`course-TKT20001`).click()
-      await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify(['TKT20001']))
+      await searchByCourseCode(page, TKT20001.code)
+      await page.getByTestId(`course-${TKT20001.code}`).click()
+      await expect(page).toHaveURL(url => url.searchParams.get('courses') === TKT20001.groupId)
 
       // This step should *not* be needed to make this test work (but it is)
       await expect(page.getByRole('heading', { name: 'Selected course' })).toBeVisible()
@@ -404,7 +414,7 @@ test.describe('Basic user', () => {
 
       await expect(page.getByText('Tietorakenteet ja algoritmit')).toBeVisible()
       await expect(page.getByText('Class of 2019-2020, 33 students')).toBeVisible()
-      await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+      await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
 
       // Check grade field in Students tab
       await page.getByText('Students (33)').click()
@@ -434,7 +444,7 @@ test.describe('Basic user', () => {
 
       await expect(page.getByText('Tietorakenteet ja algoritmit')).toBeVisible()
       await expect(page.getByText('Class of 2019-2020, 33 students')).toBeVisible()
-      await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+      await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
 
       await page.getByText('Students (33)').click()
       await expect(page.locator('tbody > tr').filter({ hasText: 'Hidden' })).toHaveCount(9)
@@ -451,7 +461,7 @@ test.describe('Basic user', () => {
 
     await expect(page.getByText('Käyttöjärjestelmät')).toBeVisible()
     await expect(page.getByText('Class of 2021-2022')).toBeVisible()
-    await expect(page.getByText('Include substitutions, Open and normal')).toBeVisible()
+    await expect(page.getByText('Substitutions included, open and normal')).toBeVisible()
 
     // Check count column of Language distribution
     await page.getByText('Language distribution').click()
@@ -1185,7 +1195,7 @@ test.describe('Basic user', () => {
     })
 
     test('Has right to see all the students, because course provider is TKT', async ({ page }) => {
-      await page.goto('coursestatistics?courseCodes=%5B%22TKT10004%22%5D&separate=false')
+      await page.goto(`coursestatistics?courses=${encodeURIComponent(TKT10004.groupId)}&separate=false`)
       await page.getByTestId('course-population-for-2021-2022').click()
       await expect(page.getByText('Students (28)')).toBeVisible()
     })
@@ -1201,11 +1211,10 @@ test.describe('Only course statistics', () => {
   test('Some features of Course Statistics are hidden for courseStatistics-users without other rights', async ({
     page,
   }) => {
-    const coursecode = 'TKT10002'
-    await searchByCourseCode(page, coursecode)
-    await page.getByTestId(`course-${coursecode}`).click()
+    await searchByCourseCode(page, TKT10002.code)
+    await page.getByTestId(`course-${TKT10002.code}`).click()
 
-    await expect(page).toHaveURL(url => url.searchParams.get('courseCodes') === JSON.stringify([coursecode]))
+    await expect(page).toHaveURL(url => url.searchParams.get('courses') === TKT10002.groupId)
 
     await expect(page.getByText('Filter statistics by degree programmes')).not.toBeVisible()
     await expect(page.getByText('Show population')).not.toBeVisible()

e2e/Degree_programme_overview.spec.ts

@@ -650,10 +650,14 @@ test.describe('Degree programme overview', () => {
     })
 
     test.describe('Curriculum section', () => {
-      test('is shown', async ({ page }) => {
-        await expect(page.getByTestId('curriculum-section')).toBeVisible()
-        await expect(page.getByText('Select curriculum to edit:')).toBeVisible()
-        await expect(page.getByTestId('curriculum-picker').getByText('2023–2026')).toBeVisible()
+      test('is shown in both settings tabs', async ({ page }) => {
+        for (const tab of ['criteria', 'visibility']) {
+          await page.getByTestId(`${tab}-tab`).click()
+          const panel = page.getByTestId(`${tab}-panel`)
+          await expect(panel.getByTestId('curriculum-section')).toBeVisible()
+          await expect(panel.getByText('Select curriculum to edit:')).toBeVisible()
+          await expect(panel.getByTestId('curriculum-picker').getByText('2023–2026')).toBeVisible()
+        }
       })
       test.skip('can be changed', () => {
         return
@@ -664,9 +668,7 @@ test.describe('Degree programme overview', () => {
       test('info box', async ({ page }) => {
         await expect(page.getByTestId('credit-criteria-section')).toBeVisible()
         await page.getByTestId('credit-criteria-info-box-button').click()
-        await expect(
-          page.getByTestId('credit-criteria-info-box-content').getByText('Here you can change')
-        ).toBeVisible()
+        await expect(page.getByTestId('credit-criteria-info-box-content').getByText('Tässä asetetut')).toBeVisible()
       })
 
       test.skip('changing the credit criteria works', () => {
@@ -677,7 +679,9 @@ test.describe('Degree programme overview', () => {
     })
 
     test('Degree course table', async ({ page }) => {
-      await expect(page.getByTestId('degree-course-table')).toBeVisible()
+      await expect(page.getByTestId('criteria-panel').getByTestId('degree-course-table')).toBeVisible()
+      await page.getByTestId('visibility-tab').click()
+      await expect(page.getByTestId('visibility-panel').getByTestId('degree-course-table')).toBeVisible()
       // TODO: Test clicking the arrow buttons
       // TODO: Test toggling visibility for MODULES and COURSES
       // TODO: Test changing criterion labels

e2e/Filters.spec.ts

@@ -190,7 +190,8 @@ test.describe('Custom Population Statistics', () => {
     await page.getByRole('combobox').fill('MAT11005 - Integraalilaskenta')
     await page.getByRole('option', { name: 'MAT11005 - Integraalilaskenta' }).click()
     await checkFilteredStudentCount(page, 7)
-    await page.getByTestId('courseFilter-MAT11005-selector').click()
+    // MAT11005 = hy-CU-117375607
+    await page.getByTestId('courseFilter-hy-CU-117375607-selector').click()
     await page.getByRole('option', { name: 'Passed' }).click()
     await checkFilteredStudentCount(page, 6)
   })

e2e/Language_center.spec.ts

@@ -4,7 +4,8 @@ import { init } from './support/commands'
 const chooseSemester = async (page: Page, semester: string, fromOrTo: 'from' | 'to') => {
   await page.getByTestId(`semester-${fromOrTo}`).click()
   await page.getByTestId(`select-opt-${semester}`).click()
-  await expect(page.getByTestId(`select-opt-${semester}`)).not.toHaveClass(/visible/)
+  // The dropdown unmounts its options on close, so wait for it to be gone before reading the table
+  await expect(page.getByRole('listbox')).toHaveCount(0)
 }
 
 const checkNumbers = async (page: Page, numbers: number[], numberOfColumns: number, mode: string) => {
@@ -36,18 +37,20 @@ test.describe('When language center is opened', () => {
         await expect(page.getByText('AYKK-RUKIRJ')).toBeVisible()
       })
 
-      test('Faculties tab shows numbers', async ({ page }) => {
-        await checkNumbers(page, [2076, 36, 9, 34, 759, 15, 40, 6, 51, 1, 24, 1059, 42, 0], 15, 'faculties')
+      // FIXME: This changes per updater run
+      // Example case: 	550789 - KK-JAOK15
+      test.skip('Faculties tab shows numbers', async ({ page }) => {
+        await checkNumbers(page, [2074, 36, 9, 33, 758, 15, 40, 6, 51, 1, 24, 1059, 42, 0], 15, 'faculties')
       })
 
-      test('Faculties tab "exceeding" button works', async ({ page }) => {
+      test.skip('Faculties tab "exceeding" button works', async ({ page }) => {
         await page.getByTestId('difference-button').click()
         await checkNumbers(page, [66, 3, 0, 4, 18, 0, 1, 1, 4, 0, 0, 31, 3, 1], 15, 'faculties')
       })
 
-      test('Faculties tab semester selector changes numbers', async ({ page }) => {
+      test.skip('Faculties tab semester selector changes numbers', async ({ page }) => {
         await chooseSemester(page, 'Syksy 2020', 'from')
-        await checkNumbers(page, [1184, 28, 6, 28, 499, 4, 10, 2, 36, 0, 9, 535, 27, 0], 15, 'faculties')
+        await checkNumbers(page, [1182, 28, 6, 27, 498, 4, 10, 2, 36, 0, 9, 535, 27, 0], 15, 'faculties')
       })
     })
 
@@ -58,8 +61,8 @@ test.describe('When language center is opened', () => {
         await chooseSemester(page, 'Kevät 2024', 'to')
       })
 
-      test('Semester tab shows numbers', async ({ page }) => {
-        await checkNumbers(page, [2076, 69, 26, 298, 58, 343, 98, 438, 138, 310, 90, 123, 74, 10, 1], 16, 'semesters')
+      test.skip('Semester tab shows numbers', async ({ page }) => {
+        await checkNumbers(page, [2074, 69, 26, 298, 58, 343, 98, 437, 138, 310, 90, 123, 73, 10, 1], 16, 'semesters')
       })
     })
   })

e2e/Numbers_match.spec.ts

@@ -4,6 +4,7 @@ import { init } from './support/commands'
 type CourseData = [
   courseOrModule: string,
   code: string,
+  groupId: string,
   name: string,
   total: number,
   passed: number,
@@ -13,20 +14,22 @@ type CourseData = [
 ]
 
 const courseData: CourseData[] = [
-  ['Course', 'MAT11001', 'Johdatus yliopistomatematiikkaan', 272, 253, 1 + 18, 1, 18],
-  ['Course', 'MAT11003', 'Raja-arvot', 270, 249, 1 + 20, 1, 20],
-  ['Course', 'MAT11004', 'Differentiaalilaskenta', 262, 248, 1 + 13, 1, 13],
-  ['Course', 'MAT21002', 'Sarjat', 256, 250, 0 + 6, 0, 6],
-  ['Course', 'MAT11005', 'Integraalilaskenta', 251, 234, 0 + 17, 0, 17],
-  ['Course', 'MAT11002', 'Lineaarialgebra ja matriisilaskenta I', 247, 242, 1 + 4, 1, 4],
-  ['Course', 'MAT12003', 'Todennäköisyyslaskenta I', 234, 227, 3 + 4, 3, 4],
-  ['Course', 'MAT21001', 'Lineaarialgebra ja matriisilaskenta II', 230, 216, 4 + 10, 4, 10],
-  ['Course', 'MAT21003', 'Vektorianalyysi I', 228, 202, 3 + 23, 3, 23],
-  ['Course', 'MAT20005', 'Akateemiset taidot', 201, 178, 0 + 23, 0, 23],
-  ['Module', 'MAT110', 'Matematiikka, perusopinnot', 195, 195, 0 + 0, 0, 0],
-  ['Course', 'MAT21014', 'Johdatus logiikkaan I', 188, 170, 0 + 18, 0, 18],
+  ['Course', 'MAT11001', 'hy-CU-117375151', 'Johdatus yliopistomatematiikkaan', 272, 253, 1 + 18, 1, 18],
+  ['Course', 'MAT11003', 'hy-CU-117375413', 'Raja-arvot', 270, 249, 1 + 20, 1, 20],
+  ['Course', 'MAT11004', 'hy-CU-117375487', 'Differentiaalilaskenta', 262, 248, 1 + 13, 1, 13],
+  ['Course', 'MAT21002', 'hy-CU-117375793', 'Sarjat', 256, 250, 0 + 6, 0, 6],
+  ['Course', 'MAT11005', 'hy-CU-117375607', 'Integraalilaskenta', 251, 234, 0 + 17, 0, 17],
+  ['Course', 'MAT11002', 'hy-CU-117375394', 'Lineaarialgebra ja matriisilaskenta I', 247, 242, 1 + 4, 1, 4],
+  ['Course', 'MAT12003', 'hy-CU-117376940', 'Todennäköisyyslaskenta I', 234, 227, 3 + 4, 3, 4],
+  ['Course', 'MAT21001', 'hy-CU-117375754', 'Lineaarialgebra ja matriisilaskenta II', 230, 216, 4 + 10, 4, 10],
+  ['Course', 'MAT21003', 'hy-CU-117375829', 'Vektorianalyysi I', 228, 202, 3 + 23, 3, 23],
+  ['Course', 'MAT20005', 'hy-CU-120230575', 'Akateemiset taidot', 201, 178, 0 + 23, 0, 23],
+  ['Module', 'MAT110', 'hy-SM-117412902', 'Matematiikka, perusopinnot', 195, 195, 0 + 0, 0, 0],
+  ['Course', 'MAT21014', 'hy-CU-117377242', 'Johdatus logiikkaan I', 188, 170, 0 + 18, 0, 18],
 ]
 
+const MAT21003_GROUP_ID = 'hy-CU-117375829'
+
 const yearlyData = [
   [2017, 1, 0, 1, 0, 1 + 0],
   [2018, 31, 27, 4, 0, 4 + 0],
@@ -45,8 +48,8 @@ const getPercentage = (value: any, total: any) => {
   return `${((value / total) * 100).toFixed(2)} %`
 }
 
-const selectCourseStatus = async (page: Page, courseCode: string, status: string) => {
-  await page.getByTestId(`courseFilter-${courseCode}-selector`).click()
+const selectCourseStatus = async (page: Page, courseGroupId: string, status: string) => {
+  await page.getByTestId(`courseFilter-${courseGroupId}-selector`).click()
   await page.getByRole('listbox').getByRole('option', { name: status }).click()
 }
 
@@ -71,18 +74,32 @@ test.describe('Numbers should match between', () => {
       const rows = page.locator('tbody tr')
       for (const [index, course] of courseData.entries()) {
         const row = rows.nth(index)
+        const [courseOrModule, code, , name, total, passed, notCompleted] = course
         // Not optimal, but not checking all locator("td") matches directly with toHaveText is hard.
-        for (const [fieldIndex, field] of course.slice(0, 6).entries()) {
+        for (const [fieldIndex, field] of [courseOrModule, code, name, total, passed, notCompleted].entries()) {
           await expect(row.locator('td').nth(fieldIndex)).toHaveText(String(field))
         }
       }
     })
 
     test('in Course statistics', async ({ page }) => {
-      for (const [_courseOrModule, code, name, total, passed, _notCompleted, failed, enrolledNoGrade] of courseData) {
-        await init(page, `/coursestatistics?courseCodes=%5B%22${code}%22%5D&combineSubstitutions=false`, 'basic')
+      for (const [
+        _courseOrModule,
+        _code,
+        groupId,
+        name,
+        total,
+        passed,
+        _notCompleted,
+        failed,
+        enrolledNoGrade,
+      ] of courseData) {
+        await init(page, `/coursestatistics?courses=${groupId}&substitutions=false`, 'basic')
         await expect(page).toHaveURL(
-          url => url.pathname === '/coursestatistics' && url.searchParams.get('courseCodes') === JSON.stringify([code])
+          url =>
+            url.pathname === '/coursestatistics' &&
+            url.searchParams.get('courses') === groupId &&
+            url.searchParams.get('substitutions') === 'false'
         )
         await expect(page.getByText('Course statistics')).toBeVisible()
         await expect(page.getByText(name)).toBeVisible()
@@ -105,12 +122,28 @@ test.describe('Numbers should match between', () => {
     })
 
     test('in Course population', async ({ page }) => {
-      for (const [_courseOrModule, code, name, total, passed, _notCompleted, failed, enrolledNoGrade] of courseData) {
+      for (const [
+        _courseOrModule,
+        code,
+        groupId,
+        name,
+        total,
+        passed,
+        _notCompleted,
+        failed,
+        enrolledNoGrade,
+      ] of courseData) {
         await init(
           page,
-          `/coursepopulation?from=${2017 - 1949}&to=${2023 - 1949}&coursecodes=%5B%22${code}%22%5D&includeSubstitutions=false`,
+          `/coursepopulation?from=${2017 - 1949}&to=${2023 - 1949}&courses=${groupId}&substitutions=false`,
           'basic'
         )
+        await expect(page).toHaveURL(
+          url =>
+            url.pathname === '/coursepopulation' &&
+            url.searchParams.get('courses') === groupId &&
+            url.searchParams.get('substitutions') === 'false'
+        )
         await expect(page.getByText(name)).toBeVisible()
 
         await page.getByTestId('courseFilter-filter-card').click()
@@ -126,7 +159,7 @@ test.describe('Numbers should match between', () => {
           ['Failed', failed],
           ['Enrolled, No Grade', enrolledNoGrade],
         ] as const) {
-          await selectCourseStatus(page, code, status)
+          await selectCourseStatus(page, groupId, status)
           await expect(page.getByText(`Students (${expected})`)).toBeVisible()
         }
       }
@@ -152,7 +185,13 @@ test.describe('Numbers should match between', () => {
 
     test('in Course statistics', async ({ page }) => {
       for (const [year, total, passed, failed, enrolledNoGrade] of yearlyData) {
-        await init(page, '/coursestatistics?courseCodes=%5B%22MAT21003%22%5D&combineSubstitutions=false', 'basic')
+        await init(page, `/coursestatistics?courses=${MAT21003_GROUP_ID}&substitutions=false`, 'basic')
+        await expect(page).toHaveURL(
+          url =>
+            url.pathname === '/coursestatistics' &&
+            url.searchParams.get('courses') === MAT21003_GROUP_ID &&
+            url.searchParams.get('substitutions') === 'false'
+        )
         const yearString = `${year}-${year + 1}`
 
         await selectYear(page, 'ToYearSelector', `ToYearSelectorOption${yearString}`)
@@ -180,9 +219,15 @@ test.describe('Numbers should match between', () => {
       for (const [year, _total, passed, failed, enrolledNoGrade] of yearlyData) {
         await init(
           page,
-          `/coursepopulation?from=${year - 1949}&to=${year - 1949}&coursecodes=%5B%22MAT21003%22%5D&includeSubstitutions=false`,
+          `/coursepopulation?from=${year - 1949}&to=${year - 1949}&courses=${MAT21003_GROUP_ID}&substitutions=false`,
           'basic'
         )
+        await expect(page).toHaveURL(
+          url =>
+            url.pathname === '/coursepopulation' &&
+            url.searchParams.get('courses') === MAT21003_GROUP_ID &&
+            url.searchParams.get('substitutions') === 'false'
+        )
         await expect(page.getByText('Vektorianalyysi I')).toBeVisible()
 
         await page.getByTestId('courseFilter-filter-card').click()
@@ -198,7 +243,7 @@ test.describe('Numbers should match between', () => {
           ['Failed', failed],
           ['Enrolled, No Grade', enrolledNoGrade],
         ] as const) {
-          await selectCourseStatus(page, 'MAT21003', status)
+          await selectCourseStatus(page, MAT21003_GROUP_ID, status)
           await expect(page.getByText(`Students (${expected})`)).toBeVisible()
         }
       }

ok.sh

@@ -0,0 +1,399 @@
+#!/usr/bin/env bash
+
+# Oodikone CLI tool (revision 2.0)
+# Bash template provided by:
+# https://betterdev.blog/minimal-safe-bash-script-template/
+
+set -Eueo pipefail
+trap cleanup SIGINT SIGTERM ERR EXIT
+
+# Constants
+# Resolve relative to the script's own location (not the caller's cwd), so
+# `ok.sh` behaves the same whether it's run as `./ok.sh` from the repo or
+# invoked with an absolute/symlinked path from anywhere (e.g. the `ok`
+# shell function installed via `./ok.sh install`).
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel)"
+
+cd "$PROJECT_ROOT"
+
+# Include scripts from other sources
+source "$PROJECT_ROOT"/scripts/functions_for_setup.sh
+
+usage() {
+  cat << EOF # remove the space between << and EOF, this is due to web plugin issue
+Usage: $(basename "${BASH_SOURCE[0]}") [command]
+
+Run without a command to open the interactive menu.
+
+Available commands:
+
+  up, u, r, real        Start Oodikone with real data
+  rs, rl                Restart Oodikone with real data (down, then up)
+  anon, a               Start Oodikone with anonymous data
+  down, d               Stop Oodikone
+
+  logs, l               Follow oodikone service logs
+  logs:updater, lu      Follow sis-updater service logs
+  lint, li              Lint the project or the path given as parameter
+  format, fmt, f        Format the project or the path given as parameter
+
+  test:backend, tb      Run backend unit tests
+  test:frontend, tf     Run frontend unit tests
+  test:updater, tu      Run updater (worker) unit tests
+  test:services, ts     Run backend service tests
+  test:api, ta          Run backend api tests
+  test:component, tc    Run frontend component tests tests
+  test:e2e, te          Run all e2e tests (add --ui to run in headed mode)
+
+  importer, i           Open a psql/pgcli shell to the importer database
+  updater               Open a psql/pgcli shell to the updater (sis) database
+  install               Add the 'ok' shell function to your shell rc file
+  -h, --help            Print this help and exit
+EOF
+  exit
+}
+
+cleanup() {
+  trap - SIGINT SIGTERM ERR EXIT
+  msg ""
+  errormsg "(╯°□°)╯︵ ┻━┻"
+}
+
+setup_colors() {
+  if [[ -t 2 ]] && [[ -z "${NO_COLOR-}" ]] && [[ "${TERM-}" != "dumb" ]]; then
+    NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+  else
+    NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+  fi
+}
+
+die() {
+  local msg=$1
+  local code=${2-1} # default exit status 1
+  msg "$msg"
+  exit "$code"
+}
+
+get_pgclient() {
+  if command -v pgcli >/dev/null 2>&1; then
+    echo "pgcli"
+  else
+    warningmsg "Couldn't verify pgcli installation, using psql as fallback" >&2
+    echo "psql"
+  fi
+}
+
+show_welcome() {
+  local cashmoneyyellow
+  cashmoneyyellow=$(tput setaf 221)
+  local normal
+  normal=$(tput sgr0)
+  if [ "$(tput cols)" -gt "76" ]; then
+    while IFS="" read -r p || [ -n "$p" ]; do
+      printf '%40s\n' "${cashmoneyyellow}$p${normal}"
+    done < "$PROJECT_ROOT"/scripts/assets/logo.txt
+  fi
+  msg "Welcome to Oodikone CLI!"
+  msg ""
+}
+
+main_menu_opts() {
+  msg "1) Start Oodikone    2) Databases    3) Data refresh    4) Setup    q) Quit\n"
+}
+
+database_menu_opts() {
+  msg "\nView local databases ($datasource data)"
+  msg "suffix: $dbsuffix"
+  msg "1) Oodikone"
+  msg "2) Importer"
+  msg "3) Kone"
+  msg "4) Toggle data source (test/real)"
+  msg "5) Go back"
+  msg ""
+}
+
+database_menu() {
+  # If a single mandatory service is down, an assumption is made that they all are.
+  if ! docker compose ps 2>/dev/null | grep -q 'oodikone-backend'; then
+    msg ""
+    warningmsg "Can't access databases, oodikone is not running!\nPlease run option 1 first with preferred data source."
+    return
+  fi
+
+
+  declare dbsuffix="-real"
+  declare datasource="real"
+
+  toggle_environment() {
+    if [[ $dbsuffix = "" ]];
+      then
+        dbsuffix="-real"
+        datasource="real"
+      else
+        dbsuffix=""
+        datasource="test"
+    fi
+  }
+
+  local pgclient
+  pgclient="$(get_pgclient)"
+  msg "${GREEN}\nUsing $pgclient for database connections${NOFORMAT}"
+  database_menu_opts $datasource
+
+  local userinput
+  while true; do
+    read -r -s -n 1 -p "Please select an option: " userinput
+
+    case ${userinput} in
+      1|o|O)
+        msg "Oodikone-db"
+        $pgclient -h localhost -p 12345 -U postgres -d "sis-db$dbsuffix" -w
+        database_menu_opts
+        ;;
+      2|i|I)
+        msg "Importer-db"
+        $pgclient -h localhost -p 12346 -U postgres -d "sis-importer-db$dbsuffix" -w
+        database_menu_opts
+        ;;
+      3|k|K)
+        msg "Kone-db"
+        $pgclient -h localhost -p 4321 -U postgres -d "kone-db$dbsuffix" -w
+        database_menu_opts
+        ;;
+      4|t|T)
+        msg "Toggling environment..."
+        database_menu_opts
+        ;;
+      5|q|Q)
+        msg ""
+        break
+        ;;
+      *)
+        true
+        ;;
+    esac
+  done
+}
+
+main_loop() {
+  local userinput
+
+  while true; do
+    main_menu_opts
+    read -r -s -n 1 -p "Please select an option: " userinput
+
+    case ${userinput} in
+      1)
+        msg "\nTODO: todo\n";;
+      2)
+        database_menu;;
+      3)
+        msg "\nTODO: todo\n";;
+      4)
+        msg "\nTODO: todo\n";;
+      5|q|Q)
+        break 2;;
+      *)
+        errormsg "${RED}Invalid option:${NOFORMAT} $userinput";;
+    esac
+  done
+}
+
+# === Direct (non-interactive) commands ===
+run_up() {
+  infomsg "Starting Oodikone with real data..."
+  npm run both:real "$@"
+}
+
+run_restart() {
+  infomsg "Restarting Oodikone with real data..."
+  npm run docker:down
+  npm run both:real
+}
+
+run_anon() {
+  infomsg "Starting Oodikone with anonymous data..."
+  npm run both "$@"
+}
+
+run_down() {
+  infomsg "Stopping Oodikone..."
+  npm run docker:down "$@"
+}
+
+run_logs() {
+  npm run logs "$@"
+}
+
+run_lint() {
+  npm run lint -- "$@"
+}
+
+run_format() {
+  npm run fmt -- "$@"
+}
+
+run_unit_tests() {
+  npm run test:unit --prefix "$@"
+}
+
+run_component_tests() {
+  npm run test:component --prefix services/frontend "$@"
+}
+
+run_service_tests() {
+  npm run test:services --prefix services/backend "$@"
+}
+
+run_api_tests() {
+  npm run test:api --prefix services/backend "$@"
+}
+
+run_e2e_tests() {
+  npm run playwright -- "$@"
+}
+
+run_logs_updater() {
+  docker compose --file docker-compose.yml logs sis-updater-worker sis-updater-scheduler -f
+}
+
+
+run_importer_db() {
+  local pgclient
+  pgclient="$(get_pgclient)"
+  msg "Importer-db"
+  $pgclient -h localhost -p 12346 -U postgres -d sis-importer-db-real -w
+}
+
+run_updater_db() {
+  local pgclient
+  pgclient="$(get_pgclient)"
+  msg "Updater-db"
+  $pgclient -h localhost -p 12345 -U postgres -d sis-db-real -w
+}
+
+# Add (or refresh) an `ok` shell function in the user's shell rc file, which
+# simply forwards all arguments to this script. This is the one-time setup
+# step referenced above the shell function no longer needs its own copy of
+# oodikone-specific logic.
+cmd_install() {
+  local shell_name
+  shell_name="$(basename "${SHELL:-bash}")"
+
+  local shell_rc
+  case "$shell_name" in
+    zsh) shell_rc="$HOME/.zshrc" ;;
+    *) shell_rc="$HOME/.bashrc" ;;
+  esac
+
+  local ok_path="$PROJECT_ROOT/ok.sh"
+  local marker_start="# >>> oodikone ok.sh >>>"
+  local marker_end="# <<< oodikone ok.sh <<<"
+
+  if [ -f "$shell_rc" ] && grep -qF "$marker_start" "$shell_rc"; then
+    warningmsg "ok.sh integration already present in $shell_rc, skipping."
+    infomsg "To reinstall (e.g. after moving the repo), remove the block between\n'$marker_start' and '$marker_end' in $shell_rc and re-run 'ok.sh install'."
+    return 0
+  fi
+
+  {
+    echo ""
+    echo "$marker_start"
+    echo "ok() { \"$ok_path\" \"\$@\"; }"
+    echo "$marker_end"
+  } >> "$shell_rc"
+
+  successmsg "Added the 'ok' function to $shell_rc"
+  infomsg "Run 'source $shell_rc' or restart your shell, then try 'ok up'."
+}
+
+# Main
+setup_colors
+
+case "${1-}" in
+  up|u|r|real)
+    shift
+    run_up "$@"
+    ;;
+  rs|rl)
+    shift
+    run_restart "$@"
+    ;;
+  anon|a)
+    shift
+    run_anon "$@"
+    ;;
+  down|d)
+    shift
+    run_down "$@"
+    ;;
+  logs|l)
+    shift
+    run_logs "$@"
+    ;;
+  lint|li)
+    shift
+    run_lint "$@"
+    ;;
+  format|fmt|f)
+    shift
+    run_format "$@"
+    ;;
+  test:backend|tb)
+    shift
+    run_unit_tests services/backend "$@"
+    ;;
+  test:frontend|tf)
+    shift
+    run_unit_tests services/frontend "$@"
+    ;;
+  test:updater|tu)
+    shift
+    run_unit_tests updater/sis-updater-worker "$@"
+    ;;
+  test:services|ts)
+    shift
+    run_service_tests "$@"
+    ;;
+  test:api|ta)
+    shift
+    run_api_tests "$@"
+    ;;
+  test:component|tc)
+    shift
+    run_component_tests "$@"
+    ;;
+  test:e2e|te)
+    shift
+    run_e2e_tests "$@"
+    ;;
+  logs:updater|lu)
+    shift
+    run_logs_updater "$@"
+    ;;
+  importer|i)
+    shift
+    run_importer_db
+    ;;
+  updater)
+    shift
+    run_updater_db
+    ;;
+  install)
+    shift
+    cmd_install
+    ;;
+  -h|--help)
+    usage
+    ;;
+  "")
+    show_welcome
+    get_s3_config
+    main_loop
+    ;;
+  *)
+    errormsg "Unknown command: $1"
+    usage
+    ;;
+esac

services/.dockerignore

@@ -114,3 +114,4 @@ cypress/
 **/*Dockerfile
 **/build
 **/dist
+**/.npm

services/backend/src/config/programmeCodes.ts

@@ -68,8 +68,8 @@ export const programmeCodes = {
   MH50_012: 'GEOG',
   MH50_013: 'USP',
   MH50_014: 'EnCHiL',
-  MH50_015: 'ATM-MP-CE"',
-  MH50_015_ATMCE: 'ATM-MP-CE"',
+  MH50_015: 'ATM-MP-CE',
+  MH50_015_ATMCE: 'ATM-MP-CE',
   KH55_001: 'FARM',
   MH55_001: 'PROV',
   MH55_002: 'MPHARM',
@@ -103,13 +103,13 @@ export const programmeCodes = {
   KH74_001: 'KSV',
   KH80_001: 'MAATK',
   KH80_002: 'METSAK',
-  KH80_003: 'ETK',
+  KH80_003: 'ERK',
   KH80_004: 'YET',
   MH80_001: 'AGRI',
   MH80_002: 'AGERE',
   MH80_003: 'FOR',
   MH80_004: 'FOOD',
-  MH80_005: 'HNFB',
+  MH80_005: 'NUTR',
   MH80_006: 'EKM',
   MH80_007: 'MMB',
   MH80_008: 'emPlant',

services/backend/src/database/migrations_kone/20260929_00_add_course_group_id_to_excluded_courses.js

@@ -0,0 +1,14 @@
+const { STRING } = require('sequelize')
+
+// Migration step 1 of 2
+module.exports = {
+  up: async ({ context: queryInterface }) => {
+    await queryInterface.addColumn('excluded_courses', 'course_group_id', {
+      type: STRING,
+    })
+  },
+
+  down: async ({ context: queryInterface }) => {
+    await queryInterface.removeColumn('excluded_courses', 'course_group_id')
+  },
+}

services/backend/src/database/migrations_kone/20260929_01_add_course_group_ids_to_progress_criteria.js

@@ -0,0 +1,20 @@
+const { ARRAY, STRING } = require('sequelize')
+
+const YEARS = ['one', 'two', 'three', 'four', 'five', 'six']
+
+// Migration step 1 of 2
+module.exports = {
+  up: async ({ context: queryInterface }) => {
+    for (const year of YEARS) {
+      await queryInterface.addColumn('progress_criteria', `course_group_ids_year_${year}`, {
+        type: ARRAY(STRING),
+      })
+    }
+  },
+
+  down: async ({ context: queryInterface }) => {
+    for (const year of YEARS) {
+      await queryInterface.removeColumn('progress_criteria', `course_group_ids_year_${year}`)
+    }
+  },
+}

services/backend/src/database/migrations_kone/20260929_02_drop_course_codes_from_kone_tables.js

@@ -0,0 +1,94 @@
+/* eslint-disable no-console */
+const { ARRAY, STRING } = require('sequelize')
+
+const YEARS = ['one', 'two', 'three', 'four', 'five', 'six']
+const UNIQUE_CONSTRAINT = 'excluded_courses_programme_version_group_key'
+
+// Migration step 2 of 2
+// Rows the backfill could not map to a course group id are removed, as they no longer reference a course.
+// NOTE: down only restores the schema, the dropped course codes cannot be restored.
+module.exports = {
+  up: async ({ context: queryInterface }) => {
+    await queryInterface.sequelize.transaction(async transaction => {
+      const [, unmapped] = await queryInterface.sequelize.query(
+        'DELETE FROM excluded_courses WHERE course_group_id IS NULL',
+        { transaction }
+      )
+      console.log(`Deleted ${unmapped.rowCount} excluded courses without a course group id`)
+
+      const [, duplicates] = await queryInterface.sequelize.query(
+        `DELETE FROM excluded_courses a
+          USING excluded_courses b
+          WHERE a.id > b.id
+            AND a.programme_code = b.programme_code
+            AND a.curriculum_version IS NOT DISTINCT FROM b.curriculum_version
+            AND a.course_group_id = b.course_group_id`,
+        { transaction }
+      )
+      console.log(`Deleted ${duplicates.rowCount} duplicate excluded courses`)
+
+      await queryInterface.removeColumn('excluded_courses', 'course_code', { transaction })
+      await queryInterface.changeColumn(
+        'excluded_courses',
+        'course_group_id',
+        { type: STRING, allowNull: false },
+        { transaction }
+      )
+      await queryInterface.addConstraint('excluded_courses', {
+        fields: ['programme_code', 'curriculum_version', 'course_group_id'],
+        type: 'unique',
+        name: UNIQUE_CONSTRAINT,
+        transaction,
+      })
+
+      for (const year of YEARS) {
+        const [[{ count }]] = await queryInterface.sequelize.query(
+          `SELECT COUNT(*) AS count FROM progress_criteria
+            WHERE course_group_ids_year_${year} IS NULL AND cardinality(courses_year_${year}) > 0`,
+          { transaction }
+        )
+        console.log(`Emptied year ${year} of ${count} progress criteria without course group ids`)
+
+        await queryInterface.sequelize.query(
+          `UPDATE progress_criteria SET course_group_ids_year_${year} = '{}' WHERE course_group_ids_year_${year} IS NULL`,
+          { transaction }
+        )
+        await queryInterface.removeColumn('progress_criteria', `courses_year_${year}`, { transaction })
+        await queryInterface.changeColumn(
+          'progress_criteria',
+          `course_group_ids_year_${year}`,
+          { type: ARRAY(STRING), allowNull: false, defaultValue: [] },
+          { transaction }
+        )
+      }
+    })
+  },
+
+  down: async ({ context: queryInterface }) => {
+    await queryInterface.sequelize.transaction(async transaction => {
+      await queryInterface.removeConstraint('excluded_courses', UNIQUE_CONSTRAINT, { transaction })
+      await queryInterface.changeColumn(
+        'excluded_courses',
+        'course_group_id',
+        { type: STRING, allowNull: true },
+        { transaction }
+      )
+      await queryInterface.addColumn('excluded_courses', 'course_code', { type: STRING }, { transaction })
+
+      for (const year of YEARS) {
+        await queryInterface.changeColumn(
+          'progress_criteria',
+          `course_group_ids_year_${year}`,
+          { type: ARRAY(STRING), allowNull: true, defaultValue: null },
+          { transaction }
+        )
+        await queryInterface.addColumn(
+          'progress_criteria',
+          `courses_year_${year}`,
+          { type: ARRAY(STRING) },
+          { transaction }
+        )
+      }
+    })
+  },
+}

services/backend/src/models/course.ts

@@ -21,15 +21,19 @@ import { EnrollmentModel } from './enrollment'
 import { OrganizationModel } from './organization'
 
 @Table({
-  underscored: false,
+  underscored: true,
   modelName: 'course',
   tableName: 'course',
 })
 export class CourseModel extends Model implements Course {
+  /* COLUMNS */
   @PrimaryKey
   @Column(DataType.STRING)
   declare id: Course['id']
 
+  @Column(DataType.STRING)
+  declare groupId: Course['id']
+
   @ForeignKey(() => CourseModel)
   @Column(DataType.STRING)
   declare code: Course['code']
@@ -38,29 +42,17 @@ export class CourseModel extends Model implements Course {
   @Column(DataType.STRING)
   declare coursetypecode: Course['coursetypecode']
 
-  @BelongsTo(() => CourseTypeModel, { foreignKey: 'coursetypecode', targetKey: 'coursetypecode' })
-  declare courseType: Course['courseType']
-
-  @HasMany(() => CreditModel, { foreignKey: 'course_id', sourceKey: 'id' })
-  declare credits: Course['credits']
-
-  @HasMany(() => EnrollmentModel, { foreignKey: 'course_id' })
-  declare enrollments: Course['enrollments']
-
-  @BelongsToMany(() => OrganizationModel, () => CourseProviderModel, 'coursecode')
-  declare organizations: Course['organizations']
-
   @Column(DataType.BOOLEAN)
-  declare is_study_module: Course['is_study_module']
+  declare isStudyModule: Course['isStudyModule']
 
   @Column(DataType.JSONB)
   declare name: Course['name']
 
   @Column(DataType.DATE)
-  declare max_attainment_date: Course['max_attainment_date']
+  declare maxAttainmentDate: Course['maxAttainmentDate']
 
   @Column(DataType.DATE)
-  declare min_attainment_date: Course['min_attainment_date']
+  declare minAttainmentDate: Course['minAttainmentDate']
 
   @CreatedAt
   @Column({ field: 'created_at', type: DataType.DATE })
@@ -71,14 +63,31 @@ export class CourseModel extends Model implements Course {
   declare updatedAt: Course['updatedAt']
 
   @Column(DataType.JSONB)
-  declare substitutions: Course['substitutions']
+  declare validityPeriod: Course['validityPeriod']
 
   @Column(DataType.JSONB)
-  declare substitution_groups: Course['substitution_groups']
+  declare substitutionGroups: Course['substitutionGroups']
+
+  @Column(DataType.BOOLEAN)
+  declare isPrimary: Course['isPrimary']
 
   @Column(DataType.STRING)
-  declare course_unit_type: Course['course_unit_type']
+  declare courseUnitType: Course['courseUnitType']
 
-  @Column({ field: 'main_course_code', type: DataType.STRING })
-  declare mainCourseCode: Course['mainCourseCode']
+  /* RELATIONS */
+  @BelongsTo(() => CourseTypeModel, { foreignKey: 'coursetypecode', targetKey: 'coursetypecode' })
+  declare courseType: Course['courseType']
+
+  @HasMany(() => CreditModel, { foreignKey: 'course_id', sourceKey: 'id' })
+  declare credits: Course['credits']
+
+  @HasMany(() => EnrollmentModel, { foreignKey: 'course_id' })
+  declare enrollments: Course['enrollments']
+
+  @BelongsToMany(() => OrganizationModel, {
+    through: () => CourseProviderModel,
+    foreignKey: 'coursecode',
+    sourceKey: 'groupId',
+  })
+  declare organizations: Course['organizations']
 }

services/backend/src/models/courseProvider.ts

@@ -13,6 +13,7 @@ import { OrganizationModel } from './organization'
 export class CourseProviderModel extends Model implements CourseProvider {
   @ForeignKey(() => CourseModel)
   @Column(DataType.STRING)
+  // TODO: RENAME: this is courseGroupId, not code
   declare coursecode: CourseProvider['coursecode']
 
   @Column(DataType.JSONB)
@@ -20,6 +21,7 @@ export class CourseProviderModel extends Model implements CourseProvider {
 
   @ForeignKey(() => OrganizationModel)
   @Column(DataType.STRING)
+  // TODO: RENAME: this is organizationId, not code
   declare organizationcode: CourseProvider['organizationcode']
 
   @CreatedAt

services/backend/src/models/kone/excludedCourse.ts

@@ -17,7 +17,7 @@ export class ExcludedCourseModel extends Model implements ExcludedCourse {
   declare programme_code: ExcludedCourse['programme_code']
 
   @Column(DataType.STRING)
-  declare course_code: ExcludedCourse['course_code']
+  declare course_group_id: ExcludedCourse['course_group_id']
 
   @Column(DataType.STRING)
   declare curriculum_version: ExcludedCourse['curriculum_version']

services/backend/src/models/kone/progressCriteria.ts

@@ -17,22 +17,22 @@ export class ProgressCriteriaModel extends Model implements ProgressCriteria {
   declare curriculumVersion: ProgressCriteria['curriculumVersion']
 
   @Column(DataType.ARRAY(DataType.STRING))
-  declare coursesYearOne: ProgressCriteria['coursesYearOne']
+  declare courseGroupIdsYearOne: ProgressCriteria['courseGroupIdsYearOne']
 
   @Column(DataType.ARRAY(DataType.STRING))
-  declare coursesYearTwo: ProgressCriteria['coursesYearTwo']
+  declare courseGroupIdsYearTwo: ProgressCriteria['courseGroupIdsYearTwo']
 
   @Column(DataType.ARRAY(DataType.STRING))
-  declare coursesYearThree: ProgressCriteria['coursesYearThree']
+  declare courseGroupIdsYearThree: ProgressCriteria['courseGroupIdsYearThree']
 
   @Column(DataType.ARRAY(DataType.STRING))
-  declare coursesYearFour: ProgressCriteria['coursesYearFour']
+  declare courseGroupIdsYearFour: ProgressCriteria['courseGroupIdsYearFour']
 
   @Column(DataType.ARRAY(DataType.STRING))
-  declare coursesYearFive: ProgressCriteria['coursesYearFive']
+  declare courseGroupIdsYearFive: ProgressCriteria['courseGroupIdsYearFive']
 
   @Column(DataType.ARRAY(DataType.STRING))
-  declare coursesYearSix: ProgressCriteria['coursesYearSix']
+  declare courseGroupIdsYearSix: ProgressCriteria['courseGroupIdsYearSix']
 
   @Column(DataType.INTEGER)
   declare creditsYearOne: ProgressCriteria['creditsYearOne']

services/backend/src/models/organization.ts

@@ -45,7 +45,11 @@ export class OrganizationModel extends Model implements Organization {
   @HasMany(() => SISStudyRightModel, { foreignKey: 'facultyCode', sourceKey: 'code' })
   declare SISStudyRights: Organization['SISStudyRights']
 
-  @BelongsToMany(() => CourseModel, () => CourseProviderModel, 'organizationcode')
+  @BelongsToMany(() => CourseModel, {
+    through: () => CourseProviderModel,
+    foreignKey: 'organizationcode',
+    targetKey: 'groupId',
+  })
   declare courses: Organization['courses']
 
   @CreatedAt

services/backend/src/routes/courseExclusions.ts

@@ -7,7 +7,7 @@ import { hasFullAccessToStudentData } from '../util'
 const router = Router()
 
 router.post<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async (req, res) => {
-  const { programmeCode, courseCodes, curriculumVersion } = req.body
+  const { programmeCode, courseGroupIds, curriculumVersion } = req.body
   const { roles, programmeRights } = req.user
 
   const hasFullAccess = hasFullAccessToStudentData(roles)
@@ -15,7 +15,7 @@ router.post<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async (r
 
   if (!hasFullAccess && !hasAccessToProgramme) return res.status(403).end()
 
-  const result = await addExcludedCourses(courseCodes, curriculumVersion, programmeCode)
+  const result = await addExcludedCourses(courseGroupIds, curriculumVersion, programmeCode)
   if (!result) {
     res.status(400).end()
     return
@@ -24,7 +24,7 @@ router.post<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async (r
 })
 
 router.delete<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async (req, res) => {
-  const { programmeCode, courseCodes, curriculumVersion } = req.body
+  const { programmeCode, courseGroupIds, curriculumVersion } = req.body
   const { roles, programmeRights } = req.user
 
   const hasFullAccess = hasFullAccessToStudentData(roles)
@@ -32,7 +32,7 @@ router.delete<never, ExcludedCoursesResBody, ExcludedCoursesReqBody>('/', async
 
   if (!hasFullAccess && !hasAccessToProgramme) return res.status(403).end()
 
-  const result = await removeExcludedCourses(courseCodes, curriculumVersion, programmeCode)
+  const result = await removeExcludedCourses(courseGroupIds, curriculumVersion, programmeCode)
   if (!result) {
     res.status(400).end()
     return

services/backend/src/routes/courses.ts

@@ -8,8 +8,9 @@ import {
   CoursesMultiQuery,
   CourseYearlyStatsReqBody,
   CourseYearlyStatsQuery,
+  CourseDetails,
+  CourseDetailsQuery,
 } from '@oodikone/shared/routes/courses'
-import type { CourseWithSubsDetails } from '@oodikone/shared/types/course'
 import { getCourseDetails, getCourseYearlyStats } from '../services/courses'
 import { getCoursesByNameAndOrCode } from '../services/courses/courseFinders'
 import {
@@ -18,49 +19,31 @@ import {
   hasFullAccessToStudentData,
   validateParamLength,
 } from '../util'
+import { CourseYearlyStats } from '@oodikone/shared/types/courseYearlyStats'
 
 const router = Router()
 
 router.get<never, CanError<CoursesMultiResBody>, CoursesMultiReqBody, CoursesMultiQuery>(
   '/coursesmulti',
   async (req, res) => {
-    const { name, code, includeSpecial } = req.query
+    const { name, code } = req.query
     if (!(validateParamLength(name, 5) || validateParamLength(code, 2))) {
       return res.status(400).json({ error: 'Query parameter name or code is invalid' })
     }
 
-    const courses = await getCoursesByNameAndOrCode(name, code, includeSpecial === 'true')
-
-    const mergedCourses = new Map<string, CourseWithSubsDetails>()
-
-    for (const course of courses) {
-      if (!(course.max_attainment_date && course.min_attainment_date)) continue
-
-      if (!mergedCourses.has(course.code)) mergedCourses.set(course.code, structuredClone(course))
-      const mergedCourse = mergedCourses.get(course.code)!
-
-      if (mergedCourse.max_attainment_date < course.max_attainment_date) {
-        mergedCourse.max_attainment_date = course.max_attainment_date
-      }
-      if (mergedCourse.min_attainment_date > course.min_attainment_date) {
-        mergedCourse.min_attainment_date = course.min_attainment_date
-      }
-    }
-
-    res.json({ courses: Array.from(mergedCourses.values()) })
+    const courses = await getCoursesByNameAndOrCode(name, code)
+    res.json({ courses }).end()
   }
 )
 
-export type CourseYearlyStatsResBody = Awaited<ReturnType<typeof getCourseYearlyStats>>
+export type CourseYearlyStatsResBody = CourseYearlyStats[]
 
 router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody, CourseYearlyStatsQuery>(
   '/courseyearlystats',
   async (req, res) => {
-    const { codes, combineSubstitutions, separate, fromYearCode, toYearCode } = req.query
+    const { courses, substitutions, separate, fromYearCode, toYearCode } = req.query
 
-    const courseCodes = handleQueryArrays(codes)
-
-    if (!courseCodes?.length) {
+    if (!courses?.length) {
       return res.status(422).send({ error: 'Missing required query parameters' })
     }
 
@@ -79,14 +62,13 @@ router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody,
     const anonymize = !userHasFullAccessToStudentData && fullStudyProgrammeRights.length === 0
     const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
 
-    const useCombineSubstitutions = combineSubstitutions !== 'false'
     const useSeparate = separate === 'true'
 
     const results = await getCourseYearlyStats(
-      courseCodes,
+      handleQueryArrays(courses),
       useSeparate,
       anonymizationSalt,
-      useCombineSubstitutions,
+      substitutions === 'true',
       fromYearCode,
       toYearCode
     )
@@ -94,12 +76,12 @@ router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody,
   }
 )
 
-router.get<never, CanError<unknown>, never, { codes: string | string[] }>('/coursedetails', async (req, res) => {
-  const { codes: coursecodes } = req.query
-  const codes = handleQueryArrays(coursecodes)
+router.get<never, CanError<CourseDetails>, never, CourseDetailsQuery>('/coursedetails', async (req, res) => {
+  const { courses } = req.query
+  const courseIds = handleQueryArrays(courses)
 
-  if (!codes?.length) return res.status(500).end()
-  const details = await getCourseDetails(codes)
+  if (!courseIds.length) return res.status(422).send({ error: 'Missing required parameters' })
+  const details = await getCourseDetails(courseIds)
 
   return res.json(details)
 })

services/backend/src/routes/population.ts

@@ -14,17 +14,11 @@ import {
   type CustomPopulationByProgrammesQuery,
   type CustomPopulationByProgrammesResBody,
   type PopulationstatisticsStudyprogrammesResBody,
-  PopulationstatisticsMaxYearsToCreatePopulationFormQuery,
-  PopulationstatisticsMaxYearsToCreatePopulationFormResBody,
 } from '@oodikone/shared/routes/populations'
 import { Unification } from '@oodikone/shared/types'
-import { keyBy, mapToProviders } from '@oodikone/shared/util'
+import { keyBy, mapToProviders, MAX_POPULATION_SIZE } from '@oodikone/shared/util'
 import { rootOrgId } from '../config'
-import {
-  getCourseProvidersForCourses,
-  maxYearsToCreatePopulationFrom,
-  searchAndCombineSubstitutionGroupsToCodes,
-} from '../services/courses'
+import { getCourseProvidersForCourses, getRelevantCourseIdMap } from '../services/courses'
 import { encrypt } from '../services/encrypt'
 import { getDegreeProgrammesOfOrganization } from '../services/faculty/faculty'
 import { getStudentTagMap } from '../services/populations/getStudentData'
@@ -33,6 +27,8 @@ import { statisticsOf } from '../services/populations/statisticsOf'
 import { getStudentNumbersWithStudyRights } from '../services/populations/studentNumbersWithStudyRights'
 import { findByCourseAndSemesters, getStartAndEndDates } from '../services/students'
 import { getFullStudyProgrammeRights, handleQueryArrays, hasFullAccessToStudentData } from '../util'
+import { parseUnification } from '@oodikone/shared/types/unification'
+import logger from '../util/logger'
 
 const router = Router()
 
@@ -91,6 +87,7 @@ const obfuscateStuff = ({
     }),
     coursestatistics: {
       courses: coursestatistics.courses,
+      idToGroupIdMap: coursestatistics.idToGroupIdMap,
       enrollments: coursestatistics.enrollments.map(enrollment => ({
         ...enrollment,
         studentnumber: encryptionMap.get(enrollment.studentnumber) ?? enrollment.studentnumber,
@@ -182,42 +179,49 @@ router.get<
     programmeRights: userProgrammeRights,
     studentsUserCanAccess: allStudentsUserCanAccess,
   } = req.user
-  const { coursecodes: coursecodeJSON, from, to, separate, unifyCourses, ...restParams } = req.query
-  const includeSubstitutions = restParams.includeSubstitutions === 'true'
+  const { courses, from, to, separate, unifyCourses, substitutions } = req.query
 
-  if (!coursecodeJSON || !from || !to) {
-    return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
+  const coursesArray = handleQueryArrays(courses) as string[]
+
+  if (!coursesArray.length || !from || !to) {
+    return res.status(400).json({ error: 'Missing required parameters.' })
   }
 
+  const includeSubstitutions = substitutions === 'true'
   const isSeparate = separate === 'true'
-  const coursecodes = JSON.parse(coursecodeJSON)
-  const codes = includeSubstitutions ? await searchAndCombineSubstitutionGroupsToCodes(coursecodes) : coursecodes
+  const unification = parseUnification(unifyCourses) ?? Unification.UNIFY
 
-  const toFromDiff = Math.abs(Number(to) - Number(from) + 1)
-  const requestedYearsToCreatePopulationFrom = Math.ceil(isSeparate ? toFromDiff / 2 : toFromDiff) // 2 semesters = 1 year
+  const idToGroupIdMap = await getRelevantCourseIdMap(coursesArray, includeSubstitutions)
+  const relevantCourseIds = Object.keys(idToGroupIdMap)
 
-  const maxYearsForPopulation = await maxYearsToCreatePopulationFrom(codes, Unification.REGULAR)
-  if (requestedYearsToCreatePopulationFrom > maxYearsForPopulation) {
-    return res.status(400).json({ error: `Max years to create population from is ${maxYearsForPopulation}` })
+  const studentNumbers = await findByCourseAndSemesters(
+    relevantCourseIds,
+    Number(from),
+    Number(to),
+    isSeparate,
+    unification
+  )
+
+  if (studentNumbers.size > MAX_POPULATION_SIZE) {
+    logger.warn(`Attempted to load course population beyond allowed limits. ${JSON.stringify(req.query)}`)
+    return res.status(400).json({ error: 'Population size too large.' }).end()
   }
 
-  const studentNumbers = await findByCourseAndSemesters(codes, Number(from), Number(to), isSeparate, unifyCourses)
-
-  const courseProviders: string[] = await getCourseProvidersForCourses(coursecodes)
+  const courseProviders = await getCourseProvidersForCourses(relevantCourseIds)
   const fullStudyProgrammeRights = getFullStudyProgrammeRights(userProgrammeRights)
   const rightsMappedToProviders = mapToProviders(fullStudyProgrammeRights)
 
   const studentsUserCanAccess = courseProviders.some(provider => rightsMappedToProviders.includes(provider))
-    ? new Set(studentNumbers)
+    ? studentNumbers
     : new Set(allStudentsUserCanAccess)
 
-  const { startDate: startDate, endDate: endDate } = await getStartAndEndDates(Number(from), Number(to), isSeparate)
+  const { startDate, endDate } = await getStartAndEndDates(Number(from), Number(to), isSeparate)
 
   const studyRights = []
-  const tagMap = await getStudentTagMap(studyRights, studentNumbers, userId)
+  const tagMap = await getStudentTagMap(studyRights, [...studentNumbers], userId)
   const filterCreditsAndEnrollmentsByDate = true
   const result = await statisticsOf(
-    studentNumbers,
+    [...studentNumbers],
     studyRights,
     tagMap,
     filterCreditsAndEnrollmentsByDate,
@@ -232,7 +236,7 @@ router.get<
     studentsUserCanAccess,
   })
 
-  return res.json({ ...processed, mainCourseCodes: coursecodes, allCourseCodes: codes })
+  return res.json({ ...processed, idToGroupIdMap }).end()
 })
 
 // Used in custom population and single study guidance groups
@@ -355,21 +359,4 @@ router.get<never, PopulationstatisticsStudyprogrammesResBody>(
   }
 )
 
-router.get<
-  never,
-  PopulationstatisticsMaxYearsToCreatePopulationFormResBody,
-  never,
-  PopulationstatisticsMaxYearsToCreatePopulationFormQuery
->('/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
-  const courseCodes = JSON.parse(req.query.courseCodes) as string[]
-  const maxYearsToCreatePopulationFromBoth = await maxYearsToCreatePopulationFrom(courseCodes, Unification.UNIFY)
-  const maxYearsToCreatePopulationFromUni = await maxYearsToCreatePopulationFrom(courseCodes, Unification.REGULAR)
-  const maxYearsToCreatePopulationFromOpen = await maxYearsToCreatePopulationFrom(courseCodes, Unification.OPEN)
-  return res.json({
-    unifyCourses: maxYearsToCreatePopulationFromBoth,
-    uniCourses: maxYearsToCreatePopulationFromUni,
-    openCourses: maxYearsToCreatePopulationFromOpen,
-  })
-})
-
 export default router

services/backend/src/routes/studyProgrammeCriteria.ts

@@ -26,17 +26,17 @@ router.get('/', async (req: StudyProgrammeCriteriaRequest, res: Response) => {
 interface CriteriaCoursesRequest extends Request {
   body: {
     code: string
-    courses: string[]
+    courseGroupIds: string[]
     year: number
   }
 }
 
 router.post('/courses', async (req: CriteriaCoursesRequest, res: Response) => {
-  const { code, courses, year } = req.body
-  if (!code || !courses || !year) {
+  const { code, courseGroupIds, year } = req.body
+  if (!code || !courseGroupIds || !year) {
     return res.status(400).end()
   }
-  const studyProgrammeCriteria = await saveYearlyCourseCriteria(code, courses, year)
+  const studyProgrammeCriteria = await saveYearlyCourseCriteria(code, courseGroupIds, year)
   if (!studyProgrammeCriteria) {
     return res.status(404).end()
   }

services/backend/src/services/completedCoursesSearch.ts

@@ -23,39 +23,58 @@ type StudentWithCourses = Omit<StudentWithStudyplanNested, 'studyplans'> & {
 }
 
 const getCourses = async (courseCodes: string[]) => {
-  const courses: CompletedCoursesCourse[] = (
-    await CourseModel.findAll({
-      attributes: ['code', 'name', 'substitution_groups'],
-      where: where(fn('LOWER', col('code')), {
-        [Op.in]: courseCodes.map(code => code.toLowerCase()),
-      }),
-    })
-  ).map(course => course.toJSON())
+  if (courseCodes.length === 0) return []
+
+  const matchedCourses: Pick<CourseModel, 'groupId'>[] = await CourseModel.findAll({
+    attributes: ['groupId'],
+    where: where(fn('LOWER', col('code')), {
+      [Op.in]: courseCodes.map(code => code.toLowerCase()),
+    }),
+    raw: true,
+  })
+  const groupIds = [...new Set(matchedCourses.map(({ groupId }) => groupId))]
+  if (groupIds.length === 0) return []
+
+  const courses: CompletedCoursesCourse[] = await CourseModel.findAll({
+    attributes: ['code', 'name', 'groupId', 'substitutionGroups'],
+    where: { groupId: { [Op.in]: groupIds }, isPrimary: true },
+    raw: true,
+  })
   return courses
 }
 
+// Every course-position a student must hold a credit/enrollment for, in order to satisfy a substitution
+// combination: the main course's own groupId, plus each substitution group's member groupIds.
+const getGroupIdCombinationsFor = (course: CompletedCoursesCourse): string[][] => [
+  [course.groupId],
+  ...(course.substitutionGroups ?? []),
+]
+
 const getPassedCredits = async (
   courses: CompletedCoursesCourse[],
-  fullCourseCodes: string[],
-  studentNumbers: string[]
+  courseIds: string[],
+  studentNumbers: string[],
+  idToGroupId: Record<string, string>
 ) => {
-  const credits: Pick<CreditModel, 'course_code' | 'student_studentnumber' | 'credittypecode' | 'attainment_date'>[] =
-    await CreditModel.findAll({
-      raw: true,
-      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'attainment_date'],
-      order: [['attainment_date', 'DESC']],
-      where: {
-        course_code: {
-          [Op.in]: fullCourseCodes,
-        },
-        student_studentnumber: {
-          [Op.in]: studentNumbers,
-        },
-        credittypecode: {
-          [Op.not]: CreditTypeCode.FAILED,
-        },
+  const credits: Pick<
+    CreditModel,
+    'course_code' | 'course_id' | 'student_studentnumber' | 'credittypecode' | 'attainment_date'
+  >[] = await CreditModel.findAll({
+    raw: true,
+    attributes: ['course_code', 'course_id', 'student_studentnumber', 'credittypecode', 'attainment_date'],
+    order: [['attainment_date', 'DESC']],
+    where: {
+      course_id: {
+        [Op.in]: courseIds,
       },
-    })
+      student_studentnumber: {
+        [Op.in]: studentNumbers,
+      },
+      credittypecode: {
+        [Op.not]: CreditTypeCode.FAILED,
+      },
+    },
+  })
 
   const creditsByStudentNumber = Object.groupBy(credits, ({ student_studentnumber }) => student_studentnumber)
 
@@ -69,26 +88,28 @@ const getPassedCredits = async (
 
   Object.entries(creditsByStudentNumber).map(([studentNumber, credits]) => {
     // We know that credits will exist, because of the way we created the previous object
-    const studentCreditCourseCodes = credits!.map(({ course_code }) => course_code)
+    const studentCreditGroupIds = credits!.map(credit => idToGroupId[credit.course_id])
     for (const course of courses) {
-      // Also handle the main course code by adding it as a group
-      for (const group of [[course.code]].concat(course.substitution_groups)) {
-        if (group.every(code => studentCreditCourseCodes.includes(code))) {
-          // We just checked that (group) course code exists in credits so .find(...)! is ok
-          const groupCredits = group.map(code => credits!.find(credit => credit.course_code === code)!)
+      // The first combination is always the main course itself; the rest are substitution groups
+      getGroupIdCombinationsFor(course).forEach((positions, index) => {
+        const isMainCourse = index === 0
+        if (positions.every(groupId => studentCreditGroupIds.includes(groupId))) {
+          // We just checked that each position has a matching credit so .find(...)! is ok
+          const groupCredits = positions.map(
+            groupId => credits!.find(credit => idToGroupId[credit.course_id] === groupId)!
+          )
           if (groupCredits.length) {
             const groupCreditCodes = groupCredits.map(({ course_code }) => course_code)
             formattedCredits.push({
               courseCode: course.code,
-              substitution:
-                groupCreditCodes.includes(course.code) && groupCreditCodes.length === 1 ? null : groupCreditCodes,
+              substitution: isMainCourse ? null : groupCreditCodes,
               studentNumber,
               creditType: groupCredits.length > 1 ? CreditTypeCode.PASSED : groupCredits.at(0)!.credittypecode,
               date: groupCredits?.at(0)?.attainment_date ?? now(), // Credits are sorted by date in desc. order
             })
           }
         }
-      }
+      })
     }
   })
 
@@ -97,26 +118,28 @@ const getPassedCredits = async (
 
 const getEnrollments = async (
   courses: CompletedCoursesCourse[],
-  fullCourseCodes: string[],
-  studentNumbers: string[]
+  courseIds: string[],
+  studentNumbers: string[],
+  idToGroupId: Record<string, string>
 ) => {
-  const enrollments: Array<Pick<EnrollmentModel, 'course_code' | 'enrollment_date_time' | 'studentnumber'>> =
-    await EnrollmentModel.findAll({
-      attributes: ['course_code', 'enrollment_date_time', 'studentnumber'],
-      order: [['enrollment_date_time', 'DESC']],
-      where: {
-        course_code: {
-          [Op.in]: fullCourseCodes,
-        },
-        studentnumber: {
-          [Op.in]: studentNumbers,
-        },
-        state: {
-          [Op.eq]: EnrollmentState.ENROLLED,
-        },
-        enrollment_date_time: { [Op.gte]: enrollmentTimeDateThreshold },
+  const enrollments: Array<
+    Pick<EnrollmentModel, 'course_code' | 'course_id' | 'enrollment_date_time' | 'studentnumber'>
+  > = await EnrollmentModel.findAll({
+    attributes: ['course_code', 'course_id', 'enrollment_date_time', 'studentnumber'],
+    order: [['enrollment_date_time', 'DESC']],
+    where: {
+      course_id: {
+        [Op.in]: courseIds,
       },
-    })
+      studentnumber: {
+        [Op.in]: studentNumbers,
+      },
+      state: {
+        [Op.eq]: EnrollmentState.ENROLLED,
+      },
+      enrollment_date_time: { [Op.gte]: enrollmentTimeDateThreshold },
+    },
+  })
 
   const enrollmentsByStudents = Object.groupBy(enrollments, ({ studentnumber }) => studentnumber)
 
@@ -129,27 +152,27 @@ const getEnrollments = async (
 
   Object.entries(enrollmentsByStudents).map(([studentNumber, enrollments]) => {
     // We know that enrollments will exist, because of the way we created the previous object
-    const studentEnrollmentCourseCodes = enrollments!.map(({ course_code }) => course_code)
+    const studentEnrollmentGroupIds = enrollments!.map(enrollment => idToGroupId[enrollment.course_id])
     for (const course of courses) {
-      // Also handle the main course code by adding it as a group
-      for (const group of [[course.code]].concat(course.substitution_groups)) {
-        if (group.every(code => studentEnrollmentCourseCodes.includes(code))) {
-          // We just checked that (group) course code exists in enrollments so .find(...)! is ok
-          const groupEnrollments = group.map(code => enrollments!.find(enrollment => enrollment.course_code === code)!)
+      // The first combination is always the main course itself; the rest are substitution groups
+      getGroupIdCombinationsFor(course).forEach((positions, index) => {
+        const isMainCourse = index === 0
+        if (positions.every(groupId => studentEnrollmentGroupIds.includes(groupId))) {
+          // We just checked that each position has a matching enrollment so .find(...)! is ok
+          const groupEnrollments = positions.map(
+            groupId => enrollments!.find(enrollment => idToGroupId[enrollment.course_id] === groupId)!
+          )
           if (groupEnrollments.length) {
             const groupEnrollmentCodes = groupEnrollments.map(({ course_code }) => course_code)
             formattedEnrollments.push({
               courseCode: course.code,
-              substitution:
-                groupEnrollmentCodes.includes(course.code) && groupEnrollmentCodes.length === 1
-                  ? null
-                  : groupEnrollmentCodes,
+              substitution: isMainCourse ? null : groupEnrollmentCodes,
               studentNumber,
               date: groupEnrollments?.at(0)?.enrollment_date_time ?? now(), // Enrollments are sorted by date in desc. order
             })
           }
         }
-      }
+      })
     }
   })
 
@@ -173,9 +196,26 @@ const getStudents = async (studentNumbers: string[]): Promise<StudentWithCourses
     ],
   })
 
-  return students.map(student => {
-    const { studyplans, ...rest }: StudentWithStudyplanNested = student.toJSON()
-    const coursesInStudyPlan = studyplans.flatMap(studyplan => studyplan.included_courses)
+  const plainStudents = students.map((student): StudentWithStudyplanNested => student.toJSON())
+
+  // included_courses holds course ids, with a rare fallback to a raw code for "custom" (non-catalogued)
+  // entries. Resolve ids to their current code for display - anything missing from the map is already a code.
+  const includedCourseIds = [
+    ...new Set(
+      plainStudents.flatMap(student => student.studyplans.flatMap(({ included_courses }) => included_courses))
+    ),
+  ]
+  const courses: Pick<CourseModel, 'id' | 'code'>[] = await CourseModel.findAll({
+    attributes: ['id', 'code'],
+    where: { id: { [Op.in]: includedCourseIds } },
+    raw: true,
+  })
+  const idToCode = Object.fromEntries(courses.map(({ id, code }) => [id, code]))
+
+  return plainStudents.map(({ studyplans, ...rest }) => {
+    const coursesInStudyPlan = studyplans.flatMap(studyplan =>
+      studyplan.included_courses.map(idOrCode => idToCode[idOrCode] ?? idOrCode)
+    )
     return { ...rest, coursesInStudyPlan }
   })
 }
@@ -185,22 +225,21 @@ export const getCompletedCourses = async (
   courseCodes: string[]
 ): Promise<{ students: Omit<CompletedCoursesStudent, 'allEnrollments'>[]; courses: CompletedCoursesCourse[] }> => {
   const courses = await getCourses(courseCodes)
-  const courseCodesSet = new Set(courseCodes)
 
-  // Get *ALL* courses including any courses in any substitution groups
-  for (const course of courses) {
-    courseCodesSet.add(course.code)
-    for (const group of course.substitution_groups) {
-      for (const code of group) {
-        courseCodesSet.add(code)
-      }
-    }
-  }
+  // Get *ALL* courses' ids, including any courses in any substitution groups
+  const substitutionGroupIds = courses.flatMap(course => (course.substitutionGroups ?? []).flat())
+  const allGroupIds = [...new Set([...courses.map(course => course.groupId), ...substitutionGroupIds])]
 
-  const fullCourseCodes = Array.from(courseCodesSet)
+  const relevantCourses: Pick<CourseModel, 'id' | 'groupId'>[] = await CourseModel.findAll({
+    attributes: ['id', 'groupId'],
+    where: { groupId: { [Op.in]: allGroupIds } },
+    raw: true,
+  })
+  const courseIds = relevantCourses.map(({ id }) => id)
+  const idToGroupId = Object.fromEntries(relevantCourses.map(({ id, groupId }) => [id, groupId]))
 
-  const credits = await getPassedCredits(courses, fullCourseCodes, studentNumbers)
-  const enrollments = await getEnrollments(courses, fullCourseCodes, studentNumbers)
+  const credits = await getPassedCredits(courses, courseIds, studentNumbers, idToGroupId)
+  const enrollments = await getEnrollments(courses, courseIds, studentNumbers, idToGroupId)
   const studentInfo = await getStudents(studentNumbers)
 
   const studentCredits: StudentCredits = {}
@@ -276,7 +315,7 @@ export const getCompletedCourses = async (
   )
 
   students.forEach(student => {
-    courseCodes.forEach(courseCode => {
+    courses.forEach(({ code: courseCode }) => {
       const [latestEnrollment] = student.allEnrollments
         .filter(enrollment => enrollment.courseCode === courseCode)
         .sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime())

services/backend/src/services/courses/courseFinders.ts

@@ -1,70 +1,116 @@
-import { Op } from 'sequelize'
+import { col, fn, Op, QueryTypes } from 'sequelize'
 
 import { CourseWithSubsDetails } from '@oodikone/shared/types/course'
 import { CourseModel } from '../../models'
+import { dbConnections } from '../../database/connection'
 
-const nameLikeTerm = (name: string) => {
-  if (!name) {
-    return undefined
-  }
-  const term = `%${name.trim()}%`
-  return {
-    name: {
-      [Op.or]: {
-        fi: {
-          [Op.iLike]: term,
+const likeTerm = (userInput: string | undefined, type: 'name' | 'code') => {
+  const searchTerm = userInput?.trim().replace(/[\\%_]/g, '')
+  if (!searchTerm) return undefined
+
+  return type === 'name'
+    ? {
+        name: {
+          [Op.or]: {
+            fi: {
+              [Op.iLike]: `%${searchTerm}%`,
+            },
+            sv: {
+              [Op.iLike]: `%${searchTerm}%`,
+            },
+            en: {
+              [Op.iLike]: `%${searchTerm}%`,
+            },
+          },
         },
-        sv: {
-          [Op.iLike]: term,
-        },
-        en: {
-          [Op.iLike]: term,
+      }
+    : {
+        code: {
+          [Op.iLike]: `${searchTerm}%`,
         },
+      }
+}
+
+export const getCoursesByNameAndOrCode = async (name: string, code: string): Promise<CourseWithSubsDetails[]> => {
+  // Get groupIds for the course units (CU) the user is querying
+  // This step allows users to search with old or upcoming course codes and names
+  // Example: searching for GEOK_2011 (old code) returns results for GEOK2011 (new code)
+  const groupIds = (
+    await CourseModel.findAll({
+      attributes: [[fn('DISTINCT', col('group_id')), 'groupId']],
+      where: {
+        ...likeTerm(name, 'name'),
+        ...likeTerm(code, 'code'),
+        groupId: { [Op.ne]: null },
       },
-    },
-  }
-}
-
-const escapeRegExp = (string: string) => string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
-
-const codeLikeTerm = (code: string) => {
-  if (!code) {
-    return undefined
-  }
-  return {
-    code: {
-      [Op.iRegexp]: `^(AY|A)?${escapeRegExp(code)}`,
-    },
-  }
-}
-
-export const getCoursesByNameAndOrCode = async (name: string, code: string, includeSpecial: boolean) => {
-  const courses = await CourseModel.findAll({
-    where: {
-      ...(includeSpecial ? {} : { mainCourseCode: { [Op.ne]: null } }),
-      ...nameLikeTerm(name),
-      ...codeLikeTerm(code),
-    },
-    raw: true,
-  })
-
-  const coursesWithSubstitutionGroupDetails = await Promise.all(
-    courses.map<Promise<CourseWithSubsDetails>>(async course => {
-      const groupDetails = await Promise.all(
-        course?.substitution_groups?.map(async group => {
-          return await CourseModel.findAll({
-            attributes: ['name', 'code'],
-            where: { code: { [Op.in]: group } },
-            raw: true,
-          })
-        })
-      )
-
-      return { ...course, substitution_groups: groupDetails }
+      raw: true,
     })
+  ).map(({ groupId }) => groupId)
+
+  if (!groupIds.length) return []
+
+  // Gets name/code from the "primary" CU, and attainment dates from all CUs with the same groupId
+  const primaryCourses = await dbConnections.sequelize.query<CourseModel>(
+    `
+      WITH filtered AS (
+        SELECT *
+        FROM course
+        WHERE group_id IN (:groupIds)
+      ),
+      grouped AS (
+        SELECT
+          group_id,
+          MIN(min_attainment_date) AS min_attainment_date,
+          MAX(max_attainment_date) AS max_attainment_date
+        FROM filtered
+        GROUP BY group_id
+      ),
+      primary_course AS (
+        SELECT code, name, group_id, substitution_groups
+        FROM filtered
+        where is_primary = true
+      )
+      SELECT
+        g.group_id AS "groupId",
+        g.min_attainment_date AS "minAttainmentDate",
+        g.max_attainment_date AS "maxAttainmentDate",
+        p.name,
+        p.code,
+        p.substitution_groups AS "substitutionGroups"
+      FROM grouped g
+      JOIN primary_course p
+        ON p.group_id = g.group_id
+    `,
+    {
+      replacements: {
+        groupIds,
+      },
+      type: QueryTypes.SELECT,
+    }
   )
 
-  return coursesWithSubstitutionGroupDetails
+  const substitutionGroupIds = [...new Set(primaryCourses.map(course => course.substitutionGroups).flat(2))]
+  const substitutionCourses = substitutionGroupIds.length
+    ? Object.groupBy(
+        await CourseModel.findAll({
+          attributes: ['name', 'groupId', 'code'],
+          where: { groupId: { [Op.in]: substitutionGroupIds }, isPrimary: true },
+          raw: true,
+        }),
+        ({ groupId }) => groupId
+      )
+    : {}
+
+  const coursesWithSubstitutionDetails = primaryCourses.map(course => ({
+    ...course,
+    // NOTE: There are some substitutions that just don't exist in importer -> don't send undefined to client
+    substitutionGroups:
+      course.substitutionGroups
+        ?.map(groupIds => groupIds.flatMap(groupId => substitutionCourses[groupId]).filter(sub => !!sub))
+        .filter(group => group.length) ?? [],
+  }))
+
+  return coursesWithSubstitutionDetails
 }
 
 export const getCoursesByCodes = (codes: string[]) => {

services/backend/src/services/courses/courseYearlyStatsCounter.ts

@@ -1,5 +1,13 @@
 import { CreditTypeCode, type Name } from '@oodikone/shared/types'
-import type { Programme, FacultyYearStats, Grades, Group, Student } from '@oodikone/shared/types/courseYearlyStats'
+import type {
+  Programme,
+  FacultyYearStats,
+  Grades,
+  Group,
+  Student,
+  CourseStat,
+  Realisation,
+} from '@oodikone/shared/types/courseYearlyStats'
 import { getSemesterNamesByCode } from '../semesters'
 import type { OrganizationDetails } from './helpers'
 import { yearCodeToYear } from '@oodikone/shared/util'
@@ -215,7 +223,7 @@ export class CourseYearlyStatsCounter {
     })
   }
 
-  private async parseGroupStatistics(anonymizationSalt: string | null) {
+  private async parseGroupStatistics(anonymizationSalt: string | null): Promise<Realisation[]> {
     await this.insertEmptyRows()
     for (const [studentNumber, data] of this.students) {
       this.groups[data.code].students.studentNumbers.push(studentNumber)
@@ -230,8 +238,8 @@ export class CourseYearlyStatsCounter {
           coursecode: group.coursecode,
           attempts: {
             categories: {
-              failed: [] as string[],
-              passed: [] as string[],
+              failed: [],
+              passed: [],
             },
             grades: Object.keys(group.attempts.grades).reduce<Grades>((grades, grade) => {
               grades[grade] = []
@@ -240,8 +248,11 @@ export class CourseYearlyStatsCounter {
           },
           yearCode: group.yearCode,
           students: {
-            studentNumbers: [] as string[],
+            studentNumbers: [],
+            grades: {},
           },
+          allEnrollments: [],
+          enrollments: [],
         }
       }
 
@@ -293,7 +304,9 @@ export class CourseYearlyStatsCounter {
     return this.facultyStats
   }
 
-  public async getFinalStatistics(anonymizationSalt: string | null) {
+  public async getFinalStatistics(
+    anonymizationSalt: string | null
+  ): Promise<Pick<CourseStat, 'obfuscated' | 'statistics' | 'programmes' | 'facultyStats'>> {
     const statistics = await this.parseGroupStatistics(anonymizationSalt)
     const obfuscated = statistics.some(row => row.obfuscated)
 

services/backend/src/services/courses/creditsAndEnrollmentsOfCourse.ts

@@ -4,86 +4,90 @@ import { Credit, Enrollment } from '@oodikone/shared/models'
 import { EnrollmentState, Unification } from '@oodikone/shared/types'
 import { enrollmentTimeDateThreshold } from '@oodikone/shared/util'
 import {
-  StudentModel,
   CreditModel,
   SemesterModel,
   OrganizationModel,
   EnrollmentModel,
   SISStudyRightModel,
   SISStudyRightElementModel,
+  CourseModel,
 } from '../../models'
 import { now } from '../../util/clock'
 import { getIsOpen } from './helpers'
 
-export const getCreditsForCourses = async (codeGroups: string[][], unification: Unification, from: Date, to: Date) => {
-  const allCourseCodes = codeGroups.flatMap(group => group)
-
+export const getCreditsForCourses = async (
+  groupIdGroups: string[][],
+  allCourseIds: string[],
+  unification: Unification,
+  from: Date,
+  to: Date
+) => {
   // We need the credits grouped by student numbers so that we can check if a student has
   // completed a substitution group
-  const students = await StudentModel.findAll({
+  const attainments = await CreditModel.findAll({
     raw: true,
     nest: true,
-    attributes: ['studentnumber'],
+    attributes: [
+      'grade',
+      'course_code',
+      'course_id',
+      'credits',
+      'attainment_date',
+      'student_studentnumber',
+      'studyright_id',
+      'credittypecode',
+    ],
+    where: {
+      course_id: {
+        [Op.in]: allCourseIds,
+      },
+      is_open: getIsOpen(unification),
+      attainment_date: { [Op.between]: [from, to] },
+    },
+    order: [['attainment_date', 'ASC']],
     include: [
       {
-        model: CreditModel,
-        attributes: [
-          'grade',
-          'course_code',
-          'credits',
-          'attainment_date',
-          'student_studentnumber',
-          'studyright_id',
-          'credittypecode',
-        ],
+        model: SemesterModel,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
         where: {
-          course_code: {
-            [Op.in]: allCourseCodes,
+          startdate: {
+            [Op.lte]: now(),
           },
-          is_open: getIsOpen(unification),
-          attainment_date: { [Op.between]: [from, to] },
         },
-        order: [['attainment_date', 'ASC']],
-        include: [
-          {
-            model: SemesterModel,
-            attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
-            where: {
-              startdate: {
-                [Op.lte]: now(),
-              },
-            },
-          },
-        ],
+      },
+      {
+        model: CourseModel,
+        attributes: ['groupId'],
+        required: true,
       },
     ],
   })
 
-  const studentNumberToCredits = students.reduce<Record<string, Credit[]>>((acc, student) => {
-    acc[student.studentnumber] ??= []
-    acc[student.studentnumber] = acc[student.studentnumber].concat(student.credits)
-
+  const studentNumberToCredits = attainments.reduce<Record<string, Credit[]>>((acc, att) => {
+    acc[att.student_studentnumber] ??= []
+    acc[att.student_studentnumber].push(att)
     return acc
   }, {})
 
   const completedGroups: Credit[][] = []
   for (const studentCredits of Object.values(studentNumberToCredits)) {
-    const codesOfPassedCredits = studentCredits
+    const groupIdsOfPassedCredits = studentCredits
       .filter(credit => CreditModel.passed(credit))
-      .map(credit => credit.course_code)
-    for (const group of codeGroups) {
+      .map(credit => credit.course.groupId)
+
+    for (const group of groupIdGroups) {
       if (group.length === 1) {
         // Failed courses are only calculated for the original course and 1-to-1 substitutions
-        const credits = studentCredits.filter(credit => credit.course_code === group.at(0))
+        const credits = studentCredits.filter(credit => credit.course.groupId === group[0])
         if (credits.length) {
           completedGroups.push(credits)
         }
       }
       // For substitution groups we must only get passed groups
-      else if (group.every(code => codesOfPassedCredits.includes(code))) {
+      else if (group.every(groupId => groupIdsOfPassedCredits.includes(groupId))) {
         // The credit in question should always exist because we just checked that it does
         completedGroups.push(
-          studentCredits.filter(credit => group.includes(credit.course_code) && CreditModel.passed(credit))
+          studentCredits.filter(credit => group.includes(credit.course.groupId) && CreditModel.passed(credit))
         )
       } else {
         // "Partially completed substitution, do not count towards anything"
@@ -136,68 +140,63 @@ export const getStudentNumberToSrElementsMap = async (studentNumbers: string[])
 }
 
 export const getEnrollmentsForCourses = async (
-  codeGroups: string[][],
+  groupIdGroups: string[][],
+  allCourseIds: string[],
   unification: Unification,
   from: Date,
   to: Date
 ) => {
-  const allCourseCodes = codeGroups.flatMap(group => group)
-
-  const students = await StudentModel.findAll({
+  const enrollments = await EnrollmentModel.findAll({
     raw: true,
     nest: true,
-    attributes: ['studentnumber'],
+    attributes: ['studentnumber', 'enrollment_date_time', 'course_code', 'course_id', 'studyright_id'],
+    where: {
+      course_id: {
+        [Op.in]: allCourseIds,
+      },
+      enrollment_date_time: {
+        [Op.between]: [from, to],
+        // Date when OK changed from Oodi to Sisu data, studyright_id is null before that date
+        [Op.gte]: enrollmentTimeDateThreshold,
+      },
+      state: EnrollmentState.ENROLLED,
+      is_open: getIsOpen(unification),
+    },
     include: [
       {
-        model: EnrollmentModel,
-        attributes: ['studentnumber', 'enrollment_date_time', 'course_code', 'studyright_id'],
-        include: [
-          {
-            model: SemesterModel,
-            attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
-            where: {
-              startdate: {
-                [Op.lte]: now(),
-              },
-            },
-          },
-        ],
+        model: SemesterModel,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
         where: {
-          course_code: {
-            [Op.in]: allCourseCodes,
+          startdate: {
+            [Op.lte]: now(),
           },
-          enrollment_date_time: {
-            [Op.and]: {
-              [Op.between]: [from, to],
-              // Date when OK changed from Oodi to Sisu data, studyright_id is null before that date
-              [Op.gte]: enrollmentTimeDateThreshold,
-            },
-          },
-          state: EnrollmentState.ENROLLED,
-          is_open: getIsOpen(unification),
         },
       },
+      {
+        model: CourseModel,
+        attributes: ['groupId'],
+        required: true,
+      },
     ],
   })
 
-  const studentNumberToEnrollments = students.reduce<Record<string, Enrollment[]>>((acc, student) => {
-    acc[student.studentnumber] ??= []
-    acc[student.studentnumber] = acc[student.studentnumber].concat(student.enrollments)
-
+  const studentNumberToEnrollments = enrollments.reduce<Record<string, Enrollment[]>>((acc, enrollment) => {
+    acc[enrollment.studentnumber] ??= []
+    acc[enrollment.studentnumber].push(enrollment)
     return acc
   }, {})
 
   // Calculate all code groups that have enrollments for all of the courses in the group
   const completedEnrollments: Enrollment[][] = []
   for (const studentEnrollments of Object.values(studentNumberToEnrollments)) {
-    const codesOfEnrollments = studentEnrollments.map(enrollment => enrollment.course_code)
+    const groupIdsOfEnrollments = studentEnrollments.map(enrollment => enrollment.course.groupId)
 
-    for (const group of codeGroups) {
-      if (group.every(code => codesOfEnrollments.includes(code))) {
+    for (const group of groupIdGroups) {
+      if (group.every(groupId => groupIdsOfEnrollments.includes(groupId))) {
         completedEnrollments.push(
           studentEnrollments
-            .filter(enrollment => group.includes(enrollment.course_code))
-            .sort((a, b) => new Date(b.enrollment_date_time).getTime() - new Date(a.enrollment_date_time).getTime())
+            .filter(enrollment => group.includes(enrollment.course.groupId))
+            .sort((a, b) => b.enrollment_date_time.getTime() - a.enrollment_date_time.getTime())
         )
       } else {
         // Partially completed enrollments for a group, skipping...
@@ -205,5 +204,5 @@ export const getEnrollmentsForCourses = async (
     }
   }
 
-  return completedEnrollments
+  return completedEnrollments.filter(enrollmentGroup => enrollmentGroup.length)
 }

services/backend/src/services/courses/helpers.ts

@@ -1,6 +1,7 @@
 import { Op } from 'sequelize'
 
 import { Name, Unification } from '@oodikone/shared/types'
+import { CourseModel } from '../../models'
 
 export type FormattedProgramme = {
   code: string
@@ -23,3 +24,13 @@ export type OrganizationDetails = {
   code?: string
   name: Name
 }
+
+/** Gets all course ids for a course given course groupId(s) */
+export const getAllCourseIds = async (groupIds: string | string[]) =>
+  (
+    await CourseModel.findAll({
+      attributes: ['id'],
+      where: { groupId: { [Op.in]: Array.isArray(groupIds) ? groupIds : [groupIds] } },
+      raw: true,
+    })
+  ).map(course => course.id)

services/backend/src/services/courses/index.ts

@@ -1,5 +1,5 @@
 import crypto from 'crypto'
-import { Op, fn as dbFn, col as dbCol } from 'sequelize'
+import { Op } from 'sequelize'
 
 import { Credit, Enrollment } from '@oodikone/shared/models'
 import { Name, EnrollmentState, Unification, CreditTypeCode } from '@oodikone/shared/types'
@@ -7,7 +7,6 @@ import { enrollmentTimeDateThreshold, getSemesterCodeAt, yearCodeToYear, yearToY
 import { dateIsBetween } from '@oodikone/shared/util/datetime'
 import logger from '../../../src/util/logger'
 import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, SISStudyRightElementModel } from '../../models'
-import { isOpenUniCourseCode } from '../../util'
 import { now } from '../../util/clock'
 import { getSemestersAndYears, SemestersAndYears } from '../semesters'
 import { CourseYearlyStatsCounter } from './courseYearlyStatsCounter'
@@ -16,7 +15,8 @@ import {
   getEnrollmentsForCourses,
   getStudentNumberToSrElementsMap,
 } from './creditsAndEnrollmentsOfCourse'
-import { FormattedProgramme, getIsOpen } from './helpers'
+import { FormattedProgramme } from './helpers'
+import { CourseYearlyStats } from '@oodikone/shared/types/courseYearlyStats'
 
 const formatStudyRightElement = (studyRightElement: SISStudyRightElementModel): FormattedProgramme => ({
   code: studyRightElement.code,
@@ -46,7 +46,7 @@ type FormattedCredit = {
 
 // Is group in question for a single course (original or 1-to-1 substitution) or a substitution group with multiple courses
 const isSingleCourse = (group: Credit[] | Enrollment[]): boolean =>
-  group?.length === 1 || [...new Set(group?.map((course: Credit | Enrollment) => course.course_code))].length === 1
+  group?.length === 1 || [...new Set(group?.map((course: Credit | Enrollment) => course.course.groupId))].length === 1
 
 export const parseCredit = (
   creditGroup: Credit[],
@@ -153,17 +153,20 @@ const parseEnrollment = (
   }
 }
 
-const getSubstitutionGroupDetails = async (codeGroups: string[][]) => {
+const getSubstitutionGroupDetails = async (
+  groupIdGroups: string[][]
+): Promise<Pick<CourseModel, 'code' | 'name' | 'groupId'>[][]> => {
   const substitutionGroupDetails = await CourseModel.findAll({
-    attributes: ['code', 'name'],
+    attributes: ['code', 'name', 'groupId'],
     where: {
-      code: { [Op.in]: codeGroups.flatMap(group => group) },
+      groupId: { [Op.in]: groupIdGroups.flatMap(group => group) },
+      isPrimary: true,
     },
     raw: true,
   })
 
-  return codeGroups.map(group =>
-    group.map(code => substitutionGroupDetails.find(subCourse => subCourse.code === code)!)
+  return groupIdGroups.map(group =>
+    group.map(groupId => substitutionGroupDetails.find(subCourse => subCourse.groupId === groupId)!)
   )
 }
 
@@ -180,223 +183,177 @@ const getSemesterAndYearByDate = (
   return { year, semester }
 }
 
-export const getYearlyStatsOfNew = async (
-  course: Pick<CourseModel, 'code' | 'name' | 'substitution_groups'> | null,
-  courseCode: string,
-  separate: boolean,
-  unification: Unification,
-  anonymizationSalt: string | null,
-  combineSubstitutions: boolean,
-  studentNumberToSrElementsMap: Record<string, SISStudyRightElementModel[]>,
-  from: Date,
-  to: Date
-) => {
-  // Includes main course code and substitutions (if enabled)
-  const creditGroupCodes =
-    combineSubstitutions && course?.substitution_groups
-      ? [[courseCode]].concat(course.substitution_groups)
-      : [[courseCode]]
+export const getYearlyStatsOfNew =
+  (
+    primaryCourse: Pick<CourseModel, 'id' | 'groupId' | 'code' | 'name' | 'substitutionGroups'>,
+    separate: boolean,
+    anonymizationSalt: string | null,
+    substitutions: boolean,
+    studentNumberToSrElementsMap: Record<string, SISStudyRightElementModel[]>,
+    from: Date,
+    to: Date
+  ) =>
+  async (unification: Unification): Promise<CourseYearlyStats['openStats']> => {
+    const { groupId, name, code: courseCode } = primaryCourse
 
-  let filteredCreditGroupCodes: string[][]
-  switch (unification) {
-    // Include only course codes / group of course codes WITHOUT ANY AY prefix
-    case Unification.REGULAR:
-      filteredCreditGroupCodes = creditGroupCodes.filter(group => group.every(course => !isOpenUniCourseCode(course)))
-      break
-    // Include all courses / group of courses
-    case Unification.OPEN:
-    case Unification.UNIFY:
-    default:
-      filteredCreditGroupCodes = creditGroupCodes
-      break
-  }
+    // Includes main course code and substitutions (if enabled)
+    const creditGroupIds = substitutions ? [[groupId]].concat(primaryCourse.substitutionGroups ?? []) : [[groupId]]
 
-  const { semesters, years } = await getSemestersAndYears()
+    const allCourseIds = (
+      await CourseModel.findAll({
+        raw: true,
+        attributes: ['id'],
+        where: { groupId: { [Op.in]: creditGroupIds.flat() } },
+      })
+    ).map(({ id }) => id)
 
-  const [creditGroups, enrollmentGroups] = await Promise.all([
-    getCreditsForCourses(filteredCreditGroupCodes, unification, from, to),
-    getEnrollmentsForCourses(filteredCreditGroupCodes, unification, from, to),
-  ])
+    const { semesters, years } = await getSemestersAndYears()
 
-  const counter = new CourseYearlyStatsCounter()
+    const [creditGroups, enrollmentGroups] = await Promise.all([
+      getCreditsForCourses(creditGroupIds, allCourseIds, unification, from, to),
+      getEnrollmentsForCourses(creditGroupIds, allCourseIds, unification, from, to),
+    ])
 
-  // A single-course credit group can contain more than one real attempt for the same course code
-  // (e.g. failed once and passed on a retry). parseCredit collapses a group into a single
-  // attainment for student-stats (no student is counted twice), but attempt-stats (include all attempts for a course)
-  // need every attempt counted in its own year/semester.
-  // Substitution groups are left untouched
-  const splitIntoAttempts = (group: Credit[]): Credit[][] =>
-    isSingleCourse(group) && group.length > 1 ? group.map(credit => [credit]) : [group]
+    const counter = new CourseYearlyStatsCounter()
 
-  for (const creditGroup of creditGroups) {
-    if (!creditGroup?.length) continue
+    // A single-course credit group can contain more than one real attempt for the same course code
+    // (e.g. failed once and passed on a retry). parseCredit collapses a group into a single
+    // attainment for student-stats (no student is counted twice), but attempt-stats (include all attempts for a course)
+    // need every attempt counted in its own year/semester.
+    // Substitution groups are left untouched
+    const splitIntoAttempts = (group: Credit[]): Credit[][] =>
+      isSingleCourse(group) && group.length > 1 ? group.map(credit => [credit]) : [group]
 
-    const studyRightElements = studentNumberToSrElementsMap[creditGroup.at(0)?.student_studentnumber ?? 0] ?? []
+    for (const creditGroup of creditGroups) {
+      if (!creditGroup?.length) continue
 
-    const {
-      studentNumber,
-      grade,
-      passed,
-      semesterCode,
-      semesterName,
-      yearCode,
-      yearName,
-      attainmentDate,
-      programme,
-      courseCode: creditCourseCode,
-      credits,
-      creditTypeCode,
-    } = parseCredit(creditGroup, anonymizationSalt, courseCode, studyRightElements)
+      const studyRightElements = studentNumberToSrElementsMap[creditGroup.at(0)?.student_studentnumber ?? 0] ?? []
 
-    counter.markStudyProgramme(
-      studentNumber,
-      yearCode,
-      passed,
-      credits,
-      programme.code,
-      programme.name,
-      programme.facultyCode,
-      programme.organization
-    )
-
-    // Credits/attainments have quaranteed matching attainment_date and semesters/years
-    const groupCode = separate ? semesterCode : yearCode
-    const groupName = separate ? semesterName : yearName
-    counter.markStudentGradeToGroup(
-      studentNumber,
-      passed,
-      grade,
-      groupCode,
-      groupName,
-      creditCourseCode,
-      yearCode,
-      creditTypeCode
-    )
-
-    // Don't add students to student stats based on improved grades
-    if (creditTypeCode !== CreditTypeCode.IMPROVED) {
-      counter.markCreditToStudentCategories(studentNumber, attainmentDate, groupCode)
-    }
-
-    // The same group can include a failed and a passed credit in which case the parseCredit
-    // would return only the passed credit. Hence we split the failed and passed credit into
-    // two separate groups
-    for (const attemptCredits of splitIntoAttempts(creditGroup)) {
-      const attempt = parseCredit(attemptCredits, anonymizationSalt, courseCode, studyRightElements)
-      const attemptGroupCode = separate ? attempt.semesterCode : attempt.yearCode
-      const attemptGroupName = separate ? attempt.semesterName : attempt.yearName
-      counter.markAttemptToGroup(
-        attempt.studentNumber,
-        attempt.passed,
-        attempt.grade,
-        attemptGroupCode,
-        attemptGroupName,
-        attempt.courseCode,
-        attempt.yearCode
-      )
-    }
-  }
-
-  for (const enrollments of enrollmentGroups) {
-    if (!enrollments?.length) continue
-
-    for (const enrollment of enrollments) {
       const {
         studentNumber,
-        courseCode: enrollmentCourseCode,
-        enrollmentDateTime,
+        grade,
+        passed,
+        semesterCode,
+        semesterName,
+        yearCode,
+        yearName,
+        attainmentDate,
         programme,
-      } = parseEnrollment(
-        enrollment,
-        anonymizationSalt,
-        studentNumberToSrElementsMap[enrollment.studentnumber ?? 0] ?? []
-      )
-
-      // Enrollments can have conflicting enrollment_date_time and semestercode, so we need to manually select
-      // semestercode matching enrollment_date_time
-      const { semester, year } = getSemesterAndYearByDate(enrollment.enrollment_date_time, semesters, years)
+        courseCode: creditCourseCode,
+        credits,
+        creditTypeCode,
+      } = parseCredit(creditGroup, anonymizationSalt, courseCode, studyRightElements)
 
       counter.markStudyProgramme(
         studentNumber,
-        year.yearcode,
-        false, // passed
-        0, // credits
+        yearCode,
+        passed,
+        credits,
         programme.code,
         programme.name,
         programme.facultyCode,
         programme.organization
       )
 
-      const groupCode = separate ? semester.semestercode : year.yearcode
-      const groupName = separate ? semester.name : year.yearname
-
-      counter.markEnrollmentToGroup(
+      // Credits/attainments have quaranteed matching attainment_date and semesters/years
+      const groupCode = separate ? semesterCode : yearCode
+      const groupName = separate ? semesterName : yearName
+      counter.markStudentGradeToGroup(
         studentNumber,
-        enrollmentDateTime,
+        passed,
+        grade,
         groupCode,
         groupName,
-        enrollmentCourseCode,
-        year.yearcode
+        creditCourseCode,
+        yearCode,
+        creditTypeCode
       )
+
+      // Don't add students to student stats based on improved grades
+      if (creditTypeCode !== CreditTypeCode.IMPROVED) {
+        counter.markCreditToStudentCategories(studentNumber, attainmentDate, groupCode)
+      }
+
+      // The same group can include a failed and a passed credit in which case the parseCredit
+      // would return only the passed credit. Hence we split the failed and passed credit into
+      // two separate groups
+      for (const attemptCredits of splitIntoAttempts(creditGroup)) {
+        const attempt = parseCredit(attemptCredits, anonymizationSalt, courseCode, studyRightElements)
+        const attemptGroupCode = separate ? attempt.semesterCode : attempt.yearCode
+        const attemptGroupName = separate ? attempt.semesterName : attempt.yearName
+        counter.markAttemptToGroup(
+          attempt.studentNumber,
+          attempt.passed,
+          attempt.grade,
+          attemptGroupCode,
+          attemptGroupName,
+          attempt.courseCode,
+          attempt.yearCode
+        )
+      }
+    }
+
+    for (const enrollments of enrollmentGroups) {
+      for (const enrollment of enrollments) {
+        const studyRightElements = studentNumberToSrElementsMap[enrollment.studentnumber ?? 0] ?? []
+
+        const {
+          studentNumber,
+          courseCode: enrollmentCourseCode,
+          enrollmentDateTime,
+          programme,
+        } = parseEnrollment(enrollment, anonymizationSalt, studyRightElements)
+
+        // Enrollments can have conflicting enrollment_date_time and semestercode, so we need to manually select
+        // semestercode matching enrollment_date_time
+        const { semester, year } = getSemesterAndYearByDate(enrollment.enrollment_date_time, semesters, years)
+
+        counter.markStudyProgramme(
+          studentNumber,
+          year.yearcode,
+          false, // passed
+          0, // credits
+          programme.code,
+          programme.name,
+          programme.facultyCode,
+          programme.organization
+        )
+
+        const groupCode = separate ? semester.semestercode : year.yearcode
+        const groupName = separate ? semester.name : year.yearname
+
+        counter.markEnrollmentToGroup(
+          studentNumber,
+          enrollmentDateTime,
+          groupCode,
+          groupName,
+          enrollmentCourseCode,
+          year.yearcode
+        )
+      }
+    }
+
+    const statistics = await counter.getFinalStatistics(anonymizationSalt)
+
+    const substitutionGroups =
+      substitutions && primaryCourse.substitutionGroups?.length
+        ? await getSubstitutionGroupDetails(primaryCourse.substitutionGroups)
+        : [[{ code: courseCode, name, groupId }]]
+
+    return {
+      ...statistics,
+      courseCode,
+      groupId,
+      substitutionGroups,
+      name,
     }
   }
 
-  const statistics = await counter.getFinalStatistics(anonymizationSalt)
-
-  const substitutionGroups: Pick<CourseModel, 'code' | 'name'>[][] =
-    combineSubstitutions && course?.substitution_groups?.length
-      ? await getSubstitutionGroupDetails(course.substitution_groups)
-      : [[{ code: course!.code, name: course!.name }]]
-
-  return {
-    ...statistics,
-    coursecode: courseCode,
-    alternatives: substitutionGroups,
-    name: course?.name,
-  }
-}
-
-/**
-  Mostly-legacy function that prevents user from opening too big of a populations.
-  At the moment the threshold to prevent a population is so high it will never trigger.
-
- */
-export const maxYearsToCreatePopulationFrom = async (courseCodes: string[], unification: Unification) => {
-  const lastAttainmentDate = (await CourseModel.findOne({
-    attributes: [[dbFn('MAX', dbCol('max_attainment_date')), 'date']],
-    where: {
-      code: { [Op.in]: courseCodes },
-    },
-    raw: true,
-  })) as { date: Date } | null
-
-  if (lastAttainmentDate?.date == null) return 0
-
-  /** Amount of years before latest attainment that the attainment count is calcucated for */
-  const yearRange = 6
-
-  const newestAttainmentYear = lastAttainmentDate?.date.getFullYear()
-  const attainmentDateThreshold = new Date(newestAttainmentYear - yearRange, 0, 1)
-
-  /** Amount of attainments between latest-attainment-year - yearRange and latest attainment date */
-  const attainmentsWithinThreshold = await CreditModel.count({
-    where: {
-      course_code: { [Op.in]: courseCodes },
-      attainment_date: { [Op.gt]: attainmentDateThreshold },
-      is_open: getIsOpen(unification),
-    },
-  })
-
-  /** Limit the allowed attainments to 15_000 during the $yearRange years */
-  const maxAllowedAttainments = 15_000 * yearRange
-  return Math.round(Math.max(1, maxAllowedAttainments / attainmentsWithinThreshold))
-}
-
 export const getCourseYearlyStats = async (
-  courseCodes: string[],
+  courseGroupIds: string[],
   separate: boolean,
   anonymizationSalt: string | null,
-  combineSubstitutions: boolean,
+  substitutions: boolean,
   fromYearCode = yearToYearCode(1950).toString(),
   toYearCode: string = yearToYearCode(now().getFullYear()).toString()
 ) => {
@@ -404,26 +361,36 @@ export const getCourseYearlyStats = async (
   const from = new Date(`${yearCodeToYear(fromYearCode)}-08-01`) // FALL
   const to = new Date(`${yearCodeToYear(toYearCode) + 1}-07-31`) // SPRING next year
 
+  const relevantCourseIds = (
+    await CourseModel.findAll({
+      attributes: ['id'],
+      raw: true,
+      where: {
+        groupId: { [Op.in]: courseGroupIds },
+      },
+    })
+  ).map(({ id }) => id)
+
   const [credits, enrollments] = await Promise.all([
     CreditModel.findAll({
       attributes: ['student_studentnumber'],
+      raw: true,
       where: {
-        course_code: { [Op.in]: courseCodes },
+        course_id: { [Op.in]: relevantCourseIds },
         attainment_date: { [Op.between]: [from, to] },
       },
     }),
     EnrollmentModel.findAll({
       attributes: ['studentnumber'],
+      raw: true,
       where: {
-        course_code: {
-          [Op.in]: courseCodes,
+        course_id: {
+          [Op.in]: relevantCourseIds,
         },
         state: EnrollmentState.ENROLLED,
         enrollment_date_time: {
-          [Op.and]: {
-            [Op.between]: [from, to],
-            [Op.gte]: enrollmentTimeDateThreshold,
-          },
+          [Op.between]: [from, to],
+          [Op.gte]: enrollmentTimeDateThreshold,
         },
       },
     }),
@@ -441,71 +408,52 @@ export const getCourseYearlyStats = async (
 
   const studentNumberToSrElementsMap = await getStudentNumberToSrElementsMap([...studentNumbers])
 
-  const statsRegular = await Promise.all(
-    courseCodes.map(async courseCode => {
-      const course: Pick<CourseModel, 'code' | 'name' | 'substitution_groups'> | null = await CourseModel.findOne({
-        raw: true,
-        attributes: ['code', 'name', 'substitution_groups'],
-        where: { code: courseCode },
-      })
+  const stats = await Promise.all(
+    courseGroupIds.map(async groupId => {
+      const primaryCourse: Pick<CourseModel, 'id' | 'groupId' | 'code' | 'name' | 'substitutionGroups'> | null =
+        await CourseModel.findOne({
+          raw: true,
+          attributes: ['groupId', 'id', 'code', 'name', 'substitutionGroups'],
+          where: { groupId, isPrimary: true },
+        })
 
-      if (!course) {
-        logger.error('Course for course stats not found with code' + courseCode)
+      if (!primaryCourse) {
+        logger.error(`Primary course for course stats not found for: ${groupId}`)
         return {}
       }
 
+      const getYearlyStatsOfUnification = getYearlyStatsOfNew(
+        primaryCourse,
+        separate,
+        anonymizationSalt,
+        substitutions,
+        studentNumberToSrElementsMap,
+        from,
+        to
+      )
+
       const [openStats, regularStats, unifyStats] = await Promise.all([
-        getYearlyStatsOfNew(
-          course,
-          courseCode,
-          separate,
-          Unification.OPEN,
-          anonymizationSalt,
-          combineSubstitutions,
-          studentNumberToSrElementsMap,
-          from,
-          to
-        ),
-        getYearlyStatsOfNew(
-          course,
-          courseCode,
-          separate,
-          Unification.REGULAR,
-          anonymizationSalt,
-          combineSubstitutions,
-          studentNumberToSrElementsMap,
-          from,
-          to
-        ),
-        getYearlyStatsOfNew(
-          course,
-          courseCode,
-          separate,
-          Unification.UNIFY,
-          anonymizationSalt,
-          combineSubstitutions,
-          studentNumberToSrElementsMap,
-          from,
-          to
-        ),
+        getYearlyStatsOfUnification(Unification.OPEN),
+        getYearlyStatsOfUnification(Unification.REGULAR),
+        getYearlyStatsOfUnification(Unification.UNIFY),
       ])
 
       return { unifyStats, regularStats, openStats }
     })
   )
 
-  return statsRegular
+  return stats as CourseYearlyStats[]
 }
 
-export const getCourseProvidersForCourses = async (codes: string[]) =>
+export const getCourseProvidersForCourses = async (courseIds: string[]) =>
   (
     await OrganizationModel.findAll({
       attributes: ['code'],
       include: {
         model: CourseModel,
         where: {
-          code: {
-            [Op.in]: codes,
+          id: {
+            [Op.in]: courseIds,
           },
         },
       },
@@ -513,28 +461,47 @@ export const getCourseProvidersForCourses = async (codes: string[]) =>
     })
   ).map(({ code }) => code)
 
-export const getCourseDetails = async (codes: string[]) =>
+export const getCourseDetails = async (courseGroupIds: string[]) =>
   CourseModel.findAll({
-    attributes: ['code', 'name', 'substitution_groups'],
-    where: { code: { [Op.in]: codes } },
+    attributes: ['id', 'groupId', 'code', 'name', 'substitutionGroups', 'isStudyModule'],
+    where: { groupId: { [Op.in]: courseGroupIds }, isPrimary: true },
     raw: true,
   })
 
-// Add all substitution_groups and coursecodes together
-export const searchAndCombineSubstitutionGroupsToCodes = async (coursecodes: string[]) => {
-  const substitutionGroups = await CourseModel.findAll({
-    raw: true,
-    attributes: ['substitution_groups'],
-    where: {
-      code: { [Op.in]: coursecodes },
-    },
-  })
+/** First gets all unique groupIds from main course and substitutions (if enabled)
+ * Then returns an id-groupId map for all courses that match above groupId
+ */
+export const getRelevantCourseIdMap = async (courseGroupIds: string[], substitutions: boolean) => {
+  const combinedGroupIds = substitutions
+    ? [
+        ...new Set(
+          (
+            await CourseModel.findAll({
+              raw: true,
+              attributes: ['groupId', 'substitutionGroups'],
+              where: {
+                groupId: { [Op.in]: courseGroupIds },
+              },
+            })
+          )
+            .flatMap(({ groupId, substitutionGroups }) => [groupId, substitutionGroups])
+            .flat(2)
+        ),
+      ]
+    : courseGroupIds
 
-  return [
-    ...new Set(
-      coursecodes.concat(
-        substitutionGroups.flatMap(({ substitution_groups }) => substitution_groups.flatMap(code => code))
-      )
-    ),
-  ]
+  const idToGroupIdMap = (
+    await CourseModel.findAll({
+      raw: true,
+      attributes: ['id', 'groupId'],
+      where: {
+        groupId: { [Op.in]: combinedGroupIds },
+      },
+    })
+  ).reduce<Record<string, string>>((acc, { id, groupId }) => {
+    acc[id] = groupId
+    return acc
+  }, {})
+
+  return idToGroupIdMap
 }

services/backend/src/services/excludedCourses.ts

@@ -2,18 +2,35 @@ import { Op } from 'sequelize'
 
 import { ExcludedCourseModel } from '../models/kone'
 
-export const addExcludedCourses = async (courseCodes: string[], curriculumVersion: string, programmeCode: string) => {
-  return ExcludedCourseModel.bulkCreate(
-    courseCodes.map(courseCode => ({
+export const addExcludedCourses = async (
+  courseGroupIds: string[],
+  curriculumVersion: string,
+  programmeCode: string
+) => {
+  const existing = await ExcludedCourseModel.findAll({
+    attributes: ['course_group_id'],
+    where: {
       programme_code: programmeCode,
       curriculum_version: curriculumVersion,
-      course_code: courseCode,
-    }))
+      course_group_id: { [Op.in]: courseGroupIds },
+    },
+    raw: true,
+  })
+  const alreadyExcluded = new Set(existing.map(({ course_group_id }) => course_group_id))
+
+  return ExcludedCourseModel.bulkCreate(
+    courseGroupIds
+      .filter(courseGroupId => !alreadyExcluded.has(courseGroupId))
+      .map(courseGroupId => ({
+        programme_code: programmeCode,
+        curriculum_version: curriculumVersion,
+        course_group_id: courseGroupId,
+      }))
   )
 }
 
 export const removeExcludedCourses = async (
-  courseCodes: string[],
+  courseGroupIds: string[],
   curriculumVersion: string,
   programmeCode: string
 ) => {
@@ -21,7 +38,7 @@ export const removeExcludedCourses = async (
     where: {
       programme_code: programmeCode,
       curriculum_version: curriculumVersion,
-      course_code: { [Op.in]: courseCodes },
+      course_group_id: { [Op.in]: courseGroupIds },
     },
   })
 }

services/backend/src/services/faculty/faculty.ts

@@ -59,7 +59,7 @@ export const getDegreeProgrammesOfOrganization = async (organizationId: string,
     progId:
       programme.code in programmeCodes ? programmeCodes[programme.code as keyof typeof programmeCodes] : programme.code,
   }))
-  const programmesGroupedByCode = groupBy(orderBy(programmesWithProgIds, ['valid_from'], ['desc']), prog => prog.code)
+  const programmesGroupedByCode = groupBy(orderBy(programmesWithProgIds, ['validFrom'], ['desc']), prog => prog.code)
   const curriculumPeriods = await getCurriculumPeriods()
   const relevantProgrammes: ProgrammeModuleWithRelevantAttributes[] = []
 

services/backend/src/services/languageCenterData.ts

@@ -4,6 +4,7 @@ import { Op } from 'sequelize'
 import { EnrollmentState } from '@oodikone/shared/types'
 import { CourseModel, CreditModel, EnrollmentModel, SISStudyRightModel } from '../models'
 import { redisClient } from './redis'
+import { getPrimaryCoursesByGroupIds } from './studyProgramme'
 
 export const LANGUAGE_CENTER_REDIS_KEY = 'LANGUAGE_CENTER_DATA'
 
@@ -25,8 +26,8 @@ const getDifference = stats => {
 }
 
 const getLanguageCenterCourses = async () => {
-  const courses: Pick<CourseModel, 'code' | 'name'>[] = await CourseModel.findAll({
-    attributes: ['code', 'name'],
+  const courses: Pick<CourseModel, 'id' | 'groupId' | 'code' | 'name'>[] = await CourseModel.findAll({
+    attributes: ['id', 'groupId', 'code', 'name'],
     where: {
       [Op.or]: [{ code: { [Op.like]: 'KK%' } }, { code: { [Op.like]: 'AYKK%' } }],
     },
@@ -111,30 +112,54 @@ type Attempt = {
 export const computeLanguageCenterData = async () => {
   const courses = await getLanguageCenterCourses()
   const autumnSemester2017 = 135
+  const courseIds = courses.map(course => course.id)
+  const idToGroupId = new Map(courses.map(course => [course.id, course.groupId]))
+
+  const primaryCourses = await getPrimaryCoursesByGroupIds([...new Set(courses.map(course => course.groupId))])
+  const groupIdToPrimaryCode = new Map(primaryCourses.map(course => [course.groupId, course.code]))
+
+  const courseCodeOf = (courseId: string) => {
+    const groupId = idToGroupId.get(courseId)
+    return groupId ? groupIdToPrimaryCode.get(groupId) : undefined
+  }
 
   const credits = await CreditModel.findAll({
     raw: true,
-    attributes: ['course_code', 'student_studentnumber', 'semestercode', 'attainment_date', 'studyright_id'],
+    attributes: ['course_id', 'student_studentnumber', 'semestercode', 'attainment_date', 'studyright_id'],
     where: {
-      [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
+      course_id: { [Op.in]: courseIds },
       semestercode: { [Op.gte]: autumnSemester2017 },
       credittypecode: 4,
     },
+    order: [
+      ['attainment_date', 'DESC NULLS LAST'],
+      ['student_studentnumber', 'ASC'],
+      ['course_id', 'ASC'],
+    ], // Add a fallbacks sorting so that runs would be more deterministic
   })
 
   const enrollments = await EnrollmentModel.findAll({
     raw: true,
-    attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'studyright_id', 'state'],
+    attributes: ['studentnumber', 'semestercode', 'course_id', 'enrollment_date_time', 'studyright_id', 'state'],
     where: {
-      [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
+      course_id: { [Op.in]: courseIds },
       state: { [Op.in]: ['ENROLLED', 'REJECTED'] },
       // EnrollmentDateTimeThreshold need not be used here as we are not counting failed courses
     },
+    order: [
+      ['enrollment_date_time', 'DESC NULLS LAST'],
+      ['studentnumber', 'ASC'],
+      ['course_id', 'ASC'], // Last fallback
+    ],
   })
 
   const studyRights = await SISStudyRightModel.findAll({
     raw: true,
-    order: [['end_date', 'DESC NULLS LAST']],
+    order: [
+      ['end_date', 'DESC NULLS LAST'],
+      ['start_date', 'DESC'],
+      ['id', 'DESC'],
+    ],
   })
 
   const attemptStudyRightToFacultyMap = studyRights.reduce<
@@ -149,12 +174,14 @@ export const computeLanguageCenterData = async () => {
   const attemptsByStudents: Record<string, Attempt[]> = {}
 
   credits.forEach(credit => {
+    const courseCode = courseCodeOf(credit.course_id)
+    if (!courseCode) return
     const studentNumber = credit.student_studentnumber
     studentNumbers.add(studentNumber)
     attemptsByStudents[studentNumber] ??= []
     attemptsByStudents[studentNumber].push({
       studentNumber,
-      courseCode: credit.course_code,
+      courseCode,
       completed: true,
       date: credit.attainment_date,
       faculty: attemptStudyRightToFacultyMap[credit.studyright_id],
@@ -163,15 +190,15 @@ export const computeLanguageCenterData = async () => {
   })
 
   enrollments.forEach(enrollment => {
+    const courseCode = courseCodeOf(enrollment.course_id)
+    if (!courseCode) return
     const studentNumber = enrollment.studentnumber
     studentNumbers.add(studentNumber)
     attemptsByStudents[studentNumber] ??= []
     if (
       attemptsByStudents[studentNumber].find(
         attempt =>
-          !attempt.completed &&
-          attempt.semestercode === enrollment.semestercode &&
-          attempt.courseCode === enrollment.course_code
+          !attempt.completed && attempt.semestercode === enrollment.semestercode && attempt.courseCode === courseCode
       )
     ) {
       return
@@ -179,7 +206,7 @@ export const computeLanguageCenterData = async () => {
 
     attemptsByStudents[studentNumber].push({
       studentNumber,
-      courseCode: enrollment.course_code,
+      courseCode,
       completed: false,
       date: enrollment.enrollment_date_time,
       faculty: attemptStudyRightToFacultyMap[enrollment.studyright_id],
@@ -226,7 +253,7 @@ export const computeLanguageCenterData = async () => {
     }
   })
 
-  const unorderedTableData = createArrayOfCourses(attempts, courses)
+  const unorderedTableData = createArrayOfCourses(attempts, primaryCourses)
   const tableData = orderBy(unorderedTableData, 'code')
 
   const faculties = [...new Set(attempts.map(({ faculty }) => faculty))]

services/backend/src/services/populations/closeToGraduation.ts

@@ -24,7 +24,8 @@ const findThesisAndLatestAndEarliestAttainments = (
   studyPlan: Studyplan,
   attainments: Credit[],
   degreeProgrammeType: DegreeProgrammeType,
-  studyRightId: string
+  studyRightId: string,
+  idToGroupIdMap: Record<string, string>
 ) => {
   let thesisData: Credit | undefined
   const attainmentDates: AttainmentDates = {}
@@ -32,16 +33,19 @@ const findThesisAndLatestAndEarliestAttainments = (
     [DegreeProgrammeType.BACHELOR]: 'urn:code:course-unit-type:bachelors-thesis',
     [DegreeProgrammeType.MASTER]: 'urn:code:course-unit-type:masters-thesis',
   }
+  const includedCourseGroupIds = new Set(
+    studyPlan.included_courses.map(id => idToGroupIdMap[id]).filter((groupId): groupId is string => !!groupId)
+  )
 
   for (const attainment of attainments) {
     if (
-      attainment.course?.course_unit_type === thesisCodes[degreeProgrammeType] &&
+      attainment.course?.courseUnitType === thesisCodes[degreeProgrammeType] &&
       attainment.studyright_id === studyRightId
     ) {
       thesisData = attainment
     }
     attainmentDates.latestTotal ??= attainment.attainment_date
-    if (studyPlan.included_courses.includes(attainment.course?.code)) {
+    if (attainment.course?.groupId && includedCourseGroupIds.has(attainment.course.groupId)) {
       attainmentDates.latestHops ??= attainment.attainment_date
       if (
         !attainmentDates.earliestHops ||
@@ -55,7 +59,12 @@ const findThesisAndLatestAndEarliestAttainments = (
   return { attainmentDates, thesisData }
 }
 
-const formatStudent = (student: Student, facultyMap: Record<string, Name>, currentSemesterCode: number) => {
+const formatStudent = (
+  student: Student,
+  facultyMap: Record<string, Name>,
+  currentSemesterCode: number,
+  idToGroupIdMap: Record<string, string>
+) => {
   const {
     studentnumber: studentNumber,
     abbreviatedname: name,
@@ -101,7 +110,8 @@ const formatStudent = (student: Student, facultyMap: Record<string, Name>, curre
       studyPlan,
       student.credits,
       degreeProgrammeType,
-      studyRightId
+      studyRightId,
+      idToGroupIdMap
     )
 
     acc.push({
@@ -230,12 +240,12 @@ export const findStudentsCloseToGraduation = async (studentNumbers?: string[]) =
         },
         {
           model: CreditModel,
-          attributes: ['attainment_date', 'grade', 'studyright_id'],
+          attributes: ['attainment_date', 'grade', 'studyright_id', 'course_id'],
           where: { credittypecode: CreditTypeCode.PASSED },
           include: [
             {
               model: CourseModel,
-              attributes: ['code', 'course_unit_type'],
+              attributes: ['code', 'courseUnitType', 'groupId'],
             },
           ],
         },
@@ -264,10 +274,20 @@ export const findStudentsCloseToGraduation = async (studentNumbers?: string[]) =
     facultyMap[programme.code] = programme['organization.name']
   }
 
+  const includedCourseIds = [
+    ...new Set(students.flatMap(student => student.studyplans.flatMap(sp => sp.included_courses))),
+  ]
+  const includedCourses = await CourseModel.findAll({
+    attributes: ['id', 'groupId'],
+    where: { id: { [Op.in]: includedCourseIds } },
+    raw: true,
+  })
+  const idToGroupIdMap = Object.fromEntries(includedCourses.map(({ id, groupId }) => [id, groupId]))
+
   const { semestercode: currentSemesterCode } = await getCurrentSemester()
 
   return students
-    .flatMap(student => formatStudent(student, facultyMap, currentSemesterCode))
+    .flatMap(student => formatStudent(student, facultyMap, currentSemesterCode, idToGroupIdMap))
     .reduce(
       (acc, student) => {
         if (student.programme.degreeProgrammeType === DegreeProgrammeType.BACHELOR) {

services/backend/src/services/populations/getStudentData.ts

@@ -1,6 +1,6 @@
-import { Op } from 'sequelize'
+import { col, Op } from 'sequelize'
 
-import { Credit, Enrollment, SISStudyRight, SISStudyRightElement, Student, Studyplan } from '@oodikone/shared/models'
+import { SISStudyRight, SISStudyRightElement, Student, Studyplan } from '@oodikone/shared/models'
 import { Tag, TagStudent } from '@oodikone/shared/models/kone'
 import { EnrollmentState } from '@oodikone/shared/types'
 import { enrollmentTimeDateThreshold } from '@oodikone/shared/util'
@@ -12,8 +12,10 @@ import {
   StudyplanModel,
   SISStudyRightModel,
   SISStudyRightElementModel,
+  CourseModel,
 } from '../../models'
 import { TagModel, TagStudentModel } from '../../models/kone'
+import { StudentCredit, StudentEnrollment } from '@oodikone/shared/types/studentData'
 
 export type StudentTags = TagStudent & {
   tag: Pick<Tag, 'tag_id' | 'tagname' | 'personal_user_id'>
@@ -46,17 +48,21 @@ export const getStudentTagMap = async (studyRights: string[], studentNumbers: st
   return studentTagMap
 }
 
-export type StudentEnrollment = Pick<
-  Enrollment,
-  'course_code' | 'state' | 'enrollment_date_time' | 'semestercode' | 'studentnumber' | 'studyright_id'
->
 export const getEnrollments = (
   studentNumbers: string[],
   startDate: string,
   endDate: string
-): Promise<Array<StudentEnrollment>> =>
+): Promise<StudentEnrollment[]> =>
   EnrollmentModel.findAll({
-    attributes: ['course_code', 'state', 'enrollment_date_time', 'semestercode', 'studentnumber', 'studyright_id'],
+    attributes: [
+      'course_id',
+      'course_code',
+      'state',
+      'enrollment_date_time',
+      'semestercode',
+      'studentnumber',
+      'studyright_id',
+    ],
     where: {
       studentnumber: {
         [Op.in]: studentNumbers,
@@ -185,24 +191,7 @@ export const getStudents = (studentNumbers: string[]): Promise<Array<StudentData
     },
   }).then(data => data.map(x => x.get({ plain: true })))
 
-export type StudentCredit = Pick<
-  Credit,
-  | 'grade'
-  | 'credits'
-  | 'credittypecode'
-  | 'attainment_date'
-  | 'isStudyModule'
-  | 'student_studentnumber'
-  | 'course_code'
-  | 'language'
-  | 'studyright_id'
->
-
-export const getCredits = async (
-  studentNumbers: string[],
-  startDate: string,
-  endDate: string
-): Promise<Array<StudentCredit>> =>
+export const getCredits = async (studentNumbers: string[], startDate: string, endDate: string) =>
   CreditModel.findAll({
     raw: true,
     attributes: [
@@ -212,15 +201,23 @@ export const getCredits = async (
       'attainment_date',
       'isStudyModule',
       'student_studentnumber',
+      'course_id',
       'course_code',
       'language',
       'studyright_id',
+      [col('course.group_id'), 'courseGroupId'],
     ],
+    include: {
+      model: CourseModel,
+      attributes: [],
+      as: 'course',
+      required: true,
+    },
     where: {
       student_studentnumber: { [Op.in]: studentNumbers },
       attainment_date: { [Op.between]: [startDate, endDate] },
     },
-  })
+  }) as unknown as Promise<StudentCredit[]>
 
 export const getStudyRightElementsForStudyRight = async (studentNumbers: string[], code: string) =>
   SISStudyRightElementModel.findAll({

services/backend/src/services/populations/shared.ts

@@ -77,11 +77,11 @@ export const getOptionsForStudents = (
   return optionMap
 }
 
-export const getCourses = (courses: string[]): Promise<Array<CourseStats>> =>
+export const getCourseGroupIds = (courseIds: string[]): Promise<Array<Pick<CourseStats, 'id' | 'groupId'>>> =>
   CourseModel.findAll({
-    attributes: ['code', 'name', 'substitution_groups', 'is_study_module'],
+    attributes: ['id', 'groupId'],
     where: {
-      code: { [Op.in]: courses },
+      id: { [Op.in]: courseIds },
     },
     raw: true,
   })

services/backend/src/services/populations/statisticsOf.ts

@@ -1,17 +1,17 @@
 import { getDegreeProgrammeType } from '../../util'
 import { now } from '../../util/clock'
 import { getCriteria } from '../studyProgramme/studyProgrammeCriteria'
+import { getCourseDetails } from '../courses'
 import { formatStudentForAPI } from './formatStatisticsForApi'
 import {
-  type StudentEnrollment,
-  type StudentCredit,
   type StudentTags,
   getStudents,
   getEnrollments,
   getCredits,
   getStudyRightElementsForStudyRight,
 } from './getStudentData'
-import { getCourses, getOptionsForStudents } from './shared'
+import { getCourseGroupIds, getOptionsForStudents } from './shared'
+import { StudentCredit, StudentEnrollment } from '@oodikone/shared/types/studentData'
 
 export type OptimizedStatisticsQuery = {
   userId: string
@@ -63,10 +63,13 @@ export const statisticsOf = async (
       getStudyRightElementsForStudyRight(studentNumbers, code),
     ])
 
-  const courseCodes = new Set<string>()
-  for (const { course_code } of credits) courseCodes.add(course_code)
-  for (const { course_code } of enrollments) courseCodes.add(course_code)
-  const courses = await getCourses(Array.from(courseCodes))
+  const courseIds = new Set<string>()
+  for (const { course_id } of credits) courseIds.add(course_id)
+  for (const { course_id } of enrollments) courseIds.add(course_id)
+  const idToGroupIdRows = await getCourseGroupIds([...courseIds])
+  const idToGroupIdMap = Object.fromEntries(idToGroupIdRows.map(({ id, groupId }) => [id, groupId]))
+  const groupIds = [...new Set(idToGroupIdRows.map(({ groupId }) => groupId))]
+  const courses = await getCourseDetails(groupIds)
 
   const optionData = getOptionsForStudents(studyRightElementsForStudyRight, degreeProgrammeType)
   const formattedStudents = students.map(student => {
@@ -79,6 +82,6 @@ export const statisticsOf = async (
   return {
     students: formattedStudents,
     criteria,
-    coursestatistics: { courses, enrollments, credits },
+    coursestatistics: { courses, idToGroupIdMap, enrollments, credits },
   }
 }

services/backend/src/services/programmeModules.ts

@@ -47,7 +47,7 @@ export const getCurriculumOptions = async (code: string) => {
   }
 }
 
-type ModuleWithChildren = Pick<ProgrammeModule, 'id' | 'type' | 'code' | 'name'> & {
+type ModuleWithChildren = Pick<ProgrammeModule, 'id' | 'group_id' | 'type' | 'code' | 'name'> & {
   // TODO: This is supposed to be in the ProgrammeModule
   degree_programme_type: DegreeProgrammeType | null
 
@@ -90,6 +90,7 @@ const recursivelyGetModuleAndChildren = async (code: string, curriculum_period_i
         GROUP BY pm.id, c.name, c.code, c.order, c.id
       ) SELECT
         id,
+        group_id,
         type,
         module_order,
         parent_code,
@@ -141,7 +142,7 @@ const labelProgrammes = (modules: ModuleWithChildren[], excludedCourses: Exclude
       label: `${module.parent_code}\n${module.parent_name?.fi}`,
       orderNumber: module.module_order,
     }
-    const foundCourse = excludedCourses.find(course => course.course_code === module.code)
+    const foundCourse = excludedCourses.find(course => course.course_group_id === module.group_id)
     const visible = { visibility: !foundCourse, id: foundCourse?.id ?? null }
 
     return { ...module, label, visible }
@@ -180,6 +181,7 @@ const getCoursesAndModulesForProgramme = async (code: string, periodIds: string)
 
   return {
     courses: labelProgrammes(modifiedCourses, excludedCourses).map(mod => ({
+      group_id: mod.group_id,
       code: mod.code,
       name: mod.name,
       degree_programme_type: mod.degree_programme_type,

services/backend/src/services/providerCredits.ts

@@ -3,7 +3,7 @@ import { EnrollmentType, ExtentCode } from '@oodikone/shared/types'
 import { CreditStats } from '@oodikone/shared/types/studyProgramme'
 import { mapToProviders } from '@oodikone/shared/util'
 import { getCreditStats, setCreditStats } from './analyticsService'
-import { getCourseCodesOfProvider } from './providers'
+import { getCourseIdsOfProvider } from './providers'
 import { getCreditsForProvider, getTransferredCredits } from './studyProgramme/creditGetters'
 import { getStudentHetuStateMap } from './studyProgramme/studentGetters'
 import { defineYear } from './studyProgramme/studyProgrammeHelpers'
@@ -107,8 +107,8 @@ const getBasicDegreeStudyRight = (
 export const computeCreditsProduced = async (providerCode: string, isAcademicYear: boolean): Promise<CreditStats> => {
   const since = new Date('2017-01-01')
   const rapoFormattedProviderCode = mapToProviders([providerCode])[0]
-  const courses = await getCourseCodesOfProvider(rapoFormattedProviderCode)
-  const credits = await getCreditsForProvider(rapoFormattedProviderCode, courses, since)
+  const courseIds = await getCourseIdsOfProvider(rapoFormattedProviderCode)
+  const credits = await getCreditsForProvider(rapoFormattedProviderCode, courseIds, since)
 
   // This part also adds oikis Vaasa which is provided by a different organization.
   // Unknown if there are other similar cases!
@@ -119,8 +119,8 @@ export const computeCreditsProduced = async (providerCode: string, isAcademicYea
   const vaasaProvider =
     rapoFormattedProviderCode in vaasaCodes ? vaasaCodes[rapoFormattedProviderCode as keyof typeof vaasaCodes] : null
   if (vaasaProvider) {
-    const courses = await getCourseCodesOfProvider(vaasaProvider)
-    const vaasaCredits = await getCreditsForProvider(vaasaProvider, courses, since)
+    const vaasaCourseIds = await getCourseIdsOfProvider(vaasaProvider)
+    const vaasaCredits = await getCreditsForProvider(vaasaProvider, vaasaCourseIds, since)
     credits.push(...vaasaCredits)
   }
 

services/backend/src/services/providers.ts

@@ -1,4 +1,5 @@
-import { getOpenUniCourseCode } from '../../src/util'
+import { Op } from 'sequelize'
+
 import { CourseModel, OrganizationModel } from '../models'
 
 export const getAllProviders = async () => {
@@ -9,10 +10,10 @@ export const getAllProviders = async () => {
   return providers
 }
 
-export const getCourseCodesOfProvider = async (provider: string) => {
+export const getCourseIdsOfProvider = async (provider: string): Promise<string[]> => {
   const coursesByProvider = await CourseModel.findAll({
     raw: true,
-    attributes: ['id', 'code', 'substitution_groups'],
+    attributes: ['id', 'substitutionGroups'],
     include: {
       model: OrganizationModel,
       required: true,
@@ -25,14 +26,28 @@ export const getCourseCodesOfProvider = async (provider: string) => {
     },
   })
 
-  const coursesWithOpenUniSubstitutions = coursesByProvider.map(({ code, substitution_groups }) => {
-    if (!substitution_groups?.length) {
-      return [code]
-    }
-    // This is ok to flatten, because we only want all credits completed by students, not what courses are
-    // completed and how => we don't care about substitutions
-    return [code].concat(substitution_groups.filter(group => group.some(code => getOpenUniCourseCode(code))).flat())
+  const directIds = coursesByProvider.map(({ id }) => id)
+  const substitutionGroupIds = [
+    ...new Set(coursesByProvider.flatMap(({ substitutionGroups }) => substitutionGroups ?? []).flat()),
+  ]
+
+  if (substitutionGroupIds.length === 0) return directIds
+
+  const substitutionCourses = await CourseModel.findAll({
+    raw: true,
+    attributes: ['id'],
+    where: { groupId: { [Op.in]: substitutionGroupIds } },
+    include: {
+      model: OrganizationModel,
+      required: true,
+      where: {
+        code: provider,
+      },
+      through: {
+        attributes: [],
+      },
+    },
   })
 
-  return coursesWithOpenUniSubstitutions.flat()
+  return [...new Set([...directIds, ...substitutionCourses.map(({ id }) => id)])]
 }

services/backend/src/services/students.ts

@@ -1,7 +1,7 @@
 import { Op } from 'sequelize'
 
-import type { Credit, Student } from '@oodikone/shared/models'
-import { CreditTypeCode, EnrollmentState, UnifyStatus } from '@oodikone/shared/types'
+import type { Credit, Student, Studyplan } from '@oodikone/shared/models'
+import { CreditTypeCode, EnrollmentState, Unification } from '@oodikone/shared/types'
 import { FormattedStudentForSearch, StudentPageStudent } from '@oodikone/shared/types/studentData'
 import { enrollmentTimeDateThresholdAcademicYear, splitByEmptySpace } from '@oodikone/shared/util'
 import { dateMaxFromList } from '@oodikone/shared/util/datetime'
@@ -43,7 +43,7 @@ const byStudentNumber = async (studentNumber: string) => {
           include: [
             {
               model: CourseModel,
-              attributes: ['code', 'name'],
+              attributes: ['code', 'name', 'groupId'],
               required: true,
             },
           ],
@@ -110,13 +110,13 @@ const byStudentNumber = async (studentNumber: string) => {
   }
 }
 
-const getUnifyStatus = (unifyCourses: UnifyStatus): [boolean] | [true, false] => {
+const getUnifyStatus = (unifyCourses: Unification): [boolean] | [true, false] => {
   switch (unifyCourses) {
-    case 'unifyStats':
+    case Unification.UNIFY:
       return [true, false]
-    case 'openStats':
+    case Unification.OPEN:
       return [true]
-    case 'regularStats':
+    case Unification.REGULAR:
       return [false]
     default:
       return [true, false]
@@ -156,104 +156,47 @@ export const getStartAndEndDates = async (from: number, to: number, separate: bo
   Returns students who have an attainment/enrollment between to and from.
  */
 export const findByCourseAndSemesters = async (
-  codes: string[],
+  courseIds: string[],
   from: number,
   to: number,
   separate: boolean,
-  unifyCourses: UnifyStatus = 'unifyStats'
+  unifyCourses: Unification = Unification.UNIFY
 ) => {
   /* from & to are semestercodes if separate = false, or yearcodes in case separate is true. */
-
-  const { startDate: startDate, endDate: endDate } = await getStartAndEndDates(from, to, separate)
+  const { startDate, endDate } = await getStartAndEndDates(from, to, separate)
+  if (!courseIds.length || !startDate || !endDate) return new Set<string>()
 
   const unifyStatus = getUnifyStatus(unifyCourses)
 
-  const credits = (await CreditModel.findAll({
-    raw: true,
-    attributes: [
-      ['student_studentnumber', 'studentnumber'],
-      'course_code',
-      ['attainment_date', 'date'],
-      'credittypecode',
-    ],
-    where: {
-      course_code: { [Op.in]: codes },
-      is_open: unifyStatus,
-      attainment_date: { [Op.between]: [startDate, endDate] },
-      credittypecode: { [Op.not]: CreditTypeCode.IMPROVED }, // We do not care about improved grades
-    },
-    order: [
-      ['attainment_date', 'DESC'],
-      ['credittypecode', 'ASC'],
-    ],
-  })) as unknown as Array<{
-    studentnumber: CreditModel['student_studentnumber']
-    course_code: CreditModel['course_code']
-    date: CreditModel['attainment_date']
-    credittypecode: CreditModel['credittypecode']
-  }>
-  // HACK: Sequelize doesn't like renaming attributes so we have to force-cast
-
-  const enrollments = (await EnrollmentModel.findAll({
-    raw: true,
-    attributes: ['studentnumber', 'course_code', ['enrollment_date_time', 'date']],
-    where: {
-      course_code: { [Op.in]: codes },
-      is_open: unifyStatus,
-      enrollment_date_time: {
-        [Op.between]: [dateMaxFromList(startDate ?? new Date(0), enrollmentTimeDateThresholdAcademicYear), endDate],
+  const creditStudents = (
+    await CreditModel.findAll({
+      raw: true,
+      attributes: ['student_studentnumber'],
+      where: {
+        course_id: { [Op.in]: courseIds },
+        is_open: unifyStatus,
+        attainment_date: { [Op.between]: [startDate, endDate] },
+        credittypecode: { [Op.not]: CreditTypeCode.IMPROVED }, // We do not care about improved grades
       },
-      state: EnrollmentState.ENROLLED,
-    },
-    order: [['enrollment_date_time', 'DESC']],
-  })) as unknown as Array<{
-    studentnumber: EnrollmentModel['studentnumber']
-    course_code: EnrollmentModel['course_code']
-    date: EnrollmentModel['enrollment_date_time']
-  }>
-  // HACK: Sequelize doesn't like renaming attributes so we have to force-cast
+    })
+  ).map(({ student_studentnumber }) => student_studentnumber)
 
-  const studentCreditsAndEnrollments: Record<
-    string,
-    Array<{
-      studentnumber: string
-      course_code: string
-      date: Date
-      type: 'credit' | 'enrollment'
-      credittypecode?: CreditTypeCode
-    }>
-  > = {}
+  const enrollmentStudents = (
+    await EnrollmentModel.findAll({
+      raw: true,
+      attributes: ['studentnumber'],
+      where: {
+        course_id: { [Op.in]: courseIds },
+        is_open: unifyStatus,
+        enrollment_date_time: {
+          [Op.between]: [dateMaxFromList(startDate ?? new Date(0), enrollmentTimeDateThresholdAcademicYear), endDate],
+        },
+        state: EnrollmentState.ENROLLED,
+      },
+    })
+  ).map(({ studentnumber }) => studentnumber)
 
-  for (const credit of credits) {
-    studentCreditsAndEnrollments[credit.studentnumber] ??= []
-    studentCreditsAndEnrollments[credit.studentnumber].push({ ...credit, type: 'credit' })
-  }
-  for (const enrollment of enrollments) {
-    studentCreditsAndEnrollments[enrollment.studentnumber] ??= []
-    studentCreditsAndEnrollments[enrollment.studentnumber].push({ ...enrollment, type: 'enrollment' })
-  }
-
-  const passedCreditTypes = [CreditTypeCode.PASSED, CreditTypeCode.APPROVED]
-
-  const filteredStudentNumbers = new Set<string>()
-  Object.entries(studentCreditsAndEnrollments).map(([studentNumber, creditsOrEnrollments]) => {
-    const latestCredit = creditsOrEnrollments.find(
-      entry => entry.type === 'credit' && codes.includes(entry.course_code)
-    )
-    const latestEnrollment = creditsOrEnrollments.find(
-      entry => entry.type === 'enrollment' && codes.includes(entry.course_code)
-    )
-
-    if (latestCredit && passedCreditTypes.includes(latestCredit.credittypecode!)) {
-      filteredStudentNumbers.add(studentNumber)
-    } else if (latestCredit?.credittypecode === CreditTypeCode.FAILED) {
-      filteredStudentNumbers.add(studentNumber)
-    } else if (latestEnrollment) {
-      filteredStudentNumbers.add(studentNumber)
-    }
-  })
-
-  return [...filteredStudentNumbers]
+  return new Set([...creditStudents, ...enrollmentStudents])
 }
 
 const formatSharedStudentData = ({
@@ -270,12 +213,13 @@ const formatSharedStudentData = ({
   updatedAt,
   createdAt,
   sis_person_id,
-}: Student): Omit<StudentPageStudent, 'tags'> => {
+}: Student): Omit<StudentPageStudent, 'tags' | 'idToGroupIdMap'> => {
   const toCourse = ({ grade, credits, credittypecode, is_open, attainment_date, course, isStudyModule }: Credit) => {
     return {
       course: {
         code: course.code,
         name: course.name,
+        groupId: course.groupId,
       },
       date: attainment_date,
       passed: CreditModel.passed({ credittypecode }) || CreditModel.improved({ credittypecode }),
@@ -307,13 +251,25 @@ const formatSharedStudentData = ({
   }
 }
 
+const getIdToGroupIdMap = async (studyplans: Pick<Studyplan, 'included_courses' | 'includedModules'>[]) => {
+  const ids = [...new Set(studyplans.flatMap(plan => [...plan.included_courses, ...plan.includedModules]))]
+  const courses = await CourseModel.findAll({
+    attributes: ['id', 'groupId'],
+    where: { id: { [Op.in]: ids } },
+    raw: true,
+  })
+  return Object.fromEntries(courses.map(({ id, groupId }) => [id, groupId]))
+}
+
 export const withStudentNumber = async (studentNumber: string, userId: string): Promise<StudentPageStudent | null> => {
   const student = await byStudentNumber(studentNumber)
   if (student == null) {
     return null
   }
+  const idToGroupIdMap = await getIdToGroupIdMap(student.studyplans)
   return {
     ...formatSharedStudentData(student),
+    idToGroupIdMap,
     tags: student.tags
       .filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
       .map(tag => ({

services/backend/src/services/studyProgramme/creditGetters.ts

@@ -4,22 +4,27 @@ import { CreditTypeCode } from '@oodikone/shared/types'
 import { CourseModel, CourseProviderModel, CreditModel, OrganizationModel } from '../../models'
 import { formatCredit } from './format'
 
-export const getCreditsForProvider = async (provider: string, codes: string[], since: Date) => {
+export const getCreditsForProvider = async (provider: string, courseIds: string[], since: Date) => {
   const credits = await CreditModel.findAll({
+    include: {
+      model: CourseModel,
+      attributes: ['groupId'],
+    },
     where: {
-      course_code: { [Op.in]: codes },
+      course_id: { [Op.in]: courseIds },
       attainment_date: { [Op.gte]: since },
       isStudyModule: { [Op.not]: true },
       credittypecode: CreditTypeCode.PASSED,
     },
     raw: true,
+    nest: true,
   })
 
-  const courseIds = credits.map(cr => cr.course_id)
+  const courseGroupIds = credits.map(cr => cr.course.groupId)
 
   const providers = await CourseProviderModel.findAll({
     where: {
-      coursecode: { [Op.in]: courseIds },
+      coursecode: { [Op.in]: courseGroupIds },
     },
     raw: true,
   })
@@ -38,14 +43,14 @@ export const getCreditsForProvider = async (provider: string, codes: string[], s
     return obj
   }, {})
 
-  const courseIdToShareMap = providers.reduce<Record<string, CourseProviderModel[]>>((obj, provider) => {
+  const courseGroupIdToShareMap = providers.reduce<Record<string, CourseProviderModel[]>>((obj, provider) => {
     obj[provider.coursecode] ??= []
     obj[provider.coursecode].push(provider)
     return obj
   }, {})
 
-  const courseIdToShare = (courseId: string, attainmentDate: Date) => {
-    const providers = courseIdToShareMap[courseId]
+  const courseGroupIdToShare = (courseGroupId: string, attainmentDate: Date) => {
+    const providers = courseGroupIdToShareMap[courseGroupId]
     const relevantProvider = providers?.find(p => organizationIdToCodeMap[p.organizationcode] === provider)
     if (!relevantProvider?.shares) return 0
     const relevantShare = relevantProvider.shares
@@ -71,7 +76,7 @@ export const getCreditsForProvider = async (provider: string, codes: string[], s
         return bTime - aTime
       })[0]
 
-    // Odd logic, but if there are multiple providers for same course code, if no fitting dates are
+    // Odd logic, but if there are multiple providers for same course group, if no fitting dates are
     // not found for our relevant provider, we can assume the share of that date is of some other provider.
     // But if only 1 provider exists, we can assume it has share of 1.
     if (!relevantShare) {
@@ -82,7 +87,7 @@ export const getCreditsForProvider = async (provider: string, codes: string[], s
   }
 
   return credits
-    .map(cr => ({ ...cr, credits: cr.credits * courseIdToShare(cr.course_id, cr.attainment_date) }))
+    .map(cr => ({ ...cr, credits: cr.credits * courseGroupIdToShare(cr.course.groupId, cr.attainment_date) }))
     .map(formatCredit)
 }
 

services/backend/src/services/studyProgramme/index.ts

@@ -1,7 +1,7 @@
 import { memoize } from 'lodash-es'
 import { Op, QueryTypes } from 'sequelize'
 
-import { Name, EnrollmentState } from '@oodikone/shared/types'
+import { EnrollmentState } from '@oodikone/shared/types'
 import { enrollmentTimeDateThresholdAcademicYear } from '@oodikone/shared/util'
 import { dateIsBetween } from '@oodikone/shared/util/datetime'
 import { dbConnections } from '../../database/connection'
@@ -9,9 +9,9 @@ import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, Programme
 import logger from '../../util/logger'
 
 export const getAllProgrammeCourses = async (providerCode: string) => {
-  const courses: Pick<CourseModel, 'id' | 'code' | 'name' | 'substitution_groups'>[] = await CourseModel.findAll({
+  const courses: Pick<CourseModel, 'id' | 'groupId' | 'code' | 'name'>[] = await CourseModel.findAll({
     raw: true,
-    attributes: ['id', 'code', 'name', 'substitution_groups'],
+    attributes: ['id', 'groupId', 'code', 'name'],
     include: [
       {
         model: OrganizationModel,
@@ -29,193 +29,210 @@ export const getAllProgrammeCourses = async (providerCode: string) => {
   return courses
 }
 
+export const getCoursesByGroupIds = async (groupIds: string[]) => {
+  if (groupIds.length === 0) return []
+  return (
+    await CourseModel.findAll({
+      raw: true,
+      attributes: ['id'],
+      where: { groupId: { [Op.in]: groupIds } },
+    })
+  ).map(course => course.id)
+}
+
+export const getPrimaryCoursesByGroupIds = async (
+  groupIds: string[]
+): Promise<Pick<CourseModel, 'code' | 'name' | 'groupId'>[]> => {
+  if (groupIds.length === 0) return []
+  return CourseModel.findAll({
+    raw: true,
+    attributes: ['code', 'name', 'groupId'],
+    where: { groupId: { [Op.in]: groupIds }, isPrimary: true },
+  })
+}
+
+type NotCompletedCourseRow = {
+  groupId: string
+  isStudyModule: boolean
+  allNotPassed: number
+}
+
 export const getNotCompletedForProgrammeCourses = async (
-  from: Date,
-  to: Date,
-  programmeCourses: string[],
+  programmeCourseIds: string[],
+  yearRanges: Array<{ year: number; from: Date; to: Date }>,
   start: Date,
   end: Date
-) => {
+): Promise<Map<number, NotCompletedCourseRow[]>> => {
+  const emptyResult = new Map<number, NotCompletedCourseRow[]>(yearRanges.map(({ year }) => [year, []]))
+
   try {
-    const enrollments: Array<
-      Pick<EnrollmentModel, 'studentnumber' | 'course_code' | 'enrollment_date_time' | 'course'>
-    > = await EnrollmentModel.findAll({
-      raw: true,
-      nest: true,
-      attributes: ['studentnumber', 'course_code', 'enrollment_date_time'],
-      include: {
-        model: CourseModel,
-        attributes: ['is_study_module'],
-      },
-      where: {
-        course_code: {
-          [Op.in]: programmeCourses,
+    const [enrollments, credits]: [
+      Pick<EnrollmentModel, 'studentnumber' | 'enrollment_date_time' | 'course'>[],
+      Pick<CreditModel, 'student_studentnumber' | 'credittypecode' | 'isStudyModule' | 'attainment_date' | 'course'>[],
+    ] = await Promise.all([
+      EnrollmentModel.findAll({
+        raw: true,
+        nest: true,
+        attributes: ['studentnumber', 'enrollment_date_time'],
+        include: {
+          model: CourseModel,
+          attributes: ['isStudyModule', 'groupId'],
         },
-        enrollment_date_time: {
-          [Op.and]: {
+        where: {
+          course_id: {
+            [Op.in]: programmeCourseIds,
+          },
+          enrollment_date_time: {
+            [Op.and]: {
+              [Op.between]: [start, end],
+              [Op.gte]: enrollmentTimeDateThresholdAcademicYear, // This has to be Academic year to match with Course statistics which doesn't display enrollments for ...-2021
+            },
+          },
+          state: EnrollmentState.ENROLLED,
+        },
+        order: [['enrollment_date_time', 'DESC']],
+      }),
+      CreditModel.findAll({
+        raw: true,
+        nest: true,
+        attributes: ['student_studentnumber', 'credittypecode', 'isStudyModule', 'attainment_date'],
+        include: {
+          model: CourseModel,
+          attributes: ['groupId'],
+        },
+        where: {
+          course_id: {
+            [Op.in]: programmeCourseIds,
+          },
+          attainment_date: {
             [Op.between]: [start, end],
-            [Op.gte]: enrollmentTimeDateThresholdAcademicYear, // This has to be Academic year to match with Course statistics which doesn't display enrollments for ...-2021
           },
         },
-        state: EnrollmentState.ENROLLED,
-      },
-      order: [['enrollment_date_time', 'DESC']],
-    })
+        order: [
+          ['attainment_date', 'DESC'],
+          ['credittypecode', 'ASC'],
+        ], // Passed credits for a student have to be handled first (only PASSED, and FAILED matter)
+      }),
+    ])
 
-    // TODO: This is being called for each year, could be optimized to call only once
-    const credits: Array<
-      Pick<
-        CreditModel,
-        'course_code' | 'student_studentnumber' | 'credittypecode' | 'isStudyModule' | 'attainment_date'
-      >
-    > = await CreditModel.findAll({
-      raw: true,
-      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'isStudyModule', 'attainment_date'],
-      where: {
-        course_code: {
-          [Op.in]: programmeCourses,
-        },
-        attainment_date: {
-          [Op.between]: [start, end],
-        },
-      },
-      order: [
-        ['attainment_date', 'DESC'],
-        ['credittypecode', 'ASC'],
-      ], // Passed credits for a student have to be handled first (only PASSED, and FAILED matter)
-    })
+    const creditKey = (studentNumber: string, courseGroupId: string) => `${studentNumber}::${courseGroupId}`
 
-    const enrolledStudentsByCourseCode = new Map<string, Set<string>>()
+    const everPassedByStudentCourse = new Set<string>()
+    const latestFailedDateByStudentCourse = new Map<string, Date>()
 
-    // Pick only this (between to, from) year's credits
-    const filteredCredits = credits.filter(credit => dateIsBetween(credit.attainment_date, from, to))
+    for (const credit of credits) {
+      const groupId = credit.course.groupId
 
-    const filteredEnrollments = enrollments
-      // Only include the first enrollment for a course for a student.
-      // This assumes that enrollments are sorted in DESC order by enrollment_date_time
-      .filter(({ course_code: courseCode, studentnumber: studentNumber }) => {
-        const courseEnrollments = enrolledStudentsByCourseCode.get(courseCode)
-        const hasEnrolled = courseEnrollments?.has(studentNumber) ?? false
-
-        if (!courseEnrollments) enrolledStudentsByCourseCode.set(courseCode, new Set<string>())
-        if (!hasEnrolled) enrolledStudentsByCourseCode.get(courseCode)!.add(studentNumber)
-
-        return !hasEnrolled
-      })
-      .filter(enrollment => dateIsBetween(enrollment.enrollment_date_time, from, to))
-
-    const courseCodeToName = (
-      await CourseModel.findAll({
-        raw: true,
-        attributes: ['code', 'name'],
-        where: {
-          code: programmeCourses,
-        },
-      })
-    ).reduce<Map<string, Name>>((acc, val) => {
-      acc.set(val.code, val.name)
-      return acc
-    }, new Map())
-
-    const creditCourses = filteredCredits.map(credit => ({
-      code: credit.course_code,
-      studentNumber: credit.student_studentnumber,
-      creditTypeCode: credit.credittypecode,
-      courseName: courseCodeToName.get(credit.course_code)!,
-      isStudyModule: credit.isStudyModule,
-      attainmentDate: credit.attainment_date,
-    }))
-
-    const passedByCourseCodes = new Map<string, Set<string>>()
-    const failedByCourseCodes = new Map<string, Set<string>>()
-    const notCompletedByCourseCodes = new Map<string, Set<string>>()
-    const courses = new Map<string, { name: Name; isStudyModule: boolean }>()
-
-    const studentHasPassedCourse = (studentNumber: string, courseCode: string): boolean =>
-      (passedByCourseCodes.get(courseCode)?.has(studentNumber) ?? false) ||
-      !!credits.find(
-        credit =>
-          credit.student_studentnumber === studentNumber &&
-          credit.course_code === courseCode &&
-          CreditModel.passed(credit)
-      )
-
-    /** Student has already been counted as failed or there is a newer failed credit */
-    const studentHasFailedCourse = (
-      studentNumber: string,
-      courseCode: string,
-      attainmentDate: Date = new Date(0)
-    ): boolean =>
-      (failedByCourseCodes.get(courseCode)?.has(studentNumber) ?? false) ||
-      !!credits.find(
-        credit =>
-          credit.student_studentnumber === studentNumber &&
-          credit.course_code === courseCode &&
-          CreditModel.failed(credit) &&
-          credit.attainment_date > attainmentDate
-      )
-
-    const studentHasCredit = (studentNumber: string, courseCode: string): boolean =>
-      studentHasPassedCourse(studentNumber, courseCode) || studentHasFailedCourse(studentNumber, courseCode)
-
-    for (const {
-      code: courseCode,
-      courseName,
-      creditTypeCode,
-      isStudyModule,
-      studentNumber,
-      attainmentDate,
-    } of creditCourses) {
-      if (!courses.has(courseCode)) {
-        courses.set(courseCode, {
-          isStudyModule,
-          name: courseName,
-        })
-        passedByCourseCodes.set(courseCode, new Set())
-        failedByCourseCodes.set(courseCode, new Set())
-        notCompletedByCourseCodes.set(courseCode, new Set())
+      if (CreditModel.passed(credit)) {
+        everPassedByStudentCourse.add(creditKey(credit.student_studentnumber, groupId))
       }
 
-      if (CreditModel.passed({ credittypecode: creditTypeCode })) {
-        passedByCourseCodes.get(courseCode)!.add(studentNumber)
-      }
-      if (
-        CreditModel.failed({ credittypecode: creditTypeCode }) &&
-        !studentHasPassedCourse(studentNumber, courseCode) &&
-        !studentHasFailedCourse(studentNumber, courseCode, attainmentDate)
-      ) {
-        failedByCourseCodes.get(courseCode)!.add(studentNumber)
+      if (CreditModel.failed(credit)) {
+        const mapKey = creditKey(credit.student_studentnumber, groupId)
+        const latestFailedDate = latestFailedDateByStudentCourse.get(mapKey)
+        if (!latestFailedDate || credit.attainment_date > latestFailedDate) {
+          latestFailedDateByStudentCourse.set(mapKey, credit.attainment_date)
+        }
       }
     }
 
-    // Add course details to courseMap if there are only enrollments
-    for (const { studentnumber: studentNumber, course_code: courseCode, course } of filteredEnrollments) {
-      if (!courses.has(courseCode)) {
-        courses.set(courseCode, {
-          isStudyModule: course.is_study_module,
-          name: courseCodeToName.get(courseCode)!,
-        })
-        passedByCourseCodes.set(courseCode, new Set())
-        failedByCourseCodes.set(courseCode, new Set())
-        notCompletedByCourseCodes.set(courseCode, new Set())
+    const enrolledStudentsByCourseGroupId = new Map<string, Set<string>>()
+
+    // Only include the latest enrollment for a course per student per given time range.
+    // This assumes that enrollments are sorted in DESC order by enrollment_date_time
+    const dedupedEnrollments = enrollments.filter(({ course, studentnumber: studentNumber }) => {
+      const groupId = course.groupId
+      const courseEnrollments = enrolledStudentsByCourseGroupId.get(groupId)
+      const hasEnrolled = !!courseEnrollments?.has(studentNumber)
+
+      if (!courseEnrollments) enrolledStudentsByCourseGroupId.set(groupId, new Set<string>())
+      if (!hasEnrolled) enrolledStudentsByCourseGroupId.get(groupId)!.add(studentNumber)
+
+      return !hasEnrolled
+    })
+
+    const result = new Map<number, NotCompletedCourseRow[]>()
+
+    for (const { year, from, to } of yearRanges) {
+      const filteredCredits = credits.filter(credit => dateIsBetween(credit.attainment_date, from, to))
+      const filteredEnrollments = dedupedEnrollments.filter(enrollment =>
+        dateIsBetween(enrollment.enrollment_date_time, from, to)
+      )
+
+      // Keys are groupIds
+      const failedByCourse = new Map<string, Set<string>>()
+      const notCompletedByCourse = new Map<string, Set<string>>()
+      const courses = new Map<string, { isStudyModule: boolean }>()
+
+      const studentHasPassedCourse = (studentNumber: string, courseGroupId: string): boolean =>
+        everPassedByStudentCourse.has(creditKey(studentNumber, courseGroupId))
+
+      // Whether there is a newer failed credit
+      const studentHasFailedCourse = (
+        studentNumber: string,
+        courseGroupId: string,
+        attainmentDate: Date = new Date(0)
+      ): boolean => {
+        const latestFailedDate = latestFailedDateByStudentCourse.get(creditKey(studentNumber, courseGroupId))
+        return !!latestFailedDate && latestFailedDate > attainmentDate
       }
 
-      if (!studentHasCredit(studentNumber, courseCode)) {
-        notCompletedByCourseCodes.get(courseCode)!.add(studentNumber)
-      }
-    }
+      const studentHasCredit = (studentNumber: string, courseGroupId: string): boolean =>
+        studentHasPassedCourse(studentNumber, courseGroupId) || studentHasFailedCourse(studentNumber, courseGroupId)
 
-    return [...courses.entries()].map(([code, { isStudyModule, name }]) => {
-      return {
-        code,
-        name,
+      const ensureCourseExists = (groupId: string, isStudyModule: boolean) => {
+        if (!courses.has(groupId)) {
+          courses.set(groupId, {
+            isStudyModule,
+          })
+          failedByCourse.set(groupId, new Set())
+          notCompletedByCourse.set(groupId, new Set())
+        }
+      }
+
+      for (const {
+        credittypecode: creditTypeCode,
         isStudyModule,
-        allNotPassed: (notCompletedByCourseCodes.get(code)?.size ?? 0) + (failedByCourseCodes.get(code)?.size ?? 0),
+        student_studentnumber: studentNumber,
+        attainment_date: attainmentDate,
+        course,
+      } of filteredCredits) {
+        const groupId = course.groupId
+        ensureCourseExists(groupId, isStudyModule)
+
+        if (
+          CreditModel.failed({ credittypecode: creditTypeCode }) &&
+          !studentHasPassedCourse(studentNumber, groupId) &&
+          !studentHasFailedCourse(studentNumber, groupId, attainmentDate)
+        ) {
+          failedByCourse.get(groupId)!.add(studentNumber)
+        }
       }
-    })
+
+      // Add course details to courseMap if there are only enrollments
+      for (const { studentnumber: studentNumber, course } of filteredEnrollments) {
+        const groupId = course.groupId
+        ensureCourseExists(groupId, course.isStudyModule)
+
+        if (!studentHasCredit(studentNumber, groupId)) {
+          notCompletedByCourse.get(groupId)!.add(studentNumber)
+        }
+      }
+
+      result.set(
+        year,
+        [...courses.entries()].map(([groupId, { isStudyModule }]) => ({
+          groupId,
+          isStudyModule,
+          allNotPassed: (notCompletedByCourse.get(groupId)?.size ?? 0) + (failedByCourse.get(groupId)?.size ?? 0),
+        }))
+      )
+    }
+
+    return result
   } catch (error) {
     logger.error(`getNotCompletedForProgrammeCourses failed ${error}`)
-    return []
+    return emptyResult
   }
 }
 

services/backend/src/services/studyProgramme/studentGetters.ts

@@ -34,6 +34,7 @@ const openUniversityExtentCodes = [ExtentCode.OPEN_UNIVERSITY_STUDIES]
 export type ProgrammeCourseCreditRow = {
   courseCode: string
   courseName: Name
+  groupId: string
   studentNumber: string
   attainmentDate: string
   credits: number
@@ -46,6 +47,7 @@ export type ProgrammeCourseCreditRow = {
 export type TransferCourseCreditRow = {
   courseCode: string
   courseName: Name
+  groupId: string
   studentNumber: string
   attainmentDate: string
   credits: number
@@ -57,7 +59,7 @@ const { sequelize } = dbConnections
 /**
 NOTE:
  1. There are some attainments that are linked to a valid study right, but the attainmentdate
-    is outside the study right's validity period. (maybe human error when the credits were registered)
+    is outside the study right's validity period.
  2. There are some attainments with no attached study right at all.
  3. Possibly some other edge cases.
 
@@ -67,39 +69,34 @@ NOTE:
    degree student -> has any degree (bsc / msc) study right in the university
    openuni -> only if no other study rights
 **/
-export const getProgrammeCourseAggregates = async (params: {
-  courseCodes: string[]
-  from: Date
+export const getProgrammeCourseAggregates = async (
+  courseIds: string[],
+  from: Date,
   to: Date
-}): Promise<ProgrammeCourseCreditRow[]> => {
-  const { courseCodes, from, to } = params
-
-  if (courseCodes.length === 0) {
-    return []
-  }
+): Promise<ProgrammeCourseCreditRow[]> => {
+  if (!courseIds.length) return []
 
   try {
     return await sequelize.query<ProgrammeCourseCreditRow>(
       `
         WITH filtered_credits AS (
           SELECT
-            cr.course_code AS "courseCode",
-            co.name AS "courseName",
+            co.group_id AS "groupId",
             cr.student_studentnumber AS "studentNumber",
             cr.attainment_date AS "attainmentDate",
             cr.credits AS credits,
             cr."isStudyModule" AS "isStudyModule"
           FROM
             credit cr
-            INNER JOIN course co ON co.code = cr.course_code
+            INNER JOIN course co ON co.id = cr.course_id
           WHERE
-            cr.course_code IN (:courseCodes)
+            cr.course_id IN (:courseIds)
+            AND co.group_id is NOT NULL
             AND cr.attainment_date BETWEEN :from AND :to
             AND cr.credittypecode IN (:creditTypes)
         )
         SELECT
-          fc."courseCode",
-          fc."courseName",
+          fc."groupId",
           fc."studentNumber",
           fc."attainmentDate",
           fc.credits,
@@ -173,7 +170,7 @@ export const getProgrammeCourseAggregates = async (params: {
       {
         type: QueryTypes.SELECT,
         replacements: {
-          courseCodes,
+          courseIds,
           from,
           to,
           creditTypes: [CreditTypeCode.PASSED, CreditTypeCode.APPROVED],
@@ -192,39 +189,35 @@ export const getProgrammeCourseAggregates = async (params: {
   }
 }
 
-export const getTransferCourseAggregates = async (params: {
-  courseCodes: string[]
-  from: Date
+export const getTransferCourseAggregates = async (
+  courseIds: string[],
+  from: Date,
   to: Date
-}): Promise<TransferCourseCreditRow[]> => {
-  const { courseCodes, from, to } = params
-
-  if (courseCodes.length === 0) {
-    return []
-  }
+): Promise<TransferCourseCreditRow[]> => {
+  if (!courseIds.length) return []
 
   try {
     return await sequelize.query<TransferCourseCreditRow>(
       `
         SELECT
-          cr.course_code AS "courseCode",
-          co.name AS "courseName",
+          co.group_id AS "groupId",
           cr.student_studentnumber AS "studentNumber",
           cr.attainment_date AS "attainmentDate",
           cr.credits AS credits,
           cr."isStudyModule" AS "isStudyModule"
         FROM
           credit cr
-          INNER JOIN course co ON co.code = cr.course_code
+          INNER JOIN course co ON co.id = cr.course_id
         WHERE
-          cr.course_code IN (:courseCodes)
+          cr.course_id IN (:courseIds)
+          AND co.group_id IS NOT NULL
           AND cr.attainment_date BETWEEN :from AND :to
           AND cr.credittypecode = :creditType
       `,
       {
         type: QueryTypes.SELECT,
         replacements: {
-          courseCodes,
+          courseIds,
           from,
           to,
           creditType: CreditTypeCode.APPROVED,

services/backend/src/services/studyProgramme/studyProgrammeCourses.ts

@@ -1,10 +1,17 @@
-import { Name, StudyProgrammeCourse, YearType } from '@oodikone/shared/types'
+import { StudyProgrammeCourse, YearType } from '@oodikone/shared/types'
 import { mapToProviders, range, yearCodeToYear } from '@oodikone/shared/util'
 import { createEmptyStats, YearStats } from '@oodikone/shared/util/studyProgramme'
 import { now } from '../../util/clock'
 
-import { getAllProgrammeCourses, getCurrentStudyYearStartDate, getNotCompletedForProgrammeCourses } from '.'
+import {
+  getAllProgrammeCourses,
+  getCoursesByGroupIds,
+  getCurrentStudyYearStartDate,
+  getNotCompletedForProgrammeCourses,
+  getPrimaryCoursesByGroupIds,
+} from '.'
 import { getStudentHetuStateMap, getProgrammeCourseAggregates, getTransferCourseAggregates } from './studentGetters'
+import logger from '../../util/logger'
 
 const START_YEAR = 2017
 const JULY = 6
@@ -18,22 +25,12 @@ const getFrom = (yearType: YearType, year: number) =>
 const getTo = (yearType: YearType, year: number) =>
   yearType === 'ACADEMIC_YEAR' ? new Date(year + 1, JULY, 31, 23, 59, 59) : new Date(year, 11, 31, 23, 59, 59)
 
-const getAllStudyProgrammeCourses = async (studyProgramme: string) => {
+const getAllStudyProgrammeCourseGroupIds = async (studyProgramme: string): Promise<string[]> => {
   const providerCode = mapToProviders([studyProgramme])[0]
   if (!providerCode) return []
 
   const normalCourses = await getAllProgrammeCourses(providerCode)
-  const courseCodes = new Set<string>()
-
-  for (const course of normalCourses) {
-    courseCodes.add(course.code)
-    // We want all AY course codes variants, even from partially completed susbtitution groups
-    for (const code of course?.substitution_groups?.flat() ?? []) {
-      if (code === 'AY' + course.code) courseCodes.add(code)
-    }
-  }
-
-  return [...courseCodes]
+  return [...new Set(normalCourses.map(course => course.groupId))]
 }
 
 const getYearKey = (date: Date, isAcademicYear: boolean) => {
@@ -58,8 +55,7 @@ type YearAccumulator = {
 }
 
 type CourseAccumulator = {
-  code: string
-  name: Name
+  groupId: string
   isStudyModule: boolean
   years: Map<number, YearAccumulator>
 }
@@ -80,23 +76,18 @@ const createYearAccumulator = (isStudyModule: boolean): YearAccumulator => ({
 
 const ensureCourse = (
   courseMap: Map<string, CourseAccumulator>,
-  code: string,
-  name: Name,
+  groupId: string,
   isStudyModule: boolean
 ): CourseAccumulator => {
-  if (!courseMap.has(code)) {
-    courseMap.set(code, {
-      code,
-      name,
+  if (!courseMap.has(groupId)) {
+    courseMap.set(groupId, {
+      groupId,
       isStudyModule,
       years: new Map(),
     })
   }
 
-  const course = courseMap.get(code)!
-  course.isStudyModule = course.isStudyModule || isStudyModule
-  course.name = course.name ?? name
-  return course
+  return courseMap.get(groupId)!
 }
 
 const ensureYear = (course: CourseAccumulator, year: number, isStudyModule: boolean) => {
@@ -154,30 +145,35 @@ export const getStudyProgrammeCoursesForStudyTrack = async (
     return []
   }
 
-  const mainProgrammeCourses = await getAllStudyProgrammeCourses(studyProgramme)
-  const secondProgrammeCourses = combinedProgramme ? await getAllStudyProgrammeCourses(combinedProgramme) : []
-  const programmeCourses = [...new Set([...mainProgrammeCourses, ...secondProgrammeCourses])]
+  const [mainProgrammeGroupIds, secondProgrammeGroupIds] = await Promise.all([
+    getAllStudyProgrammeCourseGroupIds(studyProgramme),
+    combinedProgramme ? getAllStudyProgrammeCourseGroupIds(combinedProgramme) : [],
+  ])
 
-  if (programmeCourses.length === 0) {
-    return []
-  }
+  const programmeGroupIds = [...new Set([...mainProgrammeGroupIds, ...secondProgrammeGroupIds])]
+
+  if (!programmeGroupIds.length) return []
+
+  const programmeCourseIds = [...new Set(await getCoursesByGroupIds(programmeGroupIds))]
 
   const firstYear = yearRange[0]
   const lastYear = yearRange[yearRange.length - 1]
   const from = getFrom(academicYear, firstYear)
   const to = getTo(academicYear, lastYear)
 
+  const notCompletedByYearPromise = getNotCompletedForProgrammeCourses(
+    programmeCourseIds,
+    yearRange.map(year => ({ year, from: getFrom(academicYear, year), to: getTo(academicYear, year) })),
+    from,
+    to
+  )
+  const primaryByGroupIdPromise = getPrimaryCoursesByGroupIds(programmeGroupIds).then(
+    rows => new Map(rows.map(({ groupId, code, name }) => [groupId, { code, name }]))
+  )
+
   const [programmeCredits, transferCredits] = await Promise.all([
-    getProgrammeCourseAggregates({
-      courseCodes: programmeCourses,
-      from,
-      to,
-    }),
-    getTransferCourseAggregates({
-      courseCodes: programmeCourses,
-      from,
-      to,
-    }),
+    getProgrammeCourseAggregates(programmeCourseIds, from, to),
+    getTransferCourseAggregates(programmeCourseIds, from, to),
   ])
 
   const openUniStudentHetuMap = await getStudentHetuStateMap(
@@ -194,7 +190,8 @@ export const getStudyProgrammeCoursesForStudyTrack = async (
     if (!yearRange.includes(year)) continue
     maxYear = Math.max(maxYear, year)
 
-    const course = ensureCourse(courseMap, row.courseCode, row.courseName, row.isStudyModule)
+    const course = ensureCourse(courseMap, row.groupId, row.isStudyModule)
+
     const yearAccumulator = ensureYear(course, year, row.isStudyModule)
     const { stats } = yearAccumulator
 
@@ -242,53 +239,56 @@ export const getStudyProgrammeCoursesForStudyTrack = async (
     if (!yearRange.includes(year)) continue
     maxYear = Math.max(maxYear, year)
 
-    const course = ensureCourse(courseMap, row.courseCode, row.courseName, row.isStudyModule)
+    const course = ensureCourse(courseMap, row.groupId, row.isStudyModule)
     const yearAccumulator = ensureYear(course, year, row.isStudyModule)
     yearAccumulator.transferStudents.add(row.studentNumber)
     yearAccumulator.stats.transferStudentsCredits += row.credits
   }
 
-  const notCompletedResults = await Promise.all(
-    yearRange.map(year => {
-      const yearFrom = getFrom(academicYear, year)
-      const yearTo = getTo(academicYear, year)
-      return getNotCompletedForProgrammeCourses(yearFrom, yearTo, programmeCourses, from, to)
-    })
-  )
+  const notCompletedByYear = await notCompletedByYearPromise
 
-  notCompletedResults.forEach((results, index) => {
-    const year = yearRange[index]
-    if (!results) return
+  for (const year of yearRange) {
+    const results = notCompletedByYear.get(year)
+    if (!results) continue
     if (results.length > 0) {
       maxYear = Math.max(maxYear, year)
     }
     for (const result of results) {
-      const course = ensureCourse(courseMap, result.code, result.name, result.isStudyModule ?? false)
+      const course = ensureCourse(courseMap, result.groupId, result.isStudyModule ?? false)
       const yearAccumulator = ensureYear(course, year, result.isStudyModule ?? false)
       yearAccumulator.stats.allNotPassed += result.allNotPassed
     }
-  })
+  }
 
   if (maxYear === 0) {
     maxYear = lastYear
   }
 
+  const primaryByGroupId = await primaryByGroupIdPromise
+
   const coursesRecord: Record<string, StudyProgrammeCourse> = {}
 
-  for (const [code, course] of courseMap.entries()) {
+  // Code and name for each CU group is fetched from the "primary" instance
+  for (const [groupId, course] of courseMap.entries()) {
     const years: Record<number, YearStats> = {}
     for (const [year, yearAccumulator] of course.years.entries()) {
       if (year > maxYear) continue
       years[year] = finalizeYearStats(yearAccumulator)
     }
 
-    if (Object.keys(years).length === 0) {
+    if (!Object.keys(years).length) continue
+
+    const primary = primaryByGroupId.get(groupId)
+
+    if (!primary?.code || !primary?.name) {
+      logger.warn(`Warning: Programme courses: No primary course for groupId: ${groupId ?? '(groupId is undefined)'}`)
       continue
     }
 
-    coursesRecord[code] = {
-      code,
-      name: course.name,
+    coursesRecord[groupId] = {
+      code: primary.code,
+      groupId,
+      name: primary.name,
       isStudyModule: course.isStudyModule,
       years,
     }

services/backend/src/services/studyProgramme/studyProgrammeCoursesBySemester.ts

@@ -5,7 +5,7 @@ import { CreditTypeCode, EnrollmentState } from '@oodikone/shared/types'
 import { mapToProviders } from '@oodikone/shared/util'
 import { CreditModel, EnrollmentModel } from '../../models'
 import { createArrayOfCourses } from '../languageCenterData'
-import { getAllProgrammeCourses } from '.'
+import { getAllProgrammeCourses, getPrimaryCoursesByGroupIds } from '.'
 
 type Attempt = {
   studentNumber: string
@@ -19,12 +19,21 @@ type Attempt = {
 export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgramme: string) => {
   const courses = await getAllProgrammeCourses(mapToProviders([studyProgramme])[0])
   const autumnSemester2017 = 135
-  const courseCodes = courses.map(course => course.code)
+  const courseIds = courses.map(course => course.id)
+  const idToGroupId = new Map(courses.map(course => [course.id, course.groupId]))
+
+  const primaryCourses = await getPrimaryCoursesByGroupIds([...new Set(courses.map(course => course.groupId))])
+  const groupIdToPrimaryCode = new Map(primaryCourses.map(course => [course.groupId, course.code]))
+
+  const courseCodeOf = (courseId: string) => {
+    const groupId = idToGroupId.get(courseId)
+    return groupId ? groupIdToPrimaryCode.get(groupId) : undefined
+  }
 
   const credits = await CreditModel.findAll({
-    attributes: ['course_code', 'student_studentnumber', 'semestercode', 'attainment_date'],
+    attributes: ['course_id', 'student_studentnumber', 'semestercode', 'attainment_date'],
     where: {
-      course_code: { [Op.in]: courseCodes },
+      course_id: { [Op.in]: courseIds },
       semestercode: { [Op.gte]: autumnSemester2017 },
       credittypecode: CreditTypeCode.PASSED,
     },
@@ -32,9 +41,9 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
   })
 
   const enrollments = await EnrollmentModel.findAll({
-    attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'state'],
+    attributes: ['studentnumber', 'semestercode', 'course_id', 'enrollment_date_time', 'state'],
     where: {
-      course_code: { [Op.in]: courseCodes },
+      course_id: { [Op.in]: courseIds },
       semestercode: { [Op.gte]: autumnSemester2017 }, // It is ok to use enrollments before OK sisu-migration, ColorizedTableData doesn't need it
       state: { [Op.in]: [EnrollmentState.ENROLLED, EnrollmentState.REJECTED] },
     },
@@ -45,12 +54,14 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
   const attemptsByStudents = {} as Record<string, Attempt[]>
 
   credits.forEach(credit => {
+    const courseCode = courseCodeOf(credit.course_id)
+    if (!courseCode) return
     const studentNumber = credit.student_studentnumber
     studentList.add(studentNumber)
     attemptsByStudents[studentNumber] ??= []
     attemptsByStudents[studentNumber].push({
       studentNumber,
-      courseCode: credit.course_code,
+      courseCode,
       completed: true,
       date: credit.attainment_date,
       semestercode: credit.semestercode,
@@ -58,22 +69,22 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
   })
 
   enrollments.forEach(enrollment => {
+    const courseCode = courseCodeOf(enrollment.course_id)
+    if (!courseCode) return
     const studentNumber = enrollment.studentnumber
     attemptsByStudents[studentNumber] ??= []
     studentList.add(studentNumber)
     if (
       attemptsByStudents[studentNumber].find(
         attempt =>
-          !attempt.completed &&
-          attempt.semestercode === enrollment.semestercode &&
-          attempt.courseCode === enrollment.course_code
+          !attempt.completed && attempt.semestercode === enrollment.semestercode && attempt.courseCode === courseCode
       )
     ) {
       return
     }
     attemptsByStudents[studentNumber].push({
       studentNumber,
-      courseCode: enrollment.course_code,
+      courseCode,
       completed: false,
       date: enrollment.enrollment_date_time,
       semestercode: enrollment.semestercode,
@@ -84,7 +95,7 @@ export const getStudyProgrammeStatsForColorizedCoursesTable = async (studyProgra
   const attemptsArray = [] as Attempt[]
   studentList.forEach(studentNumber => attemptsArray.push(...attemptsByStudents[studentNumber]))
 
-  const unorderedTableData = createArrayOfCourses(attemptsArray, courses)
+  const unorderedTableData = createArrayOfCourses(attemptsArray, primaryCourses)
   const tableData = orderBy(unorderedTableData, 'code')
 
   return { tableData }

services/backend/src/services/studyProgramme/studyProgrammeCriteria.ts

@@ -1,3 +1,5 @@
+import { Op } from 'sequelize'
+
 import { ProgressCriteria } from '@oodikone/shared/types'
 import { CourseModel } from '../../models'
 import { ProgressCriteriaModel } from '../../models/kone'
@@ -11,31 +13,42 @@ const getCriteriaByStudyProgramme = async (code: string): Promise<CriteriaWithou
     where: { code },
   })
 
-/** Construct course_code => substitution_groups object */
-const getSubstitutions = async (codes: string[]) => {
-  const courses: Array<Pick<CourseModel, 'code' | 'substitution_groups'>> = await CourseModel.findAll({
-    attributes: ['code', 'substitution_groups'],
-    where: { code: codes },
+/** Construct course_group_id => substitutionGroups object */
+const getSubstitutions = async (courseGroupIds: string[]) => {
+  const courses: Array<Pick<CourseModel, 'groupId' | 'substitutionGroups' | 'isPrimary'>> = await CourseModel.findAll({
+    attributes: ['groupId', 'substitutionGroups', 'isPrimary'],
+    where: { groupId: { [Op.in]: courseGroupIds } },
     raw: true,
   })
 
-  // Sort substitution_groups by length, because (usually) the shortest substitution is the "correct" one
+  // A group can have several rows (one per course code), prefer the primary one
+  const substitutionsByGroupId = new Map<string, string[][]>()
+  for (const { groupId, substitutionGroups, isPrimary } of courses) {
+    if (isPrimary || !substitutionsByGroupId.has(groupId)) {
+      substitutionsByGroupId.set(groupId, substitutionGroups)
+    }
+  }
+
+  // Sort substitutionGroups by length, because (usually) the shortest substitution is the "correct" one
   return Object.fromEntries(
-    courses.map(({ code, substitution_groups }) => [code, substitution_groups.sort((a, b) => b.length - a.length)])
+    [...substitutionsByGroupId].map(([groupId, substitutionGroups]) => [
+      groupId,
+      [...substitutionGroups].sort((a, b) => b.length - a.length),
+    ])
   )
 }
 
 const formatCriteria = async (criteria: CriteriaWithoutCurriculumVersion | null) => {
-  const yearOne = criteria?.coursesYearOne ?? []
-  const yearTwo = criteria?.coursesYearTwo ?? []
-  const yearThree = criteria?.coursesYearThree ?? []
-  const yearFour = criteria?.coursesYearFour ?? []
-  const yearFive = criteria?.coursesYearFive ?? []
-  const yearSix = criteria?.coursesYearSix ?? []
-  const courseCodes = [...yearOne, ...yearTwo, ...yearThree, ...yearFour, ...yearFive, ...yearSix]
+  const yearOne = criteria?.courseGroupIdsYearOne ?? []
+  const yearTwo = criteria?.courseGroupIdsYearTwo ?? []
+  const yearThree = criteria?.courseGroupIdsYearThree ?? []
+  const yearFour = criteria?.courseGroupIdsYearFour ?? []
+  const yearFive = criteria?.courseGroupIdsYearFive ?? []
+  const yearSix = criteria?.courseGroupIdsYearSix ?? []
+  const courseGroupIds = [...yearOne, ...yearTwo, ...yearThree, ...yearFour, ...yearFive, ...yearSix]
 
   const formattedCriteria: ProgressCriteria = {
-    allCourseGroups: await getSubstitutions(courseCodes),
+    allCourseGroups: await getSubstitutions(courseGroupIds),
     courses: { yearOne, yearTwo, yearThree, yearFour, yearFive, yearSix },
     credits: {
       yearOne: criteria?.creditsYearOne ?? 0,
@@ -56,12 +69,12 @@ const createCriteria = async (
 ) => {
   const newProgrammeCriteria = {
     code: studyProgramme,
-    coursesYearOne: courses.year1,
-    coursesYearTwo: courses.year2,
-    coursesYearThree: courses.year3,
-    coursesYearFour: courses.year4,
-    coursesYearFive: courses.year5,
-    coursesYearSix: courses.year6,
+    courseGroupIdsYearOne: courses.year1,
+    courseGroupIdsYearTwo: courses.year2,
+    courseGroupIdsYearThree: courses.year3,
+    courseGroupIdsYearFour: courses.year4,
+    courseGroupIdsYearFive: courses.year5,
+    courseGroupIdsYearSix: courses.year6,
     creditsYearOne: credits.year1,
     creditsYearTwo: credits.year2,
     creditsYearThree: credits.year3,
@@ -109,35 +122,35 @@ export const saveYearlyCreditCriteria = async (studyProgramme: string, credits:
   }
 }
 
-export const saveYearlyCourseCriteria = async (studyProgramme: string, courses: string[], year: number) => {
+export const saveYearlyCourseCriteria = async (studyProgramme: string, courseGroupIds: string[], year: number) => {
   const studyProgrammeToUpdate = await getCriteriaByStudyProgramme(studyProgramme)
   if (!studyProgrammeToUpdate) {
     const courseObj: Record<string, string[]> = { year1: [], year2: [], year3: [], year4: [], year5: [], year6: [] }
     const creditObj = { year1: 0, year2: 0, year3: 0, year4: 0, year5: 0, year6: 0 }
     if (year === 1) {
-      courseObj.year1 = courses
+      courseObj.year1 = courseGroupIds
     } else if (year === 2) {
-      courseObj.year2 = courses
+      courseObj.year2 = courseGroupIds
     } else if (year === 3) {
-      courseObj.year3 = courses
+      courseObj.year3 = courseGroupIds
     } else if (year === 4) {
-      courseObj.year4 = courses
+      courseObj.year4 = courseGroupIds
     } else if (year === 5) {
-      courseObj.year5 = courses
+      courseObj.year5 = courseGroupIds
     } else {
-      courseObj.year6 = courses
+      courseObj.year6 = courseGroupIds
     }
     return await createCriteria(studyProgramme, courseObj, creditObj)
   }
 
   try {
     const years = {
-      1: 'coursesYearOne',
-      2: 'coursesYearTwo',
-      3: 'coursesYearThree',
-      4: 'coursesYearFour',
-      5: 'coursesYearFive',
-      6: 'coursesYearSix',
+      1: 'courseGroupIdsYearOne',
+      2: 'courseGroupIdsYearTwo',
+      3: 'courseGroupIdsYearThree',
+      4: 'courseGroupIdsYearFour',
+      5: 'courseGroupIdsYearFive',
+      6: 'courseGroupIdsYearSix',
     } as const
     if (!(year in years)) {
       throw new Error(`Invalid year: ${year}`)
@@ -145,7 +158,7 @@ export const saveYearlyCourseCriteria = async (studyProgramme: string, courses:
 
     const yearToUpdate = years[year as keyof typeof years]
 
-    const updatedCriteria = await studyProgrammeToUpdate.update({ [yearToUpdate]: courses })
+    const updatedCriteria = await studyProgrammeToUpdate.update({ [yearToUpdate]: courseGroupIds })
     return await formatCriteria(updatedCriteria)
   } catch (error) {
     logger.error(`Updating yearly credit criteria failed: ${error}`)

services/backend/src/services/studyProgramme/studyProgrammeHelpers.ts

@@ -139,17 +139,19 @@ export const getYearlyMonthlyCreditsObj = () => {
   return yearlyMonthlyCredits
 }
 
-const getMonthlyCreditsObj = (academicStartYear?: number) => {
+const getMonthlyCreditsObj = (academicStartYear = 2017) => {
   const today = now()
-  const time = getStartDate(true, academicStartYear)
+
+  /* Compare whole months instead of dates, so the result does not depend on the time of day.
+    Academic year starts in August (month index 7). */
+  const firstMonth = academicStartYear * 12 + 7
+  const lastMonth = today.getFullYear() * 12 + today.getMonth()
 
   const monthlyCredits: Record<string, number[]> = {}
-  today.setDate(1)
 
   /* NB: JS months start at 0. We need them to start at 1. */
-  while (time < today) {
-    monthlyCredits[`${time.getFullYear()}-${time.getMonth() + 1}`] = []
-    time.setMonth(time.getMonth() + 1)
+  for (let month = firstMonth; month <= lastMonth; month++) {
+    monthlyCredits[`${Math.floor(month / 12)}-${(month % 12) + 1}`] = []
   }
 
   return monthlyCredits
@@ -191,8 +193,7 @@ export const computePercentiles = (
   return values
 }
 
-// In the object programmes should be {bachelorCode: masterCode}
-export const combinedStudyProgrammes = { KH90_001: 'MH90_001' } as const
+export { combinedStudyProgrammes } from '@oodikone/shared/util/studyProgramme'
 
 // There are 9 course_unit_types
 // 1. urn:code:course-unit-type:regular
@@ -312,7 +313,9 @@ export const getGoal = async (programme?: string) => {
   return ((minimumCredits / 60) * 12) / 6 // 60 Credits per year divided to semesters
 }
 
-export const isRelevantProgramme = (code: string) => /^(KH|MH)\d{2}_\d{3}$/.test(code) || /^T\d{6}$/.test(code)
+/** Returns true for all programme codes that are determined to be newer than 2017 */
+export const isRelevantProgramme = (code: string) =>
+  /^(KH|MH)\d{2}_\d{3}$/.test(code) || /^T\d{6}$/.test(code) || /^DP\d{3}$/.test(code)
 
 export const getStudyRightElementsWithPhase = (studyRight: Pick<SISStudyRight, 'studyRightElements'>, phase: Phase) => {
   return orderBy(

services/backend/src/services/studyProgramme/studyRightFinders.ts

@@ -43,7 +43,7 @@ export const getStudyRightsInProgramme = async (
         include: [
           {
             model: CreditModel,
-            attributes: ['attainment_date', 'credits', 'course_code'],
+            attributes: ['attainment_date', 'credits', 'course_code', 'course_id'],
             required: false,
             where: {
               isStudyModule: false,

services/backend/src/services/studyProgramme/studyTrackStats.ts

@@ -247,7 +247,7 @@ const getMainStatsByTrackAndYear = async (
 
     const latestAttainmentWithinYearAndInHops = dayjs(
       studyRight.student.credits
-        .filter(credit => studyRight.studyPlans.some(sp => sp.included_courses.includes(credit.course_code)))
+        .filter(credit => studyRight.studyPlans.some(sp => sp.included_courses.includes(credit.course_id)))
         .reduce((latest, { attainment_date }) => {
           return attainment_date > latest ? attainment_date : latest
         }, new Date('1900-01-01'))

services/backend/src/services/teachers/index.ts

@@ -34,7 +34,7 @@ const findTeacherCredits = async (teacherId: string) => {
       include: [
         {
           model: CourseModel,
-          attributes: ['name', 'code'],
+          attributes: ['name', 'code', 'groupId'],
           required: true,
         },
         {
@@ -105,9 +105,9 @@ const markCreditForYear = (years: Record<number, any>, credit: Credit) => {
 
 const markCreditForCourse = (courses: Record<string, any>, credit: Credit) => {
   const { passed, failed, credits, course, semester, transferred } = parseCreditInfo(credit)
-  const { code, name } = course
+  const { code, name, groupId } = course
   const { semestercode } = semester
-  const { stats, semesters = {}, ...rest } = courses[code] ?? { id: code, name }
+  const { stats, semesters = {}, ...rest } = courses[code] ?? { id: code, groupId, name }
   return {
     ...courses,
     [code]: {
@@ -151,7 +151,7 @@ const getActiveTeachers = async (providers: string[], startSemester: number, end
     JOIN credit_teachers ON teacher.id = credit_teachers.teacher_id
     JOIN credit ON credit.id = credit_teachers.credit_id
     JOIN course ON credit.course_id = course.id
-    JOIN course_providers ON course.id = course_providers.coursecode
+    JOIN course_providers ON course.group_id = course_providers.coursecode
     JOIN organization ON organization.id = course_providers.organizationcode
     WHERE credit.semestercode BETWEEN :startSemester AND :endSemester
       AND organization.code IN (:providers)

services/backend/src/util/index.ts

@@ -51,8 +51,6 @@ export const hasFullAccessToStudentData = (roles?: Role[]): boolean => {
 
 // Only open uni courses
 export const isOpenUniCourseCode = (code: string): boolean => /^AY.+?(?:en|fi|sv)?$/.test(code)
-// Open uni AND normal courses
-export const getOpenUniCourseCode = (code: string): string[] | null => /^AY?(.+?)(?:en|fi|sv)?$/.exec(code)
 
 /**
  * Returns the keys of the given object as an array of strings, typed as the keys of the object.

services/backend/tests/api/courses/courseYearlyStats/attemptStatistics/mat11001.test.ts

@@ -4,12 +4,12 @@ import { describe, it, beforeAll, assert } from 'vitest'
 import { initTests } from '../../../../utils'
 import { getCourseYearlyStats, CourseYearlyStats } from '../helpers'
 
-void describe('Course yearly attempt statistics - MAT11001 (no substitutions)', () => {
+void describe('Course yearly attempt statistics - MAT11001 (hy-CU-117375151, no substitutions)', () => {
   let app: Express
   let body: CourseYearlyStats
   beforeAll(async () => {
     app = await initTests()
-    body = await getCourseYearlyStats(app, 'codes=MAT11001&combineSubstitutions=false')
+    body = await getCourseYearlyStats(app, 'courses=hy-CU-117375151&substitutions=false')
   })
 
   it('should include every attempt of a student who has only failed the course', () => {

services/backend/tests/api/courses/courseYearlyStats/attemptStatistics/mat11002.test.ts

@@ -4,12 +4,12 @@ import { describe, it, beforeAll, assert } from 'vitest'
 import { initTests } from '../../../../utils'
 import { getCourseYearlyStats, CourseYearlyStats } from '../helpers'
 
-void describe('Course yearly attempt statistics - MAT11002 (no substitutions)', () => {
+void describe('Course yearly attempt statistics - MAT11002 (hy-CU-117375394, no substitutions)', () => {
   let app: Express
   let body: CourseYearlyStats
   beforeAll(async () => {
     app = await initTests()
-    body = await getCourseYearlyStats(app, 'codes=MAT11002&combineSubstitutions=false')
+    body = await getCourseYearlyStats(app, 'courses=hy-CU-117375394&substitutions=false')
   })
 
   // FIXME: What is this even testing??

services/backend/tests/api/courses/courseYearlyStats/attemptStatistics/mat21003.test.ts

@@ -5,7 +5,9 @@ import { yearToYearCode } from '@oodikone/shared/util'
 import { initTests } from '../../../../utils'
 import { getCourseYearlyStats, CourseYearlyStats } from '../helpers'
 
-void describe('Course yearly attempt statistics - MAT21003 (no substitutions)', () => {
+const MAT21003 = 'hy-CU-117375829'
+
+void describe('Course yearly attempt statistics - MAT21003 (hy-CU-117375394, no substitutions)', () => {
   let app: Express
   beforeAll(async () => {
     app = await initTests()
@@ -14,7 +16,7 @@ void describe('Course yearly attempt statistics - MAT21003 (no substitutions)',
   describe('duplicate failed attempts across a large timespan are counted separately', () => {
     let body: CourseYearlyStats
     beforeAll(async () => {
-      body = await getCourseYearlyStats(app, 'codes=MAT21003&combineSubstitutions=false')
+      body = await getCourseYearlyStats(app, `courses=${MAT21003}&substitutions=false`)
     })
 
     it('2017-2018 should include a single failed attempt (539036)', () => {
@@ -42,7 +44,7 @@ void describe('Course yearly attempt statistics - MAT21003 (no substitutions)',
     ])('should include correct years for %s', async (name, from, to) => {
       const body = await getCourseYearlyStats(
         app,
-        `codes=MAT21003&combineSubstitutions=false&fromYearCode=${yearToYearCode(from)}&toYearCode=${yearToYearCode(to)}`
+        `courses=hy-CU-117375394&substitutions=false&fromYearCode=${yearToYearCode(from)}&toYearCode=${yearToYearCode(to)}`
       )
 
       const years: string[] = []
@@ -50,7 +52,7 @@ void describe('Course yearly attempt statistics - MAT21003 (no substitutions)',
         years.push(`${i}-${i + 1}`)
       }
       assert.deepStrictEqual(
-        body.unifyStats!.statistics.map(({ name }) => name),
+        body.unifyStats.statistics.map(({ name }) => name),
         years,
         `${name} included incorrect years`
       )
@@ -61,9 +63,9 @@ void describe('Course yearly attempt statistics - MAT21003 (no substitutions)',
     it('2017-2018 should include a failed attempt (539036)', async () => {
       const body = await getCourseYearlyStats(
         app,
-        `codes=MAT21003&combineSubstitutions=false&fromYearCode=${yearToYearCode(2017)}&toYearCode=${yearToYearCode(2017)}`
+        `courses=${MAT21003}&substitutions=false&fromYearCode=${yearToYearCode(2017)}&toYearCode=${yearToYearCode(2017)}`
       )
-      const year = body.unifyStats!.statistics.find(year => year.name === '2017-2018')
+      const year = body.unifyStats.statistics.find(year => year.name === '2017-2018')
       assert(year && 'enrollments' in year, 'Missing stats for 2017-2018')
       assert.deepStrictEqual(
         year.attempts.categories.failed,
@@ -75,9 +77,9 @@ void describe('Course yearly attempt statistics - MAT21003 (no substitutions)',
     it('2018-2019 should include a failed attempt (539036)', async () => {
       const body = await getCourseYearlyStats(
         app,
-        `codes=MAT21003&combineSubstitutions=false&fromYearCode=${yearToYearCode(2018)}&toYearCode=${yearToYearCode(2018)}`
+        `courses=${MAT21003}&substitutions=false&fromYearCode=${yearToYearCode(2018)}&toYearCode=${yearToYearCode(2018)}`
       )
-      const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
+      const year = body.unifyStats.statistics.find(year => year.name === '2018-2019')
       assert(year && 'enrollments' in year, 'Missing stats for 2018-2019')
       assert.include(year.attempts.categories.failed, '539036', 'Failed attempts included the incorrect students')
     })

services/backend/tests/api/courses/courseYearlyStats/attemptStatistics/tkt10002.test.ts

@@ -5,7 +5,7 @@ import { yearToYearCode } from '@oodikone/shared/util'
 import { initTests } from '../../../../utils'
 import { getCourseYearlyStats, CourseYearlyStats } from '../helpers'
 
-void describe('Course yearly attempt statistics - TKT10002 (no substitutions)', () => {
+void describe('Course yearly attempt statistics - TKT10002 (hy-CU-118023867, no substitutions)', () => {
   let app: Express
   beforeAll(async () => {
     app = await initTests()
@@ -16,14 +16,14 @@ void describe('Course yearly attempt statistics - TKT10002 (no substitutions)',
     beforeAll(async () => {
       body = await getCourseYearlyStats(
         app,
-        `codes=TKT10002&combineSubstitutions=false&fromYearCode=${yearToYearCode(2016)}&toYearCode=${yearToYearCode(2023)}`
+        `courses=hy-CU-118023867&substitutions=false&fromYearCode=${yearToYearCode(2016)}&toYearCode=${yearToYearCode(2023)}`
       )
     })
 
     it('has one entry per statistic year', () => {
-      assert.strictEqual(body.unifyStats!.statistics.length, 8, 'unifyStats years')
-      assert.strictEqual(body.regularStats!.statistics.length, 8, 'regularStats years')
-      assert.strictEqual(body.openStats!.statistics.length, 4, 'openStats years')
+      assert.strictEqual(body.unifyStats.statistics.length, 8, 'unifyStats years')
+      assert.strictEqual(body.regularStats.statistics.length, 8, 'regularStats years')
+      assert.strictEqual(body.openStats.statistics.length, 4, 'openStats years')
     })
 
     it.each([
@@ -60,7 +60,7 @@ void describe('Course yearly attempt statistics - TKT10002 (no substitutions)',
       ],
       ['2023-2024', { total: 1, passed: 1, failed: 0 }, { 5: 1, 4: 0, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyl.': 0, 'Hyv.': 0 }],
     ])('for attempt statistic content during (%s)', (year, categories, grades) => {
-      const stats = body.unifyStats!.statistics.find(yearStats => yearStats.name === year)!
+      const stats = body.unifyStats.statistics.find(yearStats => yearStats.name === year)!
       assert('enrollments' in stats, 'Missing field enrollment in statsitics')
 
       assert.strictEqual(
@@ -89,10 +89,10 @@ void describe('Course yearly attempt statistics - TKT10002 (no substitutions)',
     })
 
     it('should include every attempt of a student who has failed a course in 2017 and later completed it', () => {
-      const year2017 = body.unifyStats!.statistics.find(year => year.name === '2017-2018')
-      const year2018 = body.unifyStats!.statistics.find(year => year.name === '2018-2019')
-      const year2020 = body.unifyStats!.statistics.find(year => year.name === '2020-2021')
-      const year2022 = body.unifyStats!.statistics.find(year => year.name === '2022-2023')
+      const year2017 = body.unifyStats.statistics.find(year => year.name === '2017-2018')
+      const year2018 = body.unifyStats.statistics.find(year => year.name === '2018-2019')
+      const year2020 = body.unifyStats.statistics.find(year => year.name === '2020-2021')
+      const year2022 = body.unifyStats.statistics.find(year => year.name === '2022-2023')
       assert(year2017 && 'enrollments' in year2017, 'Missing stats for 2017-2018')
       assert(year2018 && 'enrollments' in year2018, 'Missing stats for 2018-2019')
       assert(year2020 && 'enrollments' in year2020, 'Missing stats for 2020-2021')
@@ -124,9 +124,9 @@ void describe('Course yearly attempt statistics - TKT10002 (no substitutions)',
     ])('should include correct attempt stats for one academic year (%s)', async (year, categories) => {
       const body = await getCourseYearlyStats(
         app,
-        `codes=TKT10002&combineSubstitutions=false&fromYearCode=${yearToYearCode(year.split('-').at(0))}&toYearCode=${yearToYearCode(parseInt(year.split('-').at(1)!)) - 1}`
+        `courses=hy-CU-118023867&substitutions=false&fromYearCode=${yearToYearCode(year.split('-').at(0))}&toYearCode=${yearToYearCode(parseInt(year.split('-').at(1)!)) - 1}`
       )
-      const stats = body.unifyStats!.statistics.find(yearStats => yearStats.name === year)!
+      const stats = body.unifyStats.statistics.find(yearStats => yearStats.name === year)!
       assert('enrollments' in stats, 'Missing field enrollment in statsitics')
 
       assert.strictEqual(

services/backend/tests/api/courses/courseYearlyStats/studentStatistics/mat11001.test.ts

@@ -4,16 +4,16 @@ import { describe, it, beforeAll, assert } from 'vitest'
 import { initTests } from '../../../../utils'
 import { calculatePassedAndFailed, getCourseYearlyStats, CourseYearlyStats } from '../helpers'
 
-void describe('Course yearly statistics - MAT11001 (no substitutions)', () => {
+void describe('Course yearly statistics - MAT11001 (hy-CU-117375151, no substitutions)', () => {
   let app: Express
   let body: CourseYearlyStats
   beforeAll(async () => {
     app = await initTests()
-    body = await getCourseYearlyStats(app, 'codes=MAT11001&combineSubstitutions=false')
+    body = await getCourseYearlyStats(app, 'courses=hy-CU-117375151&substitutions=false')
   })
 
   it('should include a student who has only failed the course', () => {
-    const stats = body.unifyStats?.statistics!.find(year => year.name === '2018-2019')
+    const stats = body.unifyStats.statistics.find(year => year.name === '2018-2019')
     assert(stats && 'enrollments' in stats, 'Missing field enrollment in statsitics')
 
     const studentCategories = calculatePassedAndFailed(stats.students.grades)

services/backend/tests/api/courses/courseYearlyStats/studentStatistics/mat11002.test.ts

@@ -4,23 +4,25 @@ import { describe, it, beforeAll, assert } from 'vitest'
 import { initTests } from '../../../../utils'
 import { calculatePassedAndFailed, getCourseYearlyStats, CourseYearlyStats } from '../helpers'
 
-void describe('Course yearly statistics - MAT11002 (no substitutions)', () => {
+const MAT11002 = 'hy-CU-117375394'
+
+void describe('Course yearly statistics - MAT11002 (hy-CU-117375394, no substitutions)', () => {
   let app: Express
   let body: CourseYearlyStats
   beforeAll(async () => {
     app = await initTests()
-    body = await getCourseYearlyStats(app, 'codes=MAT11002&combineSubstitutions=false')
+    body = await getCourseYearlyStats(app, `courses=${MAT11002}&combineSubstitutions=false`)
   })
 
   it('should not include students for AY code', () => {
-    const stats = body.unifyStats?.statistics!.find(year => year.name === '2017-2018')
+    const stats = body.unifyStats.statistics.find(year => year.name === '2017-2018')
     assert(stats && 'enrollments' in stats, 'Missing field enrollment in statsitics')
     const studentCategories = calculatePassedAndFailed(stats.students.grades)
     assert(!studentCategories.failed.includes('534980'))
   })
 
   it('should not include student with failed grade after passed grade', () => {
-    const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
+    const year = body.unifyStats.statistics.find(year => year.name === '2018-2019')
     assert(year && 'enrollments' in year, 'Stats missing completely')
     const studentCategories = calculatePassedAndFailed(year.students.grades)
     assert(!studentCategories.failed.includes('501716'), 'Failed students included incorrectly the student in question')
@@ -30,7 +32,7 @@ void describe('Course yearly statistics - MAT11002 (no substitutions)', () => {
 
   // FIXME: Doesn't test anything acually. Student doesn't have a passed AY grade
   it.skip('should include student with failed grade and passed AY grade', () => {
-    const year = body.unifyStats?.statistics.find(year => year.name === '2020-2021')
+    const year = body.unifyStats.statistics.find(year => year.name === '2020-2021')
     assert(year && 'enrollments' in year, 'Stats missing completely')
     const studentCategories = calculatePassedAndFailed(year.students.grades)
     assert(
@@ -45,7 +47,7 @@ void describe('Course yearly statistics - MAT11002 (no substitutions)', () => {
   })
 
   it('should include student with only approved grade', () => {
-    const year = body.unifyStats?.statistics.find(year => year.name === '2022-2023')
+    const year = body.unifyStats.statistics.find(year => year.name === '2022-2023')
     assert(year && 'enrollments' in year, 'Stats missing completely')
     const studentCategories = calculatePassedAndFailed(year.students.grades)
     assert(!studentCategories.failed.includes('543385'), 'Failed students included incorrectly the student in question')
@@ -54,7 +56,7 @@ void describe('Course yearly statistics - MAT11002 (no substitutions)', () => {
   })
 
   it('should not count a student with only improved grades as passed (509770)', () => {
-    const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
+    const year = body.unifyStats.statistics.find(year => year.name === '2018-2019')
     assert(year && 'enrollments' in year, 'Missing stats for 2018-2019')
 
     const studentCategories = calculatePassedAndFailed(year.students.grades)

services/backend/tests/api/courses/courseYearlyStats/studentStatistics/mat21003.test.ts

@@ -5,6 +5,8 @@ import { yearToYearCode } from '@oodikone/shared/util'
 import { initTests } from '../../../../utils'
 import { calculatePassedAndFailed, getCourseYearlyStats, CourseYearlyStats } from '../helpers'
 
+const MAT21003 = 'hy-CU-117375829'
+
 void describe('Course yearly statistics - MAT21003 (no substitutions)', () => {
   let app: Express
   beforeAll(async () => {
@@ -14,11 +16,11 @@ void describe('Course yearly statistics - MAT21003 (no substitutions)', () => {
   describe('duplicate failed grades across a large timespan are not double-counted', () => {
     let body: CourseYearlyStats
     beforeAll(async () => {
-      body = await getCourseYearlyStats(app, 'codes=MAT21003&combineSubstitutions=false')
+      body = await getCourseYearlyStats(app, `courses=${MAT21003}&substitutions=false`)
     })
 
     it('2017-2018 should not include a failed grade', () => {
-      const year = body.unifyStats?.statistics.find(year => year.name === '2017-2018')
+      const year = body.unifyStats.statistics.find(year => year.name === '2017-2018')
       assert(year && 'enrollments' in year, 'Missing stats for 2017-2018')
       const studentCategories = calculatePassedAndFailed(year.students.grades)
       assert.strictEqual(studentCategories.failed.length, 0, 'Failed stats should not include any students')
@@ -26,7 +28,7 @@ void describe('Course yearly statistics - MAT21003 (no substitutions)', () => {
     })
 
     it('2018-2019 should include a failed grade', () => {
-      const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
+      const year = body.unifyStats.statistics.find(year => year.name === '2018-2019')
       assert(year && 'enrollments' in year, 'Missing stats for 2018-2019')
       const studentCategories = calculatePassedAndFailed(year.students.grades)
       assert.strictEqual(studentCategories.failed.length, 1, 'Failed stats should include only one student')
@@ -42,7 +44,7 @@ void describe('Course yearly statistics - MAT21003 (no substitutions)', () => {
     ])('should include correct years for %s', async (name, from, to) => {
       const body = await getCourseYearlyStats(
         app,
-        `codes=MAT21003&combineSubstitutions=false&fromYearCode=${yearToYearCode(from)}&toYearCode=${yearToYearCode(to)}`
+        `courses=${MAT21003}&substitutions=false&fromYearCode=${yearToYearCode(from)}&toYearCode=${yearToYearCode(to)}`
       )
 
       const years: string[] = []
@@ -50,7 +52,7 @@ void describe('Course yearly statistics - MAT21003 (no substitutions)', () => {
         years.push(`${i}-${i + 1}`)
       }
       assert.deepStrictEqual(
-        body.unifyStats!.statistics.map(({ name }) => name),
+        body.unifyStats.statistics.map(({ name }) => name),
         years,
         `${name} included incorrect years`
       )
@@ -61,9 +63,9 @@ void describe('Course yearly statistics - MAT21003 (no substitutions)', () => {
     it('2017-2018 should include a failed grade', async () => {
       const body = await getCourseYearlyStats(
         app,
-        `codes=MAT21003&combineSubstitutions=false&fromYearCode=${yearToYearCode(2017)}&toYearCode=${yearToYearCode(2017)}`
+        `courses=${MAT21003}&substitutions=false&fromYearCode=${yearToYearCode(2017)}&toYearCode=${yearToYearCode(2017)}`
       )
-      const year = body.unifyStats!.statistics.find(year => year.name === '2017-2018')
+      const year = body.unifyStats.statistics.find(year => year.name === '2017-2018')
       assert(year && 'enrollments' in year, 'Missing stats for 2017-2018')
       const studentCategories = calculatePassedAndFailed(year.students.grades)
 
@@ -74,9 +76,9 @@ void describe('Course yearly statistics - MAT21003 (no substitutions)', () => {
     it('2018-2019 should include a failed grade', async () => {
       const body = await getCourseYearlyStats(
         app,
-        `codes=MAT21003&combineSubstitutions=false&fromYearCode=${yearToYearCode(2018)}&toYearCode=${yearToYearCode(2018)}`
+        `courses=${MAT21003}&substitutions=false&fromYearCode=${yearToYearCode(2018)}&toYearCode=${yearToYearCode(2018)}`
       )
-      const year = body.unifyStats?.statistics.find(year => year.name === '2018-2019')
+      const year = body.unifyStats.statistics.find(year => year.name === '2018-2019')
       assert(year && 'enrollments' in year, 'Missing stats for 2018-2019')
       const studentCategories = calculatePassedAndFailed(year.students.grades)
       assert.deepStrictEqual(

services/backend/tests/api/courses/courseYearlyStats/studentStatistics/tkt10002.test.ts

@@ -5,6 +5,8 @@ import { yearToYearCode } from '@oodikone/shared/util'
 import { initTests } from '../../../../utils'
 import { calculatePassedAndFailed, getCourseYearlyStats, CourseYearlyStats } from '../helpers'
 
+const TKT10002 = 'hy-CU-118023867'
+
 void describe('Course yearly statistics - TKT10002 (no substitutions)', () => {
   let app: Express
   beforeAll(async () => {
@@ -16,19 +18,19 @@ void describe('Course yearly statistics - TKT10002 (no substitutions)', () => {
     beforeAll(async () => {
       body = await getCourseYearlyStats(
         app,
-        `codes=TKT10002&combineSubstitutions=false&fromYearCode=${yearToYearCode(2016)}&toYearCode=${yearToYearCode(2023)}`
+        `courses=${TKT10002}&substitutions=false&fromYearCode=${yearToYearCode(2016)}&toYearCode=${yearToYearCode(2023)}`
       )
     })
 
     it('has one entry per statistic year', () => {
-      assert.strictEqual(body.unifyStats!.statistics.length, 8, 'unifyStats years')
-      assert.strictEqual(body.regularStats!.statistics.length, 8, 'regularStats years')
-      assert.strictEqual(body.openStats!.statistics.length, 4, 'openStats years')
+      assert.strictEqual(body.unifyStats.statistics.length, 8, 'unifyStats years')
+      assert.strictEqual(body.regularStats.statistics.length, 8, 'regularStats years')
+      assert.strictEqual(body.openStats.statistics.length, 4, 'openStats years')
 
       // Faculties should have the same amount of years as normal statistics (above)
-      assert.strictEqual(Object.keys(body.unifyStats!.facultyStats).length, 8, 'unifyStats years')
-      assert.strictEqual(Object.keys(body.regularStats!.facultyStats).length, 8, 'regularStats years')
-      assert.strictEqual(Object.keys(body.openStats!.facultyStats).length, 4, 'openStats years')
+      assert.strictEqual(Object.keys(body.unifyStats.facultyStats).length, 8, 'unifyStats years')
+      assert.strictEqual(Object.keys(body.regularStats.facultyStats).length, 8, 'regularStats years')
+      assert.strictEqual(Object.keys(body.openStats.facultyStats).length, 4, 'openStats years')
     })
 
     // TODO: Rewrite /courseyearlystats so that total = passed + failed + enrolledNoGrade
@@ -76,7 +78,7 @@ void describe('Course yearly statistics - TKT10002 (no substitutions)', () => {
         { 5: 1, 4: 0, 3: 0, 2: 0, 1: 0, 0: 0, 'Hyv.': 0 },
       ],
     ])('for statistic content during ($0)', (year, categories, grades) => {
-      const stats = body.unifyStats!.statistics.find(yearStats => yearStats.name === year)!
+      const stats = body.unifyStats.statistics.find(yearStats => yearStats.name === year)!
       assert('enrollments' in stats, 'Missing field enrollment in statsitics')
 
       const studentCategories = calculatePassedAndFailed(stats.students.grades)
@@ -115,24 +117,18 @@ void describe('Course yearly statistics - TKT10002 (no substitutions)', () => {
     })
 
     it('should include students who have failed a course and then completed it only once', () => {
-      const stats2017 = body.unifyStats!.statistics.find(year => year.name === '2017-2018')!
+      const stats2017 = body.unifyStats.statistics.find(year => year.name === '2017-2018')!
       const { studentNumbers } = stats2017.students
-
-      assert(
-        !(
-          (
-            '457686' in studentNumbers || // SPRING 2019
-            '455478' in studentNumbers || // FALL 2020
-            '547994' in studentNumbers
-          ) // FALL 2022
-        ),
+      assert.notIncludeMembers(
+        studentNumbers,
+        ['457686' /* SPRING 2019*/, '455478' /* FALL 2020 */, '547994' /* FALL 2022*/],
         "Students that have completed course later should not be included in the previous year's stats"
       )
 
-      const year2017 = body.unifyStats!.statistics.find(year => year.name === '2017-2018')
-      const year2018 = body.unifyStats!.statistics.find(year => year.name === '2018-2019')
-      const year2020 = body.unifyStats!.statistics.find(year => year.name === '2020-2021')
-      const year2022 = body.unifyStats!.statistics.find(year => year.name === '2022-2023')
+      const year2017 = body.unifyStats.statistics.find(year => year.name === '2017-2018')
+      const year2018 = body.unifyStats.statistics.find(year => year.name === '2018-2019')
+      const year2020 = body.unifyStats.statistics.find(year => year.name === '2020-2021')
+      const year2022 = body.unifyStats.statistics.find(year => year.name === '2022-2023')
       assert(year2017 && 'enrollments' in year2017, 'Missing stats for 2017-2018')
       assert(year2018 && 'enrollments' in year2018, 'Missing stats for 2018-2019')
       assert(year2020 && 'enrollments' in year2020, 'Missing stats for 2020-2021')
@@ -172,8 +168,8 @@ void describe('Course yearly statistics - TKT10002 (no substitutions)', () => {
     })
 
     it("should mark student's enrollment time correctly with mismatched semestercode and enrollment_date_time", () => {
-      const stats2021 = body.unifyStats!.statistics.find(year => year.name === '2021-2022')!
-      const stats2022 = body.unifyStats!.statistics.find(year => year.name === '2022-2023')!
+      const stats2021 = body.unifyStats.statistics.find(year => year.name === '2021-2022')!
+      const stats2022 = body.unifyStats.statistics.find(year => year.name === '2022-2023')!
       assert('enrollments' in stats2021, 'Missing field enrollment in statsitics (2021)')
       assert('enrollments' in stats2022, 'Missing field enrollment in statsitics (2022)')
       // Student with enrollment_date_time outside of semestert start and end dates
@@ -238,9 +234,9 @@ void describe('Course yearly statistics - TKT10002 (no substitutions)', () => {
     ])('should include correct stats for one academic year ($0)', async (year, categories, grades) => {
       const body = await getCourseYearlyStats(
         app,
-        `codes=TKT10002&combineSubstitutions=false&fromYearCode=${yearToYearCode(year.split('-').at(0))}&toYearCode=${yearToYearCode(parseInt(year.split('-').at(1)!)) - 1}`
+        `courses=${TKT10002}&substitutions=false&fromYearCode=${yearToYearCode(year.split('-').at(0))}&toYearCode=${yearToYearCode(parseInt(year.split('-').at(1)!)) - 1}`
       )
-      const stats = body.unifyStats!.statistics.find(yearStats => yearStats.name === year)!
+      const stats = body.unifyStats.statistics.find(yearStats => yearStats.name === year)!
       assert('enrollments' in stats, 'Missing field enrollment in statsitics')
 
       const studentCategories = calculatePassedAndFailed(stats.students.grades)

services/backend/tests/api/population/populationStatisticsByCourse.test.ts

@@ -8,7 +8,10 @@ import { PopulationstatisticsbycourseResBody } from '@oodikone/shared/routes/pop
 import { CreditModel } from '@/models'
 import { EnrollmentState } from '@oodikone/shared/types'
 
-void describe('Population statistics by course', () => {
+const MAT21003 = 'hy-CU-117375829'
+const MAT21005 = 'hy-CU-117376344'
+
+void describe('Population statistics by course MAT21003 + MAT21005 (hy-CU-117375829 + hy-CU-117376344)', () => {
   let app: Express
   beforeAll(async () => {
     app = await initTests()
@@ -22,7 +25,7 @@ void describe('Population statistics by course', () => {
       .set('hygroupcn', 'grp-oodikone-basic-users')
 
     assert.strictEqual(res.status, 400)
-    assert.deepStrictEqual(res.body.error, 'The body should have a yearcode and coursecode defined')
+    assert.deepStrictEqual(res.body.error, 'Missing required parameters.')
   })
 
   // Vektorianalyysi I, MAT21003
@@ -40,9 +43,7 @@ void describe('Population statistics by course', () => {
     [2023, 21, 1, 0, 36],
   ])('should return correct amount of students for single year ($0)', async (year, total, passed, failed, enrolled) => {
     const res = await request(app)
-      .get(
-        `/populationstatisticsbycourse?coursecodes=["MAT21003"]&from=${yearToYearCode(year)}&to=${yearToYearCode(year)}`
-      )
+      .get(`/populationstatisticsbycourse?courses=${MAT21003}&from=${yearToYearCode(year)}&to=${yearToYearCode(year)}`)
       .set('shib-session-id', 'test')
       .set('uid', 'basic')
       .set('hygroupcn', 'grp-oodikone-basic-users')
@@ -52,20 +53,23 @@ void describe('Population statistics by course', () => {
 
     assert.strictEqual(body.students.length, total, 'Incorrect amount of students')
     assert.strictEqual(
-      body.coursestatistics.credits.filter(credit => credit.course_code === 'MAT21003' && CreditModel.passed(credit))
-        .length,
+      body.coursestatistics.credits.filter(
+        credit => body.idToGroupIdMap[credit.course_id] === MAT21003 && CreditModel.passed(credit)
+      ).length,
       passed,
       'Incorrect amount of passed credits'
     )
     assert.strictEqual(
-      body.coursestatistics.credits.filter(credit => credit.course_code === 'MAT21003' && CreditModel.failed(credit))
-        .length,
+      body.coursestatistics.credits.filter(
+        credit => body.idToGroupIdMap[credit.course_id] === MAT21003 && CreditModel.failed(credit)
+      ).length,
       failed,
       'Incorrect amount of failed credits'
     )
     assert.strictEqual(
       body.coursestatistics.enrollments.filter(
-        enrollment => enrollment.course_code === 'MAT21003' && enrollment.state === EnrollmentState.ENROLLED
+        enrollment =>
+          body.idToGroupIdMap[enrollment.course_id] === MAT21003 && enrollment.state === EnrollmentState.ENROLLED
       ).length,
       enrolled,
       'Incorrect amount of enrollments'
@@ -74,7 +78,7 @@ void describe('Population statistics by course', () => {
 
   it('should return correct amount of students for a course', async () => {
     const res = await request(app)
-      .get('/populationstatisticsbycourse?coursecodes=["MAT21005"]&from=70&to=70')
+      .get(`/populationstatisticsbycourse?courses=${MAT21005}&from=70&to=70`)
       .set('shib-session-id', 'test')
       .set('uid', 'basic')
       .set('hygroupcn', 'grp-oodikone-basic-users')

services/backend/tests/api/studyProgramme/studyProgrammeCourseStats.test.ts

@@ -97,13 +97,12 @@ void describe("Study programme's course stats (all years)", () => {
       .set('hygroupcn', 'grp-oodikone-basic-users')) as ResponseWithBody<StudyProgrammeCourse[]>
 
     assert.strictEqual(res.status, 200)
-    assert.strictEqual(res.body.length, 102, 'Response included incorrect amount of courses') // Not checked number
+    assert.strictEqual(res.body.length, 88, 'Response included incorrect amount of courses') // Not checked number
 
     const courseCodes = res.body.map(({ code }) => code)
-    assert(
-      ['MAT11003', 'MAT11001', 'MAT11004', 'AYMAT11003', 'AYMAT11001', 'AYMAT11004'].every(code =>
-        courseCodes.includes(code)
-      ),
+    assert.includeMembers(
+      courseCodes,
+      ['MAT11003', 'MAT11001', 'MAT11004' /* , 'AYMAT11003', 'AYMAT11001', 'AYMAT11004' */], // AY codes were previously required
       'Required courses or their open uni variants missing'
     )
 

services/backend/tests/services/courses/index.test.ts

@@ -1,161 +1,154 @@
-import { describe, it, assert, beforeAll, vi } from 'vitest'
-import { Unification } from '@oodikone/shared/types'
+import { describe, it, assert, beforeAll } from 'vitest'
 import { initializeDatabaseConnection } from '@/database/connection'
-import { maxYearsToCreatePopulationFrom, searchAndCombineSubstitutionGroupsToCodes } from '@/services/courses'
 import { findByCourseAndSemesters } from '@/services/students'
 import { yearToYearCode } from '@oodikone/shared/util'
+import { Unification } from '@oodikone/shared/types'
+import { getRelevantCourseIdMap } from '@/services/courses'
+
+const CSM14204 = 'hy-CU-119366777'
+const MAT21018 = 'hy-CU-117377764'
 
 void describe.concurrent('Search and complete substitution groups to codes', () => {
   beforeAll(async () => {
     await initializeDatabaseConnection()
   })
 
-  it('should return correct course codes for MAT11001 (only single-length groups)', async () => {
-    const res = await searchAndCombineSubstitutionGroupsToCodes(['MAT11001'])
-    assert.deepStrictEqual(res, ['MAT11001', '57033', 'AYMAT11001', 'A57033'])
-  })
+  type TestList = [string, number, number][]
 
-  it('should return correct course codes for MAT21016 (also multicourse groups)', async () => {
-    const res = await searchAndCombineSubstitutionGroupsToCodes(['MAT21016'])
-    assert.deepStrictEqual(res, ['MAT21016', '57282', 'MFK-M310', 'MAT21023'])
-  })
-  it('should return correct course codes for MAT11001 + MAT21016', async () => {
-    const res = await searchAndCombineSubstitutionGroupsToCodes(['MAT11001', 'MAT21016'])
-    assert.deepStrictEqual(res, [
-      'MAT11001',
-      'MAT21016',
-      '57033',
-      'AYMAT11001',
-      'A57033',
-      '57282',
-      'MFK-M310',
-      'MAT21023',
-    ])
-  })
-  it.todo('should return correct course codes for X + Y (duplicate courses in substitution groups)')
-})
+  const semesterToSemesterCode = (semesterYear: number, springOrFall: 'spring' | 'fall') => {
+    return (semesterYear - 1950) * 2 + (springOrFall === 'spring' ? 0 : 1)
+  }
 
-void describe.skip('Max years to generate population from', () => {
-  beforeAll(async () => {
-    await initializeDatabaseConnection()
-  })
+  void describe.concurrent('Find by course and semester', () => {
+    it.each([
+      ['semester', semesterToSemesterCode(2019, 'fall'), semesterToSemesterCode(2023, 'spring')],
+      ['year', yearToYearCode(2017), yearToYearCode(2023)],
+    ] as TestList)(
+      'should return correct student numbers for a short course (CSM14204, 2019-2023) by $0',
+      async (separate, to, from) => {
+        const courseIds = await getRelevantCourseIdMap([CSM14204], false)
+        const res = await findByCourseAndSemesters(
+          Object.keys(courseIds),
+          to,
+          from,
+          separate === 'semester',
+          Unification.UNIFY
+        )
 
-  it('should return no years without a course code', async () => {
-    const res = await maxYearsToCreatePopulationFrom([], Unification.UNIFY)
-    assert.strictEqual(res, 0, 'Got something else than 0 years')
-  })
+        assert.deepStrictEqual([...res].sort(), ['458079', '478837', '483126'])
+      }
+    )
 
-  it('should return correct years for MAT11001', async () => {
-    const res = await maxYearsToCreatePopulationFrom(['MAT11001'], Unification.UNIFY)
-    assert.strictEqual(res, 6)
-  })
+    it.each([
+      ['semester', semesterToSemesterCode(2020, 'fall'), semesterToSemesterCode(2023, 'spring')],
+      ['year', yearToYearCode(2020), yearToYearCode(2023)],
+    ] as TestList)('should filter out students outside of 2020-2023 by $0', async (separate, to, from) => {
+      const courseIds = await getRelevantCourseIdMap([CSM14204], false)
+      const res = await findByCourseAndSemesters(
+        Object.keys(courseIds),
+        to,
+        from,
+        separate === 'semester',
+        Unification.UNIFY
+      )
+      assert.deepStrictEqual([...res].sort(), ['478837', '483126'])
+    })
 
-  it.todo('should should return the same max years as before', () => {
-    const date = new Date(2020)
-    vi.useFakeTimers()
-    vi.setSystemTime(date)
+    it.each([
+      ['semester', semesterToSemesterCode(2017, 'fall'), semesterToSemesterCode(2021, 'spring')],
+      ['year', yearToYearCode(2017), yearToYearCode(2020)],
+    ] as TestList)(
+      'should return correct student numbers for a longer course (MAT21018, 2017-2021) by $0',
+      async (separate, to, from) => {
+        const courseIds = await getRelevantCourseIdMap([MAT21018], false)
+        const res = await findByCourseAndSemesters(
+          Object.keys(courseIds),
+          to,
+          from,
+          separate === 'semester',
+          Unification.UNIFY
+        )
+        const passed = ['457686', '495976', '484997', '491970', '461485', '501442', '508370']
+        const failed = [
+          '484541',
+          '487566',
+          '493344',
+          '520906',
+          '541350',
+          '544750',
+          '550840',
+          '495398',
+          '511089',
+          '538399',
+        ]
+        assert.deepStrictEqual([...res].sort(), passed.concat(failed).sort())
+      }
+    )
 
-    vi.useRealTimers()
-  })
-})
+    it.each([
+      ['semester', semesterToSemesterCode(2021, 'fall'), semesterToSemesterCode(2023, 'spring')],
+      ['year', yearToYearCode(2021), yearToYearCode(2022)],
+    ] as TestList)(
+      'should return correct student numbers for a longer course (MAT21018, 2021-2023) by $0',
+      async (separate, to, from) => {
+        const courseIds = await getRelevantCourseIdMap([MAT21018], false)
+        const res = await findByCourseAndSemesters(
+          Object.keys(courseIds),
+          to,
+          from,
+          separate === 'semester',
+          Unification.UNIFY
+        )
+        const passed = [
+          '509745',
+          '504315',
+          '520805',
+          '509881',
+          '511089',
+          '519527',
+          '522321',
+          '527445',
+          '538399',
+          '534980',
+          '474789',
+          '478837',
+          '482406',
+          '486809',
+          '547552',
+          '493345',
+          '495398',
+          '498558',
+        ]
+        const enrolledNoGrade = ['529866', '488481', '479440', '518062']
 
-type TestList = [string, number, number][]
+        assert.deepStrictEqual([...res].sort(), passed.concat(enrolledNoGrade).sort())
+        assert.strictEqual(res.size, passed.length + enrolledNoGrade.length)
+      }
+    )
 
-const semesterToSemesterCode = (semesterYear: number, springOrFall: 'spring' | 'fall') => {
-  return (semesterYear - 1950) * 2 + (springOrFall === 'spring' ? 0 : 1)
-}
+    it.todo('should return correct student numbers for multiple courses (CSM14204+MAT21018)')
 
-void describe.concurrent('Find by course and semester', () => {
-  it.each([
-    ['semester', semesterToSemesterCode(2019, 'fall'), semesterToSemesterCode(2023, 'spring')],
-    ['year', yearToYearCode(2017), yearToYearCode(2023)],
-  ] as TestList)(
-    'should return correct student numbers for a short course (CSM14204, 2019-2023) by $0',
-    async (separate, to, from) => {
-      const res = (await findByCourseAndSemesters(['CSM14204'], to, from, separate === 'semester', 'unifyStats')).sort()
-      assert.deepStrictEqual(res, ['458079', '478837', '483126'])
-    }
-  )
+    describe('should work with specific cases', () => {
+      it('Student with enrollment in -21 and passed grade in -23 should be included in both stats when querying one year at a time (522321)', async () => {
+        const courseIds = await getRelevantCourseIdMap([MAT21018], false)
+        const res21 = await findByCourseAndSemesters(
+          Object.keys(courseIds),
+          yearToYearCode(2021),
+          yearToYearCode(2021),
+          false,
+          Unification.UNIFY
+        )
+        const res23 = await findByCourseAndSemesters(
+          Object.keys(courseIds),
+          yearToYearCode(2022),
+          yearToYearCode(2022),
+          false,
+          Unification.UNIFY
+        )
 
-  it.each([
-    ['semester', semesterToSemesterCode(2020, 'fall'), semesterToSemesterCode(2023, 'spring')],
-    ['year', yearToYearCode(2020), yearToYearCode(2023)],
-  ] as TestList)('should filter out students outside of 2020-2023 by $0', async (separate, to, from) => {
-    const res = (await findByCourseAndSemesters(['CSM14204'], to, from, separate === 'semester', 'unifyStats')).sort()
-    assert.deepStrictEqual(res, ['478837', '483126'])
-  })
-
-  it.each([
-    ['semester', semesterToSemesterCode(2017, 'fall'), semesterToSemesterCode(2021, 'spring')],
-    ['year', yearToYearCode(2017), yearToYearCode(2020)],
-  ] as TestList)(
-    'should return correct student numbers for a longer course (MAT21018, 2017-2021) by $0',
-    async (separate, to, from) => {
-      const res = (await findByCourseAndSemesters(['MAT21018'], to, from, separate === 'semester', 'unifyStats')).sort()
-      const passed = ['457686', '495976', '484997', '491970', '461485', '501442', '508370']
-      const failed = [
-        '484541',
-        '487566',
-        '493344',
-        '520906',
-        '541350',
-        '544750',
-        '550840',
-        '495398',
-        '511089',
-        '538399',
-      ]
-      assert.deepStrictEqual(res, passed.concat(failed).sort())
-    }
-  )
-
-  it.each([
-    ['semester', semesterToSemesterCode(2021, 'fall'), semesterToSemesterCode(2023, 'spring')],
-    ['year', yearToYearCode(2021), yearToYearCode(2022)],
-  ] as TestList)(
-    'should return correct student numbers for a longer course (MAT21018, 2021-2023) by $0',
-    async (separate, to, from) => {
-      const res = (await findByCourseAndSemesters(['MAT21018'], to, from, separate === 'semester', 'unifyStats')).sort()
-      const passed = [
-        '509745',
-        '504315',
-        '520805',
-        '509881',
-        '511089',
-        '519527',
-        '522321',
-        '527445',
-        '538399',
-        '534980',
-        '474789',
-        '478837',
-        '482406',
-        '486809',
-        '547552',
-        '493345',
-        '495398',
-        '498558',
-      ]
-      const enrolledNoGrade = ['529866', '488481', '479440', '518062']
-
-      assert.deepStrictEqual(res, passed.concat(enrolledNoGrade).sort())
-      assert.strictEqual(res.length, passed.length + enrolledNoGrade.length)
-    }
-  )
-
-  it.todo('should return correct student numbers for multiple courses (CSM14204+MAT21018)')
-
-  describe('should work with specific cases', () => {
-    it('Student with enrollment in -21 and passed grade in -23 should be included in both stats when querying one year at a time (522321)', async () => {
-      const res21 = (
-        await findByCourseAndSemesters(['MAT21018'], yearToYearCode(2021), yearToYearCode(2021), false, 'unifyStats')
-      ).sort()
-      const res23 = (
-        await findByCourseAndSemesters(['MAT21018'], yearToYearCode(2022), yearToYearCode(2022), false, 'unifyStats')
-      ).sort()
-
-      assert.include(res21, '522321', '522321 incorrectly missing in -21 stats')
-      assert.include(res23, '522321', '522321 incorrectly missing in -23 stats')
+        assert.include([...res21], '522321', '522321 incorrectly missing in -21 stats')
+        assert.include([...res23], '522321', '522321 incorrectly missing in -23 stats')
+      })
     })
   })
 })

services/backend/tests/unit/services/courses/getCreditsForCourses.test.ts

@@ -2,125 +2,153 @@ import { describe, it, assert, vi, afterEach } from 'vitest'
 import { Op } from 'sequelize'
 
 import { CreditTypeCode, Unification } from '@oodikone/shared/types'
+import { Credit } from '@oodikone/shared/models'
 import { createCredit as credit } from '@oodikone/shared/test/utils'
-import { StudentModel } from '@/models'
+import { CreditModel } from '@/models'
 import { getCreditsForCourses } from '@/services/courses/creditsAndEnrollmentsOfCourse'
 
-const mockStudents = (students: Array<{ studentnumber: string; credits: Array<Record<string, unknown>> }>) =>
-  vi.spyOn(StudentModel, 'findAll').mockResolvedValue(students as any)
+const MAT11002 = 'hy-CU-117375394'
+const MAT21001 = 'hy-CU-117375754'
+const MAT21002 = 'hy-CU-117375793'
+const OTHER101 = 'hy-CU-000000000'
+
+const mockCredits = (credits: Array<Record<string, unknown>>) =>
+  vi.spyOn(CreditModel, 'findAll').mockResolvedValue(credits as any)
 
 const from = new Date(2000, 0, 1)
 const to = new Date(2030, 0, 1)
 
-// Tests share a spy on the same StudentModel.findAll, so they must not run concurrently
+// Tests share a spy on the same CreditModel.findAll, so they must not run concurrently
 // (the suite config default is sequence.concurrent: true).
 void describe('Get credits for courses (unit, mocked db)', { concurrent: false }, () => {
   afterEach(() => {
     vi.restoreAllMocks()
   })
 
-  it('should return no groups without any course codes, without querying credits by course', async () => {
-    const findAll = mockStudents([{ studentnumber: '111', credits: [credit()] }])
+  it('should return no groups when there are no groupId groups to match', async () => {
+    const findAll = mockCredits([credit()])
 
-    const res = await getCreditsForCourses([], Unification.UNIFY, from, to)
+    const res = await getCreditsForCourses([], [], Unification.UNIFY, from, to)
 
     assert.deepStrictEqual(res, [])
     assert.strictEqual(findAll.mock.calls.length, 1)
   })
 
-  it('should query StudentModel with course codes, date range and openness filters', async () => {
-    const findAll = mockStudents([])
+  it('should query CreditModel with course ids, date range and openness filters', async () => {
+    const findAll = mockCredits([])
 
-    await getCreditsForCourses([['MAT11002'], ['MAT21001', 'MAT21002']], Unification.OPEN, from, to)
+    await getCreditsForCourses([[MAT11002], [MAT21001, MAT21002]], ['course-1', 'course-2'], Unification.OPEN, from, to)
 
     const call = findAll.mock.calls[0][0] as any
-    const creditInclude = call.include[0]
 
-    assert.deepStrictEqual(creditInclude.where.course_code, { [Op.in]: ['MAT11002', 'MAT21001', 'MAT21002'] })
-    assert.deepStrictEqual(creditInclude.where.attainment_date, { [Op.between]: [from, to] })
-    assert.deepStrictEqual(creditInclude.where.is_open, { [Op.eq]: true })
+    assert.deepStrictEqual(call.where.course_id, { [Op.in]: ['course-1', 'course-2'] })
+    assert.deepStrictEqual(call.where.attainment_date, { [Op.between]: [from, to] })
+    assert.deepStrictEqual(call.where.is_open, { [Op.eq]: true })
   })
 
   it('should include both passed and failed credits for a single-code group as one group', async () => {
-    mockStudents([
-      {
-        studentnumber: '111',
-        credits: [credit({ credittypecode: CreditTypeCode.FAILED }), credit({ credittypecode: CreditTypeCode.PASSED })],
-      },
-    ])
+    mockCredits([credit({ credittypecode: CreditTypeCode.FAILED }), credit({ credittypecode: CreditTypeCode.PASSED })])
 
-    const res = await getCreditsForCourses([['MAT11002']], Unification.UNIFY, from, to)
+    const res = await getCreditsForCourses([[MAT11002]], ['course-1'], Unification.UNIFY, from, to)
 
     assert.strictEqual(res.length, 1)
     assert.strictEqual(res.flat().length, 2)
   })
 
   it('should create a separate group per student for a single-code group', async () => {
-    mockStudents([
-      { studentnumber: '111', credits: [credit({ student_studentnumber: '111' })] },
-      { studentnumber: '222', credits: [credit({ student_studentnumber: '222' })] },
-    ])
+    mockCredits([credit({ student_studentnumber: '111' }), credit({ student_studentnumber: '222' })])
 
-    const res = await getCreditsForCourses([['MAT11002']], Unification.UNIFY, from, to)
+    const res = await getCreditsForCourses([[MAT11002]], ['course-1'], Unification.UNIFY, from, to)
 
     assert.strictEqual(res.length, 2)
     assert.deepStrictEqual(res.map(group => group.map(c => c.student_studentnumber)).sort(), [['111'], ['222']])
   })
 
   it('should not create a group for a single course code the student has no credit for', async () => {
-    mockStudents([{ studentnumber: '111', credits: [credit({ course_code: 'OTHER101' })] }])
+    mockCredits([credit({ course_code: 'OTHER101', course: { groupId: OTHER101 } as Credit['course'] })])
 
-    const res = await getCreditsForCourses([['MAT11002']], Unification.UNIFY, from, to)
+    const res = await getCreditsForCourses([[MAT11002]], ['course-1'], Unification.UNIFY, from, to)
 
     assert.deepStrictEqual(res, [])
   })
 
   it('should only include the passed credits of a substitution group when every code was passed', async () => {
-    mockStudents([
-      {
-        studentnumber: '111',
-        credits: [
-          credit({ course_code: 'MAT11002', credittypecode: CreditTypeCode.PASSED }),
-          credit({ course_code: 'MAT21001', credittypecode: CreditTypeCode.APPROVED }),
-        ],
-      },
+    mockCredits([
+      credit({
+        course_code: 'MAT11002',
+        course: { groupId: MAT11002 } as Credit['course'],
+        credittypecode: CreditTypeCode.PASSED,
+      }),
+      credit({
+        course_code: 'MAT21001',
+        course: { groupId: MAT21001 } as Credit['course'],
+        credittypecode: CreditTypeCode.APPROVED,
+      }),
     ])
 
-    const res = await getCreditsForCourses([['MAT11002', 'MAT21001']], Unification.UNIFY, from, to)
+    const res = await getCreditsForCourses(
+      [[MAT11002, MAT21001]],
+      ['course-1', 'course-2'],
+      Unification.UNIFY,
+      from,
+      to
+    )
 
     assert.strictEqual(res.length, 1)
     assert.deepStrictEqual(res[0].map(c => c.course_code).sort(), ['MAT11002', 'MAT21001'])
   })
 
   it('should exclude a partially completed substitution group entirely', async () => {
-    mockStudents([
-      {
-        studentnumber: '111',
-        credits: [
-          credit({ course_code: 'MAT11002', credittypecode: CreditTypeCode.PASSED }),
-          credit({ course_code: 'MAT21001', credittypecode: CreditTypeCode.FAILED }),
-        ],
-      },
+    mockCredits([
+      credit({
+        course_code: 'MAT11002',
+        course: { groupId: MAT11002 } as Credit['course'],
+        credittypecode: CreditTypeCode.PASSED,
+      }),
+      credit({
+        course_code: 'MAT21001',
+        course: { groupId: MAT21001 } as Credit['course'],
+        credittypecode: CreditTypeCode.FAILED,
+      }),
     ])
 
-    const res = await getCreditsForCourses([['MAT11002', 'MAT21001']], Unification.UNIFY, from, to)
+    const res = await getCreditsForCourses(
+      [[MAT11002, MAT21001]],
+      ['course-1', 'course-2'],
+      Unification.UNIFY,
+      from,
+      to
+    )
 
     assert.deepStrictEqual(res, [])
   })
 
   it('should exclude an earlier failed attempt from a substitution group even when it later passed', async () => {
-    mockStudents([
-      {
-        studentnumber: '111',
-        credits: [
-          credit({ course_code: 'MAT11002', credittypecode: CreditTypeCode.FAILED }),
-          credit({ course_code: 'MAT11002', credittypecode: CreditTypeCode.PASSED }),
-          credit({ course_code: 'MAT21001', credittypecode: CreditTypeCode.PASSED }),
-        ],
-      },
+    mockCredits([
+      credit({
+        course_code: 'MAT11002',
+        course: { groupId: MAT11002 } as Credit['course'],
+        credittypecode: CreditTypeCode.FAILED,
+      }),
+      credit({
+        course_code: 'MAT11002',
+        course: { groupId: MAT11002 } as Credit['course'],
+        credittypecode: CreditTypeCode.PASSED,
+      }),
+      credit({
+        course_code: 'MAT21001',
+        course: { groupId: MAT21001 } as Credit['course'],
+        credittypecode: CreditTypeCode.PASSED,
+      }),
     ])
 
-    const res = await getCreditsForCourses([['MAT11002', 'MAT21001']], Unification.UNIFY, from, to)
+    const res = await getCreditsForCourses(
+      [[MAT11002, MAT21001]],
+      ['course-1', 'course-2'],
+      Unification.UNIFY,
+      from,
+      to
+    )
 
     assert.strictEqual(res.length, 1)
     assert.strictEqual(res.flat().length, 2)
@@ -131,18 +159,33 @@ void describe('Get credits for courses (unit, mocked db)', { concurrent: false }
   })
 
   it('should handle independent groups and students together', async () => {
-    mockStudents([
-      { studentnumber: '111', credits: [credit({ student_studentnumber: '111', course_code: 'MAT11002' })] },
-      {
-        studentnumber: '222',
-        credits: [
-          credit({ student_studentnumber: '222', course_code: 'MAT21001', credittypecode: CreditTypeCode.PASSED }),
-          credit({ student_studentnumber: '222', course_code: 'MAT21002', credittypecode: CreditTypeCode.PASSED }),
-        ],
-      },
+    mockCredits([
+      credit({
+        student_studentnumber: '111',
+        course_code: 'MAT11002',
+        course: { groupId: MAT11002 } as Credit['course'],
+      }),
+      credit({
+        student_studentnumber: '222',
+        course_code: 'MAT21001',
+        course: { groupId: MAT21001 } as Credit['course'],
+        credittypecode: CreditTypeCode.PASSED,
+      }),
+      credit({
+        student_studentnumber: '222',
+        course_code: 'MAT21002',
+        course: { groupId: MAT21002 } as Credit['course'],
+        credittypecode: CreditTypeCode.PASSED,
+      }),
     ])
 
-    const res = await getCreditsForCourses([['MAT11002'], ['MAT21001', 'MAT21002']], Unification.UNIFY, from, to)
+    const res = await getCreditsForCourses(
+      [[MAT11002], [MAT21001, MAT21002]],
+      ['course-1', 'course-2', 'course-3'],
+      Unification.UNIFY,
+      from,
+      to
+    )
 
     assert.strictEqual(res.length, 2)
   })

services/backend/tests/unit/services/courses/parseCredit.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, assert } from 'vitest'
 
 import { CreditTypeCode } from '@oodikone/shared/types'
+import { Credit } from '@oodikone/shared/models'
 import { createCredit as credit, createStudyRightElementModel } from '@oodikone/shared/test/utils'
 import { parseCredit } from '@/services/courses'
 import { SISStudyRightElementModel } from '@/models'
@@ -30,7 +31,10 @@ void describe('parseCredit', () => {
 
   it('should combine a substitution group into one credit with summed credits', () => {
     const res = parseCredit(
-      [credit({ course_code: 'MAT11002', credits: 5 }), credit({ course_code: 'MAT21001', credits: 5 })],
+      [
+        credit({ course_code: 'MAT11002', course: { groupId: 'hy-CU-117375394' } as Credit['course'], credits: 5 }),
+        credit({ course_code: 'MAT21001', course: { groupId: 'hy-CU-117375754' } as Credit['course'], credits: 5 }),
+      ],
       null,
       'MAT11002',
       [studyRightElement()]

services/backend/tests/unit/services/studyProgramme/studyProgrammeHelpers.test.ts

@@ -1,5 +1,12 @@
 import { describe, it, assert, vi, beforeAll } from 'vitest'
-import { getPercentage, getYearsArray, getYearsObject } from '@/services/studyProgramme/studyProgrammeHelpers'
+import {
+  getMonthlyCredits,
+  getPercentage,
+  getYearlyMonthlyCreditsObj,
+  getYearsArray,
+  getYearsObject,
+} from '@/services/studyProgramme/studyProgrammeHelpers'
+import { clockStorage } from '@/util/clock'
 
 void describe('Get years object', () => {
   it('should return nothing with empty years', () => {
@@ -95,3 +102,36 @@ void describe('Get percentage', () => {
     assert.strictEqual(getPercentage(-5, 10), '-50.0 %')
   })
 })
+
+void describe('Get monthly credits', () => {
+  const atTime = <T>(date: string, fn: () => T) => clockStorage.run({ now: new Date(date) }, fn)
+
+  it('should include the current month regardless of the time of day', () => {
+    const lastKey = (date: string) => Object.keys(atTime(date, () => getYearlyMonthlyCreditsObj())[2024]).at(-1)
+
+    assert.strictEqual(lastKey('2026-10-07T00:05:00'), '2026-10')
+    assert.strictEqual(lastKey('2026-10-07T23:40:00'), '2026-10')
+  })
+
+  it('should not throw when computation continues past midnight', () => {
+    const monthlyCreditsByStartingYear = atTime('2026-10-06T23:40:00', () => getYearlyMonthlyCreditsObj())
+
+    assert.doesNotThrow(() =>
+      atTime('2026-10-07T00:05:00', () =>
+        getMonthlyCredits([], new Date('2024-08-01'), 2024, monthlyCreditsByStartingYear[2024])
+      )
+    )
+  })
+
+  it('should accumulate credits cumulatively by month', () => {
+    const monthlyCredits = atTime('2024-10-15T12:00:00', () => getYearlyMonthlyCreditsObj())[2024]
+    const credits = [
+      { attainment_date: new Date('2024-08-20'), credits: 5 },
+      { attainment_date: new Date('2024-10-01'), credits: 3 },
+    ] as Parameters<typeof getMonthlyCredits>[0]
+
+    atTime('2024-10-15T12:00:00', () => getMonthlyCredits(credits, new Date('2024-08-01'), 2024, monthlyCredits))
+
+    assert.deepStrictEqual(monthlyCredits, { '2024-8': [5], '2024-9': [5], '2024-10': [8] })
+  })
+})

services/frontend/dev.Dockerfile

@@ -1,6 +1,9 @@
 FROM registry.access.redhat.com/ubi10/nodejs-24-minimal
 
 ENV TZ="Europe/Helsinki"
+# Default install location is node_modules/.vite which is not writable
+#  by user 1001 when binding the local files to the container
+ENV VITE_CACHE_DIR=/tmp/vite
 
 WORKDIR /opt/app-root/shared
 COPY ./shared .

services/frontend/src/common/InfoToolTips/studyProgramme.ts

@@ -190,16 +190,35 @@ However this is optional and you can create a new tag without selecting year.
   },
 
   degreeCoursesTab: {
-    creditCriteria: `
-Here you can change visibility of degree courses as and set course and credits criteria, for each year their
-own. Credits criteria is computed as follows: for the first academic year the credits are taken into account
-if they are completed during the first 12 months. For the second year, we take into account the completions
-during the first 24 months, for the third year the first 36 months.
+    creditCriteria: {
+      en: `The targets set here will affect the Class Statistics → Students tab → Progress tab view.
 
-The progress of the students by these criteria will be shown in class statistics view.
+Credit criteria is computed as follows: for the first academic year, the credits are counted
+if they are completed during the first 12 months. For the second year, completions
+within the first 24 months. Third year, the first 36 months, and so on.
 `,
-  },
+      fi: `Tässä asetetut tavoitteet vaikuttavat Class Statistics → Students tab → Progress tab näkymään.
 
+Laskenta toimii seuraavasti: ensimmäisen akateemisen vuoden opintopisteet lasketaan, jos ne ovat suoritettu ensimmäisen 12 kuukauden ajalla. Toisen vuoden opintopisteet, jos ne ovat suoritettu ensimmäisen 24 kuukauden aikana. Kolmas, ensimmäiset 36kk, ja niin edelleen.
+`,
+    },
+    requiredCourses: {
+      en: `Click a module to see its courses in the selected curriculum.
+
+Tick a year to make the course a required course for that academic year in Class Statistics → Students → Progress tab. A course can be required for several years. The numbers on a module row show how many of its courses are required for each year.`,
+      fi: `Klikkaa opintokokonaisuutta nähdäksesi sen kurssit valitussa opetussuunnitelmassa.
+
+Valitse vuosi, jolloin kurssi on tarkoitus suorittaa. Kurssi näytetään kyseisen akateemisen vuoden alla Class Statistics → Students → Progress tab -näkymässä. Kurssi voi olla pakollinen useammalle vuodelle. Opintokokonaisuuden rivillä olevat luvut kertovat, kuinka moni sen kursseista on pakollinen kunakin vuonna. Huom. kurssi voi kuulua useaan opintokokonaisuuteen.`,
+    },
+    courseVisibility: {
+      en: `Click a module to see its courses in the selected curriculum.
+
+Hiding a course also hides it in the Courses of Class view and Students → Courses tab for all users. Hiding a module hides all of its courses.`,
+      fi: `Klikkaa opintokokonaisuutta nähdäksesi sen kurssit valitussa opetussuunnitelmassa.
+
+Kurssin piilottaminen piilottaa sen kaikilta käyttäjiltä myös Courses of Class ja Students → Courses tab -näkymissä. Opintokokonaisuuden piilottaminen piilottaa kaikki sen kurssit.`,
+    },
+  },
   // Not used
   notCompleted: `
 Sisältää opiskelijat, jotka ovat ilmoittautuneet kurssille, mutta eivät ole suorittaneet sitä ja opiskelijat, jotka ovat saaneet hylätyn arvosanan.

services/frontend/src/common/index.ts

@@ -106,28 +106,29 @@ export const getTextInWithOpen = (
   return courseName
 }
 
-export const getUnifyTextIn = (unifyCourses: string) => {
+export const getUnifyTextIn = (unifyCourses?: string) => {
   switch (unifyCourses) {
     case 'regularStats':
-      return 'Normal'
+      return 'normal'
     case 'openStats':
-      return 'Open'
+      return 'open'
     case 'unifyStats':
-      return 'Open and normal'
+      return 'open and normal'
     default:
       return ''
   }
 }
 
-// Gives students course completion date
-export const getStudentToTargetCourseDateMap = (
-  students: Pick<FormattedStudent, 'studentNumber' | 'courses'>[],
-  codes: string[]
-) => {
-  const uniqueCourses = new Set(codes)
+export const getStudentToCourseCompletionDateTimeMap = (
+  students: Pick<FormattedStudent, 'studentNumber' | 'courses'>[] | undefined,
+  courseIds: string[] | undefined
+): Record<string, Date | null> => {
+  if (!students?.length || !courseIds?.length) return {}
+
+  const uniqueCourseIds = new Set(courseIds)
   return students.reduce((acc, student) => {
     const targetCourse = student.courses
-      .filter(course => uniqueCourses.has(course.course_code))
+      .filter(course => uniqueCourseIds.has(course.course_id))
       .sort((a, b) => +new Date(b.date) - +new Date(a.date))[0]
     acc[student.studentNumber] = targetCourse ? targetCourse.date : null
     return acc

services/frontend/src/components/AccessDenied/index.tsx

@@ -6,7 +6,7 @@ import { ErrorBackground } from '@/components/ErrorBoundary/ErrorBackground'
 export const AccessDenied = () => {
   const [easterEggVisible, setEasterEggVisible] = useState(false)
   useEffect(() => {
-    setTimeout(() => setEasterEggVisible(true), Math.floor(Math.random() * 25000 + 5000))
+    setTimeout(() => setEasterEggVisible(true), Math.floor(Math.random() * (20 * 60 * 1000) + 30 * 1000))
   }, [])
 
   const header = 'Welcome to Oodikone!'

services/frontend/src/components/ColorizedCoursesTable/FacultiesTab/index.tsx

@@ -52,9 +52,7 @@ export const FacultiesTab = () => {
       const facultiesTotal = { ...emptyFields }
       selectedSemesters.forEach(semestercode => {
         data.faculties.forEach(faculty => {
-          if (!course.bySemesters.cellStats[faculty]) {
-            course.bySemesters.cellStats[faculty] = { ...emptyFields }
-          }
+          course.bySemesters.cellStats[faculty] ??= { ...emptyFields }
           const stats = course.bySemesters[semestercode]?.[faculty]
           if (!stats) return
           course.bySemesters.cellStats[faculty].completions += stats.completions

services/frontend/src/components/ColorizedCoursesTable/SemestersTab/logic.jsx

@@ -34,11 +34,11 @@ export const useColumns = (getTextIn, semesters, numberMode, colorMode, allTotal
               sx={{
                 'td:has(> &)':
                   colorMode !== 'course'
-                    ? getColor(row.original[numberMode], semesters.length, colorMode, numberMode, null, allTotal)
+                    ? getColor(row.original.bySemesters, semesters.length, colorMode, numberMode, null, allTotal)
                     : {},
               }}
             >
-              {row.original[numberMode] ?? 0}
+              {row.original.bySemesters[numberMode] ?? 0}
             </Box>
           )
         },
@@ -56,6 +56,8 @@ export const useColumns = (getTextIn, semesters, numberMode, colorMode, allTotal
                 ),
               0
             ),
+        sortingFn: (rowA, rowB) =>
+          (rowA.original.bySemesters[numberMode] ?? 0) - (rowB.original.bySemesters[numberMode] ?? 0),
       }),
       ...semesters.map(({ name, semestercode }) =>
         columnHelper.accessor(() => undefined, {

services/frontend/src/components/CoursePopulation/CoursePopulationCreditGainTable/index.tsx

@@ -29,6 +29,7 @@ const CreditGainTable = ({
   totalStudents: number
   headerText: string
 }) => {
+  'use memo'
   const { getTextIn } = useLanguage()
 
   return (
@@ -64,12 +65,12 @@ const CreditGainTable = ({
 
 export const CoursePopulationCreditGainTable = ({
   students,
-  codes,
+  courseIds,
   from,
   to,
 }: {
   students: FormattedStudent[]
-  codes: string[]
+  courseIds: string[]
   from: string
   to: string
 }) => {
@@ -89,11 +90,11 @@ export const CoursePopulationCreditGainTable = ({
   let totalStudents = 0
 
   students.forEach(student => {
-    const courses = student.courses.filter(course => codes.includes(course.course_code))
+    const courses = student.courses.filter(course => courseIds.includes(course.course_id))
 
     const programme = findCorrectProgramme(
       student,
-      codes,
+      courseIds,
       semesters,
       new Date(from),
       new Date(to),

services/frontend/src/components/CoursePopulation/CoursePopulationGradeDist/index.tsx

@@ -16,12 +16,12 @@ import { FormattedStudent } from '@oodikone/shared/types'
 
 export const CoursePopulationGradeDist = ({
   students,
-  courseCodes,
+  courseIds,
   from,
   to,
 }: {
   students: FormattedStudent[]
-  courseCodes: string[]
+  courseIds: string[]
   from: string
   to: string
 }) => {
@@ -31,7 +31,7 @@ export const CoursePopulationGradeDist = ({
     const grades = {}
 
     students.forEach(student => {
-      const courses = student.courses.filter(course => courseCodes.includes(course.course_code))
+      const courses = student.courses.filter(course => courseIds.includes(course.course_id))
       const highestGrade = getHighestGradeOfCourseBetweenRange(courses, from, to)
       if (!highestGrade) {
         grades['No grade'] ??= []

services/frontend/src/components/CoursePopulation/CoursePopulationLanguageDist/index.tsx

@@ -15,12 +15,12 @@ dayjsExtend(isBetween)
 
 export const CoursePopulationLanguageDist = ({
   students,
-  codes,
+  courseIds,
   from,
   to,
 }: {
   students: FormattedStudent[]
-  codes: string[]
+  courseIds: string[]
   from: string
   to: string
 }) => {
@@ -28,7 +28,7 @@ export const CoursePopulationLanguageDist = ({
 
   students.forEach(student => {
     const filteredCourse = student.courses
-      .filter(course => codes.includes(course.course_code))
+      .filter(course => courseIds.includes(course.course_id))
       .filter(course => dayjs(course.date).isBetween(dayjs(from), dayjs(to)))
       .sort((a, b) => Number(new Date(b.date) < new Date(a.date)))
       .find(({ language }) => !!language)

services/frontend/src/components/CoursePopulation/index.tsx

@@ -1,9 +1,7 @@
 import Box from '@mui/material/Box'
 import Typography from '@mui/material/Typography'
-import { useEffect, useMemo, useState } from 'react'
-import { useLocation } from 'react-router'
 
-import { getStudentToTargetCourseDateMap, getUnifyTextIn } from '@/common'
+import { getStudentToCourseCompletionDateTimeMap, getUnifyTextIn } from '@/common'
 import { populationStatisticsToolTips } from '@/common/InfoToolTips'
 import { PageTitle } from '@/components/common/PageTitle'
 import { PanelView } from '@/components/common/PanelView'
@@ -39,49 +37,40 @@ import { useSemesters } from '@/hooks/useSemesters'
 import { useGetCourseDetailsQuery } from '@/redux/courseStats'
 import { useGetPopulationStatisticsByCourseQuery } from '@/redux/populations'
 import { FilteredCourse } from '@/util/coursesOfPopulation'
-import { parseQueryParams } from '@/util/queryparams'
+import { useParseQueryParams } from '@/util/queryparams'
 import { SISStudyRightElement } from '@oodikone/shared/models'
 import { FormattedStudent } from '@oodikone/shared/types'
+import { getFromToDates, getStudentRelevantProgrammes } from './util'
 
 export const CoursePopulation = () => {
+  'use memo'
   useTitle('Course population')
 
-  const location = useLocation()
   const { getTextIn } = useLanguage()
 
-  const [codes, setCodes] = useState<{ allCodes: string[]; mainCodes: string[] }>({
-    allCodes: [],
-    mainCodes: [],
-  })
+  const params = useParseQueryParams()
+  const courses = params.courses
+  const from = params.from?.[0]
+  const to = params.to?.[0]
+  const separate = params.separate?.[0]
+  const substitutions = params.substitutions?.[0]
+  const unifyCourses = params.unifyCourses?.[0]
 
-  const { coursecodes, from, to, separate, unifyCourses, includeSubstitutions } = parseQueryParams(location.search)
-
-  const { data: courseDetails } = useGetCourseDetailsQuery(
-    { codes: codes.mainCodes },
-    { skip: !codes.mainCodes.length }
+  const { data: population, isFetching: populationFetching } = useGetPopulationStatisticsByCourseQuery(
+    {
+      courses: courses?.[0],
+      from,
+      to,
+      separate,
+      unifyCourses,
+      substitutions,
+    },
+    {
+      skip: !courses?.length || !from || !to,
+    }
   )
-  const { data: population } = useGetPopulationStatisticsByCourseQuery({
-    coursecodes,
-    from,
-    to,
-    separate,
-    unifyCourses,
-    includeSubstitutions,
-  })
 
-  useEffect(() => {
-    const parsedCourseCodes = JSON.parse(coursecodes)
-    if (parsedCourseCodes.length) setCodes({ ...codes, mainCodes: parsedCourseCodes })
-  }, [coursecodes])
-
-  useEffect(() => {
-    if (population?.allCourseCodes?.length) setCodes({ ...codes, allCodes: population.allCourseCodes })
-  }, [population])
-
-  const studentToTargetCourseDateMap = useMemo(
-    () => getStudentToTargetCourseDateMap(population?.students ?? [], codes.allCodes),
-    [population?.students, codes]
-  )
+  const { data: courseDetails } = useGetCourseDetailsQuery({ courses: courses ?? [] }, { skip: !courses?.length })
 
   const {
     semesters: allSemesters,
@@ -90,25 +79,15 @@ export const CoursePopulation = () => {
     isLoading: semestersFetching,
   } = useSemesters()
 
-  const getFromToDates = (from: number, to: number, separate: boolean) => {
-    if (semestersFetching) return {}
+  const isSeparate = separate === 'true'
 
-    const dataValues = separate ? Object.values(allSemesters) : Object.values(semesterYears)
+  if (!courses?.length) return null
+  if (populationFetching || !population || semestersFetching || !currentSemester) return <PageLoading isLoading />
 
-    const key = separate ? 'semestercode' : 'yearcode'
-
-    const findDateByCode = (code: number) => dataValues.find(item => item[key] === code)
-
-    return {
-      dateFrom: findDateByCode(Number(from))?.startdate,
-      dateTo: findDateByCode(Number(to))?.enddate,
-    }
-  }
-
-  const isSeparate = separate ? JSON.parse(separate) : false
-  const { dateFrom, dateTo } = getFromToDates(from, to, isSeparate)
+  const populationCourseIds = Object.keys(population.idToGroupIdMap)
 
   // Dates must be set
+  const { dateFrom, dateTo } = getFromToDates(Number(from), Number(to), isSeparate, allSemesters, semesterYears)
   if (!dateFrom || !dateTo) return null
 
   const singleSemester =
@@ -117,33 +96,29 @@ export const CoursePopulation = () => {
     ? (getTextIn(singleSemester.name) ?? '')
     : `${new Date(dateFrom).getFullYear()}-${new Date(dateTo).getFullYear()}`
 
-  if (!population || !currentSemester) return <PageLoading isLoading />
-
-  const getStudentRelevantProgrammes = (students: FormattedStudent[]) =>
-    students.reduce<Map<string, string>>((programmes, student) => {
-      const programme = findCorrectProgramme(
-        student,
-        coursecodes,
-        allSemesters,
-        new Date(dateFrom),
-        new Date(dateTo),
-        currentSemester?.semestercode
-      )
-
-      programmes.set(student.studentNumber, getTextIn(programme.name) ?? '')
-      return programmes
-    }, new Map())
-
   // Page title data
   const courseNames = courseDetails?.map(({ name }) => getTextIn(name)) ?? []
   const header = courseNames.length ? `${courseNames?.join(', ')}` : undefined
 
+  const getRelatedProgrammeMap = getStudentRelevantProgrammes(
+    populationCourseIds,
+    allSemesters,
+    dateFrom,
+    dateTo,
+    currentSemester.semestercode,
+    getTextIn
+  )
+
+  const studentToTargetCourseDateMap = getStudentToCourseCompletionDateTimeMap(
+    population.students ?? [],
+    populationCourseIds
+  )
   const createPanels = (filteredStudents: FormattedStudent[], filteredCourses: FilteredCourse[]) => [
     {
       title: 'Grade distribution',
       content: (
         <CoursePopulationGradeDist
-          courseCodes={codes.allCodes}
+          courseIds={populationCourseIds}
           from={dateFrom}
           students={filteredStudents}
           to={dateTo}
@@ -153,14 +128,19 @@ export const CoursePopulation = () => {
     {
       title: 'Language distribution',
       content: (
-        <CoursePopulationLanguageDist codes={codes.allCodes} from={dateFrom} students={filteredStudents} to={dateTo} />
+        <CoursePopulationLanguageDist
+          courseIds={populationCourseIds}
+          from={dateFrom}
+          students={filteredStudents}
+          to={dateTo}
+        />
       ),
     },
     {
       title: 'Programme distribution',
       content: (
         <CustomPopulationProgrammeDist
-          coursecodes={codes.allCodes}
+          courseIds={populationCourseIds}
           from={dateFrom}
           infotext={populationStatisticsToolTips.programmeDistributionCoursePopulation}
           students={filteredStudents}
@@ -176,7 +156,7 @@ export const CoursePopulation = () => {
       title: 'Credit gains',
       content: (
         <CoursePopulationCreditGainTable
-          codes={codes.allCodes}
+          courseIds={populationCourseIds}
           from={dateFrom}
           students={filteredStudents}
           to={dateTo}
@@ -202,10 +182,10 @@ export const CoursePopulation = () => {
               showBachelorAndMaster: false,
               includePrimaryProgramme: true,
 
-              coursecodes: codes.allCodes,
+              courseIds: populationCourseIds,
               from: dateFrom,
               to: dateTo,
-              relatedProgrammeMap: getStudentRelevantProgrammes(filteredStudents),
+              relatedProgrammeMap: getRelatedProgrammeMap(filteredStudents),
             })
           }
           studentToTargetCourseDateMap={studentToTargetCourseDateMap}
@@ -220,8 +200,9 @@ export const CoursePopulation = () => {
     studentNumberFilter(),
     ageFilter(),
     courseFilter({
-      courses: population?.coursestatistics.courses,
-      includeSubstitutions: includeSubstitutions === 'true',
+      courses: population?.coursestatistics.courses ?? [],
+      includeSubstitutions: substitutions === 'true',
+      idToGroupIdMap: population?.coursestatistics.idToGroupIdMap ?? {},
     }),
     creditsEarnedFilter(),
     hetuFilter(),
@@ -234,7 +215,7 @@ export const CoursePopulation = () => {
           predicate: (student: FormattedStudent, studyRightElement: SISStudyRightElement) => {
             const correctProgramme = findCorrectProgramme(
               student,
-              codes.allCodes,
+              populationCourseIds,
               allSemesters,
               new Date(dateFrom),
               new Date(dateTo),
@@ -247,12 +228,17 @@ export const CoursePopulation = () => {
       ],
     }),
     gradeFilter({
-      courseCodes: codes.allCodes,
+      courseIds: populationCourseIds,
       from: dateFrom,
       to: dateTo,
     }),
   ]
 
+  const courseCodes = courses
+    .map(course => courseDetails?.find(cd => cd.groupId === course)?.code)
+    .filter(Boolean)
+    .join(', ')
+
   return (
     <FilterView
       coursestatistics={population?.coursestatistics}
@@ -266,16 +252,14 @@ export const CoursePopulation = () => {
         <>
           <PageTitle title={header}>
             <Typography color="text.secondary" variant="h6">
-              {codes.mainCodes.join(', ')}
+              {courseCodes}
             </Typography>
             <Typography
               color="text.secondary"
               variant="h6"
             >{`Class of ${dateRange}, ${population.students.length} students`}</Typography>
             <Typography color="text.secondary" variant="h6">
-              {(includeSubstitutions === 'true' ? 'Include' : 'Exclude') +
-                ' substitutions, ' +
-                getUnifyTextIn(unifyCourses)}
+              {`Substitutions ${substitutions === 'true' ? 'included' : 'excluded'}, ${getUnifyTextIn(unifyCourses)}`}
             </Typography>
           </PageTitle>
           <PanelView panels={createPanels(filteredStudents, filteredCourses)} />

services/frontend/src/components/CoursePopulation/util.ts

@@ -0,0 +1,48 @@
+import { SemestersData } from '@/hooks/useSemesters'
+import { FormattedStudent } from '@oodikone/shared/types'
+import { findCorrectProgramme } from '../PopulationComponents/ProgrammeDist/util'
+import { GetTextIn } from '../LanguagePicker/useLanguage'
+
+export const getFromToDates = (
+  from: number,
+  to: number,
+  separate: boolean,
+  semesters: SemestersData['semesters'],
+  semesterYears: SemestersData['years']
+) => {
+  const dataValues = separate ? Object.values(semesters) : Object.values(semesterYears)
+
+  const key = separate ? 'semestercode' : 'yearcode'
+
+  const findDateByCode = (code: number) => dataValues.find(item => item[key] === code)
+
+  return {
+    dateFrom: findDateByCode(Number(from))?.startdate,
+    dateTo: findDateByCode(Number(to))?.enddate,
+  }
+}
+
+export const getStudentRelevantProgrammes =
+  (
+    courseIds: string[],
+    allSemesters: any,
+    dateFrom: string,
+    dateTo: string,
+    semesterCode: number,
+    getTextIn: GetTextIn
+  ) =>
+  (students: FormattedStudent[]) => {
+    return students.reduce<Map<string, string>>((programmes, student) => {
+      const programme = findCorrectProgramme(
+        student,
+        courseIds,
+        allSemesters,
+        new Date(dateFrom),
+        new Date(dateTo),
+        semesterCode
+      )
+
+      programmes.set(student.studentNumber, getTextIn(programme.name) ?? '')
+      return programmes
+    }, new Map())
+  }

services/frontend/src/components/CustomPopulation/CourseMatrixExport.tsx

@@ -20,8 +20,10 @@ export const CourseMatrixExport = ({
   const { getTextIn } = useLanguage()
 
   const getXlsx = () => {
-    const courseNameByCode = new Map(courses.map(({ course }) => [course.code, getTextIn(course.name) ?? '']))
-    const { completedCoursesRows, courseCounterRows } = calculateExcelData(students, courseNameByCode)
+    const courseInfoByGroupId = new Map(
+      courses.map(({ course }) => [course.groupId, { code: course.code, name: getTextIn(course.name) ?? '' }])
+    )
+    const { completedCoursesRows, courseCounterRows } = calculateExcelData(students, courseInfoByGroupId)
 
     // Columns: Student number, Name, student's attainments in the format: "course name (course code)"
     const completedCoursesSheet = utils.aoa_to_sheet([['Student number', 'Name'], ...completedCoursesRows])

services/frontend/src/components/CustomPopulation/CustomPopulationContent.tsx

@@ -55,7 +55,7 @@ export const CustomPopulationContent = ({
 
   useEffect(() => {
     setStudentAmountLimit(Math.round(filteredStudents.length ? filteredStudents.length * 0.3 : 0))
-  }, [filteredStudents.length])
+  }, [filteredStudents.length, setStudentAmountLimit])
 
   const onStudentAmountLimitChange = value => {
     setStudentAmountLimit(Number.isNaN(Number(value)) ? studentAmountLimit : Number(value))
@@ -63,7 +63,7 @@ export const CustomPopulationContent = ({
 
   const panels = [
     {
-      title: `Credit accumulation (for ${filteredStudents.length} students)`,
+      title: `Credit accumulation (for ${filteredStudents.length} student${filteredStudents.length > 1 ? 's' : ''})`,
       content: (
         <CreditAccumulationGraph
           programmeCodes={[associatedProgramme].filter(Boolean) as string[]}
@@ -123,7 +123,7 @@ export const CustomPopulationContent = ({
               showBachelorAndMaster: false,
               includePrimaryProgramme: false,
 
-              coursecodes: [],
+              courseIds: [],
               from: undefined,
               to: undefined,
             })

services/frontend/src/components/CustomPopulation/CustomPopulationWrapper.tsx

@@ -61,7 +61,10 @@ export const CustomPopulationWrapper = ({
     const filtersList = [
       genderFilter(),
       ageFilter(),
-      courseFilter({ courses: population?.coursestatistics.courses ?? [] }),
+      courseFilter({
+        courses: population?.coursestatistics.courses ?? [],
+        idToGroupIdMap: population?.coursestatistics.idToGroupIdMap ?? {},
+      }),
       creditsEarnedFilter(),
       transferredToProgrammeFilter(),
       startYearAtUniFilter(),

services/frontend/src/components/CustomPopulation/courseMatrix.ts

@@ -21,41 +21,43 @@ export type CourseMatrixAggregate = {
 
 /**
  * Passed completions of the student that are included in one of the student's study plans (HOPS).
- * Returns the latest completion per course code, sorted by course code.
+ * Returns the latest completion per course group, sorted by course group id.
  */
 export const getHopsCourses = (student: FormattedStudent): StudentCourse[] => {
-  const hopsCourseCodes = new Set(student.studyplans?.flatMap(plan => plan.included_courses ?? []) ?? [])
-  const passedCoursesInHops = student.courses.filter(course => course.passed && hopsCourseCodes.has(course.course_code))
+  const hopsCourseIds = new Set(student.studyplans?.flatMap(plan => plan.included_courses ?? []) ?? [])
+  const passedCoursesInHops = student.courses.filter(course => course.passed && hopsCourseIds.has(course.course_id))
 
-  const latestByCode = new Map<string, StudentCourse>()
+  const latestByGroupId = new Map<string, StudentCourse>()
   for (const course of passedCoursesInHops) {
-    if (!course.course_code) continue
+    if (!course.courseGroupId) continue
 
-    const existing = latestByCode.get(course.course_code)
+    const existing = latestByGroupId.get(course.courseGroupId)
     if (!existing || new Date(course.date).getTime() > new Date(existing.date).getTime()) {
-      latestByCode.set(course.course_code, course)
+      latestByGroupId.set(course.courseGroupId, course)
     }
   }
 
-  return [...latestByCode.values()].sort((a, b) => a.course_code.localeCompare(b.course_code))
+  return [...latestByGroupId.values()].sort((a, b) => a.courseGroupId.localeCompare(b.courseGroupId))
 }
 
-export const calculateExcelData = (students: FormattedStudent[], courseNameByCode: Map<string, string>) => {
-  const counters = new Map<string, { name: string; credits: number; students: Set<string> }>()
+export const calculateExcelData = (
+  students: FormattedStudent[],
+  courseInfoByGroupId: Map<string, { code: string; name: string }>
+) => {
+  // Keyed by course groupId internally so different courses are never accidentally merged,
+  // while different versions of the same course are; code/name are only for display.
+  const counters = new Map<string, { code: string; name: string; credits: number; students: Set<string> }>()
 
   const rows = students.map(student => {
     const courses = getHopsCourses(student).map(course => {
-      const code = course.course_code
-      const counter = counters.get(code) ?? {
-        name: courseNameByCode.get(code) ?? '',
-        credits: 0,
-        students: new Set(),
-      }
-      counters.set(code, counter)
+      const groupId = course.courseGroupId
+      const { code, name } = courseInfoByGroupId.get(groupId) ?? { code: course.course_code, name: '' }
+      const counter = counters.get(groupId) ?? { code, name, credits: 0, students: new Set() }
+      counters.set(groupId, counter)
       counter.credits += course.credits ?? 0
       counter.students.add(student.studentNumber)
 
-      return { code, name: courseNameByCode.get(code) ?? '', credits: course.credits ?? 0 }
+      return { code, name, credits: course.credits ?? 0 }
     })
 
     return { studentNumber: student.studentNumber, studentName: student.name, courses }
@@ -67,8 +69,8 @@ export const calculateExcelData = (students: FormattedStudent[], courseNameByCod
     ...courses.map(course => `${course.name} (${course.code})`),
   ])
 
-  const courseCounterRows = [...counters.entries()]
-    .map(([code, { name, credits, students }]) => [code, name, students.size.toString(), credits.toString()])
+  const courseCounterRows = [...counters.values()]
+    .map(({ code, name, credits, students }) => [code, name, students.size.toString(), credits.toString()])
     .sort(([_a, __a, aStudents], [_b, __b, bStudents]) => parseInt(bStudents) - parseInt(aStudents))
 
   return { completedCoursesRows, courseCounterRows }

services/frontend/src/components/FilterView/filters/courses/CourseCard.tsx

@@ -7,6 +7,7 @@ import { FilterSelect } from '@/components/FilterView/filters/common/FilterSelec
 import { FilterType } from '@/components/FilterView/filters/courses/filterType'
 import { useLanguage } from '@/components/LanguagePicker/useLanguage'
 import { ClearIcon } from '@/theme'
+import { CourseStats } from '@oodikone/shared/routes/populations'
 
 const filterTexts = {
   [FilterType.ALL]: {
@@ -26,19 +27,19 @@ const filterTexts = {
     label: 'Enrolled, No Grade',
   },
 }
-const getSubstitutionTooltip = (substitution_groups: string[][]) => (
+const SubstitutionTooltip = ({ substitutionGroupCodes }: { substitutionGroupCodes: string[][] }) => (
   <Typography fontSize="0.9rem" whiteSpace="pre-line">
-    Included course's equivalent groups:
-    <br />
-    {substitution_groups.map(group => group.join(', ')).join('\n\n')}
+    {`Included equivalent courses:\n${substitutionGroupCodes.map(group => group.join(', ')).join('\n')}`}
   </Typography>
 )
 export const CourseCard = ({
   course,
+  courses,
   filterType,
   onChange,
 }: {
-  course: any
+  course: CourseStats
+  courses: Record<string, CourseStats>
   filterType: number
   onChange: (type: number | null) => any
 }) => {
@@ -50,6 +51,11 @@ export const CourseCard = ({
     value: type,
   }))
 
+  // Only substitute courses that some student in the current population acually has
+  const substitutionGroupCodes = (course?.substitutionGroups ?? [])
+    .map(group => group.map(groupId => courses[groupId]?.code).filter((code): code is string => Boolean(code)))
+    .filter(group => group.length > 0)
+
   return (
     <Box
       sx={theme => ({
@@ -64,10 +70,10 @@ export const CourseCard = ({
       <Stack direction="row" sx={{ justifyContent: 'space-between' }}>
         <Box sx={{ mb: 2 }}>
           <Typography>{getTextIn(course?.name)}</Typography>
-          {course?.substitutions?.length ? (
-            <Tooltip title={getSubstitutionTooltip(course.substitution_groups)}>
+          {substitutionGroupCodes.length ? (
+            <Tooltip title={<SubstitutionTooltip substitutionGroupCodes={substitutionGroupCodes} />}>
               <Typography sx={{ color: 'text.secondary' }}>
-                {course?.code}... +{course?.substitutions?.length}
+                {course?.code}... +{substitutionGroupCodes.length}
               </Typography>
             </Tooltip>
           ) : (
@@ -75,7 +81,7 @@ export const CourseCard = ({
           )}
         </Box>
         <ClearIcon
-          data-cy={`courseFilter-${course?.code}-clear`}
+          data-cy={`courseFilter-${course?.groupId}-clear`}
           onClick={() => onChange(null)}
           sx={{
             color: theme => theme.palette.error.dark,
@@ -86,7 +92,7 @@ export const CourseCard = ({
         />
       </Stack>
       <FilterSelect
-        filterKey={`courseFilter-${course?.code}`}
+        filterKey={`courseFilter-${course?.groupId}`}
         label="Select course"
         onChange={({ target }) => onChange(Number(target.value))}
         options={dropdownOptions}

services/frontend/src/components/FilterView/filters/courses/index.tsx

@@ -13,8 +13,9 @@ type Options = {
   courses: Record<string, CourseStats>
   substitutedBy: Record<string, string[][]>
   includeSubstitutions: boolean
+  idToGroupIdMap: Record<string, string>
 }
-type Args = { courses: CourseStats[]; includeSubstitutions?: boolean }
+type Args = { courses: CourseStats[]; includeSubstitutions?: boolean; idToGroupIdMap: Record<string, string> }
 type Precompute = any
 
 const CourseFilterCard = ({ options, onOptionsChange }: FilterTrayProps<Options, Args, Precompute>) => {
@@ -24,18 +25,18 @@ const CourseFilterCard = ({ options, onOptionsChange }: FilterTrayProps<Options,
   const { getTextIn } = useLanguage()
 
   const dropdownOptions = Object.values(courseStats)
-    .filter(cs => !courseFilters[cs.code])
+    .filter(cs => !courseFilters[cs.groupId])
     .sort((a, b) => getSortRank(b.code) - getSortRank(a.code))
     .map(cs => ({
-      key: `courseFilter-option-${cs.code}`,
+      key: `courseFilter-option-${cs.groupId}`,
       text: `${cs.code} - ${getTextIn(cs.name)}`,
-      value: cs.code,
+      value: cs.groupId,
     }))
 
-  const setCourseFilter = (code: string, type: FTValue | null) => {
+  const setCourseFilter = (groupId: string, type: FTValue | null) => {
     const newOpts = structuredClone(options)
-    if (type === null) delete newOpts.courseFilters[code]
-    else newOpts.courseFilters[code] = type
+    if (type === null) delete newOpts.courseFilters[groupId]
+    else newOpts.courseFilters[groupId] = type
 
     onOptionsChange(newOpts)
   }
@@ -49,12 +50,13 @@ const CourseFilterCard = ({ options, onOptionsChange }: FilterTrayProps<Options,
         options={dropdownOptions}
         value={null}
       />
-      {Object.entries(courseFilters).map(([code, type]) => (
+      {Object.entries(courseFilters).map(([groupId, type]) => (
         <CourseCard
-          course={courseStats[code]}
+          course={courseStats[groupId]}
+          courses={courseStats}
           filterType={type}
-          key={`courseFilter-selected-course-${code}`}
-          onChange={type => setCourseFilter(code, type as FTValue)}
+          key={`courseFilter-selected-course-${groupId}`}
+          onChange={type => setCourseFilter(groupId, type as FTValue)}
         />
       ))}
     </>
@@ -71,25 +73,27 @@ export const courseFilter = createFilter<Options, Args, Precompute>({
     courses: {},
     substitutedBy: {},
     includeSubstitutions: true,
+    idToGroupIdMap: {},
   },
 
   precompute: ({
     args,
     options,
   }: {
-    args: { courses: CourseStats[]; includeSubstitutions?: boolean }
+    args: Args
     options: {
       courses?: Record<string, CourseStats>
       substitutedBy?: Record<string, string[][]>
       includeSubstitutions?: boolean
+      idToGroupIdMap?: Record<string, string>
     }
   }) => {
     const substitutedBy = args.courses.reduce<Record<string, string[][]>>((acc, course: CourseStats) => {
-      const { code, substitution_groups } = course
-      if (substitution_groups) {
-        for (const group of substitution_groups) {
-          acc[code] ??= []
-          acc[code].push(group)
+      const { groupId, substitutionGroups } = course
+      if (substitutionGroups) {
+        for (const group of substitutionGroups) {
+          acc[groupId] ??= []
+          acc[groupId].push(group)
         }
       }
 
@@ -98,32 +102,37 @@ export const courseFilter = createFilter<Options, Args, Precompute>({
 
     /* option.courses maybe frozen even when it should be used only within the scope of createFilter factory. */ {
       delete options.courses
-      options.courses = Object.fromEntries(args.courses.map(course => [course.code, course]))
+      options.courses = Object.fromEntries(args.courses.map(course => [course.groupId, course]))
     }
 
     options.includeSubstitutions = args.includeSubstitutions ?? true
     options.substitutedBy = substitutedBy
+    options.idToGroupIdMap = args.idToGroupIdMap
   },
 
   isActive: ({ courseFilters }) => Object.keys(courseFilters).length > 0,
 
   filter(student: FormattedStudent, { options }) {
     const { courses, enrollments } = student
-    const passedCoursesCodes = courses
-      .filter(({ credittypecode }) => credittypecode !== CreditTypeCode.FAILED)
-      .map(({ course_code }) => course_code)
-    const courseCodes = courses.map(({ course_code }) => course_code)
-    const enrollmentCodes = enrollments.map(({ course_code }) => course_code)
-    for (const [mainCode, filterType] of Object.entries(options.courseFilters)) {
+    const toGroupIds = (courseIds: string[]) => courseIds.map(id => options.idToGroupIdMap[id]).filter(Boolean)
+
+    const passedCourseGroupIds = toGroupIds(
+      courses.filter(({ credittypecode }) => credittypecode !== CreditTypeCode.FAILED).map(({ course_id }) => course_id)
+    )
+
+    const courseGroupIds = toGroupIds(courses.map(({ course_id }) => course_id))
+    const enrollmentGroupIds = toGroupIds(enrollments.map(({ course_id }) => course_id))
+
+    for (const [mainGroupId, filterType] of Object.entries(options.courseFilters)) {
       let foundPassed = false
       let foundAttainment = false
       let foundEnrollment = false
 
-      void [[mainCode], ...(options.includeSubstitutions ? (options.substitutedBy?.[mainCode] ?? []) : [])].forEach(
+      ;[[mainGroupId], ...(options.includeSubstitutions ? (options.substitutedBy?.[mainGroupId] ?? []) : [])].forEach(
         group => {
-          foundPassed = foundPassed ? true : group.every(code => passedCoursesCodes.includes(code))
-          foundAttainment = foundAttainment ? true : group.every(code => courseCodes.includes(code))
-          foundEnrollment = foundEnrollment ? true : group.every(code => enrollmentCodes.includes(code))
+          foundPassed = foundPassed ? true : group.every(groupId => passedCourseGroupIds.includes(groupId))
+          foundAttainment = foundAttainment ? true : group.every(groupId => courseGroupIds.includes(groupId))
+          foundEnrollment = foundEnrollment ? true : group.every(groupId => enrollmentGroupIds.includes(groupId))
         }
       )
 
@@ -148,16 +157,17 @@ export const courseFilter = createFilter<Options, Args, Precompute>({
 
   selectors: {
     // NOTE: Remember FilterType.ALL === 0 when checking if courseFilters[course] exists
-    isCourseSelected: ({ courseFilters }, course) => Object.values(FilterType).includes(courseFilters[course]),
-    selectedCourseName: ({ courses }, courseCodes) => courses[courseCodes[0]]?.name,
+    isCourseSelected: ({ courseFilters }, courseGroupId) =>
+      Object.values(FilterType).includes(courseFilters[courseGroupId]),
+    selectedCourseName: ({ courses }, courseGroupIds) => courses[courseGroupIds[0]]?.name,
   },
 
   actions: {
-    toggleCourseSelection: (options, code: string) => {
-      if (!Object.values(FilterType).includes(options.courseFilters[code])) {
-        options.courseFilters[code] = FilterType.ALL
+    toggleCourseSelection: (options, groupId: string) => {
+      if (!Object.values(FilterType).includes(options.courseFilters[groupId])) {
+        options.courseFilters[groupId] = FilterType.ALL
       } else {
-        delete options.courseFilters[code]
+        delete options.courseFilters[groupId]
       }
 
       return options

services/frontend/src/components/FilterView/filters/grade.tsx

@@ -6,15 +6,12 @@ import Typography from '@mui/material/Typography'
 import { getHighestGradeOfCourseBetweenRange } from '@/common'
 import { createFilter, FilterTrayProps } from '@/components/FilterView/filters/createFilter'
 import { FormattedStudent } from '@oodikone/shared/types'
+import { dateIsBetween } from '@oodikone/shared/util/datetime'
 
-type Options = any
-type Args = any
-type Precompute = any
+type Options = { selected: string[] }
+type Args = { courseIds: string[]; from: string; to: string }
+type Precompute = { grades: Record<string, string[]> }
 
-/**
- * Grade filter.
- * Only applicable to a single course.
- */
 const GradeFilterCard = ({ options, onOptionsChange, precomputed }: FilterTrayProps<Options, Args, Precompute>) => {
   const { grades } = precomputed
   const { selected } = options
@@ -24,9 +21,9 @@ const GradeFilterCard = ({ options, onOptionsChange, precomputed }: FilterTrayPr
   // Therefore this cannot be filtered by Number
   const choices = Object.keys(grades).sort((a, b) => +b - +a)
 
-  const checked = grade => selected.includes(grade)
+  const checked = (grade: string) => selected.includes(grade)
 
-  const onChange = grade => () => {
+  const onChange = (grade: string) => () => {
     if (checked(grade)) {
       onOptionsChange({
         ...options,
@@ -58,6 +55,10 @@ const GradeFilterCard = ({ options, onOptionsChange, precomputed }: FilterTrayPr
   )
 }
 
+/**
+ * Grade filter.
+ * Only applicable to a single course.
+ */
 export const gradeFilter = createFilter<Options, Args, Precompute>({
   key: 'gradeFilter',
 
@@ -76,17 +77,21 @@ export const gradeFilter = createFilter<Options, Args, Precompute>({
           student =>
             [
               student.studentNumber,
-              student.courses.filter((course: any) => args.courseCodes.includes(course.course_code)),
+              student.courses.filter(
+                course =>
+                  args.courseIds.includes(course.course_id) &&
+                  dateIsBetween(new Date(course.date), new Date(args.from), new Date(args.to))
+              ),
             ] as [string, FormattedStudent['courses']]
         )
         .map(([studentNumber, courses]) => [
           studentNumber,
           getHighestGradeOfCourseBetweenRange(courses, args.from, args.to),
         ])
-        .filter(([_, grade]) => grade !== undefined)
         .reduce((acc, [studentNumber, grade]) => {
-          acc[grade!] ??= []
-          acc[grade!].push(studentNumber)
+          const fixedGrade = grade ?? 'No grade'
+          acc[fixedGrade] ??= []
+          acc[fixedGrade].push(studentNumber)
           return acc
         }, {}),
     }
@@ -108,7 +113,7 @@ export const gradeFilter = createFilter<Options, Args, Precompute>({
 
   actions: {
     selectGrade(options, grade) {
-      if (options.selected.indexOf(grade) === -1) {
+      if (!options.selected.includes(grade)) {
         options.selected.push(grade)
       }