Rapodiff: credits diff work in progress, csv parsing added

Return date to production console logs
[Uni overview] Add link list to individual faculties
2026-10-13 03:13:40 +03:00 · 2024-01-24 12:21:06 +02:00 · 2024-01-24 12:14:32 +02:00 · 2024-01-24 12:05:11 +02:00 · 2024-01-24 12:03:40 +02:00 · 2024-01-24 11:23:27 +02:00
603 changed files with 22267 additions and 62411 deletions
@@ -15,37 +15,32 @@ inputs:
 runs:
  using: composite
  steps:
-    - name: 'Download images'
-      uses: actions/download-artifact@v3
+    - name: 'Login to toska Docker registry'
+      uses: docker/login-action@v3
      with:
-        path: /tmp/
-    - name: 'Load built images'
-      shell: bash
-      run: |
-        docker load --input /tmp/artifact/backend.tar
-        docker load --input /tmp/artifact/frontend.tar
-    - name: 'Login to toska docker registry'
-      uses: docker/login-action@v2
-      with:
-        registry: registry.toska.cs.helsinki.fi
+        registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
        username: ${{ inputs.username }}
        password: ${{ inputs.password }}
+
    - name: 'Run services, pull rest of the needed images'
      shell: bash
-      run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      run: docker-compose --file docker-compose.ci.yml up --detach
+
    - name: 'Run cypress'
-      uses: cypress-io/github-action@v4
+      uses: cypress-io/github-action@v6
      with:
        browser: chrome
        spec: ${{ inputs.spec }}
      env:
        CYPRESS_baseUrl: http://localhost:1337
+
    - name: 'Upload video if tests fail'
      uses: actions/upload-artifact@v3
      if: failure()
      with:
        name: Cypress videos
        path: ./cypress/videos/*
+
    - name: 'Print logs on failure'
      shell: bash
      if: failure()
@@ -8,6 +8,8 @@ updates:
    labels:
      - 'dependencies'
      - 'infra'
+    allow:
+      - dependency-type: 'production'
    ignore:
      - dependency-name: '*'
        update-types: ['version-update:semver-major']
@@ -20,6 +22,8 @@ updates:
    labels:
      - 'dependencies'
      - 'updater'
+    allow:
+      - dependency-type: 'production'
    ignore:
      - dependency-name: '*'
        update-types: ['version-update:semver-major']
@@ -32,6 +36,8 @@ updates:
    labels:
      - 'dependencies'
      - 'updater'
+    allow:
+      - dependency-type: 'production'
    ignore:
      - dependency-name: '*'
        update-types: ['version-update:semver-major']
@@ -44,6 +50,8 @@ updates:
    labels:
      - 'dependencies'
      - 'back'
+    allow:
+      - dependency-type: 'production'
    ignore:
      - dependency-name: '*'
        update-types: ['version-update:semver-major']
@@ -56,6 +64,8 @@ updates:
    labels:
      - 'dependencies'
      - 'front'
+    allow:
+      - dependency-type: 'production'
    ignore:
      - dependency-name: '*'
        update-types: ['version-update:semver-major']
@@ -0,0 +1,149 @@
+name: 'Release to production'
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  release_frontend:
+    name: 'Build and release frontend'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: 'Get current time'
+        id: get_current_time
+        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
+
+      - name: Extract release version
+        id: extract_version
+        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+
+      - name: Build frontend image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: oodikone-frontend
+          tags: production ${{ github.sha }}
+          context: services/frontend
+          containerfiles: |
+            services/frontend/Dockerfile
+          build-args: |
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
+            SENTRY_ENVIRONMENT=production
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_backend:
+    name: 'Build and release backend'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Extract release version
+        id: extract_version
+        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+
+      - name: Build backend image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: oodikone-backend
+          tags: production ${{ github.sha }}
+          context: services/backend
+          containerfiles: |
+            services/backend/Dockerfile
+          build-args: |
+            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
+            SENTRY_ENVIRONMENT=production
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  sentry_release:
+    name: 'Create Sentry release'
+    runs-on: ubuntu-latest
+    needs:
+      - release_frontend
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Extract release version
+        id: extract_version
+        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+
+      - name: Pull image and create a container
+        run: docker create --name frontend quay.io/toska/oodikone-frontend:production
+
+      - name: Copy assets from the container
+        run: docker cp frontend:/opt/app-root/src/build ./build
+
+      - name: Create Sentry release
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: https://toska.cs.helsinki.fi/
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: oodikone
+        with:
+          environment: production
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/static/js'
+          version: ${{ steps.extract_version.outputs.VERSION }}
+
+  report_success:
+    name: 'Report deployment success to oodikone notifications'
+    runs-on: ubuntu-latest
+    needs:
+      - release_frontend
+      - release_backend
+      - sentry_release
+
+    if: success()
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone.cs.helsinki.fi/
+          deployment-target: production
+
+  report_failure:
+    name: 'Report deployment failure to oodikone notifications'
+    runs-on: ubuntu-latest
+    needs:
+      - release_frontend
+      - release_backend
+      - sentry_release
+
+    if: failure()
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment-failure
+          deployment-target: production
@@ -1,122 +0,0 @@
-name: 'Release oodikone to production'
-
-on:
-  release:
-    types: [published]
-
-jobs:
-  build_and_release_backend:
-    name: 'Build and release backend'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - name: 'Login to DockerHub'
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: 'Extract release version'
-        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
-        # This steps extracts release version from GITHUB_REF env var
-        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
-        id: extract_version
-        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
-      - name: 'Build and push'
-        uses: docker/build-push-action@v3
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
-          cache-from: |
-            type=registry,ref=toska/oodikone2-backend:staging
-            type=registry,ref=toska/oodikone2-backend:latest
-          cache-to: type=inline
-          context: services/backend
-          push: true
-          tags: toska/oodikone2-backend:latest
-          target: production
-
-  build_and_release_frontend:
-    name: 'Build and release frontend'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - name: 'Login to DockerHub'
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: 'Get current time'
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-      - name: 'Extract release version'
-        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
-        # This steps extracts release version from GITHUB_REF env var
-        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
-        id: extract_version
-        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
-      - name: 'Build and push'
-        uses: docker/build-push-action@v3
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-          cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:staging
-            type=registry,ref=toska/oodikone2-frontend:latest
-          cache-to: type=inline
-          context: services/frontend
-          push: true
-          tags: toska/oodikone2-frontend:latest
-          target: production
-      - name: 'Create a container from built image '
-        run: docker create --name frontend toska/oodikone2-frontend:latest
-      - name: 'Copy assets from the container'
-        run: docker cp frontend:/usr/src/app/build ./build
-      - name: 'Create Sentry release'
-        uses: getsentry/action-release@v1
-        env:
-          SENTRY_URL: ${{ secrets.SENTRY_URL }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_ORG: toska
-          SENTRY_PROJECT: oodikone
-        with:
-          environment: production
-          set_commits: 'skip'
-          sourcemaps: './build/static/js'
-          url_prefix: '~/static/js'
-          version: ${{ steps.extract_version.outputs.VERSION }}
-
-  report_success:
-    runs-on: ubuntu-latest
-    name: 'Report deployment success to oodikone notifications'
-    needs:
-      - build_and_release_backend
-      - build_and_release_frontend
-    if: success()
-    steps:
-      - name: 'Send notification to slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment
-          softa-url: https://oodikone.cs.helsinki.fi/
-          deployment-target: production
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: 'Report deployment failure to oodikone notifications'
-    needs:
-      - build_and_release_backend
-      - build_and_release_frontend
-    if: failure()
-    steps:
-      - name: 'Send notification to slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: production
@@ -0,0 +1,128 @@
+name: 'Test and release to staging'
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - '.github/workflows/test.yaml'
+      - '.github/workflows/staging.yaml'
+      - '.github/actions/**'
+      - 'cypress/**'
+      - 'cypress.config.json'
+      - 'services/**'
+      - '!services/backend/src/rapodiff.js'
+      - 'docker-compose.ci.yml'
+      - 'package*.json'
+      - 'updater/sis-updater-worker/src/db/migrations/**'
+
+jobs:
+  test:
+    uses: ./.github/workflows/test.yaml
+    secrets:
+      username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+      password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+
+  release_frontend:
+    name: 'Build and release frontend'
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get current time
+        id: get_current_time
+        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
+
+      - name: Build frontend image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: oodikone-frontend
+          tags: staging ${{ github.sha }}
+          context: services/frontend
+          containerfiles: |
+            services/frontend/Dockerfile
+          build-args: |
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+            BASE_PATH=/oodikone
+            GIT_SHA=${{ github.sha }}
+            STAGING=true
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_backend:
+    name: 'Build and release backend'
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build backend image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: oodikone-backend
+          tags: staging ${{ github.sha }}
+          context: services/backend
+          containerfiles: |
+            services/backend/Dockerfile
+          build-args: |
+            BASE_PATH=/oodikone
+            GIT_SHA=${{ github.sha }}
+            STAGING=true
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  report_success:
+    name: 'Report deployment success to oodikone notifications'
+    runs-on: ubuntu-latest
+    needs:
+      - release_frontend
+      - release_backend
+
+    if: success()
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://toska-staging.cs.helsinki.fi/oodikone
+          deployment-target: staging
+
+  report_failure:
+    name: 'Report deployment failure to oodikone notifications'
+    runs-on: ubuntu-latest
+    needs:
+      - release_frontend
+      - release_backend
+
+    if: failure()
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment-failure
+          deployment-target: staging
@@ -1,162 +0,0 @@
-name: 'Test and release updater'
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/test-and-release-updater.yaml'
-      - 'updater/sis-updater-worker/**'
-      - 'updater/sis-updater-scheduler/**'
-
-jobs:
-  build_worker:
-    name: 'Build sis-updater-worker'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - name: 'Build the image'
-        uses: docker/build-push-action@v3
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=production
-          context: updater/sis-updater-worker/
-          target: production
-          push: false
-          outputs: type=docker,dest=/tmp/worker.tar
-          tags: toska/sis-updater-worker:latest
-          cache-from: type=registry,ref=toska/sis-updater-worker:latest
-          cache-to: type=inline
-      - name: 'Store built image'
-        uses: actions/upload-artifact@v3
-        with:
-          path: /tmp/worker.tar
-
-  build_scheduler:
-    name: 'Build sis-updater-scheduler'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - name: 'Build the image'
-        uses: docker/build-push-action@v3
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=production
-          context: updater/sis-updater-scheduler/
-          target: production
-          push: false
-          outputs: type=docker,dest=/tmp/scheduler.tar
-          tags: toska/sis-updater-scheduler:latest
-          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
-          cache-to: type=inline
-      - name: 'Store built image'
-        uses: actions/upload-artifact@v3
-        with:
-          path: /tmp/scheduler.tar
-
-  test_updater:
-    name: 'Test updater'
-    runs-on: ubuntu-latest
-    needs:
-      - build_worker
-      - build_scheduler
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v2
-        with:
-          registry: registry.toska.cs.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: 'Download built images'
-        uses: actions/download-artifact@v3
-        with:
-          path: /tmp/
-      - name: 'Load worker image'
-        run: docker load --input /tmp/artifact/worker.tar
-      - name: 'Load scheduler image'
-        run: docker load --input /tmp/artifact/scheduler.tar
-      - name: 'Start worker'
-        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
-      - name: 'Run hourly jobs'
-        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-        env:
-          SCHEDULE_IMMEDIATE: hourly
-          EXIT_AFTER_IMMEDIATES: yes
-      - name: 'Run daily jobs'
-        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-        env:
-          SCHEDULE_IMMEDIATE: daily
-          EXIT_AFTER_IMMEDIATES: yes
-      - name: 'Run weekly jobs'
-        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-        env:
-          SCHEDULE_IMMEDIATE: weekly
-          EXIT_AFTER_IMMEDIATES: yes
-      - name: 'Run prepurge'
-        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-        env:
-          SCHEDULE_IMMEDIATE: prepurge
-          EXIT_AFTER_IMMEDIATES: yes
-      - name: 'Run purge'
-        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-        env:
-          SCHEDULE_IMMEDIATE: purge
-          EXIT_AFTER_IMMEDIATES: yes
-
-  release_updater:
-    name: 'Release updater'
-    runs-on: ubuntu-latest
-    needs:
-      - test_updater
-    steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - name: 'Login to DockerHub'
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: 'Build and push scheduler'
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
-          cache-to: type=inline
-          context: updater/sis-updater-scheduler/
-          push: true
-          tags: toska/sis-updater-scheduler:latest
-          target: production
-      - name: 'Build and push worker'
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=registry,ref=toska/sis-updater-worker:latest
-          cache-to: type=inline
-          context: updater/sis-updater-worker/
-          push: true
-          tags: toska/sis-updater-worker:latest
-          target: production
-      - name: 'Create Sentry release'
-        uses: getsentry/action-release@v1
-        env:
-          SENTRY_URL: ${{ secrets.SENTRY_URL }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_ORG: toska
-          SENTRY_PROJECT: updater
-        with:
-          environment: production
-          set_commits: 'skip'
-          sourcemaps: './build/static/js'
-          url_prefix: '~/static/js'
-          version: ${{ github.sha }}
-      - name: 'Send notification to slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment
-          softa-url: https://oodikone.cs.helsinki.fi
-          deployment-target: production
-          softa-to-deploy: sis-updater
@@ -1,515 +0,0 @@
-name: 'Test oodikone and release to staging'
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/test-oodikone.yaml'
-      - '.github/actions/**'
-      - 'cypress/**'
-      - 'cypress.config.json'
-      - 'services/**'
-      - '!services/backend/src/rapodiff.js'
-      - 'docker-compose.ci.yml'
-      - 'docker-compose.test.yml'
-      - 'nginx.conf'
-      - 'package*.json'
-      - 'updater/sis-updater-worker/src/db/migrations/**'
-
-jobs:
-  # === Static tests / linting ===
-
-  lint_backend:
-    name: 'Lint backend'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3.4.1
-        with:
-          node-version: 14
-      - name: 'Install dependencies'
-        run: npm ci && cd services/backend && npm ci
-      - name: 'Run backend linter'
-        run: npm run eslint services/backend
-
-  lint_frontend:
-    name: 'Lint frontend'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3.4.1
-        with:
-          node-version: 14
-      - name: 'Install dependencies'
-        run: npm ci && cd services/frontend && npm ci
-      - name: 'Run frontend linter'
-        run: npm run concurrently \
-          'eslint --ext .jsx,.js services/frontend' \
-          'stylelint services/frontend/src/**/*.css'
-
-  # === Build images ===
-  build_backend:
-    name: 'Build backend'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - name: 'Build image and export it'
-        uses: docker/build-push-action@v3
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-backend:staging
-            type=registry,ref=toska/oodikone2-backend:latest
-          cache-to: type=inline
-          context: services/backend
-          push: false
-          tags: toska/oodikone2-backend:staging
-          target: production
-          outputs: type=docker,dest=/tmp/backend.tar
-      - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v3
-        with:
-          path: /tmp/backend.tar
-
-  build_frontend:
-    name: 'Build frontend'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - name: 'Get current time'
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-      - name: 'Build image and export it'
-        uses: docker/build-push-action@v3
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-          cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:staging
-            type=registry,ref=toska/oodikone2-frontend:latest
-          cache-to: type=inline
-          context: services/frontend
-          push: false
-          tags: toska/oodikone2-frontend:staging
-          target: production
-          outputs: type=docker,dest=/tmp/frontend.tar
-      - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v3
-        with:
-          path: /tmp/frontend.tar
-
-  build_and_push_anon_sis_db:
-    name: 'Build anon sis-db and push to toska hub if new migrations have been added'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
-        id: check_migrations
-        with:
-          filters: |
-            migrations:
-              - 'updater/sis-updater-worker/src/db/migrations/**'
-              - '.github/workflows/test-oodikone.yaml'
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v2
-        with:
-          registry: registry.toska.cs.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
-        if: steps.check_migrations.outputs.migrations == 'true'
-        run: |
-          docker-compose --file docker-compose.ci.yml up --detach sis-updater-worker
-          sleep 30 # wait for sis-updater-worker to start and run migrations
-          ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
-
-  build_and_push_anon_kone_db:
-    name: 'Build anon kone-db and push to toska hub if new migrations have been added'
-    needs:
-      - build_backend
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
-        id: check_migrations
-        with:
-          filters: |
-            migrations:
-              - 'services/backend/src/database/migrations_kone/**'
-              - '.github/workflows/test-oodikone.yaml'
-      - name: 'Download images'
-        uses: actions/download-artifact@v3
-        with:
-          path: /tmp/
-      - name: 'Load built backend image'
-        run: docker load --input /tmp/artifact/backend.tar
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v2
-        with:
-          registry: registry.toska.cs.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: 'Let backend run migrations, then push new version of image to toska hub'
-        if: steps.check_migrations.outputs.migrations == 'true'
-        run: |
-          docker-compose --file docker-compose.ci.yml up --detach backend
-          sleep 30 # wait for sis-updater-worker to start and run migrations
-          ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
-
-  build_and_push_anon_user_db:
-    name: 'Build anon user-db and push to toska hub if new migrations have been added'
-    needs:
-      - build_backend
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
-        id: check_migrations
-        with:
-          filters: |
-            migrations:
-              - 'services/backend/src/database/migrations_user/**'
-              - '.github/workflows/test-oodikone.yaml'
-      - name: 'Download images'
-        uses: actions/download-artifact@v3
-        with:
-          path: /tmp/
-      - name: 'Load built backend image'
-        run: docker load --input /tmp/artifact/backend.tar
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v2
-        with:
-          registry: registry.toska.cs.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: 'Let backend run migrations, then push new version of image to toska hub'
-        if: steps.check_migrations.outputs.migrations == 'true'
-        run: |
-          docker-compose --file docker-compose.ci.yml up --detach backend
-          sleep 30 # wait for sis-updater-worker to start and run migrations
-          ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
-
-  # === End-to-end tests ===
-  test_course_statistics:
-    name: 'Test course statistics'
-    needs:
-      - build_backend
-      - build_frontend
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-          spec: cypress/e2e/Course_statistics.js
-
-  test_custom_population:
-    name: 'Test custom population'
-    needs:
-      - build_backend
-      - build_frontend
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-          spec: cypress/e2e/Custom_population.js
-
-  test_population_statistics:
-    name: 'Test population statistics'
-    needs:
-      - build_backend
-      - build_frontend
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-          spec: cypress/e2e/Population_statistics.js
-
-  test_filters:
-    name: 'Test filters'
-    needs:
-      - build_backend
-      - build_frontend
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-          spec: cypress/e2e/Filters.js
-
-  test_student_statistics:
-    name: 'Test student statistics'
-    needs:
-      - build_backend
-      - build_frontend
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-          spec: cypress/e2e/Student_statistics.js
-  test_completed_courses:
-    name: 'Test completed courses'
-    needs:
-      - build_backend
-      - build_frontend
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-          spec: cypress/e2e/Completed_courses.js
-  test_studyprogramme_overview:
-    name: 'Test studyprogramme overview'
-    needs:
-      - build_backend
-      - build_frontend
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-          spec: cypress/e2e/Studyprogramme_overview.js
-
-  test_teachers_and_users:
-    name: 'Test teachers and users'
-    needs:
-      - build_backend
-      - build_frontend
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-          spec: |
-            cypress/e2e/Teachers.js
-            cypress/e2e/Users.js
-
-  test_faculty_statistics:
-    name: 'Test faculty statistics'
-    needs:
-      - build_backend
-      - build_frontend
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-          spec: cypress/e2e/Faculty_statistics.js
-
-  test_open_uni_search:
-    name: 'Test open uni search'
-    needs:
-      - build_backend
-      - build_frontend
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-          spec: cypress/e2e/Openuni.js
-
-  # === Other tests ===
-
-  test_service_exit_statuses:
-    name: 'Check services run correctly'
-    needs:
-      - build_backend
-      - build_frontend
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Download images'
-        uses: actions/download-artifact@v3
-        with:
-          path: /tmp/
-      - name: 'Load built images'
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v2
-        with:
-          registry: registry.toska.cs.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: 'Run services, pull rest of the needed images'
-        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: 'Check that exit statuses are not errors'
-        run: |
-          docker ps -a
-          if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
-            echo 'Some process had nonzero exit code';
-            exit 1;
-          fi
-      - name: 'Print logs on failure'
-        if: failure()
-        run: docker-compose --file docker-compose.ci.yml logs
-
-  # === Release ===
-
-  release_images:
-    name: 'Release built images'
-    runs-on: ubuntu-latest
-    needs:
-      - lint_backend
-      - lint_frontend
-      - test_course_statistics
-      - test_custom_population
-      - test_filters
-      - test_population_statistics
-      - test_student_statistics
-      - test_completed_courses
-      - test_studyprogramme_overview
-      - test_teachers_and_users
-      - test_faculty_statistics
-      - test_open_uni_search
-      - test_service_exit_statuses
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Download images'
-        uses: actions/download-artifact@v3
-        with:
-          path: /tmp/
-      - name: 'Load built images'
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-      - name: 'Login to DockerHub'
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: 'Create a container from frontend'
-        run: docker create --name frontend toska/oodikone2-frontend:staging
-      - name: 'Copy assets from the container'
-        run: docker cp frontend:/usr/src/app/build ./build
-      - name: 'Create Sentry release'
-        uses: getsentry/action-release@v1
-        env:
-          SENTRY_URL: ${{ secrets.SENTRY_URL }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_ORG: toska
-          SENTRY_PROJECT: oodikone
-        with:
-          environment: staging
-          set_commits: 'skip'
-          sourcemaps: './build/static/js'
-          url_prefix: '~/static/js'
-          version: ${{ github.sha }}
-      - name: 'Push to docker hub'
-        run: |
-          docker push toska/oodikone2-backend:staging
-          docker push toska/oodikone2-frontend:staging
-      - name: 'Send deployment success notification to slack'
-        if: success()
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment
-          softa-url: https://oodikone-staging.cs.helsinki.fi/
-          deployment-target: staging
-      - name: 'Send deployment failure notification to slack'
-        if: failure()
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: staging
-
-  report_test_failure:
-    runs-on: ubuntu-latest
-    name: 'Report test failure to oodikone notifications'
-    needs:
-      - lint_backend
-      - lint_frontend
-      - test_course_statistics
-      - test_custom_population
-      - test_filters
-      - test_population_statistics
-      - test_student_statistics
-      - test_completed_courses
-      - test_studyprogramme_overview
-      - test_teachers_and_users
-      - test_faculty_statistics
-      - test_open_uni_search
-      - test_service_exit_statuses
-    if: failure()
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Send notification to slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: test-failure
@@ -0,0 +1,232 @@
+name: 'Lint and test'
+
+on:
+  workflow_call:
+    secrets:
+      username:
+        required: true
+        description: 'Toska registry username'
+      password:
+        required: true
+        description: 'Toska registry password'
+
+jobs:
+  # === Static tests / linting ===
+  lint_backend:
+    name: 'Lint backend'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+      - name: 'Install dependencies'
+        run: npm ci && cd services/backend && npm ci
+      - name: 'Run backend linter'
+        run: npm run eslint services/backend
+
+  lint_frontend:
+    name: 'Lint frontend'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+      - name: 'Install dependencies'
+        run: npm ci && cd services/frontend && npm ci
+      - name: 'Run frontend linter'
+        run: npm run concurrently \
+          'eslint --ext .jsx,.js services/frontend' \
+          'stylelint services/frontend/src/**/*.css'
+
+  # === Build database images ===
+  build_backend:
+    name: 'Build backend'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - name: 'Build image and export it'
+        uses: docker/build-push-action@v5
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
+          context: services/backend
+          push: false
+          tags: oodikone-backend:staging
+          outputs: type=docker,dest=/tmp/backend.tar
+      - name: 'Upload image for other jobs'
+        uses: actions/upload-artifact@v3
+        with:
+          path: /tmp/backend.tar
+
+  build_and_push_anon_sis_db:
+    name: 'Build anon sis-db and push to toska hub if new migrations have been added'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'updater/sis-updater-worker/src/db/migrations/**'
+              - '.github/workflows/test.yaml'
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v3
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+      - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach sis-updater-worker
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
+
+  build_and_push_anon_kone_db:
+    name: 'Build anon kone-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_kone/**'
+              - '.github/workflows/test.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v3
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v3
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
+
+  build_and_push_anon_user_db:
+    name: 'Build anon user-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_user/**'
+              - '.github/workflows/test.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v3
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v3
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
+
+  # === End-to-end tests ===
+  test_end_to_end:
+    runs-on: ubuntu-latest
+    needs:
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
+    strategy:
+      matrix:
+        include:
+          - name: 'Test course statistics'
+            spec: cypress/e2e/Course_statistics.js
+          - name: 'Test custom population'
+            spec: cypress/e2e/Custom_population.js
+          - name: 'Test population statistics'
+            spec: cypress/e2e/Population_statistics.js
+          - name: 'Test filters'
+            spec: cypress/e2e/Filters.js
+          - name: 'Test student statistics'
+            spec: cypress/e2e/Student_statistics.js
+          - name: 'Test completed courses'
+            spec: cypress/e2e/Completed_courses.js
+          - name: 'Test studyprogramme overview'
+            spec: cypress/e2e/Studyprogramme_overview.js
+          - name: 'Test teachers and users'
+            spec: |
+              cypress/e2e/Teachers.js
+              cypress/e2e/Users.js
+          - name: 'Test faculty statistics'
+            spec: cypress/e2e/Faculty_statistics.js
+          - name: 'Test open uni search'
+            spec: cypress/e2e/Openuni.js
+          - name: 'Test language center view'
+            spec: cypress/e2e/Language_center.js
+          - name: 'Test study guidance groups'
+            spec: cypress/e2e/Study_guidance_groups.js
+    name: ${{ matrix.name }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: 'Run tests'
+        uses: ./.github/actions/run-tests
+        with:
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+          spec: ${{ matrix.spec }}
+
+  # === Other tests ===
+  test_service_exit_statuses:
+    name: 'Check services run correctly'
+    needs:
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v3
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+
+      - name: 'Run services, pull rest of the needed images'
+        run: docker-compose --file docker-compose.ci.yml up --detach
+
+      - name: 'Check that exit statuses are not errors'
+        run: |
+          docker ps -a
+          if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
+            echo 'Some process had nonzero exit code';
+            exit 1;
+          fi
+
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
@@ -0,0 +1,150 @@
+name: 'Test and release Updater'
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - '.github/workflows/updater.yaml'
+      - 'updater/sis-updater-worker/**'
+      - 'updater/sis-updater-scheduler/**'
+
+jobs:
+  build_worker:
+    name: 'Build Updater worker'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - name: 'Build the image'
+        uses: docker/build-push-action@v5
+        with:
+          context: updater/sis-updater-worker/
+          push: false
+          outputs: type=docker,dest=/tmp/worker.tar
+          tags: toska/sis-updater-worker:latest
+      - name: 'Store built image'
+        uses: actions/upload-artifact@v3
+        with:
+          path: /tmp/worker.tar
+
+  build_scheduler:
+    name: 'Build Updater scheduler'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - name: 'Build the image'
+        uses: docker/build-push-action@v5
+        with:
+          context: updater/sis-updater-scheduler/
+          push: false
+          outputs: type=docker,dest=/tmp/scheduler.tar
+          tags: toska/sis-updater-scheduler:latest
+      - name: 'Store built image'
+        uses: actions/upload-artifact@v3
+        with:
+          path: /tmp/scheduler.tar
+
+  test_updater:
+    name: 'Test Updater'
+    runs-on: ubuntu-latest
+    needs:
+      - build_worker
+      - build_scheduler
+    steps:
+      - uses: actions/checkout@v4
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v3
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Download built images'
+        uses: actions/download-artifact@v3
+        with:
+          path: /tmp/
+      - name: 'Load worker image'
+        run: docker load --input /tmp/artifact/worker.tar
+      - name: 'Load scheduler image'
+        run: docker load --input /tmp/artifact/scheduler.tar
+      - name: 'Start worker'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
+      - name: 'Run hourly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: hourly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run daily jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: daily
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run weekly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: weekly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run prepurge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: prepurge
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run purge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: purge
+          EXIT_AFTER_IMMEDIATES: yes
+
+  release_worker:
+    name: 'Build and release Updater worker'
+    runs-on: ubuntu-latest
+    needs: test_updater
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build worker image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: updater-worker
+          tags: staging production ${{ github.sha }}
+          context: updater/sis-updater-worker
+          containerfiles: |
+            updater/sis-updater-worker/Dockerfile
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_scheduler:
+    name: 'Build and release Updater scheduler'
+    runs-on: ubuntu-latest
+    needs: test_updater
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build scheduler image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: updater-scheduler
+          tags: staging production ${{ github.sha }}
+          context: updater/sis-updater-scheduler
+          containerfiles: |
+            updater/sis-updater-scheduler/Dockerfile
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
@@ -7,7 +7,7 @@ module.exports = {
  '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
  '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
  '*.css': files => `stylelint --fix ${files.join(' ')}`,
-  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0 hadolint ${relativeFilePaths(files)}`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.12.0 hadolint ${relativeFilePaths(files)}`,
  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck:v0.7.2 ${relativeFilePaths(files)} -x`,
  '.github/{workflows,actions}/*': files => `npm run actionlint ${relativeFilePaths(files)}`,
  'docker-compose*': files => {
@@ -1,6 +1,6 @@
 # Oodikone

-An application for analyzing university data, running at [https://oodikone.cs.helsinki.fi/](https://oodikone.cs.helsinki.fi/).
+An application for analyzing university data, running at [https://oodikone.helsinki.fi/](https://oodikone.helsinki.fi/).

 ## ✔️ Requirements

@@ -8,7 +8,7 @@ To run Oodikone locally, you will need the following:

 1. Applications:
   - [Docker](https://docs.docker.com/install/) (version 20.10+),
-   - [Docker Compose](https://docs.docker.com/compose/install/)(version 1.29+) and
+   - [Docker Compose](https://docs.docker.com/compose/install/) (version 1.29+) and
   - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
 2. Access to Toska Docker Hub,
 3. For real data setup, access to Toskas production servers.
@@ -45,12 +45,57 @@ What different CLI options do:
   - Creates real data databases and populates them with downloaded dumps.
 3. Reset single database
   - Like option 2, but only for single database. Leaves other databases untouched.
+4. Restore data from dumps
+   - Like option 2, but skips downloading, and simply restores databases from the dumps downloaded earlier.
+5. Docker system prune
+   - Runs `docker system prune -a && docker volume prune -a`. This will remove all your docker data, also related to any other applications than oodikone! It is sometimes necessary after strange errors due to caching.

 ## ⌨️ Development

 ### Architecture

-![Oodikone architecture](./documentation/Oodikone.png)
+```mermaid
+graph TB
+    subgraph Oodikone & Updater
+        subgraph Oodikone
+            client[React app] --> server[NodeJS server]
+            server --> ok_redis(Redis)
+            konedb[(kone-db)]
+        end
+        server -->|Read| sisdb[(sis-db)]
+        subgraph Updater
+            updaterscheduler[Scheduler] --> |Push jobs| nats
+            nats(NATS server)
+            updaterworker --> |Fetch jobs| nats
+            updaterworker[Worker] -->|Write| sisdb
+            updaterscheduler --> redis(Redis)
+            updaterworker --> redis
+        end
+    end
+
+    subgraph Common Toska services
+        jami[JAMI]
+        pate[Pate]
+        importer[Importer]
+    end
+
+    updaterworker -->|Read db| importer
+    updaterscheduler -->|Read db| importer
+    importer -->|Fetch data| sisu[Sisu export APIs]
+    server -->|Get IAM access| jami
+    server -->|Send mail| pate
+
+    subgraph Analytics
+        direction TB
+        sentry[Sentry]
+        graylog[Graylog]
+        grafana[Grafana]
+    end
+```
+
+In development, backend and updater use the same redis, but in production they are separate.
+
+See also [older graph](documentation/Oodikone.png), which presents more information.

 ### Documentation

@@ -71,6 +116,9 @@ Some useful commands are defined in `package.json` and can be run with `npm run
 - `npm run both`: starts oodikone and updater with anonymized data
 - `npm run both:real`: starts oodikone and updater with real data
 - `npm run docker:down`: stops the whole environment
+- `npm run testupdater`: runs tests for updater
+- `npm run flushredis`: Clears redis, which forces big calculations to be redone in e.g. studyprogramme overview and faculty views.
+- `npm run rapodiff`: runs [Rapodiff](./services/backend/src/rapodiff/)

 Once you have ran setup for oodikone , you can just execute the first one (`npm run oodikone`). After starting and waiting for a while for containers to compile, oodikone can be accessed at [http://localhost:3000/](http://localhost:3000/) and Adminer (database investigation tool) at [http://localhost:5050/](http://localhost:5050/). Adminer requires you to login with username `postgres` and with any password you choose (for example `p`).

@@ -80,13 +128,13 @@ As said, the development environment runs entirely inside docker containers. To

 `./run.sh` is simply a wrapper script to run oodikone, updater or both services in either anon or real mode. If you take a look at `package.json`, you can see that most of the predefined scripts above use `./run.sh` under the hood.

-It is recommended to spend some time to become familiar with `docker` and `docker-compose` cli commands. You can then use them directly or with `./run.sh` wrapper. Here is some examples for day-to-day development situations:
+It is recommended to spend some time to become familiar with `docker` and `docker compose` cli commands. You can then use them directly or with `./run.sh` wrapper. Here is some examples for day-to-day development situations:

 - `./run.sh oodikone anon pull`: Pull all images related to oodikone development
 - `./run.sh updater real up --build --force-recreate --detach`: Start updater detached (=in the background) in real data mode, but build new images before starting
- `docker-compose ps`: view the containers in the running environment
- `docker-compose logs frontend`: print logs for just frontend
- `docker-compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
+- `docker compose ps`: view the containers in the running environment
+- `docker compose logs frontend`: print logs for just frontend
+- `docker compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
 - `docker exec -it backend sh`: open bash terminal inside backend container
 - `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.

@@ -102,31 +150,30 @@ Cli script sets up pre-commit hooks that are used to lint and fix files before c

 For more information on how files are tested, take a look at "lint-staged" in `package.json`. Some files (e.g. github action files, dockerfiles, shell scripts) are checked with external tools and may require you to install those tools in case you're modifying files in question.

-There's no separate "lintfix" / "format" / "lint" command available, since pre-commit hooks will fix formatting and basic linting errors for you. In case pre-commit fails, you should fix the erroring code manually.
-
-However, if you want to run linting / formatting manually from command line, `package.json` defines `npm run eslint`, `npm run prettier` and `npm run stylelint` which you can use as an entrypoint.
+Pre-commit hooks will fix auto-fixable problems. To set up quick formatting: In VSCode you can go to command palette, select "ESLint: Fix all auto-fixable problems" and assign a hotkey for it, like `SHIFT + CTRL + '.'`.

 ## 🔨 Testing & CI

-There are three types of tests in this project: static tests with eslint/prettier/other tools, jest integration tests for single service and cypress end-to-end tests. Linting is described above and other do work as follows:
+We use cypress for end-to-end testing. No unit tests are currently used.

- Cypress
-  - can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
-  - are defined in cypress -folder and cypress.config.json
-  - whole cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
-  - There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.
- Jest
-  - are ran with `docker-compose.test.yml`
-  - test command and environment themselves are in Dockerfile for service in question
-  - can be debugged by overriding default command in `docker-compose.test.yml` and running docker container in interactive mode
+Cypress
+
+- Can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
+- Are defined in cypress -folder and cypress.config.json
+- The entire cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
+- Running tests locally sometimes causes problems after running the whole tests suite due to some login-related issue. Also for this reason, it is usually best practice to only run single views locally, and let the CI run the entire test suite.
+- There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.

 Continuous integration (CI) works with Github actions and is defined in workflow files in `.github/workflows` folder:

- oodikone setup for cypress and other tests in CI is defined in `docker-compose.ci.yml`. Take a look at this too if debugging github action workflows.
- tests are run on every push.
- after successful test run, oodikone is deployed to staging
- after creating a release, oodikone is deployed to production
- updater is deployed to production on every push, no tests included
+- Oodikone setup for cypress and other tests in CI is defined in `docker-compose.ci.yml`. Take a look at this too if debugging github action workflows.
+- Tests are run on every push
+- After a successful test run, oodikone is deployed to staging
+- After creating a release, oodikone is deployed to production
+- Updater is deployed to production when pushed if tests pass
+- Test databases can be updated, instructions in [anonyymioodi README](anonyymioodi/README.md)
+
+[Rapodiff](./services/backend/src/rapodiff/) can be used to check differences between Rapo API and Oodikone data.

 ## ❓FAQ

@@ -134,21 +181,19 @@ Continuous integration (CI) works with Github actions and is defined in workflow

 You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container.

-### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data
+### Studyguidance groups don't work on my machine

-Make sure you have your VPN on
-
-### I made a change to sis-db models aand created a migration, but tests are failing in CI!
-
-Migrations are handled by sis-importer-worker, since oodikone is using only replica. Since CI is testing only services included in oodikone, your new changes aren't migrated and thus are failing. Open anonyymioodi-folder in this repo to find more about how to to update the anon sis-db.
+Make sure you have your VPN on.

 ### Everything is broken, can't get oodikone running, data is not there etc.

-Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.
+First: Try `npm run cli` option 1: _Set up oodikone from scratch_ and then option 2 to download and restore databases (or option 4 to skip downloading, if you've done it recently).
+
+If that does not help, try option 5: _Docker system prune_. Notice that this clears up **all docker data**, including data related to other projects. If this is not ok, you'll need to identify the oodikone-specific docker volumes yourself and caches and clear those invididually.

 ## ✌🏼 Maintainers and contribution

-Toska of course.
+[Toska](https://toska.dev) of course.

 University of Helsinki

@@ -1,65 +1,18 @@
-# Anonyymioodi - anonymous database setup for oodikone
+# Anonyymioodi - test database setup

-(name anonyymioodi for :culture: reasons)
+This folder contains a guide and scripts to modify or create anonymous database images for oodikone, mainly used for testing.

-This folder contains a guide and scripts to create anonymous database images for oodikone, mainly used for testing.
+This quick guide should be enough for most purposes: Updating schema, data or postgres version. If not, a more elaborate guide is stored in [old readme](old_readme.md) (though it may have some outdated info).

-## How to create / modify database images
+### Quick guide to update a test database

-Creating / modifying images varies a bit depending on the db. Generally, to modify images, you can either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose. 
-
-By default, previous db images from Toska Hub are used as a base. If needed, comment out Toska hub image and uncomment corresponding clean postgres image in docker-compose before running other steps.
-
-See more details on how to create / modify databases below.
-
-### Sis-importer-db
-
-The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
-
- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
- Move `sis-importer-db.sqz` to this folder
- Run `./create_container_from_dump.sh sis-importer-db`
- Enrich db with command in file `enrich_data.sql` either by piping command to container or by running command inside container.
-
-### Sis-db:
-
-The following steps are for building a clean sis-db image from scratch using oodikone's updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
-
- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
- Start updater, see oodikone readme for more info.
- Populate sis-db from sis-importer-db:
-
-```
-curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
-curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
-curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
-```
-(or run these from frontend. You might also want to follow sis-updater-workers logs to ensure things are running smoothly.)
- Clean up any extra stuff you don't want to keep in sis-db.
- Bonus: check that cypress tests run okay
-
-### User-db / kone-db
-
- Temporarily replace the db service in oodikone's docker-compose.yml with the same db service from this folder's docker-compose.yml
- Ensure the newest user-db/kone-db image is pulled from the registry with `docker-compose pull`
- Modify db to wanted state. Tips:
-  - Run oodikone if you want to upgrade something from frontend (e.g. tags in kone-db, courses in sis-db etc.).
-  - Easy way to create new users to user-db is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
-
-## Publishing the image
-
-After completing the steps above and making sure your local version has the correct data, run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the db image to Toska Hub.
-
-## How to update Postgres versions
-
-When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
-
- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
- Start database with docker-compose
- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
- Close container with docker-compose down.
- Update the Postgres version in docker-compose.yml in this folder.
- Uncomment row with Postgres image and comment row with Toskahub image.
- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`
+- Make sure you have the latest test-db. Just pulling images doesn't seem to replace your already existing volume, so you may need to remove it by hand. This may work: `docker volume rm oodikone_sis-db-data`. The new image including data should be pulled next time you launch oodikone normally.
+- Do your dev stuff normally with the test database, for example with `npm run both`. Edit data however you like, run updater via frontend for example, then run tests to see if they pass.
+- When you're happy with the database, create a dump:
+- `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > anonyymioodi/<dbname>.sqz`
+- Now you can shut it down `npm run docker:down`
+- Navigate to this folder `cd anonyymioodi`
+- If you need to **update postgres version**, do it now by changing it in this directory's `docker-compose.yml`.
+- Create new container from dump `./create_container_from_dump.sh <dbname>`
+- Commit and push it to toska registry `./commit_and_push_to_toska_hub.sh <dbname>`
+- `docker compose down`
@@ -43,6 +43,6 @@ fi

 infomsg "Committing and pushing to registry"
 database=${args[0]}
-docker commit "$database" "registry.toska.cs.helsinki.fi/$database"
-docker push "registry.toska.cs.helsinki.fi/$database"
+docker commit "$database" "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
+docker push "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
 successmsg "Succesfully pushed"
@@ -9,8 +9,7 @@ services:
      PGDATA: /data
      POSTGRES_DB: kone-db
      POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry.toska.cs.helsinki.fi/kone-db
-    # image: postgres:13.4-alpine
+    image: postgres:15.3
    profiles:
      - oodikone

@@ -21,8 +20,7 @@ services:
      PGDATA: /data
      POSTGRES_DB: sis-db
      POSTGRES_HOST_AUTH_METHOD: trust
-    # image: registry.toska.cs.helsinki.fi/sis-db
-    image: postgres:13.2
+    image: postgres:15.3
    shm_size: 2gb

  sis-importer-db:
@@ -32,8 +30,7 @@ services:
      PGDATA: /data
      POSTGRES_DB: sis-importer-db
      POSTGRES_HOST_AUTH_METHOD: trust
-    # image: registry.toska.cs.helsinki.fi/sis-importer-db
-    image: postgres:13.1
+    image: postgres:15.3
    profiles:
      - updater

@@ -44,7 +41,6 @@ services:
      PGDATA: /data
      POSTGRES_DB: user-db
      POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry.toska.cs.helsinki.fi/user-db
-    # image: postgres:13.4-alpine
+    image: postgres:15.3
    profiles:
      - oodikone
@@ -0,0 +1,71 @@
+# Old readme of anonyymioodi
+
+This is mostly obsolete, but may help with something sometimes.
+
+## Anonyymioodi - anonymous database setup for oodikone
+
+(name anonyymioodi for :culture: reasons)
+
+This folder contains a guide and scripts to create anonymous database images for oodikone, mainly used for testing.
+
+## How to create / modify database images
+
+Creating / modifying images varies a bit depending on the db. Generally, to modify images, you can either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose.
+
+By default, previous db images from Toska Hub are used as a base. If needed, comment out Toska hub image and uncomment corresponding clean postgres image in docker-compose before running other steps.
+
+See more details on how to create / modify databases below.
+
+### Sis-importer-db
+
+The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
+- Move `sis-importer-db.sqz` to this folder
+- Run `./create_container_from_dump.sh sis-importer-db`
+- Enrich db with command in file `enrich_data.sql` either by piping command to container or by running command inside container.
+
+### Sis-db:
+
+The following steps are for building a clean sis-db image from scratch using oodikone's updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
+- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
+- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
+- Start updater, see oodikone readme for more info.
+- Populate sis-db from sis-importer-db:
+
+```
+curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
+```
+
+(or run these from frontend. You might also want to follow sis-updater-workers logs to ensure things are running smoothly.)
+
+- Clean up any extra stuff you don't want to keep in sis-db.
+- Bonus: check that cypress tests run okay
+
+### User-db / kone-db
+
+- Temporarily replace the db service in oodikone's docker-compose.yml with the same db service from this folder's docker-compose.yml
+- Ensure the newest user-db/kone-db image is pulled from the registry with `docker-compose pull`
+- Modify db to wanted state. Tips:
+  - Run oodikone if you want to upgrade something from frontend (e.g. tags in kone-db, courses in sis-db etc.).
+  - Easy way to create new users to user-db is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
+
+## Publishing the image
+
+After completing the steps above and making sure your local version has the correct data, run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the db image to Toska Hub.
+
+## How to update Postgres versions
+
+When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
+
+- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
+- Start database with docker-compose
+- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
+- Close container with docker-compose down.
+- Update the Postgres version in docker-compose.yml in this folder.
+- Uncomment row with Postgres image and comment row with Toskahub image.
+- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`
@@ -51,6 +51,8 @@ set_custom_select_prompt() {
    "Set up oodikone from scratch."
    "Reset all real data."
    "Reset single database."
+    "Restore data from dumps"
+    "Docker system prune"
    "Quit."
  )
 }
@@ -72,6 +74,10 @@ while true; do
        reset_single_database
        set_custom_select_prompt # Set prompt back to initial values
        ;;
+      "Restore data from dumps")
+      restore_data_from_dumps;;
+      "Docker system prune")
+        docker_prune;;
      "Quit.")
        break 2;;
      *) msg "${RED}Invalid option:${NOFORMAT} $REPLY
@@ -48,7 +48,7 @@ describe('When search modal is opened', () => {
    hasLanded()
    openCompletedCoursesModal()
    cy.contains('Search completed courses of students')
-    cy.contains('Insert one or more student numbers, separated by a space, a newline, or a comma.')
+    cy.contains('Insert one or more student numbers, separated by a space, a newline, a comma, or a semicolon.')
    cy.contains('Insert one or more courses, separated by a space, a newline, or a comma.')
    cy.contains('Insert name for this course list if you wish to save it')
  })
@@ -73,7 +73,7 @@ describe('When search modal is opened', () => {
  })

  describe('When a search with correct data is executed', () => {
-    before(() => {
+    beforeEach(() => {
      cy.init('/completedcoursessearch')
      hasLanded()
      openCompletedCoursesModal()
@@ -90,6 +90,7 @@ describe('When search modal is opened', () => {
        '/completedcoursessearch?courseList=TKT10001&courseList=TKT10002&courseList=TKT10003&courseList=TKT10004&studentList=010450938&studentList=010589388&studentList=010614509'
      )
    })
+
    it('Finds correct students and courses', () => {
      cy.fixture('completedCoursesData').then(({ studentSet1, coursesSet1 }) => {
        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', studentSet1[0])
@@ -1,27 +1,99 @@
 /// <reference types="Cypress" />

+const checkGradeTable = gradesTableContents => {
+  cy.get('[data-cy="Grade distribution"]')
+    .parent()
+    .siblings('.content.active')
+    .get('table > tbody')
+    .first()
+    .within(() => {
+      gradesTableContents.forEach((values, trIndex) => {
+        cy.get('tr')
+          .eq(trIndex)
+          .within(() => {
+            values.forEach((value, tdIndex) => {
+              if (value === null) return
+              cy.get('td').eq(tdIndex).contains(value)
+            })
+          })
+      })
+      cy.get('tr').should('have.length', gradesTableContents.length)
+    })
+}
+
 describe('Course Statistics tests', () => {
  describe('when using basic user', () => {
    beforeEach(() => {
      cy.init('/coursestatistics')
-    })
-    it('Searching single course having substitution mappings shows course statistics', () => {
      cy.url().should('include', '/coursestatistics')
+    })
+
+    describe('It shows correct statistics for courses with other grades than 0-5', () => {
+      it('Shows correct statistics for courses with scale HT-TT', () => {
+        const gradesTableContents = [
+          [null, 'TT', 190],
+          [null, 'Hyl.', 4],
+          [null, 'HT', 177],
+        ]
+        cy.contains('Search for courses')
+        cy.get("input[placeholder='Search by a course code']").type('KK-RUKIRJ')
+        cy.contains('td', 'KK-RUKIRJ').click()
+        cy.contains('Search for courses').should('not.exist')
+        cy.contains('KK-RUKIRJ, 992912, AYKK-RUKIRJ Toisen kotimaisen kielen kirjallinen taito, ruotsi (CEFR B1)')
+        cy.contains('.tabular.menu a', 'Grade distribution chart').click()
+        cy.contains('svg', 'Grades')
+        cy.contains('Show population').should('be.enabled').click()
+        checkGradeTable(gradesTableContents)
+      })
+
+      it("Shows correct statistics for old master's thesis grade scales", () => {
+        const gradesTableContents = [
+          [null, 'L', 1],
+          [null, 'CL', 1],
+        ]
+        cy.contains('Search for courses')
+        cy.get("input[placeholder='Search by a course code']").type('704220')
+        cy.contains('td', '704220').click()
+        cy.contains('Search for courses').should('not.exist')
+        cy.contains('704220 Taloustiede. Pro gradu -tutkielma')
+        cy.contains('.tabular.menu a', 'Grade distribution chart').click()
+        cy.contains('svg', 'Grades')
+        cy.contains('Show population').should('be.enabled').click()
+        checkGradeTable(gradesTableContents)
+      })
+
+      it('Shows correct statistics for courses with scale passed-failed', () => {
+        const gradesTableContents = [[null, 'Hyv.', 258]]
+        cy.contains('Search for courses')
+        cy.get("input[placeholder='Search by a course code']").type('KK-ENMALU1')
+        cy.contains('td', 'KK-ENMALU1').click()
+        cy.contains('Search for courses').should('not.exist')
+        cy.contains(
+          'KK-ENFARM1, KK-ENOIK1, KK-ENLAAK1, KK-ENMALU1, 995010Mat-lu Academic and Professional Communication in English 1 (CEFR B2)'
+        )
+        cy.contains('.tabular.menu a', 'Grade distribution chart').click()
+        cy.contains('svg', 'Grades')
+        cy.contains('Show population').should('be.enabled').click()
+        checkGradeTable(gradesTableContents)
+      })
+    })
+
+    it('Searching single course having substitution mappings shows course statistics', () => {
      cy.contains('Search for courses')
      cy.get("input[placeholder='Search by a course code']").type('TKT20001')
      // Click uni course, not avoin
      cy.contains('td', /^TKT20001/).click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.contains('Search for courses').should('not.exist')

      cy.contains('Tietorakenteet ja algoritmit')
      cy.contains('TKT20001')
      cy.contains('58131') // old mapped code

-      cy.contains('.tabular.menu a', 'Table').click()
+      cy.contains('.tabular.menu a', 'Students').click()
+      cy.contains('All')
+
+      cy.contains('.tabular.menu a', 'Attempts').click()
      cy.contains('All')
-      cy.cs('viewMode-Attempts').click()
-      cy.cs('viewMode-Students').click()

      cy.contains('.tabular.menu a', 'Pass rate chart').click()
      cy.contains('svg', 'Pass rate chart')
@@ -34,8 +106,9 @@ describe('Course Statistics tests', () => {
    })

    it('Searching multiple courses having substitution mappings shows course statistics', () => {
-      cy.url().should('include', '/coursestatistics')
      cy.contains('Search for courses')
+      cy.cs('select-multiple-courses-toggle').should('not.have.class', 'checked').click()
+      cy.cs('select-multiple-courses-toggle').should('have.class', 'checked')
      cy.get("input[placeholder='Search by a course code']").type('TKT')
      cy.contains('td', /^TKT20001/).click()
      cy.contains('td', /^TKT10002/).click()
@@ -59,11 +132,9 @@ describe('Course Statistics tests', () => {
    })

    it('On consecutive searches should not crash and search should work', () => {
-      cy.url().should('include', '/coursestatistics')
      cy.contains('Search for courses')
      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
      cy.contains('td', 'TKT20003').click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.contains('Show population')
      cy.contains('TKT20003')

@@ -71,18 +142,15 @@ describe('Course Statistics tests', () => {
      cy.contains('Search for courses')
      cy.get("input[placeholder='Search by a course code']").type('TKT20001')
      cy.contains('td', 'TKT20001').click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.contains('Show population')
      cy.contains('TKT20001')
    })

    it('Shows statistics when separating by semester', () => {
      const courseCode = 'DATA11002'
-      cy.url().should('include', '/coursestatistics')
      cy.contains('Search for courses')
      cy.get("input[placeholder='Search by a course code']").type(courseCode)
      cy.contains('td', courseCode).click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.get('[data-cy=separateToggle]', { force: true }).click({ force: true })
      cy.contains('Syksy 2019')
      cy.contains('Syksy 2017')
@@ -91,18 +159,15 @@ describe('Course Statistics tests', () => {
    })

    it('Searching course by name displays right courses', () => {
-      cy.url().should('include', '/coursestatistics')
      cy.contains('Search for courses')
      cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')

-      cy.contains('Tietokantojen perusteet')
      cy.contains('Tietokantojen perusteet')
      cy.contains('A581328')
      cy.contains('AYTKT10004')
      cy.contains('TKT10004, 581328')
      cy.contains('td', /^TKT10004/).click()

-      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.contains('Search for courses').should('not.exist')

      cy.contains('TKT10004, 581328, AYTKT10004, A581328 Tietokantojen perusteet')
@@ -110,29 +175,49 @@ describe('Course Statistics tests', () => {
      cy.contains('No results')
    })

+    it('"Select all search results" button is not showing unless "Select multiple courses" toggle is on', () => {
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT')
+      cy.get('[data-cy="select-multiple-courses-toggle"]').should('not.have.class', 'checked')
+      cy.contains('TKT10004, 581328')
+      cy.contains('Select all search results').should('not.exist')
+      cy.get('[data-cy="select-multiple-courses-toggle"]').click()
+      cy.get('[data-cy="select-multiple-courses-toggle"]').should('have.class', 'checked')
+      cy.contains('Select all search results')
+    })
+
+    // Skipped for now, due to limit being removed. Subject to change after testing in production
+    it.skip('"Fetch statistics" button is disabled if over 40 courses are selected', () => {
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT')
+      cy.get('[data-cy="select-multiple-courses-toggle"]').should('not.have.class', 'checked').click()
+      cy.get('[data-cy="select-multiple-courses-toggle"]').should('have.class', 'checked')
+      cy.contains('Select all search results').click()
+      cy.contains('Fetch statistics').should('not.be.enabled').trigger('mouseover', { force: true })
+      cy.contains('Voit valita tarkasteltavaksi enintään 40 kurssia kerrallaan.')
+      cy.get("input[placeholder='Search by a course code']").type('21')
+      cy.contains('Fetch statistics').should('be.enabled')
+    })
+
    it('Provider organization toggle works', () => {
-      cy.url().should('include', '/coursestatistics')
      cy.contains('Search for courses')
      cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')

      cy.contains('td', /^TKT10004/).click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.contains('Search for courses').should('not.exist')

      cy.contains('TKT10004, 581328, AYTKT10004, A581328 Tietokantojen perusteet')
-      cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.get('[data-cy=unify_radio_regular]').click()
      cy.contains('TKT10004, 581328 Tietokantojen perusteet')
    })

    it('Searching course by name displays right courses, 10 credit courses', () => {
-      cy.url().should('include', '/coursestatistics')
      cy.contains('Search for courses')
      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
      cy.contains('Tietorakenteet ja algoritmit')
      cy.contains('AYTKT20001')
      cy.contains('TKT20001, 58131')
      cy.contains('td', /^TKT20001/).click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.contains('Search for courses').should('not.exist')

      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
@@ -141,20 +226,16 @@ describe('Course Statistics tests', () => {
      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
      cy.contains('td', 'TKT20001, 58131, AYTKT20001').click()

-      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.contains('Search for courses').should('not.exist')
      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
    })

    it('Can find course population', () => {
-      cy.url().should('include', '/coursestatistics')
      cy.contains('Search for courses')
      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
      cy.contains('tr', 'TKT20003').click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.contains('Käyttöjärjestelmät')
      cy.contains('TKT20003')
-
      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
      cy.contains('Population of course Käyttöjärjestelmät 2019-2020 (open and normal')
      cy.contains('TKT20003')
@@ -166,22 +247,20 @@ describe('Course Statistics tests', () => {

    it('Population of course shows grades for each student', () => {
      cy.get("input[placeholder='Search by a course code']").type('TKT20001')
-      cy.contains(/^TKT20001, AYTKT20001$/).click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains(/^TKT20001, 58131, AYTKT20001$/).click({ force: true })
      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
-      cy.get('tbody >:nth-child(3) > :nth-child(2) .level').click()
+      cy.get('tbody >:nth-child(3) > :nth-child(2) .level').click({ force: true })
      cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019 (open and normal)')
-      cy.contains('Students (204)').click()
+      cy.contains('Students (204)').click({ force: true })
      cy.contains('td', '010262566').siblings().contains('4')
      cy.contains('td', '010674989').siblings().contains('1')
    })

    it('Language distribution is correct', () => {
      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
-      cy.contains(/^TKT20003$/).click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains(/^TKT20003, 582219$/).click()
      cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
-      cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.get('[data-cy=unify_radio_regular]').click()
      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
      cy.contains('Language distribution').click()
@@ -189,13 +268,85 @@ describe('Course Statistics tests', () => {
      cy.contains('td', 'english').siblings().contains('5')
    })

+    it('Shows correct statistics when course is not combined with its substitutions', () => {
+      cy.get('[data-cy="combine-substitutions-toggle"]').should('have.class', 'checked').click()
+      cy.get('[data-cy="combine-substitutions-toggle"]').should('not.have.class', 'checked')
+
+      cy.get("input[placeholder='Search by a course code']").type('TKT10002')
+      cy.contains('td', /^TKT10002$/).click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('TKT10002 Ohjelmoinnin perusteet')
+
+      // Statistics
+      const yearRange = { from: '2010-2011', to: '2020-2021' }
+      const attemptsTableContents = [
+        // [time, attempts, passed, failed]
+        ['Total', null, 290, 278, 12],
+      ]
+
+      const gradesTableContents = [
+        // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
+        ['Total', null, 290, 12, 8, 9, 12, 17, 223, 9],
+        ['2020-2021', null, 1, 0, 0, 0, 0, 0, 1, 0],
+        ['2019-2020', null, 37, 1, 2, 0, 3, 3, 28],
+        ['2018-2019', null, 109, 4, 2, 5, 4, 6, 88, 0],
+        ['2017-2018', null, 129, 7, 3, 3, 3, 4, 101, 8],
+        ['2016-2017', null, 9, 0, 0, 1, 2, 2, 4, 0],
+        ['2015-2016', null, 2, 0, 0, 0, 0, 1, 1, 0],
+        ['2014-2015', null, 1, 0, 0, 0, 0, 0, 0, 1],
+        ['2013-2014', null, 1, 0, 1, 0, 0, 0, 0, 0],
+        ['2010-2011', null, 1, 0, 0, 0, 0, 1, 0, 0],
+      ]
+      // Time range
+      cy.get("div[name='fromYear']").within(() => {
+        cy.get("div[role='option']").first().should('have.text', yearRange.to)
+        cy.contains("div[role='option']", yearRange.from).should('have.class', 'selected')
+        cy.get("div[role='option']").last().should('have.text', '2010-2011')
+        cy.get("div[role='option']").should('have.length', 11)
+      })
+      cy.get("div[name='toYear']").within(() => {
+        cy.get("div[role='option']").first().should('have.text', '2020-2021')
+        cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
+        cy.get("div[role='option']").last().should('have.text', yearRange.from)
+        cy.get("div[role='option']").should('have.length', 11)
+      })
+
+      cy.contains('#CourseStatPanes a.item', 'Attempts').click()
+      cy.get('#CourseStatPanes table>tbody').within(() => {
+        attemptsTableContents.forEach((values, trIndex) => {
+          cy.get('tr')
+            .eq(trIndex)
+            .within(() => {
+              values.forEach((value, tdIndex) => {
+                if (value === null) return
+                cy.get('td').eq(tdIndex).contains(value)
+              })
+            })
+        })
+        cy.get('tr').should('have.length', 10)
+      })
+      cy.get('[data-cy=gradeToggle]', { force: true }).click({ force: true })
+      cy.get('#CourseStatPanes table>tbody').within(() => {
+        gradesTableContents.forEach((values, trIndex) => {
+          cy.get('tr')
+            .eq(trIndex)
+            .within(() => {
+              values.forEach((value, tdIndex) => {
+                if (value === null) return
+                cy.get('td').eq(tdIndex).contains(value)
+              })
+            })
+        })
+        cy.get('tr').should('have.length', 10)
+      })
+    })
+
    describe('When searching unified course stats', () => {
      beforeEach(() => {
        cy.url().should('include', '/coursestatistics')
        cy.contains('Search for courses')
        cy.get("input[placeholder='Search by a course code']").type('TKT10002')
-        cy.contains('td', 'TKT10002, AYTKT10002').click()
-        cy.contains('Fetch statistics').should('be.enabled').click()
+        cy.contains('td', /^TKT10002, 581325, A581325, AYTKT10002$/).click()
        cy.contains('Search for courses').should('not.exist')
        cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
      })
@@ -207,16 +358,6 @@ describe('Course Statistics tests', () => {
        ['Total', null, 511, 486, 25],
      ]

-      const attemptsTableContentsOpen = [
-        // [time, passed, failed, passrate]
-        ['Total', null, 175, 175, 0],
-      ]
-
-      const attemptsTableContentsReqular = [
-        // [time, passed, failed, passrate]
-        ['Total', null, 312, 298, 14],
-      ]
-
      const gradesTableContents = [
        // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
        ['Total', null, 511, 25, 11, 17, 21, 59, 365, 13],
@@ -244,7 +385,6 @@ describe('Course Statistics tests', () => {
        })
        cy.contains('Show population').should('not.be.enabled')

-        cy.contains('#CourseStatPanes a.item', 'Table').click()
        cy.contains('#CourseStatPanes a.item', 'Attempts').click()
        cy.get('#CourseStatPanes table>tbody').within(() => {
          attemptsTableContents.forEach((values, trIndex) => {
@@ -275,44 +415,6 @@ describe('Course Statistics tests', () => {
        })
      })

-      it('Toggling course provider changes stats correctly', () => {
-        cy.get('[data-cy=unify_radio_reqular]').click()
-        cy.get("div[name='toYear']").within(() => {
-          cy.get("div[role='option']").first().should('have.text', '2020-2021')
-          cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
-          cy.get("div[role='option']").last().should('have.text', yearRange.from)
-          cy.get("div[role='option']").should('have.length', 21)
-        })
-        cy.contains('#CourseStatPanes a.item', 'Table').click()
-        cy.contains('#CourseStatPanes a.item', 'Attempts').click()
-        cy.get('#CourseStatPanes table>tbody').within(() => {
-          attemptsTableContentsReqular.forEach((values, trIndex) => {
-            cy.get('tr')
-              .eq(trIndex)
-              .within(() => {
-                values.forEach((value, tdIndex) => {
-                  if (value === null) return
-                  cy.get('td').eq(tdIndex).contains(value)
-                })
-              })
-          })
-        })
-        cy.get('[data-cy=unify_radio_open]').click()
-        cy.get('#CourseStatPanes table>tbody').within(() => {
-          attemptsTableContentsOpen.forEach((values, trIndex) => {
-            cy.get('tr')
-              .eq(trIndex)
-              .within(() => {
-                values.forEach((value, tdIndex) => {
-                  if (value === null) return
-                  cy.get('td').eq(tdIndex).contains(value)
-                })
-              })
-          })
-        })
-        cy.get('[data-cy=unify_radio_unify]').click()
-      })
-
      it('After changing time range shows same stats', () => {
        const newYearRange = { from: '2016-2017', to: '2019-2020' }
        cy.get("div[name='fromYear']")
@@ -346,13 +448,12 @@ describe('Course Statistics tests', () => {
    it('If no data available, unify toggle is disabled', () => {
      cy.get("input[placeholder='Search by a course code']").type('AYCSM14111')
      cy.contains('AYCSM14111').click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
      cy.contains('AYCSM14111 Avoin yo: Full Stack -websovelluskehitys: React Native')
-      cy.get('[data-cy=unify_radio_reqular]').find('input').should('be.disabled')
+      cy.get('[data-cy=unify_radio_regular]').find('input').should('be.disabled')
      cy.get('[data-cy=unify_radio_unify]').find('input').should('not.be.disabled')
    })

-    it('Has right to see all the sudents, because course provider is TKT', () => {
+    it('Has right to see all the students, because course provider is TKT', () => {
      cy.visit('coursestatistics?courseCodes=%5B%22TKT10004%22%5D&cs_tab=0&separate=false')
      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
      cy.contains('Students (154)').click()
@@ -360,24 +461,21 @@ describe('Course Statistics tests', () => {
    })
  })

-  it.skip('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
+  // ! This test does not pass locally but it does pass on GitHub Actions (see issue #4317)
+  it('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
    cy.init('/coursestatistics', 'onlycoursestatistics')
    cy.get('[data-cy=navbar-courseStatistics]').click()
    cy.get('[data-cy=course-code-input]').type('TKT20003')
    cy.contains('tr', 'TKT20003').click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
    cy.contains('Filter statistics by study programmes').should('not.exist')
    cy.contains('Faculty statistics').should('not.exist')
    cy.contains('Show population').should('not.exist')
-
-    cy.cs('viewMode-Attempts').click()
+    cy.contains('.tabular.menu a', 'Attempts').click()

    const attemptsTableContents = [
-      // [time, passed, failed, passrate]
-      ['Total', 288, 213, 75, '73.96 %'], //Somehow cypress in CI can't find the number even if it is present in the artifact video
-      ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
-      // ['Total', null, 295, 216, 79, '73.22 %'],
-      // ['2020-2021', null, 5, 2, 3, '40.00 %'],
+      // [time, passed, failed, passrate, total enrollments, enrolled, rejected, aborted]
+      ['Total', null, 288, 213, 75, '73.96 %', 0, 0, 0, 0],
+      ['2020-2021', null, '5 or less students', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA'],
      ['2019-2020', null, 164, 121, 43, '73.78 %'],
      ['2018-2019', null, 85, 60, 25, '70.59 %'],
      ['2017-2018', null, 39, 32, 7, '82.05 %'],
@@ -41,7 +41,7 @@ const deleteAllSearches = () => {
 }

 const fillForm = content => {
-  cy.contains('Insert studentnumbers you wish to use for population here')
+  cy.contains('Insert student numbers you wish to use for population here')
    .siblings()
    .get('textarea')
    .type(content.join('\n'))
@@ -160,7 +160,7 @@ describe('Custom population tests', () => {
      })
    })

-    it('Updates a custom population search', () => {
+    it('Updates a custom population search', { retries: 2 }, () => {
      cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
        const students1 = studentNumbersForCSStudentsSet1
        const students2 = studentNumbersForCSStudentsSet2
@@ -96,10 +96,6 @@ describe('Faculty overview', () => {
    it('Study programme credit information can be toggled', () => {
      cy.get('[data-cy="Table-CreditsProducedByTheFaculty"]').should('be.visible')

-      // I tried to fix this without cy.wait() method without effort. Sorting the programmes
-      // or something else causes rerendering of the view and yield to failing tests.
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
      cy.get('[data-cy="Button-Show-CreditsProducedByTheFaculty-0"]').click()
      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
@@ -125,8 +121,6 @@ describe('Faculty overview', () => {

    it('Graph stays open when sorted', () => {
      cy.get('[data-cy="Table-CreditsProducedByTheFaculty"]').should('be.visible')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
      cy.get('[data-cy="Button-Show-CreditsProducedByTheFaculty-0"]').click()
      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
      cy.get('[data-cy="Menu-CreditsProducedByTheFaculty-Total"]').click()
@@ -180,19 +174,20 @@ describe('Faculty overview', () => {

      cy.get('[data-cy="Section-master"]').within(() => {
        cy.get('div[class="faculty-graph"]')
-        cy.contains('1 graduated').should('have.length', 1)
-        cy.contains('1 graduated').trigger('mouseover')
-        cy.contains('1 students graduated in year 2020')
-        cy.contains('median study time: 25 months')
+        cy.contains('24 graduated').should('have.length', 1)
+        cy.contains('24 graduated').trigger('mouseover')
+        cy.contains('24 students graduated in year 2020')
+        cy.contains('median study time: 22 months')
        cy.contains('0 graduated over year late')
-        cy.contains('1 graduated max year overtime')
+        cy.contains('20 graduated on time')
+        cy.contains('4 graduated max year overtime')

-        cy.contains('1 graduated').click()
+        cy.contains('24 graduated').click()
        cy.contains('Year 2020 by graduation year')
        cy.get('div[class="programmes-graph"]').should('be.visible')
        cy.get('div[class="programmes-graph"]').within(() => {
          cy.contains('EDUM')
-          cy.contains('1 graduated').trigger('mouseover')
+          cy.contains('24 graduated').trigger('mouseover')
          cy.contains('Kasvatustieteiden maisteriohjelma')
          cy.contains('MH60_001')
        })
@@ -207,18 +202,21 @@ describe('Faculty overview', () => {

      cy.get('[data-cy="Section-master"]').within(() => {
        cy.get('div[class="faculty-graph"]')
-        cy.contains('1 graduated (100 % of class)').should('have.length', 1)
-        cy.contains('1 graduated').trigger('mouseover')
-        cy.contains('From class of 2018, 1/undefined students have graduated')
+        cy.contains('19 graduated (44.2 % of class)').should('have.length', 1)
+        cy.contains('19 graduated').trigger('mouseover')
+        cy.contains('From class of 2018, 19/43 students have graduated')
      })

      cy.get('[data-cy="GraduationTimeToggle"]').click()
      cy.get('[data-cy="Section-master"]').within(() => {
-        cy.contains('1').click()
+        cy.contains('16').click()
        cy.get('div[class="programmes-breakdown-graph"]').should('be.visible')
        cy.get('div[class="programmes-breakdown-graph"]').within(() => {
          cy.contains('Year 2018 by start year')
          cy.contains('EDUM')
+          cy.contains('16').trigger('mouseover')
+          cy.contains('Kasvatustieteiden maisteriohjelma')
+          cy.contains('MH60_001')
        })
      })
    })
@@ -269,23 +267,22 @@ describe('Faculty overview', () => {
    })

    it('Progress tables can be toggled', () => {
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-6"]').should('not.be.visible')
-      cy.get('[data-cy="Button-Show-FacultyBachelorsProgressTable-6"]').click()
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-0"]').should('not.exist')
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-1"]').should('not.exist')
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-2"]').should('not.exist')
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-3"]').should('not.exist')
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-4"]').should('not.exist')
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-6"]').should('be.visible')
-      cy.get('[data-cy="Button-Hide-FacultyBachelorsProgressTable-6"]').click()
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-6"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-5"]').should('not.be.visible')
+      cy.get('[data-cy="Button-Show-FacultyBachelorsProgressTable-5"]').click()
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-0"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-1"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-2"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-3"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-5"]').should('be.visible')
+      cy.get('[data-cy="Button-Hide-FacultyBachelorsProgressTable-5"]').click()
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-5"]').should('not.be.visible')
    })

-    it('Sudent statstable can be toggled', () => {
+    it('Studentstats table can be toggled', () => {
      cy.get('[data-cy="FacultyStudentStatsTable"]').should('not.contain', 'TKT')
-      cy.get('[data-cy="Button-FacultyStudentStatsTable-3"]').click()
+      cy.get('[data-cy="Button-FacultyStudentStatsTable-6"]').click()
      cy.get('[data-cy="FacultyStudentStatsTable"]').should('contain', 'TKT')
-      cy.get('[data-cy="Button-FacultyStudentStatsTable-3"]').click()
+      cy.get('[data-cy="Button-FacultyStudentStatsTable-6"]').click()
      cy.get('[data-cy="FacultyStudentStatsTable"]').should('not.contain', 'TKT')
    })

@@ -1,8 +1,6 @@
 /* eslint-disable cypress/no-unnecessary-waiting */
 /// <reference types="Cypress" />

-// const baseUrl = Cypress.config().baseUrl
-
 const MOCKED_DATE = new Date(2022, 6, 12).getTime()

 const checkFilteringResult = studentCount => {
@@ -68,13 +66,58 @@ const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
  }
 }

+describe('Population Statistics with CS master', () => {
+  const pathToCSMast2019 =
+    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22MH50_009%22}&tag&year=2019'
+  const defaultAmountOfStudents = 27
+  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
+  beforeEach(() => {
+    cy.init(pathToCSMast2019)
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  describe('Studyright Type filter', () => {
+    it('is visible', () => {
+      cy.contains('Studyright Type')
+    })
+
+    it('is set to all by default', () => {
+      runTestStepWithPreAndPostParts('studyright-type', () => {
+        const card = cy.cs('studyright-type-filter-card')
+        card.get('[data-cy="all"] input').should('be.checked')
+      })
+    })
+
+    it('works', () => {
+      runTestStepWithPreAndPostParts('studyright-type', () => {
+        const expectedBaMaStudents = 27
+        const expectedMasterOnlyStudents = 0
+        cy.cs('studyright-type-filter-card').cs('bama').click()
+        checkFilteringResult(expectedBaMaStudents)
+        cy.cs('studyright-type-filter-card').cs('master').click()
+        checkFilteringResult(expectedMasterOnlyStudents)
+        cy.cs('studyright-type-filter-card').cs('all').click()
+        checkFilteringResult(defaultAmountOfStudents)
+      })
+    })
+  })
+})
+
 describe('Population Statistics', () => {
  const pathToCSBach2018 =
    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2018'
  const defaultAmountOfStudents = 159
  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)

-  before(() => {
+  beforeEach(() => {
    cy.init(pathToCSBach2018)
  })

@@ -87,19 +130,7 @@ describe('Population Statistics', () => {
    })
  })

-  // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
-  // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
-  // So fix this
-  it.skip('Study track filter works', () => {
-    runTestStepWithPreAndPostParts('StudyTrack', () => {
-      const card = cy.cs('StudyTrack-filter-card')
-      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
-      checkFilteringResult(1)
-      programmeDropdown.get('i.delete').click()
-    })
-  })
-
-  it('Graduation filter works', () => {
+  it('Graduation filter works', { retries: 2 }, () => {
    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
      const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')

@@ -127,7 +158,7 @@ describe('Population Statistics', () => {
    runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
      cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(5)
-      checkFilteringResult(147)
+      checkFilteringResult(148)
      clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
    })
  })
@@ -136,7 +167,6 @@ describe('Population Statistics', () => {
    runTestStepWithPreAndPostParts('CreditsEarned', () => {
      testRangeFilter('CreditsEarned-filter-card', 50, 150, 118)
    })
-    cy.cs('reset-all-filters').click()
  })

  it('Age filter works', () => {
@@ -147,7 +177,6 @@ describe('Population Statistics', () => {
  })

  it('Gender filter works', () => {
-    cy.cs('reset-all-filters').click()
    runTestStepWithPreAndPostParts('Gender', () => {
      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
      checkFilteringResult(42)
@@ -166,8 +195,8 @@ describe('Population Statistics', () => {
      cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
      checkFilteringResult(1)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.cs('startYearAtUni-dropdown').selectFromDropdown(7)
-      checkFilteringResult(152)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(14)
+      checkFilteringResult(136)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
    })
  })
@@ -183,21 +212,20 @@ describe('Population Statistics', () => {
    })
  })

-  // plz fix
-  it.skip('Courses filter works', () => {
-    // courses takes some time to load, wait for it to complete
-    cy.wait(10000)
+  it('Courses filter works', () => {
    runTestStepWithPreAndPostParts('Courses', () => {
-      cy.cs('courseFilter-course-dropdown').click()
-      const courses = ['TKT20001', 'MAT11002']
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
-      checkFilteringResult(140)
+      const courses = ['TKT20002', 'MAT11002']
+      cy.cs('courseFilter-course-dropdown').click().contains(`${courses[0]} - Ohjelmistotekniikka`).click()
+      checkFilteringResult(133)
      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
-      checkFilteringResult(131)
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
-      checkFilteringResult(56)
+      checkFilteringResult(127)
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(`${courses[1]} - Lineaarialgebra ja matriisilaskenta I`)
+        .click()
+      checkFilteringResult(100)
      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
-      checkFilteringResult(2)
+      checkFilteringResult(1)
      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
    })
  })
@@ -213,6 +241,21 @@ describe('Population Statistics', () => {
      })
    })
  })
+
+  it('"Reset All Filters" button works', () => {
+    cy.cs('Gender-header').click()
+    cy.cs('genderFilter-dropdown').selectFromDropdown(0)
+    checkFilteringResult(42)
+    cy.cs('StartYearAtUni-header').click()
+    cy.cs('startYearAtUni-dropdown').selectFromDropdown(7)
+    checkFilteringResult(33)
+    cy.cs('reset-all-filters').click()
+    checkFilteringResult(defaultAmountOfStudents)
+  })
+
+  it('Studyright Type filter is not visible', () => {
+    cy.contains('Studyright Type').should('not.exist')
+  })
 })

 describe('Course Statistics', () => {
@@ -221,7 +264,7 @@ describe('Course Statistics', () => {
  const defaultAmountOfStudents = 118
  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)

-  before(() => {
+  beforeEach(() => {
    cy.init(pathToDSAndAlgoSpring2019)
  })

@@ -251,19 +294,7 @@ describe('Course Statistics', () => {
    })
  })

-  // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
-  // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
-  // So fix this
-  it.skip('Study track filter works', () => {
-    runTestStepWithPreAndPostParts('StudyTrack', () => {
-      const card = cy.cs('StudyTrack-filter-card')
-      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
-      checkFilteringResult(1)
-      programmeDropdown.get('i.delete').click()
-    })
-  })
-  // FIX THIS SOON!
-  it.skip('Age filter works', () => {
+  it('Age filter works', () => {
    cy.clock(MOCKED_DATE, ['Date'])
    runTestStepWithPreAndPostParts('Age', () => {
      testRangeFilter('Age-filter-card', 20, 40, 33)
@@ -271,7 +302,6 @@ describe('Course Statistics', () => {
  })

  it('Gender filter works', () => {
-    cy.cs('reset-all-filters').click()
    runTestStepWithPreAndPostParts('Gender', () => {
      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
      checkFilteringResult(27)
@@ -290,8 +320,8 @@ describe('Course Statistics', () => {
      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
      checkFilteringResult(1)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.cs('startYearAtUni-dropdown').selectFromDropdown(3)
-      checkFilteringResult(84)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(13)
+      checkFilteringResult(71)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
    })
  })
@@ -311,14 +341,13 @@ describe('Course Statistics', () => {
 describe('Custom Population Statistics', () => {
  let runTestStepWithPreAndPostParts

-  before(() => {
+  beforeEach(() => {
    cy.init('/custompopulation')
    cy.cs('custom-pop-search-button').click()
    cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
      const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
      runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
      cy.cs('student-no-input').click().type(students.join('\n'))
-      cy.wait(10000)
    })
    cy.cs('search-button').click()
  })
@@ -328,7 +357,6 @@ describe('Custom Population Statistics', () => {
    runTestStepWithPreAndPostParts('Age', () => {
      testRangeFilter('Age-filter-card', 20, 30, 1)
    })
-    cy.cs('reset-all-filters').click()
  })

  it('Programme defaults to "Active Study Right" mode', () => {
@@ -368,22 +396,18 @@ describe('Custom Population Statistics', () => {
    })
  })

-  it.skip('Courses filter works', () => {
+  it('Courses filter works', () => {
    runTestStepWithPreAndPostParts('Courses', () => {
      const courses = ['MAT11001', 'TKT20004']
-      cy.cs('courseFilter-course-dropdown')
-        .click()
-        .contains(courses[0] + ' ')
-        .click()
+      cy.cs('courseFilter-course-dropdown').click().contains(`${courses[0]} - Johdatus yliopistomatematiikkaan`).click()
      checkFilteringResult(5)
      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
      checkFilteringResult(5)
-      cy.cs('courseFilter-course-dropdown')
-        .click()
-        .contains(courses[1] + ' ')
-        .click()
+      cy.cs('courseFilter-course-dropdown').click().contains(`${courses[1]} - Tietoliikenteen perusteet`).click()
+      checkFilteringResult(5)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(1)
      checkFilteringResult(3)
-      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(3)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
      checkFilteringResult(1)
      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
    })
@@ -0,0 +1,65 @@
+/// <reference types="Cypress" />
+
+const hasLanded = () => {
+  cy.contains('This view displays amounts')
+  cy.contains('Show number of')
+}
+
+/*
+  TODO FIX
+  These use dumb numbers, because actual_studyright update in db is not yet
+  in anon data, and the faculties do not get resolved.
+*/
+
+describe('When language center is opened', () => {
+  describe('as an admin user', () => {
+    beforeEach(() => {
+      cy.init('/languagecenterview', 'admin')
+      cy.get('[data-cy="completions-button"]').click()
+      cy.get('[data-cy="semester-from"]').click()
+      cy.contains('Syksy 2017').click()
+      cy.get('[data-cy="semester-to"]').click()
+      hasLanded()
+    })
+
+    it('Initial view is correct', () => {
+      cy.contains('All courses total')
+      cy.contains('Academic and Professional')
+      cy.contains('AYKK-RUKIRJ')
+    })
+
+    it('Faculties tab shows numbers', () => {
+      cy.contains('All courses total')
+      cy.contains('1363')
+    })
+
+    it('Faculties tab "exceeding" button works', () => {
+      cy.get('[data-cy="difference-button"]').click()
+      cy.contains('0')
+      cy.contains('Total')
+      cy.contains('1363').should('not.exist')
+    })
+
+    it('Faculties tab semester selector changes numbers', () => {
+      cy.get('[data-cy="semester-from"]').click()
+      cy.contains('Syksy 2020').click()
+      cy.contains('1363').should('not.exist')
+      cy.contains('All courses total')
+    })
+
+    it('Semester tab opens and contains coloring filter', () => {
+      cy.contains('By semesters').click()
+      cy.contains('1363')
+      cy.contains('Coloring mode')
+    })
+  })
+  describe('with a user with no rights', () => {
+    it('"Access denied" is shown', () => {
+      cy.init('/languagecenterview', 'norights')
+      cy.contains('Access denied')
+      cy.contains("You're currently not allowed to see this page.")
+      cy.contains('Custom populations').click()
+      cy.contains('Language center view').should('not.exist')
+    })
+  })
+})
@@ -67,21 +67,18 @@ describe('Population Statistics tests', () => {
    it('Population statistics is usable on general level', () => {
      cy.visit(pathToCSBach2017)
      cy.cs('filtered-students')
-      cy.get('.header').within(() => {
-        cy.contains('Tietojenkäsittelytieteen kandiohjelma')
-        cy.contains('class size 170 students')
-      })
-      cy.get('.card').within(() => {
-        cy.contains('Excludes exchange students')
-        cy.contains('Excludes students with non-degree study right')
-        cy.contains('Excludes students who have transferred out of this programme')
-      })
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('class size 170 students')
+      cy.contains('Excludes exchange students')
+      cy.contains('Excludes students with non-degree study right')
+      cy.contains('Excludes students who have transferred out of this programme')
      cy.cs('filtered-students')
-      cy.contains('Courses of class').click()
+    })

+    it('Courses of class is displayed and link to individual course stats page works', () => {
+      cy.visit(pathToCSBach2017)
+      cy.contains('Courses of class').click()
      cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(25000) // a bit hacky way, wait until ui is ready
      cy.get('[data-cy=toggle-group-module-TKT1]').click()
      cy.contains('td', 'TKT10002').siblings().find('i.level.up').click()
      cy.wait('@coursePage')
@@ -89,9 +86,22 @@ describe('Population Statistics tests', () => {
      cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
    })

+    it('Courses of class curriculum selection works', () => {
+      cy.visit(pathToCSBach2017)
+      cy.contains('Courses of class').click()
+      cy.get('Opiskelijan digitaidot - Kumpula').should('not.exist')
+      cy.get('[data-cy=curriculum-picker]').click()
+      cy.contains('2020 - 2022').click()
+      cy.contains('Opiskelijan digitaidot - Kumpula')
+    })
+
    it("Empty 'tags' tab has a link to the page where tags can be created", () => {
      cy.visit(pathToCSBach2017)
-      cy.contains('Students (170)').click()
+      cy.contains('Students (170)')
+        .parent()
+        .then($parentDiv => {
+          if (!$parentDiv.hasClass('active')) cy.contains('Students (170)').click()
+        })
      cy.get('[data-cy=student-table-tabs]').contains('Tags').click()
      cy.contains('No tags defined. You can define them here.').find('a').click()
      cy.contains('Tags').click()
@@ -99,48 +109,46 @@ describe('Population Statistics tests', () => {
      cy.contains('Create new tag')
    })

-    // TODO: this fails in CI, but not locally. Investigate and fix
-    it.skip('Advanced settings work', () => {
+    // This test sometimes fails on headless mode. It seems that the click on the
+    // 'Fetch class with new settings' button doesn't always trigger history.push()
+    // so the page doesn't reload. This is why waiting also doesn't help.
+    it('Advanced settings work', { retries: 2 }, () => {
      cy.visit(pathToCSBach2017)
      cy.get('[data-cy=advanced-toggle]').click()
      // only spring
      cy.cs('toggle-fall').click()
      cy.contains('Fetch class').click()
-
-      // moar waiting hack
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
-
-      cy.contains('No statistics found for the given query.')
-
+      cy.contains('Students (0)')
      // only fall
      cy.get('[data-cy=advanced-toggle]').click()
      cy.cs('toggle-fall').click()
      cy.cs('toggle-spring').click()
-
      cy.contains('Fetch class').click()
-
      cy.contains('Credit accumulation (for 170 students)')
-
      // spring + fall
      cy.get('[data-cy=advanced-toggle]').click()
      cy.cs('toggle-spring').click()
      cy.contains('Fetch class').click()
-
      cy.contains('Credit accumulation (for 170 students)')
    })

    it('Credit Statistics, Credits Gained tab works', () => {
      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
-      cy.contains('Credit statistics').click()
+      cy.contains('Credit statistics')
+        .parent()
+        .then($parentDiv => {
+          if (!$parentDiv.hasClass('active')) cy.contains('Credit statistics').click()
+        })
      cy.contains('Credits Gained').click()
      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the class')
      const today = moment().endOf('month')
      const months = Math.round(moment.duration(moment(today).diff(moment('2017-08-01'))).asMonths())
-      const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
+      // Months are capped to goal time of the degree, 36 months is the goal time for bachelor's degree
+      const monthsForLimits = Math.min(36, months)
+      const limits = [1, ..._.range(1, 5).map(p => Math.ceil(monthsForLimits * ((p * 15) / 12))), null]
      const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))

-      cy.get('.credits-gained-table').should('contain', '(n=170)')
+      cy.get('.credits-gained-table').should('contain', '(n = 170)')

      for (const [start, end] of ranges) {
        let value = 'credits'
@@ -158,7 +166,6 @@ describe('Population Statistics tests', () => {

      cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
        const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)
-
        expect(sum).to.equal(170)
      })

@@ -169,7 +176,11 @@ describe('Population Statistics tests', () => {
      cy.visit(pathToCSMaster2019)
      cy.cs('filtered-students')
      cy.contains('Credit statistics')
-      cy.get("[data-cy='credit-statistics']").click()
+      cy.get("[data-cy='Credit statistics']")
+        .parent()
+        .then($parentDiv => {
+          if (!$parentDiv.hasClass('active')) cy.get("[data-cy='Credit statistics']").click()
+        })
      cy.contains('Credits Gained').click()
      cy.get('.credits-gained-divider').click()
      cy.get("[data-cy='credits-gained-table-Avoin väylä']").should('exist')
@@ -180,7 +191,11 @@ describe('Population Statistics tests', () => {

    it('Credit Statistics, Statistics pane works', () => {
      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
-      cy.contains('Credit statistics').click({ force: true })
+      cy.contains('Credit statistics')
+        .parent()
+        .then($parentDiv => {
+          if (!$parentDiv.hasClass('active')) cy.contains('Credit statistics').click()
+        })
      cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()

      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 170 Students')
@@ -199,7 +214,11 @@ describe('Population Statistics tests', () => {
      const existing = '010113437'
      const nonExisting = '66666666'
      cy.visit(pathToCSBach2017)
-      cy.contains('Students (170)').click()
+      cy.contains('Students (170)')
+        .parent()
+        .then($parentDiv => {
+          if (!$parentDiv.hasClass('active')) cy.contains('Students (170)').click()
+        })
      cy.contains(existing)
      cy.contains(nonExisting).should('not.exist')
      cy.contains('button', 'Check studentnumbers').click()
@@ -217,17 +236,14 @@ describe('Population Statistics tests', () => {
    })
  })

-  /*
  describe('when using IAM user', () => {
    beforeEach(() => {
      cy.init(pathToCSBach2017, 'onlyiamrights')
-    })
-    it('Population statistics is visible', () => {
-      cy.reload()
-
      cy.contains('Tietojenkäsittelytieteen kandiohjelma')
      cy.contains('class size 170 students')
+    })

+    it('Population statistics is visible', () => {
      cy.get('.card').within(() => {
        cy.contains('Excludes exchange students')
        cy.contains('Excludes students with non-degree study right')
@@ -238,10 +254,8 @@ describe('Population Statistics tests', () => {
      cy.contains('Credit statistics')
    })

-    it.skip('Students tab is not available', () => {
+    it('Students tab is not available', () => {
      cy.contains('Students (170)').should('not.exist')
-      // TODO: somehow check that obfuscation works
    })
  })
-  */
 })
@@ -39,7 +39,6 @@ describe('Student Statistics tests', () => {
    cy.url().should('include', '/students')
    cy.get('.prompt').type(student.studentnumber)
    cy.contains('td a', student.studentnumber).click()
-    cy.contains('Started: 01.08.2017')
    cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 31.07.2025')
    cy.contains(student.lastname).should('not.exist')
    cy.contains(student.firstnames).should('not.exist')
@@ -66,6 +65,7 @@ describe('Student Statistics tests', () => {
    cy.contains('td a', student.studentnumber).click()
    cy.contains('Introduction to Machine Learning (DATA11002)')
      .siblings()
+      .last()
      .within(() => {
        cy.get('.level').click()
      })
@@ -78,7 +78,7 @@ describe('Student Statistics tests', () => {
    cy.contains('td a', student.studentnumber).click()
    cy.get('[data-cy=sisulink] > a')
      .should('have.attr', 'href')
-      .and('include', 'https://sis-helsinki.funidata.fi/tutor/role/staff/student/hy-hlo-84830887/basic/basic-info')
+      .and('include', 'https://sisu.helsinki.fi/tutor/role/staff/student/hy-hlo-84830887/basic/basic-info')
  })

  it('Searching with bad inputs doesnt yield results', () => {
@@ -103,7 +103,6 @@ describe('Testing with admin rights', () => {
  it('Does not crash if student has no studyright or courses', () => {
    const studentNumber = '011825096'
    cy.init(`/students/${studentNumber}`, 'admin') // use admin to see all students
-    cy.contains('Started: Unavailable')
    cy.contains('Credits: 0')
  })
 })
@@ -0,0 +1,27 @@
+/// <reference types="Cypress" />
+
+describe('Student Statistics tests', () => {
+  beforeEach(() => {
+    cy.init('/studyguidancegroups', 'admin')
+    cy.intercept('/api/studyguidancegroups', { fixture: 'studyGuidanceGroups.json' })
+  })
+
+  it('The overview page shows the correct data', () => {
+    cy.contains('Study guidance groups')
+
+    const attemptsTableContents = [
+      ['TKT kandit 2018', 3, 'Tietojenkäsittelytieteen kandiohjelma', '2018 - 2019'],
+      ['Oma ohjausryhmä', 2, 'Add study programme', 'Add year'],
+    ]
+
+    cy.checkTableStats(attemptsTableContents, 'study-guidance-group-overview')
+  })
+
+  it('The single group page shows the correct data', () => {
+    cy.cs('study-guidance-group-link-sgg-cypress-1').click()
+    cy.contains('TKT kandit 2018')
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+    cy.contains('2018 - 2019')
+    cy.contains('Credit accumulation (for 3 students)')
+  })
+})
@@ -8,9 +8,11 @@ const deleteTag = name => {
 }

 const getEmptyYears = isAcademicYear => {
-  const thisYear = new Date().getFullYear()
+  const today = new Date()
+  const latestYear = isAcademicYear && today.getMonth() < 7 ? today.getFullYear() - 1 : today.getFullYear()
+
  const years = []
-  for (let year = thisYear; year > 2021; year--) {
+  for (let year = latestYear; year > 2021; year--) {
    if (isAcademicYear) {
      years.push(`${year} - ${year + 1}`)
    } else {
@@ -25,10 +27,7 @@ describe('Studyprogramme overview', () => {
  describe('Basic information -view works for basic user', () => {
    beforeEach(() => {
      cy.init('/study-programme')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click({ force: true })
-      // cy.viewport(1536, 960)
    })

    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
@@ -46,8 +45,8 @@ describe('Studyprogramme overview', () => {
        // [Year, Started, Graduated, Transferred Away, Transferred to]
        ...years.map(year => [year, 0, 0, 0, 0]),
        [2021, 0, 0, 1, 0],
-        [2020, 12, 35, 3, 2],
-        [2019, 87, 17, 0, 1],
+        [2020, 10, 35, 2, 2],
+        [2019, 86, 17, 1, 1],
        [2018, 161, 1, 0, 0],
        [2017, 171, 0, 0, 0],
      ]
@@ -62,8 +61,8 @@ describe('Studyprogramme overview', () => {
        ...years.map(year => [year, 0, 0, 0, 0, 0]),
        [2021, 0, 0, 0, 0, 0],
        [2020, 5860, 5591, 102, 0, 167],
-        [2019, 8696, 7737, 577, 33, 349],
-        [2018, 7818, 6584, 852, 110, 272],
+        [2019, 8696, 7737, 587, 23, 349],
+        [2018, 7818, 6584, 837, 125, 272],
        [2017, 3085, 2492, 153, 52, 388],
      ]

@@ -85,17 +84,17 @@ describe('Studyprogramme overview', () => {
      cy.checkTableStats(tableContents, 'GraduatedAndThesisWritersOfTheProgramme')
    })

-    it.skip('Special studyrights can be excluded and basic data changes accordingly', () => {
+    it('Special studyrights can be excluded and basic data changes accordingly', () => {
      cy.get('[data-cy=StudentToggle]').click()
      const years = getEmptyYears()
      const studentTableContents = [
        // [Year, Started, Graduated]
        ...years.map(year => [year, 0, 0]),
        [2021, 0, 0],
-        [2020, 22, 0],
-        [2019, 12, 0],
-        [2018, 0, 0],
-        [2017, 0, 0],
+        [2020, 10, 35],
+        [2019, 86, 17],
+        [2018, 161, 1],
+        [2017, 171, 0],
      ]

      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
@@ -104,10 +103,10 @@ describe('Studyprogramme overview', () => {
        // [Year, Total, Major students credits, Transferred credits]
        ...years.map(year => [year, 0, 0, 0]),
        [2021, 0, 0, 0],
-        [2020, 117.5, 0, 7.5],
-        [2019, 67, 0, 0],
-        [2018, 0, 0, 0],
-        [2017, 0, 0, 0],
+        [2020, 4999, 4675, 167],
+        [2019, 6182, 5699, 349],
+        [2018, 4884, 4537, 272],
+        [2017, 1988, 1600, 388],
      ]

      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
@@ -116,9 +115,9 @@ describe('Studyprogramme overview', () => {
        // [Year, Graduated, Wrote thesis]
        ...years.map(year => [year, 0, 0]),
        [2021, 0, 0],
-        [2020, 0, 0],
-        [2019, 0, 0],
-        [2018, 0, 0],
+        [2020, 35, 35],
+        [2019, 17, 25],
+        [2018, 1, 5],
        [2017, 0, 0],
      ]

@@ -127,7 +126,6 @@ describe('Studyprogramme overview', () => {
    })

    it('Year can be changed to academic year, and data changes accordingly', () => {
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
      cy.get('[data-cy=YearToggle]').click({ force: true })
      const isAcademicYear = true
      const years = getEmptyYears(isAcademicYear)
@@ -135,27 +133,24 @@ describe('Studyprogramme overview', () => {
        // [Year, Started, Graduated, Transferred away, Transferred to]
        ...years.map(year => [year, 0, 0, 0, 0]),
        ['2021 - 2022', 0, 0, 0, 0],
-        ['2020 - 2021', 12, 6, 3, 2],
-        ['2019 - 2020', 86, 41, 1, 0],
-        ['2018 - 2019', 162, 5, 0, 1],
+        ['2020 - 2021', 10, 6, 3, 2],
+        ['2019 - 2020', 86, 38, 1, 0],
+        ['2018 - 2019', 161, 8, 0, 1],
        ['2017 - 2018', 171, 1, 0, 0],
      ]
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')

      const creditTableContents = [
        // [Year, Total, Major students credits, Non-major students credits, Non-degree credits, Transferred credits]
        ...years.map(year => [year, 0, 0, 0, 0, 0]),
        ['2021 - 2022', 0, 0, 0, 0, 0],
-        ['2020 - 2021', 566, 486, 20, 0, 60],
-        ['2019 - 2020', 8595, 8271, 106, 0, 218],
-        ['2018 - 2019', 9992, 8855, 781, 53, 303],
-        ['2017 - 2018', 6055, 4792, 777, 142, 344],
+        ['2020 - 2021', 541, 471, 10, 0, 60],
+        ['2019 - 2020', 8575, 8266, 116, 0, 193],
+        ['2018 - 2019', 10013, 8851, 791, 43, 328],
+        ['2017 - 2018', 6079, 4816, 762, 157, 344],
      ]

      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
-      //cy.wait(10000)
      cy.get('[data-cy=YearToggle]').click()
    })

@@ -169,8 +164,8 @@ describe('Studyprogramme overview', () => {
        .should('contain', 'Major students credits')
        .should('contain', 'Non-major students credits')
        .should('contain', 'Transferred credits')
-        .should('contain', 852)
-        .should('contain', 577)
+        .should('contain', 7737)
+        .should('contain', 6584)

      cy.get('[data-cy=Graph-GraduatedAndThesisWritersOfTheProgramme')
        .should('contain', 'Graduated students')
@@ -181,9 +176,9 @@ describe('Studyprogramme overview', () => {
        .should('contain', 25)
        .should('contain', 35)

-      cy.get('[data-cy=graduation-times-graph-breakdown]')
+      cy.get('[data-cy=graduation-times-graph-breakdownBachelor]')
      cy.get('[data-cy=GraduationTimeToggle]').click()
-      cy.get('[data-cy=graduation-times-graph]').within(() => {
+      cy.get('[data-cy=graduation-times-graphBachelor]').within(() => {
        cy.contains('Graduation year')
        cy.contains('2019')
        cy.contains('17 graduated').trigger('mouseover')
@@ -196,17 +191,15 @@ describe('Studyprogramme overview', () => {
        .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
        .should('contain', 'Datatieteen maisteriohjelma')
        .should('contain', 'Matematiikan ja tilastotieteen maisteriohjelma')
-        .should('contain', 1)
-        .should('contain', 4)
-        .should('contain', 13)
+        .should('contain', 11)
+        .should('contain', 12)
+        .should('contain', 24)
    })
  })

  describe('Graduation times of master programmes', () => {
    it('are split into two graphs', () => {
      cy.init('/study-programme', 'admin')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(5000)
      cy.contains('a', 'Kasvatustieteiden maisteriohjelma').click({ force: true })

      cy.get('[data-cy=graduation-times-graph-breakdownMaster]')
@@ -236,8 +229,6 @@ describe('Studyprogramme overview', () => {
  describe('Studytrack overview works for basic user', () => {
    beforeEach(() => {
      cy.init('/study-programme')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
      cy.get('.attached').contains('Studytracks and class statistics').click()
    })
@@ -251,12 +242,12 @@ describe('Studyprogramme overview', () => {

    it('Students of the studyprogramme are shown correctly', () => {
      const tableContents = [
-        // [Year, All, Started studying, Currently enrolled, Absent, Cancelled, Graduated, Men, Women, Finnish]
-        ['2020 - 2021', 12, 10, 0, 0, 0, 0, 10, 2, 12],
-        ['2019 - 2020', 86, 86, 0, 0, 1, 1, 65, 21, 86],
-        ['2018 - 2019', 162, 161, 0, 0, 0, 11, 118, 44, 161],
-        ['2017 - 2018', 171, 171, 0, 0, 4, 42, 135, 36, 166],
-        ['Total', 431, 428, 0, 0, 5, 54, 328, 103, 425],
+        // [Year, All, Started studying, Currently enrolled, Absent, Inactive, Graduated, Men, Women, other/unknown Finnish, other]
+        ['2020 - 2021', 12, 10, 0, 0, 12, 0, 10, 2, 0, 12, 0],
+        ['2019 - 2020', 86, 86, 0, 0, 85, 1, 65, 21, 0, 86, 0],
+        ['2018 - 2019', 162, 161, 0, 0, 151, 11, 118, 44, 0, 161, 1],
+        ['2017 - 2018', 171, 171, 0, 0, 129, 42, 135, 36, 0, 166, 5],
+        ['Total', 431, 428, 0, 0, 377, 54, 328, 103, 0, 425, 6],
      ]

      cy.checkTableStats(tableContents, 'StudytrackOverview')
@@ -264,11 +255,14 @@ describe('Studyprogramme overview', () => {

    it('Student progress data is shown correctly', () => {
      const tableContents = [
-        ['2020 - 2021', 12, 12, 0, 0, 0, 0, 0, 0, 0],
-        ['2019 - 2020', 86, 11, 15, 35, 20, 5, 0, 0, 0],
-        ['2018 - 2019', 162, 3, 9, 28, 50, 33, 25, 11, 3],
-        ['2017 - 2018', 171, 7, 13, 22, 32, 18, 23, 32, 24],
-        ['Total', 431, 33, 37, 85, 102, 56, 48, 43, 27],
+        ['2023 - 2024', 0, 0, 0, 0, 0, 0, 0, 0],
+        ['2022 - 2023', 0, 0, 0, 0, 0, 0, 0, 0],
+        ['2021 - 2022', 0, 0, 0, 0, 0, 0, 0, 0],
+        ['2020 - 2021', 12, 12, 0, 0, 0, 0, 0, 0],
+        ['2019 - 2020', 86, 26, 34, 21, 5, 0, 0, 0],
+        ['2018 - 2019', 162, 10, 27, 47, 36, 26, 12, 4],
+        ['2017 - 2018', 171, 20, 21, 26, 22, 22, 33, 27],
+        ['Total', 431, 68, 82, 94, 63, 48, 45, 31],
      ]

      cy.checkTableStats(tableContents, 'StudytrackProgress')
@@ -276,17 +270,15 @@ describe('Studyprogramme overview', () => {

    it('Studytrack overview graphs render', () => {
      cy.get('[data-cy=Graph-StudytrackProgress]')
-        .should('contain', 'Less than 15 credits')
-        .should('contain', '30-59 credits')
-        .should('contain', '180 or more credits')
-        .should('contain', '4.1%') // The percentage for less than 15 credits in 2017-2018, to check that the graph renders
-        .should('contain', '5.6%') // The percentage for 15-29 credits in 2018-2019
-        .should('contain', '12.8%') // The percentage for less than 15 credits in 2019-2020
+        .should('contain', 'Less than 30 credits')
+        .should('contain', '30–60 credits')
+        .should('contain', 'At least 180 credits')
+        .should('contain', '100.0%') // The percentage for less than 15 credits in 2017-2018, to check that the graph renders

-      cy.get('[data-cy=graduation-times-graph-breakdown]')
+      cy.get('[data-cy=graduation-times-graph-breakdownBachelor]')
      cy.get('[data-cy=GraduationTimeToggle]').click()

-      cy.get('[data-cy=graduation-times-graph]').within(() => {
+      cy.get('[data-cy=graduation-times-graphBachelor]').within(() => {
        cy.contains('Start year')
        cy.contains('2019 - 2020')
        cy.contains('42 graduated').trigger('mouseover')
@@ -300,8 +292,6 @@ describe('Studyprogramme overview', () => {
  describe('Programme courses works for basic user', () => {
    beforeEach(() => {
      cy.init('/study-programme')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
      cy.get('.attached').contains('Programme courses').click()
    })
@@ -322,21 +312,12 @@ describe('Studyprogramme overview', () => {
    })

    it('calendar year -> academic year toggle works', () => {
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
      cy.get('[data-cy=fromYear]').click().contains('2018').click()
-
      cy.get('[data-cy=toYear').click().contains('2019').click()
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(20000)
      cy.get('[data-cy=CoursesSortableTable]').within(() => {
        cy.get('tr').eq(1).contains('684')
      })
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(20000)
      cy.get('[data-cy=calendarAcademicYearToggle]').first().click()
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(20000)
      cy.get('[data-cy=CoursesSortableTable]').within(() => {
        cy.get('tr').eq(1).contains('772')
      })
@@ -378,8 +359,6 @@ describe('Studyprogramme overview', () => {
  describe('Basic information -view works for basic user', () => {
    beforeEach(() => {
      cy.init('/study-programme')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
      cy.get('.attached').contains('Tags').click()
    })
@@ -429,8 +408,6 @@ describe('Studyprogramme overview', () => {

      cy.go('back')
      cy.go('back')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
      deleteTag(name)

      cy.contains('Student statistics').click()
@@ -440,7 +417,6 @@ describe('Studyprogramme overview', () => {
    })
  })

-  /*
  describe('IAM user', () => {
    beforeEach(() => {
      cy.init('/study-programme', 'onlyiamrights')
@@ -455,7 +431,7 @@ describe('Studyprogramme overview', () => {
      cy.contains('Studytracks and class statistics')

      cy.contains('Update statistics').should('not.exist')
-      cy.contains('Degree Courses').should('not.exist')
+      cy.contains('Degree courses').should('not.exist')
    })

    it('can access basic information', () => {
@@ -476,5 +452,4 @@ describe('Studyprogramme overview', () => {
      cy.get('[data-cy=Section-AverageGraduationTimesStudytracks]')
    })
  })
-  */
 })
@@ -42,7 +42,6 @@ describe('Users tests', () => {
      cy.cs('user-edit-button-basic').click()
      cy.get('i.spy').click()
      cy.contains('mocking as basic')
-      cy.wait(1000)
      cy.contains('Study programme').click().siblings().contains('Search by class').click()
      cy.contains('label', 'Study programme')
      cy.contains('label', 'Study programme')
@@ -0,0 +1,28 @@
+[
+  {
+    "id": "sgg-cypress-1",
+    "name": {
+      "fi": "TKT kandit 2018"
+    },
+    "members": [
+      { "personStudentNumber": "010450938" },
+      { "personStudentNumber": "010589388" },
+      { "personStudentNumber": "010262566" }
+    ],
+    "tags": {
+      "studyProgramme": "KH50_005",
+      "year": "2018"
+    }
+  },
+  {
+    "id": "sgg-cypress-2",
+    "name": {
+      "fi": "Oma ohjausryhmä"
+    },
+    "members": [{ "personStudentNumber": "010614509" }, { "personStudentNumber": "010307146" }],
+    "tags": {
+      "studyProgramme": null,
+      "year": null
+    }
+  }
+]
@@ -134,7 +134,7 @@ Cypress.Commands.add('selectStudyProgramme', name => {
 })

 Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {
-  cy.get(`[data-cy=Table-${tableName}]>tbody`).within(() => {
+  cy.get(`[data-cy=Table-${tableName}] tbody`).within(() => {
    correctStats.forEach((values, trIndex) => {
      cy.get('tr')
        .eq(trIndex)
@@ -19,12 +19,16 @@ services:
      SECRET_TOKEN: dev
      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-    image: toska/oodikone2-backend:staging
+      IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
+      JAMI_URL: 'http://jami:3003/'
+    build:
+      context: services/backend

  frontend:
    <<: *defaults
    container_name: frontend
-    image: toska/oodikone2-frontend:staging
+    build:
+      context: services/frontend

  kone-db:
    <<: *defaults
@@ -33,20 +37,20 @@ services:
      PGDATA: /data
      POSTGRES_DB: kone-db
      POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry.toska.cs.helsinki.fi/kone-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db

  redis:
    <<: *defaults
-    command: 'redis-server --appendonly yes'
-    container_name: redis
-    image: redis
+    command: >
+      sh -c 'redis-server --appendonly yes && redis-cli flushall'
+    image: redis:6.2.13
    ports:
      - 6379:6379

  sis-db:
    <<: *defaults
    container_name: sis-db
-    image: registry.toska.cs.helsinki.fi/sis-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
    environment:
      PGDATA: /data
      POSTGRES_DB: sis-db
@@ -60,18 +64,16 @@ services:
      PGDATA: /data
      POSTGRES_DB: user-db
      POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry.toska.cs.helsinki.fi/user-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db

  jami:
    <<: *defaults
-    image: toska/jami
+    image: quay.io/toska/jami:production
    environment:
+      - npm_config_cache=/tmp
      - PORT=3003
      - NODE_ENV=production
-      - POSTGRES_HOST=jami-db
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DATABASE=postgres
+      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
    ports:
      - 3003:3003
    container_name: jami
@@ -101,14 +103,13 @@ services:
  sis-importer-db:
    <<: *defaults
    container_name: sis-importer-db
-    image: registry.toska.cs.helsinki.fi/sis-importer-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
    profiles:
      - updater

  sis-updater-worker:
    <<: *defaults
    build:
-      target: development
      context: updater/sis-updater-worker
    depends_on:
      - redis
@@ -116,12 +117,14 @@ services:
      - sis-importer-db
      - sis-updater-nats
    environment:
+      CI: 'true'
      DB_URL: postgres://postgres@sis-db:5432/sis-db
      REDIS_URI: //redis:6379
      SIS_IMPORTER_DATABASE: sis-importer-db
      SIS_IMPORTER_HOST: sis-importer-db
      SIS_IMPORTER_PASSWORD: postgres
      SIS_IMPORTER_USER: postgres
+      SIS_PASSWORD: postgres
      SIS_NATS_TOKEN: dev
      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
    profiles:
@@ -130,7 +133,6 @@ services:
  sis-updater-scheduler:
    <<: *defaults
    build:
-      target: development
      context: updater/sis-updater-scheduler
    container_name: sis-updater-scheduler
    depends_on:
@@ -138,6 +140,7 @@ services:
      - sis-importer-db
      - sis-updater-nats
    environment:
+      CI: 'true'
      REDIS_URI: //redis:6379
      SECRET_TOKEN: dev
      SIS_IMPORTER_DATABASE: sis-importer-db
@@ -4,7 +4,7 @@ x-defaults: &defaults
  restart: unless-stopped

 x-build_defaults: &build_defaults
-  target: development
+  dockerfile: dev.Dockerfile

 services:
  adminer:
@@ -23,7 +23,6 @@ services:
    build:
      <<: *build_defaults
      context: services/backend
-    container_name: backend
    depends_on:
      - kone-db
      - redis
@@ -39,13 +38,15 @@ services:
      SECRET_TOKEN: dev
      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
+      IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
+      JAMI_URL: 'http://jami:3003/'
    ports:
      - 8080:8080
    profiles:
      - oodikone
    volumes:
-      - backend-nodemod:/usr/src/app/node_modules
-      - ./services/backend/:/usr/src/app
+      - backend-nodemod:/opt/app-root/src/node_modules
+      - ./services/backend/:/opt/app-root/src

  frontend:
    <<: *defaults
@@ -58,13 +59,13 @@ services:
    profiles:
      - oodikone
    volumes:
-      - frontend-nodemod:/usr/src/app/node_modules
-      - ./services/frontend/:/usr/src/app
+      - frontend-nodemod:/opt/app-root/src/node_modules
+      - ./services/frontend/:/opt/app-root/src

  kone-db:
    <<: *defaults
    container_name: kone-db
-    image: registry.toska.cs.helsinki.fi/kone-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
    profiles:
      - oodikone
    volumes:
@@ -74,7 +75,7 @@ services:
    <<: *defaults
    command: 'redis-server --appendonly yes'
    container_name: redis
-    image: redis
+    image: redis:6.2.13
    ports:
      - '6379:6379'
    volumes:
@@ -83,19 +84,23 @@ services:
  sis-db:
    <<: *defaults
    container_name: sis-db
-    image: registry.toska.cs.helsinki.fi/sis-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
    volumes:
      - sis-db-data:/data
    shm_size: 2gb
+    ports:
+      - 12345:5432

  sis-importer-db:
    <<: *defaults
    container_name: sis-importer-db
-    image: registry.toska.cs.helsinki.fi/sis-importer-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
    profiles:
      - updater
    volumes:
      - sis-importer-db-data:/data
+    ports:
+      - 12346:5432

  sis-updater-nats:
    <<: *defaults
@@ -131,8 +136,8 @@ services:
    profiles:
      - updater
    volumes:
-      - sis-updater-scheduler-nodemod:/usr/src/app/node_modules
-      - ./updater/sis-updater-scheduler/:/usr/src/app
+      - sis-updater-scheduler-nodemod:/opt/app-root/src/node_modules
+      - ./updater/sis-updater-scheduler/:/opt/app-root/src

  sis-updater-worker:
    <<: *defaults
@@ -151,18 +156,19 @@ services:
      SIS_IMPORTER_HOST: sis-importer-db
      SIS_IMPORTER_PASSWORD: postgres
      SIS_IMPORTER_USER: postgres
+      SIS_PASSWORD: postgres
      SIS_NATS_TOKEN: dev
      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
    profiles:
      - updater
    volumes:
-      - sis-updater-worker-nodemod:/usr/src/app/node_modules
-      - ./updater/sis-updater-worker/:/usr/src/app
+      - sis-updater-worker-nodemod:/opt/app-root/src/node_modules
+      - ./updater/sis-updater-worker/:/opt/app-root/src

  user-db:
    <<: *defaults
    container_name: user-db
-    image: registry.toska.cs.helsinki.fi/user-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
    profiles:
      - oodikone
    volumes:
@@ -170,14 +176,12 @@ services:

  jami:
    <<: *defaults
-    image: toska/jami
+    image: quay.io/toska/jami:production
    environment:
+      - npm_config_cache=/tmp
      - PORT=3003
      - NODE_ENV=dev
-      - POSTGRES_HOST=jami-db
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DATABASE=postgres
+      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
    ports:
      - 3003:3003
    container_name: jami
@@ -0,0 +1,96 @@
+# Combined programmes
+
+At the moment only programmes KH90_001 and MH90_001 have a combined view.
+
+The made changes do not have any effects on faculty or trends views. **Needs manual work** is added into those parts that needs visit, if this functionality is added to other programmes.
+
+Basic rule of thumb is that for the master/licentiate programme I used names `combinedProgramme` and in some rare cases `secondProgramme` as whole or part of the variable names.
+
+## Study programme
+
+**Needs manual work** in the function `getCombinedProgrammeId` in `services/frontend/src/common/index.js` if new programmes is added. For showing the both programme data correctly, the filtering mechanism is changed to add both programmes and it is not caring for which programme user has acual rights or IAMrights. However, the links for the class stats level are hidden in studytrack tables, if the user has no rights for the programme.
+
+- Search by class
+  - **Needs manual work** Combine programmes here
+- Overview
+  - **Needs manual work**
+
+**Redis nightly study programme update**
+
+- Updates automatically combined programme stats. **Needs manual work** here `services/backend/src/services/studyprogrammeHelpers.js` in the constant `combinedStudyprogrammes`. This might needs refactoring if bachelor programme can have many separate master programmes. Then remember to change function `refreshNewOverview` in `services/backend/src/events.js` accordingly to handle lists.
+
+**NOTE:** `combinedStudyprogrammes` is also used to fetch course modules (mandatory courses). If the logic of this constant is changed in `studyProgrammeHelpers.js` -> the needed changes must be done also here `services/backend/src/services/programmeModules.js`. While writing this, the cleverer idea of doing this might have been to send `code` in form `KHxx_xxx+MHxx_xxx` and do the splitting into two codes in the backend as done in Study guidance groups logic. Changes were done here 'services/backend/src/routes/population.js', acual place is near `getFilteredAssociations` function.
+
+**Basic information**
+
+All things listed below should work automatically, if new combinations are done.
+
+- Credits produced byt the studyprogramme
+  - credits are computed for both programmes separately
+- Average graduation times
+  - For bachelor + master/licentiate studyright the time is the whole study time starting from bachelor
+- _Students of the programme_ and _Graduated and thesis writers of the programme_
+  - separate fields for started, graduated and thesis writers
+
+**Studytracks and class statistics**
+
+- _Students of the studyprogramme by starting year_
+  - Graduated field separated for both programmes
+  - All = currently enrolled + absent + inactive + graduated licentiate
+- _Progress of students of the studyprogramme by starting year_
+  - computed for whole bachelor + master/licentiate studyright period
+- _Average graduation times_
+  - bachelor + master/licentiate graph: start is bachelor studyright start
+
+**Programme courses**
+
+- Fetch all courses for both programmes and show them in one table.
+
+**Degree courses**
+
+- Possibility to add criteria for whole bachelor-master study time
+- For eläinlääkis this criteria is saved by bachelor programme code. Other cases may need other approaches.
+  In eläinklääkis this done by adding the criteria for 6 years -> the whole study time criteria is saved with the bachelor programme code. That is, same criteria is found, if bachelor or combined programme view is shown. This approach might need extra thinking for different programmes.
+- Show own tables for both programmes
+  - To fetch data **needs manual work** here `services/backend/src/services/studyprogrammeHelpers.js` in the constant `combinedStudyprogrammes`.
+
+**Tags**
+
+- Possibility to add tags for combined programme.
+
+## Class statistics view
+
+**Credit accumulation**
+
+- Show graduations for both programmes
+
+**Credit statistics**
+
+- Bachelor and bachelor+master combo have own graphs.
+
+**Courses of class**
+
+- Show courses for both programmes
+
+**Students**
+
+- **General tab** - Additional fields for graduation and hops
+- **Courses tab** - show all courses for both programmes. The courses are fetched with same logic as for Degree programmes.
+- **Tags tab** - show tabs related to this combination.
+- **Progress tab** - possibility to have progress for whole bachelor-master study time. **See Degree courses approach** to add criteria. **May need future manual work**, depending on the programme id with which the data is saved on the database.
+
+## Filters
+
+- **Hops (study plan)** - possibility filter out credits separate for both programmes or for both at the same time
+
+- **Graduated from programme** - possibility to filter out students based on graduation either in bachelor's or master's.
+
+- **Studyright status** - active or inactive based on the chosen degree
+
+- **Studytrack filter** (needs programme code) is not touched at the moment, thus **may need future manual work**.
+
+## Study guidance groups
+
+- Backend: student data is fetched by bachelor study programme code in `/v3/populationstatisticsbystudentnumbers`. Separation from code 'KHxx_xxx+MHxx_xxx' is done in backend.
+- Frontend: things are done in the same way as with class stats.
+- **Need manual work**: `./oodikone/services/frontend/src/redux/elementdetails.js` add combined programme codes (both bachelor and master) to `combinedProgrammeCodes` and modify `combinedOptions` list accordinly in `useFilterAndFormattedElementDetails()` function.
@@ -19,7 +19,7 @@ These are actually defined in db table studyright_extents. Tells studyright leve

 5: Rescinded (Seemingly not in use anymore)

-6: Option
+6: Option (Bachelor+master studyright students can add a master programme to their study plan before graduation in bachelor programme: marked as option in Oodikone)

 30: Graduated

@@ -16,8 +16,10 @@
    "both": "./run.sh both anon up --detach",
    "both:real": "./run.sh both real up --detach",
    "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
+    "testupdater": "docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start",
+    "flushredis": "docker exec -it redis redis-cli flushall",
    "prepare": "husky install",
-    "rapodiff": "docker exec backend node /usr/src/app/src/rapodiff.js"
+    "rapodiff": "docker exec oodikone_backend_1 node /opt/app-root/src/src/rapodiff/rapodiff.js"
  },
  "repository": {
    "type": "git",
@@ -31,17 +33,16 @@
  "devDependencies": {
    "@babel/core": "^7.19.3",
    "@babel/eslint-parser": "^7.19.1",
+    "@babel/eslint-plugin": "^7.22.10",
    "@babel/plugin-syntax-jsx": "^7.16.7",
    "concurrently": "^7.4.0",
-    "cypress": "^10.9.0",
+    "cypress": "^12.17.4",
    "eslint": "^8.25.0",
    "eslint-config-airbnb": "^19.0.4",
    "eslint-config-prettier": "^8.5.0",
-    "eslint-plugin-babel": "^5.3.1",
    "eslint-plugin-cypress": "^2.12.1",
    "eslint-plugin-import": "^2.25.4",
    "eslint-plugin-jsx-a11y": "^6.5.1",
-    "eslint-plugin-prefer-object-spread": "^1.2.1",
    "eslint-plugin-prettier": "^4.2.1",
    "eslint-plugin-react": "^7.31.9",
    "eslint-plugin-react-hooks": "^4.3.0",
@@ -83,8 +83,6 @@ parse_profiles
 parse_env
 if [[ "$compose_command" == "" ]]; then
  final_command="docker-compose ${env}"
-elif [[ "$compose_command" == *"down"* ]]; then
-  final_command="docker-compose ${env} ${compose_command}"
 else
  final_command="docker-compose ${env} ${profiles} ${compose_command}"
 fi
@@ -21,10 +21,10 @@ USER_DB_NAME="user-db"
 DATABASES=("$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")

 ## Urls should be in same order as databases as both are iterated through by indexes.
-KONE_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-kone-db.sqz"
-SIS_DB_REAL_DUMP_URL="svm-96.cs.helsinki.fi:/home/updater_user/backups/latest-sis.sqz"
-SIS_IMPORTER_DB_REAL_DUMP_URL="svm-92.cs.helsinki.fi:/home/importer_user/importer-db/backup/importer-db.sqz"
-USER_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-user-db.sqz"
+KONE_DB_REAL_DUMP_URL="svm-116.cs.helsinki.fi:/home/toska_user/most_recent_backup_store/oodikone-kone-db.sql.gz"
+SIS_DB_REAL_DUMP_URL="svm-116.cs.helsinki.fi:/home/toska_user/most_recent_backup_store/oodikone-sis-db.sql.gz"
+SIS_IMPORTER_DB_REAL_DUMP_URL="svm-116.cs.helsinki.fi:/home/toska_user/most_recent_backup_store/importer.sql.gz"
+USER_DB_REAL_DUMP_URL="svm-116.cs.helsinki.fi:/home/toska_user/most_recent_backup_store/oodikone-user-db.sql.gz"
 REAL_DUMP_URLS=("$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")

 # Source utility functions
@@ -56,7 +56,7 @@ retry() {
 download_real_dump() {
  local database=$1
  local pannu_url=$2
-  local dump_destination="$DUMP_DIR/$database.sqz"
+  local dump_destination="$DUMP_DIR/$database.sql.gz"
  scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkki.cs.helsinki.fi" "$username@$pannu_url" "$dump_destination"
 }

@@ -76,7 +76,7 @@ reset_databases() {
  docker-compose up -d "${databases[@]}"

  for database in "${databases[@]}"; do
-    local database_dump="$database_dump_dir/$database.sqz"
+    local database_dump="$database_dump_dir/$database.sql.gz"
    local database_container="$database"
    local database_name="$database-real"

@@ -91,10 +91,8 @@ reset_databases() {
    docker exec -u postgres "$database_container" createdb "$database_name" || warningmsg "This is okay, continuing"

    infomsg "Restoring database from dump database"
-    msg "1. Copying dump..."
-    docker cp "$database_dump" "$database_container:/asd.sqz"
-    msg "2. Writing database..."
-    docker exec "$database_container" pg_restore -U postgres --no-owner -F c --dbname="$database_name" -j4 /asd.sqz
+    msg "Writing database..."
+    docker exec -i "$database_container" /bin/bash  -c "gunzip | psql -U postgres -d $database_name" < "$database_dump" 2> /dev/null
    msg ""
  done

@@ -103,8 +101,8 @@ reset_databases() {

 reset_jami_data() {
  local database="jami-db"
-  local pannu_url="svm-85.cs.helsinki.fi:/home/toska_user/most_recent_backup_store/jami.sql.gz"
-  local dump_destination="$DUMP_DIR/$database.sqz"
+  local pannu_url="svm-116.cs.helsinki.fi:/home/toska_user/most_recent_backup_store/jami.sql.gz"
+  local dump_destination="$DUMP_DIR/$database.sql.gz"

  infomsg "Downloading Jami data"
  scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkki.cs.helsinki.fi" "$username@$pannu_url" "$dump_destination"
@@ -118,7 +116,7 @@ reset_jami_data() {
  retry docker-compose -f "$DOCKER_COMPOSE" exec "$database" pg_isready --dbname="postgres"

  infomsg "Populating Jami"
-  docker exec -i "$database" /bin/bash -c "gunzip | psql -U postgres" < "$dump_destination"
+  docker exec -i "$database" /bin/bash -c "gunzip | psql -U postgres" < "$dump_destination" 2> /dev/null
  docker-compose stop "$database"
  msg ""
 }
@@ -130,8 +128,13 @@ reset_all_real_data() {
    local url="${REAL_DUMP_URLS[$i]}"
    download_real_dump "$database" "$url"
  done
-  reset_databases "${DATABASES[@]}"
  reset_jami_data
+  reset_databases "${DATABASES[@]}"
+}
+
+restore_data_from_dumps() {
+  infomsg "Restoring earlier-downloaded dumps to databases"
+  reset_databases "${DATABASES[@]}"
 }

 reset_single_database() {
@@ -229,6 +232,11 @@ linting and formatting"
  successmsg "Setup ready, oodikone can be started! See README for more info."
 }

+docker_prune() {
+  infomsg "Running: docker system prune -a && docker volume prune -a"
+  docker system prune -a && docker volume prune -a
+}
+
 init_dirs() {
  if [[ ! -d "$DUMP_DIR" ]]; then
    infomsg "Creating directory for dumps"
@@ -1,3 +1,3 @@
 .dockerignore
-Dockerfile
+*Dockerfile
 node_modules/
@@ -1,23 +1,27 @@
-FROM node:14.17-alpine3.13 as base
+FROM registry.access.redhat.com/ubi9/nodejs-18-minimal
+
+ENV TZ="Europe/Helsinki"
+
+WORKDIR /opt/app-root/src

-WORKDIR /usr/src/app
 EXPOSE 8080

-FROM base as development
-ENV NODE_ENV=development
-COPY ./package* /usr/src/app/
-RUN npm ci
-COPY . /usr/src/app/
-CMD ["npm", "run", "dev"]
-
-FROM base as production
 ENV NODE_ENV=production
+
+ARG BASE_PATH
+ENV PUBLIC_URL=$BASE_PATH
+
+ARG STAGING
+ENV REACT_APP_STAGING=$STAGING
+
 ARG SENTRY_ENVIRONMENT
 ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
+
 ARG SENTRY_RELEASE
 ENV SENTRY_RELEASE=$SENTRY_RELEASE
-COPY --chown=node:node ./package* /usr/src/app/
-RUN npm ci --production && apk add --no-cache vim curl nano
-COPY --chown=node:node . /usr/src/app/
-USER node
+
+COPY ./package* ./
+RUN npm ci --omit-dev
+COPY . .
+
 CMD ["npm", "start"]
@@ -14,6 +14,7 @@ const codes = {
  MH30_001: 'll',
  MH30_002: 'tmed',
  MH30_003: 'hll',
+  MH30_006: 'terv',
  KH40_001: 'filk',
  KH40_002: 'ttk',
  KH40_003: 'kik',
@@ -95,6 +96,7 @@ const codes = {
  MH80_005: 'hnfb',
  MH80_006: 'ekm',
  MH80_007: 'mmb',
+  MH80_008: 'emplant',
  KH90_001: 'elk',
  MH90_001: 'ell',
  T920101: 'dptheol',
@@ -0,0 +1,15 @@
+FROM node:18-alpine
+
+ENV TZ="Europe/Helsinki"
+
+WORKDIR /opt/app-root/src
+
+EXPOSE 8080
+
+ENV NODE_ENV=development
+
+COPY ./package* ./
+RUN npm ci
+COPY . .
+
+CMD ["npm", "run", "dev"]
@@ -4,15 +4,14 @@
  "scripts": {
    "start": "node --max_old_space_size=4096 index.js",
    "dev": "nodemon --max_old_space_size=4096 index.js",
-    "rapodiff": "node /usr/src/app/src/rapodiff.js"
+    "rapodiff": "node /opt/app-root/src/src/rapodiff.js"
  },
  "dependencies": {
    "@sentry/node": "^5.30.0",
    "@sentry/tracing": "^6.18.2",
-    "async": "^3.2.3",
-    "async-lock": "^1.3.1",
    "axios": "^0.21.4",
    "bluebird": "^3.7.2",
+    "bullmq": "^4.14.2",
    "cors": "^2.8.5",
    "cron": "^1.8.2",
    "express": "^4.18.2",
@@ -22,13 +21,14 @@
    "moment": "^2.29.1",
    "moment-timezone": "^0.5.37",
    "morgan": "^1.10.0",
+    "papaparse": "^5.4.1",
    "pg": "^8.8.0",
    "redis": "^3.1.2",
    "sequelize": "^6.24.0",
    "umzug": "^2.3.0",
    "unfuck-utf8-headers-middleware": "^1.0.1",
-    "uuid": "^3.4.0",
-    "winston": "^3.8.2"
+    "winston": "^3.8.2",
+    "winston-gelf-transporter": "^1.0.2"
  },
  "devDependencies": {
    "nodemon": "^1.19.4"
@@ -5,6 +5,7 @@ const routes = require('./routes')
 const { startCron } = require('./events')
 const { initializeDatabaseConnection, dbConnections } = require('./database/connection')
 const logger = require('./util/logger')
+require('./worker/worker')

 initializeDatabaseConnection()
  .then(() => {
@@ -1,7 +1,7 @@
 // Node env to use
-const isStaging = process.env.NODE_ENV === 'staging'
 const isDev = process.env.NODE_ENV === 'development'
-const isProduction = process.env.NODE_ENV === 'production'
+const isStaging = process.env.REACT_APP_STAGING === 'true'
+const isProduction = !isStaging && process.env.NODE_ENV === 'production'

 // Sentry
 const sentryRelease = process.env.SENTRY_RELEASE || ''
@@ -17,9 +17,10 @@ const requiredGroup = isStaging
 const pateToken = process.env.PATE_API_TOKEN || ''

 // Jami
-const jamiUrl = isDev || runningInCI ? 'http://jami:3003/' : 'https://importer.cs.helsinki.fi/api/auth'
+const jamiUrl = process.env.JAMI_URL || ''

 // Importer client
+const importerUrl = process.env.IMPORTER_DB_API_URL || ''
 const importerToken = process.env.IMPORTER_DB_API_TOKEN || ''

 // Networking: Urls & ports
@@ -27,16 +28,6 @@ const baseUrl = isDev ? '/api' : ''
 const frontUrl = process.env.FRONT_URL
 const backendPort = 8080

-// Analytics
-const ANALYTICS_INFLUXDB_URL = process.env.ANALYTICS_INFLUXDB_URL
-const ANALYTICS_INFLUXDB_USER = process.env.ANALYTICS_INFLUXDB_USER
-const ANALYTICS_INFLUXDB_PASSWORD = process.env.ANALYTICS_INFLUXDB_PASSWORD
-const ANALYTICS_INFLUXDB_DB = process.env.ANALYTICS_INFLUXDB_DB
-
-// Matomo
-const MATOMO_SITE_ID = process.env.MATOMO_SITE_ID
-const MATOMO_URL = process.env.MATOMO_URL
-
 // Other stuff
 const DB_URL_KONE = process.env.DB_URL_KONE
 const DB_URL_USER = process.env.DB_URL_USER
@@ -55,11 +46,9 @@ const SECRET_TOKEN = process.env.SECRET_TOKEN
 const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
 const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL

-const formatURL = url => {
-  return !!url && !url.startsWith('http') ? `http://${url}` : url
-}
-
-const addSlashToEnd = url => (url.endsWith('/') ? url : url + '/')
+const KONE_PASSWORD = process.env.KONE_PASSWORD || 'postgres'
+const USER_PASSWORD = process.env.USER_PASSWORD || 'postgres'
+const SIS_PASSWORD = process.env.SIS_PASSWORD || 'postgres'

 module.exports = {
  DB_URL_USER,
@@ -69,14 +58,11 @@ module.exports = {
  redis,
  SECRET_TOKEN,
  DB_SCHEMA_KONE,
+  KONE_PASSWORD,
+  USER_PASSWORD,
+  SIS_PASSWORD,
  requiredGroup,
  SIS_UPDATER_URL,
-  ANALYTICS_INFLUXDB_URL: formatURL(ANALYTICS_INFLUXDB_URL),
-  ANALYTICS_INFLUXDB_USER,
-  ANALYTICS_INFLUXDB_PASSWORD,
-  ANALYTICS_INFLUXDB_DB,
-  MATOMO_SITE_ID,
-  MATOMO_URL: MATOMO_URL && addSlashToEnd(MATOMO_URL),
  runningInCI,
  sentryRelease,
  sentryEnvironment,
@@ -86,5 +72,7 @@ module.exports = {
  baseUrl,
  frontUrl,
  backendPort,
+  importerUrl,
  importerToken,
+  isDev,
 }
@@ -20,6 +20,7 @@ class DbConnection extends EventEmitter {
        idle: 300000000,
      },
      logging: false,
+      password: conf.SIS_PASSWORD,
    })
  }

@@ -44,12 +45,14 @@ const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
  schema: conf.DB_SCHEMA_KONE,
  searchPath: conf.DB_SCHEMA_KONE,
  logging: false,
+  password: conf.KONE_PASSWORD,
 })

 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)

 const sequelizeUser = new Sequelize(conf.DB_URL_USER, {
  logging: false,
+  password: conf.USER_PASSWORD,
 })

 const initializeDatabaseConnection = async () => {
@@ -0,0 +1,7 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('mandatory_courses')
+    await queryInterface.dropTable('mandatory_course_labels')
+  },
+  down: async () => {},
+}
@@ -0,0 +1,7 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.sequelize.query('DROP SEQUENCE "mandatory_courses_id_seq"')
+    await queryInterface.sequelize.query('DROP SEQUENCE "mandatory_course_labels_id_seq"')
+  },
+  down: async () => {},
+}
@@ -0,0 +1,17 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.sequelize.query('TRUNCATE TABLE progress_criteria')
+    await queryInterface.sequelize.query('TRUNCATE TABLE excluded_courses')
+    await queryInterface.removeConstraint('excluded_courses', 'excluded_courses_programme_code_course_code_key')
+    await queryInterface.addColumn('progress_criteria', 'curriculum_version', {
+      type: Sequelize.STRING,
+    })
+    await queryInterface.addColumn('excluded_courses', 'curriculum_version', {
+      type: Sequelize.STRING,
+    })
+  },
+  down: async queryInterface => {
+    await queryInterface.removeColumn('progress_criteria', 'curriculum_version')
+    await queryInterface.removeColumn('excluded_courses', 'curriculum_version')
+  },
+}
@@ -3,13 +3,18 @@ const moment = require('moment')
 const { refreshStatus, refreshStatusGraduated, refreshUber, getStartYears } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
-const { updateBasicView, updateStudytrackView } = require('./services/studyprogrammeUpdates')
+const { updateBasicView, updateStudytrackView } = require('./services/studyprogramme/studyprogrammeUpdates')
 const { findAndSaveTeachers } = require('./services/topteachers')
 const { faculties } = require('./services/organisations')
+const { combinedStudyprogrammes } = require('./services/studyprogramme/studyprogrammeHelpers')
 const { updateFacultyOverview, updateFacultyProgressOverview } = require('./services/faculty/facultyUpdates')
 const { isProduction } = require('./conf-backend')
 const { getCurrentSemester } = require('./services/semesters')
 const logger = require('./util/logger')
+const { getAssociations } = require('./services/studyrights')
+const { redisClient } = require('./services/redis')
+const { computeLanguageCenterData, LANGUAGE_CENTER_REDIS_KEY } = require('./services/languageCenterData')
+const { jobMaker } = require('./worker/queue')

 const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })

@@ -19,24 +24,27 @@ const refreshStudyrightAssociations = async () => {
 }

 const refreshFaculties = async () => {
-  logger.info('Refreshing faculty overview statistics for all faculties')
+  logger.info('Adding jobs to refresh all faculties')
  const facultyList = (await faculties()).filter(f => !['Y', 'H99', 'Y01', 'H92', 'H930'].includes(f.code))
-  let ready = 0
-
  for (const faculty of facultyList) {
-    try {
-      await updateFacultyOverview(faculty.code, 'ALL')
-      await updateFacultyProgressOverview(faculty.code)
-      ready += 1
-    } catch (e) {
-      logger.error({ message: `Failed to update stats for faculty ${faculty?.code} with type ALL`, meta: e })
-    }
-    logger.info(`${ready}/${facultyList.length} faculties done`)
+    jobMaker.faculty(faculty.code)
  }
-  logger.info('Faculty stats refreshed!')
 }

-const refreshNewOverviews = async () => {
+const refreshFaculty = async code => {
+  logger.info('Started updating faculty')
+  const start = new Date().getTime()
+  try {
+    await updateFacultyOverview(code, 'ALL')
+    await updateFacultyProgressOverview(code)
+    const time = new Date().getTime() - start
+    logger.info(`Updated faculty ${code} (took ${(time / 1000 / 60).toFixed(2)} minutes)`)
+  } catch (e) {
+    logger.error({ message: `Failed to update stats for faculty ${code} with type ALL`, meta: `${e}` })
+  }
+}
+
+const refreshProgrammes = async () => {
  logger.info('Refreshing studyprogramme and studytrack overview statistics for all programmes')
  // Filters out old programmes and special ones like Fitech studies. Filters also out the programmes starting with
  // 2_ or ending with _2. Those programmes are mapped to correct programme (studentProgrammeModuleFixer.js)
@@ -48,18 +56,23 @@ const refreshNewOverviews = async () => {
        (code.includes('MH') && !code.startsWith('2_MH') && !code.endsWith('_2')) ||
        /^(T)[0-9]{6}$/.test(code)
    )
-  let ready = 0
+
+  // Ensure that studyright associations are refreshed before launching jobs, otherwise each job will do it
+  await getAssociations()
  for (const code of codes) {
-    try {
-      await updateBasicView(code)
-      await updateStudytrackView(code)
-      ready += 1
-    } catch (e) {
-      logger.error({ message: `Failed to update overview stats for programme ${code}`, meta: e })
-    }
-    logger.info(`${ready}/${codes.length} programmes done`)
+    // If combined programme is given, this updates only the bachelor programme
+    jobMaker.programme(code)
  }
-  logger.info('Studyprogramme and studytrack overview stats refreshed!')
+}
+
+const refreshProgramme = async code => {
+  const associations = await getAssociations()
+  await updateBasicView(code, '')
+  await updateStudytrackView(code, '', associations)
+
+  let combinedProgramme = combinedStudyprogrammes[code] || ''
+  await updateBasicView(code, combinedProgramme)
+  await updateStudytrackView(code, combinedProgramme, associations)
 }

 const refreshTeacherLeaderboard = async () => {
@@ -98,7 +111,7 @@ const refreshUberToRedis = async () => {
      await refreshUber(oldAttainmentsQuery)
      logger.info(`Refreshing CDS uber year ${formattedYear} doned`)
    } catch (e) {
-      logger.error({ message: `Error when refreshing CDS uber year ${formattedYear}`, meta: e })
+      logger.error({ message: `Error when refreshing CDS uber year ${formattedYear}`, meta: `${e}` })
    }
  }
  logger.info(`Refreshing CDS Uber data doned`)
@@ -122,14 +135,36 @@ const refreshTrends = async () => {
  logger.info('Trends refreshed!')
 }

+// eslint-disable-next-line no-unused-vars
+const refreshLanguageCenterData = async () => {
+  logger.info('Refreshing language center data...')
+  const freshData = await computeLanguageCenterData()
+  await redisClient.setAsync(LANGUAGE_CENTER_REDIS_KEY, JSON.stringify(freshData))
+  logger.info('Language center data refreshed!')
+}
+
+const dailyJobs = () => {
+  refreshFaculties()
+  refreshProgrammes()
+  jobMaker.languagecenter()
+  jobMaker.trends()
+  jobMaker.statistics()
+}
+
 const startCron = () => {
  if (isProduction) {
-    logger.info('Cronjob for refreshing stats started: runs at 3am every day.')
-    // refresh 3am every day
-    schedule('0 3 * * *', async () => {
-      for (const func of [refreshStatistics, refreshTrends, refreshNewOverviews, refreshFaculties]) {
-        await func()
-      }
+    logger.info('Cronjob for refreshing stats started: runs daily at 23:00.')
+    schedule('0 23 * * *', async () => {
+      logger.info('Running daily jobs from cron')
+      dailyJobs()
+    })
+    schedule('0 19 * * 1', async () => {
+      logger.info('Updating students whose studyplans have not been updated recently')
+      jobMaker.studyplansUpdate(4)
+    })
+    schedule('0 10 * * 2', async () => {
+      logger.info('Updating students whose studyplans have not been updated recently')
+      jobMaker.studyplansUpdate(5)
    })
  }
 }
@@ -139,4 +174,8 @@ module.exports = {
  refreshStatistics,
  refreshTrends,
  refreshFaculties,
+  refreshFaculty,
+  refreshProgramme,
+  refreshProgrammes,
+  refreshLanguageCenterData,
 }
@@ -1,6 +1,7 @@
 const morgan = require('morgan')
 const logger = require('../util/logger')
 const _ = require('lodash')
+const { isProduction } = require('../conf-backend')

 // So this appears to be a hack to get neatly formatted stats like response-time etc. from morgan
 // without actually using morgan what it's used for (LOGGING REQUESTS!).
@@ -16,16 +17,24 @@ const accessLogger = morgan((tokens, req, res) => {
    meta['req-route'] = req.route.path
  }

-  const message = [
-    user.name,
-    ':',
-    tokens['method'](req, res),
-    tokens['url'](req, res),
-    tokens['status'](req, res),
-    '-',
-    tokens['response-time'](req, res),
-    'ms',
-  ].join(' ')
+  const message = isProduction
+    ? [
+        user.name,
+        tokens['method'](req, res),
+        tokens['url'](req, res),
+        tokens['status'](req, res),
+        '-',
+        tokens['response-time'](req, res),
+        'ms',
+      ].join(' ')
+    : [
+        user.name,
+        tokens['response-time'](req, res).split('.')[0],
+        'ms\t',
+        tokens['status'](req, res),
+        tokens['method'](req, res),
+        tokens['url'](req, res),
+      ].join(' ')

  const usingIamRights = user.iamRights.some(programmeCode => tokens['url'](req, res).includes(programmeCode))
  const onlyIamRights = !user.isAdmin && user.rights.length === 0
@@ -4,7 +4,7 @@ const roles = requiredRoles => async (req, res, next) => {
      user: { roles },
    } = req

-    if (requiredRoles.every(r => roles.indexOf(r) >= 0) || roles.includes('admin')) {
+    if (requiredRoles.some(r => roles.includes(r)) || roles.includes('admin')) {
      return next()
    }
  }
@@ -1,4 +1,4 @@
-const { Model, STRING, DATE } = require('sequelize')
+const { Model, STRING, DATE, JSONB } = require('sequelize')
 const { dbConnections } = require('../database/connection')

 class CourseProvider extends Model {}
@@ -18,6 +18,9 @@ CourseProvider.init(
      onUpdate: 'cascade',
      onDelete: 'cascade',
    },
+    shares: {
+      type: JSONB,
+    },
    organizationcode: {
      type: STRING,
      references: {
@@ -81,6 +81,14 @@ Credit.init(
    is_open: {
      type: BOOLEAN,
    },
+    studyright_id: {
+      type: STRING,
+      allowNull: true,
+      references: {
+        model: 'studyright',
+        key: 'studyrightid',
+      },
+    },
  },
  {
    underscored: false,
@@ -57,6 +57,14 @@ Enrollment.init(
    is_open: {
      type: BOOLEAN,
    },
+    studyright_id: {
+      type: STRING,
+      allowNull: true,
+      references: {
+        model: 'studyright',
+        key: 'studyrightid',
+      },
+    },
  },
  {
    underscored: false,
@@ -16,6 +16,9 @@ ExcludedCourse.init(
    course_code: {
      type: STRING,
    },
+    curriculum_version: {
+      type: STRING,
+    },
    created_at: {
      type: DATE,
    },
@@ -50,6 +50,8 @@ Credit.passed = ({ credittypecode }) =>
  credittypecode === CREDIT_TYPE_CODES.PASSED || credittypecode === CREDIT_TYPE_CODES.APPROVED
 Credit.failed = credit => credit.credittypecode === CREDIT_TYPE_CODES.FAILED
 Credit.improved = credit => credit.credittypecode === CREDIT_TYPE_CODES.IMPROVED
+Credit.belongsTo(Studyright, { foreignKey: 'studyright_id', targetKey: 'studyrightid', constraints: false })
+Studyright.hasMany(Credit, { foreignKey: 'studyright_id', constraints: false })

 Credit.belongsTo(Student, { foreignKey: 'student_studentnumber', targetKey: 'studentnumber' })
 Student.hasMany(Credit, { foreignKey: 'student_studentnumber', sourceKey: 'studentnumber' })
@@ -97,6 +99,9 @@ Transfer.belongsTo(ElementDetail, { as: 'source', foreignKey: 'sourcecode' })
 Transfer.belongsTo(ElementDetail, { as: 'target', foreignKey: 'targetcode' })

 Enrollment.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
+Enrollment.belongsTo(Studyright, { foreignKey: 'studyright_id', targetKey: 'studyrightid', constraints: false })
+Studyright.hasMany(Enrollment, { foreignKey: 'studyright_id', constraints: false })
+
 Student.hasMany(Enrollment, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
 Enrollment.belongsTo(Course, { foreignKey: 'course_id' })
 Course.hasMany(Enrollment, { foreignKey: 'course_id' })
@@ -18,6 +18,9 @@ const ExcludedCourse = sequelizeKone.define(
    programme_code: {
      type: Sequelize.STRING,
    },
+    curriculum_version: {
+      type: STRING,
+    },
    course_code: {
      type: Sequelize.STRING,
    },
@@ -33,18 +36,6 @@ const ExcludedCourse = sequelizeKone.define(
  }
 )

-const MandatoryCourse = sequelizeKone.define('mandatory_courses', {
-  course_code: {
-    type: Sequelize.STRING,
-  },
-  studyprogramme_id: {
-    type: Sequelize.STRING,
-  },
-  label: {
-    type: Sequelize.BIGINT,
-  },
-})
-
 const TagStudent = sequelizeKone.define(
  'tag_student',
  {
@@ -88,23 +79,6 @@ const Tag = sequelizeKone.define(
  }
 )

-const MandatoryCourseLabels = sequelizeKone.define('mandatory_course_labels', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true,
-  },
-  studyprogramme_id: {
-    type: Sequelize.STRING,
-  },
-  label: {
-    type: Sequelize.STRING,
-  },
-  orderNumber: {
-    type: Sequelize.INTEGER,
-  },
-})
-
 const CustomPopulationSearch = sequelizeKone.define('custom_population_searches', {
  id: {
    primaryKey: true,
@@ -152,6 +126,9 @@ const ProgressCriteria = sequelizeKone.define(
      primaryKey: true,
      type: STRING,
    },
+    curriculumVersion: {
+      type: STRING,
+    },
    coursesYearOne: {
      type: Sequelize.ARRAY(Sequelize.STRING),
    },
@@ -214,24 +191,13 @@ const StudyGuidanceGroupTag = sequelizeKone.define(
  { underscored: true, timestamps: true }
 )

-MandatoryCourse.belongsTo(MandatoryCourseLabels, {
-  foreignKey: 'label',
-  sourceKey: 'id',
-})
-MandatoryCourseLabels.hasMany(MandatoryCourse, {
-  foreignKey: 'label',
-  sourceKey: 'id',
-})
-
 TagStudent.belongsTo(Tag, { foreignKey: 'tag_id', sourceKey: 'tag_id' })
 Tag.hasMany(TagStudent, { foreignKey: 'tag_id', sourceKey: 'tag_id' })

 module.exports = {
-  MandatoryCourse,
  TagStudent,
  Tag,
  ThesisTypeEnums,
-  MandatoryCourseLabels,
  CustomPopulationSearch,
  OpenUniPopulationSearch,
  sequelizeKone,
@@ -1,4 +1,4 @@
-const { Model, STRING, DATE, JSONB } = require('sequelize')
+const { Model, STRING, DATE, INTEGER, JSONB, ARRAY, TEXT } = require('sequelize')
 const { dbConnections } = require('../database/connection')

 class ProgrammeModule extends Model {}
@@ -9,6 +9,9 @@ ProgrammeModule.init(
      primaryKey: true,
      type: STRING,
    },
+    group_id: {
+      type: STRING,
+    },
    code: {
      type: STRING,
    },
@@ -18,6 +21,9 @@ ProgrammeModule.init(
    type: {
      type: STRING,
    },
+    order: {
+      type: INTEGER,
+    },
    studyLevel: {
      type: STRING,
    },
@@ -28,6 +34,15 @@ ProgrammeModule.init(
        key: 'id',
      },
    },
+    valid_from: {
+      type: DATE,
+    },
+    valid_to: {
+      type: DATE,
+    },
+    curriculum_period_ids: {
+      type: ARRAY(TEXT),
+    },
    createdAt: {
      type: DATE,
    },
@@ -5,19 +5,19 @@ class ProgrammeModuleChild extends Model {}

 ProgrammeModuleChild.init(
  {
-    parent_id: {
+    composite: {
+      type: STRING,
+      primaryKey: true,
+    },
+    parentId: {
      type: STRING,
      references: {
        model: 'programme_modules',
        key: 'id',
      },
    },
-    child_id: {
+    childId: {
      type: STRING,
-      references: {
-        model: 'programme_modules',
-        key: 'id',
-      },
    },
    createdAt: {
      type: DATE,
@@ -1,4 +1,4 @@
-const { Model, STRING, DATE, INTEGER, BOOLEAN } = require('sequelize')
+const { Model, STRING, DATE, INTEGER, BOOLEAN, JSONB } = require('sequelize')
 const { dbConnections } = require('../database/connection')

 class Studyright extends Model {}
@@ -9,6 +9,9 @@ Studyright.init(
      primaryKey: true,
      type: STRING,
    },
+    actual_studyrightid: {
+      type: STRING,
+    },
    startdate: {
      type: DATE,
    },
@@ -63,6 +66,12 @@ Studyright.init(
    admissionType: {
      type: STRING,
    },
+    isBaMa: {
+      type: BOOLEAN,
+    },
+    semesterEnrollments: {
+      type: JSONB,
+    },
  },
  {
    underscored: true,
@@ -0,0 +1,21 @@
+# Rapodiff
+
+Use this to check differences between oodikone and rapo.
+
+Before running:
+
+1. Depends on nodeproxy running in toska.cs.helsinki.fi
+2. Set these correctly in .env, notice to wrap in quotes ", otherwise strange characters may be misformatted:
+   RAPO_NODEPROXY="acual_token"
+   IMPORTER_DB_API_TOKEN="acual_token"
+
+Running:
+
+1. Have real oodikone running. Then run rapodiff: `npm run rapodiff -- args`
+2. place args with a mode. omit args to print possible modes.
+
+known differences in APIs
+
+- rapo does not show students that have graduated (it however takes some days for rapo to notice a graduation)
+- rapo does not handle the starting year of transfers properly (shows in old staring population)
+- rapo show some students incorrectly passive, see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3521
@@ -0,0 +1,28 @@
+// In .env, have the token wrapped in quotes! (") otherwise some signs may be mistakenly changed
+const rapoToken = process.env.RAPO_NODEPROXY
+const importerDbApiToken = process.env.IMPORTER_DB_API_TOKEN
+
+const nodeproxyUrl = `https://toska.cs.helsinki.fi`
+const importerDbApiUrl = `https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer`
+
+const bscProgrammesOfFaculties = {
+  H10: ['KH10_001'],
+  H20: ['KH20_001'],
+  H30: ['KH30_001', 'KH30_002'],
+  H40: ['KH40_001', 'KH40_002', 'KH40_003', 'KH40_004', 'KH40_005', 'KH40_006'],
+  H50: ['KH50_001', 'KH50_002', 'KH50_003', 'KH50_004', 'KH50_005', 'KH50_006', 'KH50_007', 'KH50_008'],
+  H55: ['KH55_001'],
+  H60: ['KH60_001'],
+  H70: ['KH70_001', 'KH70_002', 'KH70_003', 'KH70_004'],
+  H74: ['KH74_001'],
+  H80: ['KH80_001', 'KH80_002', 'KH80_003', 'KH80_004'],
+  H90: ['KH90_001'],
+}
+
+module.exports = {
+  rapoToken,
+  importerDbApiToken,
+  nodeproxyUrl,
+  importerDbApiUrl,
+  bscProgrammesOfFaculties,
+}
@@ -0,0 +1,63 @@
+/* eslint-disable no-console */
+const { getCreditStats } = require('../services/analyticsService')
+const { parseCsv } = require('./helpers')
+
+const diff = (rapoData, okData, code) => {
+  for (const year of Object.keys(okData)) {
+    const rapo = rapoData[year]
+    const ok = okData[year]
+    const diff = Math.abs(rapo.total - ok.total)
+    if (diff > 100) {
+      console.log(`${code} - ${year}: Total diff: ${diff}\nOk: ${ok.total} Rapo: ${rapo.total}`)
+    }
+  }
+}
+
+const parseOkData = data => {
+  return data.tableStats.reduce((obj, cur) => {
+    const [year, major, nonmajor, nondegree, transferred] = cur
+    obj[year] = { major, nonmajor, nondegree, transferred }
+    return obj
+  }, {})
+}
+
+// Acual logic in this function
+const process = async data => {
+  const rapoProgrammeData = formatData(data)
+  const allProgrammeCodes = [...new Set(Object.keys(rapoProgrammeData))]
+  for (const programmeCode of allProgrammeCodes) {
+    const okProgrammeData = await getCreditStats(programmeCode, null, '')
+    diff(rapoProgrammeData[programmeCode], parseOkData(okProgrammeData), programmeCode)
+  }
+}
+
+const programmeCreditsDiff = async fileName => {
+  console.log('Running studyprogramme credits diff')
+  await parseCsv(fileName, process)
+}
+
+// Change weird rapo programmecodes to oodikone format, example: 300-M003 => MH30_003
+const transformProgrammeCode = oldCode => `H${oldCode.slice(1, 3)}_${oldCode.slice(5, 8)}`
+
+const formatData = data =>
+  data.reduce((obj, cur) => {
+    const [year, facultyCode, programmeInfo, basic, exchange, otherUni, openUni, special, total, abroad, other] = cur
+    const programmeCode = transformProgrammeCode(programmeInfo.split(' ')[0])
+    if (!obj[programmeCode]) obj[programmeCode] = []
+    obj[programmeCode].push({
+      year,
+      facultyCode,
+      programmeInfo,
+      basic,
+      exchange,
+      otherUni,
+      openUni,
+      special,
+      total,
+      abroad,
+      other,
+    })
+    return obj
+  }, {})
+
+module.exports = { programmeCreditsDiff }
@@ -0,0 +1,50 @@
+### Esimerkki (fake data)
+
+Studycredits-route (suoritetut kreditit - kalenterivuosi?)
+
+**Route**: https://toska.cs.helsinki.fi/test/rapo/report/studyCredits?faculty=H50&education=500-K005&token=acual_token
+
+**Response**:
+
+```
+[
+  {
+    "vuosi": 2022,
+    "koulutusohjelmaKoodi": "500-K002",
+    "koulutusohjelma": "Fysikaalisten tieteiden kandiohjelma",
+    "tiedekuntaKoodi": "H50",
+    "tiedekunta": "H50 Matemaattis-luonnontieteellinen tiedekunta / Faculty of Science",
+    "perustutkinto": 12345,
+    "jatko": null,
+    "suoritetutYhteensa": 12345,
+    "eriklaakari": null,
+    "tutkAvoin": 12345,
+    "korkValYhteis": 12345,
+    "muutHyvaksiluetut": 12345,
+    "ulkomailta": null,
+    "taydennyskoul": null,
+    "avoinEiTutk": 12345,
+    "erillisoik": 12345,
+    "saapuvatVaihtoop": 12345
+  },
+  {
+    "vuosi": 2023,
+    "koulutusohjelmaKoodi": "500-M014",
+    "koulutusohjelma": "Nordic Master Programme in Environmental Changes at Higher Latitudes",
+    "tiedekuntaKoodi": "H50",
+    "tiedekunta": "H50 Matemaattis-luonnontieteellinen tiedekunta / Faculty of Science",
+    "perustutkinto": 12345,
+    "jatko": null,
+    "suoritetutYhteensa": 12345,
+    "eriklaakari": null,
+    "tutkAvoin": null,
+    "korkValYhteis": null,
+    "muutHyvaksiluetut": null,
+    "ulkomailta": 12345,
+    "taydennyskoul": null,
+    "avoinEiTutk": null,
+    "erillisoik": null,
+    "saapuvatVaihtoop": null
+  }
+]
+```
@@ -0,0 +1,21 @@
+/* eslint-disable no-console */
+const Papa = require('papaparse')
+const fs = require('fs/promises')
+
+const parseCsv = async (fileName, fun) => {
+  const data = await fs.readFile(`${__dirname}/${fileName}`, 'utf8')
+
+  await Papa.parse(data, {
+    complete: async data => {
+      if (!data?.data) {
+        console.log('Parsing file failed')
+        console.log(data.errors)
+        return
+      }
+      await fun(data?.data)
+    },
+    encoding: 'utf8',
+  })
+}
+
+module.exports = { parseCsv }
@@ -0,0 +1,39 @@
+/* eslint-disable no-unused-vars */
+/* eslint no-console: 0 */
+
+const { bscProgrammesOfFaculties } = require('./conf')
+const { facultyDiff } = require('./studyrightDiff')
+const { parseCsv, programmeCreditsDiff } = require('./creditsDiff')
+
+const runFacultyStudyrightDiffs = async () => {
+  console.log('Running faculty diffs.')
+  for (let faculty of Object.keys(bscProgrammesOfFaculties)) {
+    await facultyDiff(faculty, bscProgrammesOfFaculties[faculty], true)
+  }
+
+  // await programme_diff_year('KH55_001', 'H55', 2021, true)
+  // await programme_diff('KH60_001', 'H60', true)
+}
+
+const runCreditDiffs = async () => {
+  console.log('Running credit diffs.')
+}
+
+// ** To add new mode, just type key and function here **
+const modes = { credits: () => programmeCreditsDiff('data.csv') } // { studyrights: runFacultyStudyrightDiffs, credits: runCreditDiffs }
+
+const main = async () => {
+  console.log('**RAPODIFF**\n')
+  const mode = process.argv[2]
+  const modeFunction = modes[mode]
+  if (!modeFunction) {
+    console.log(
+      'Unknown or missing mode. Supply the mode as an argument. Supported modes:\n',
+      Object.keys(modes).join('\n ')
+    )
+    return
+  }
+  await modeFunction()
+}
+
+main().then(() => process.exit(0))
@@ -1,25 +1,15 @@
-/* eslint no-console: 0 */
-
-/*
-  depends on nodeproxy running in importer.cs.helsinki.fi, and env RAPO_NODEPROXY (see importer for a proper value)
-  needs also acces to importer db api, uses env IMPORTER_API
-
-  known differences in APIs
-    - rapo does not show students that have graduated (it however takes some days for rapo to notice a graduation)
-    - rapo does not handle the starting year of transfers properly (shows in old staring population)
-    - rapo show some students incorrectly passive, see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3521
-*/
+/* eslint-disable no-console */

 const axios = require('axios').default

 const _ = require('lodash')

-const { optimizedStatisticsOf } = require('./services/populations')
-const { Student, Transfer, StudyrightElement, Studyright } = require('./models')
-
-const token = process.env.RAPO_NODEPROXY
+const { optimizedStatisticsOf } = require('../services/populations')
+const { Student, Transfer, StudyrightElement, Studyright } = require('../models')
+const { rapoToken, tokenImporterApi, importerDbApiUrl, nodeproxyUrl } = require('./conf.js')

 const getFromRapo = async (urlStart, urlEnd) => {
+  console.log({ urlStart, urlEnd })
  const { data: rapoStuffLasna } = await axios.get(urlStart)
  const { data: rapoStuffLasnaMinus } = await axios.get(urlEnd)

@@ -83,10 +73,12 @@ const getFromOodikone = async (programme, year) => {

 const isAcuallyActive = async student => {
  const currentSemester = 144
-  const tokenImporterApi = process.env.IMPORTER_API

-  const url = `https://importer.cs.helsinki.fi/api/importer/students/${student}/rapo_semester_enrollments?token=${tokenImporterApi}`
-  const { data } = await axios.get(url)
+  const url = `${importerDbApiUrl}/students/${student}/rapo_semester_enrollments?token=${tokenImporterApi}`
+  const result = await axios.get(url)
+  console.log('result:')
+  console.log(result)
+  const { data } = result
  const terms = data.reduce((all, s) => all.concat(s.terms), []).filter(t => t.semester_code === currentSemester)
  const active = terms.filter(t => t.type !== 'NEGLECTED')
  const negl = terms.filter(t => t.type === 'NEGLECTED')
@@ -144,11 +136,11 @@ const programme_diff_year = async (programme, facultyCode, year, brief = false)

  const rapoUrlStart = presence =>
    encodeURI(
-      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+      `${nodeproxyUrl}/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${rapoToken}`
    )
  const rapoUrlEnd = presence =>
    encodeURI(
-      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+      `${nodeproxyUrl}/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${rapoToken}`
    )

  const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
@@ -247,7 +239,7 @@ const programme_diff = async (programme, faculty, brief) => {
  await programme_diff_year(programme, faculty, 2021, brief)
 }

-const faculty_diff = async (faculty, programmes, brief) => {
+const facultyDiff = async (faculty, programmes, brief) => {
  console.log('\n')
  console.log(faculty)
  console.log('==')
@@ -265,11 +257,11 @@ const msc_programme_diff_year = async (programme, facultyCode, year, brief = fal

  const rapoUrlStart = presence =>
    encodeURI(
-      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+      `${nodeproxyUrl}/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${rapoToken}`
    )
  const rapoUrlEnd = presence =>
    encodeURI(
-      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+      `${nodeproxyUrl}/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${rapoToken}`
    )

  const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
@@ -337,27 +329,4 @@ const msc_programme_diff_year = async (programme, facultyCode, year, brief = fal
  }
 }

-const bsc_priogrammes_of_faculties = {
-  H10: ['KH10_001'],
-  H20: ['KH20_001'],
-  H30: ['KH30_001', 'KH30_002'],
-  H40: ['KH40_001', 'KH40_002', 'KH40_003', 'KH40_004', 'KH40_005', 'KH40_006'],
-  H50: ['KH50_001', 'KH50_002', 'KH50_003', 'KH50_004', 'KH50_005', 'KH50_006', 'KH50_007', 'KH50_008'],
-  H55: ['KH55_001'],
-  H60: ['KH60_001'],
-  H70: ['KH70_001', 'KH70_002', 'KH70_003', 'KH70_004'],
-  H74: ['KH74_001'],
-  H80: ['KH80_001', 'KH80_002', 'KH80_003', 'KH80_004'],
-  H90: ['KH90_001'],
-}
-
-const main = async () => {
-  for (let faculty of Object.keys(bsc_priogrammes_of_faculties)) {
-    await faculty_diff(faculty, bsc_priogrammes_of_faculties[faculty], true)
-  }
-
-  //await programme_diff_year('KH55_001', 'H55', 2021, true)
-  //await programme_diff('KH60_001', 'H60', true)
-}
-
-main().then(() => process.exit(0))
+module.exports = { facultyDiff }
@@ -3,10 +3,10 @@ const express = require('express')
 const cors = require('cors')
 const { frontUrl } = require('./conf-backend')
 const shibbolethCharsetMiddleware = require('./middleware/shibbolethCharsetMiddleware')
-const matomoInit = require('./routes/matomo-init')
 const currentUserMiddleware = require('./middleware/currentUserMiddleware')
 const accessLogger = require('./middleware/accessLogger')
 const auth = require('./middleware/auth')
+const changelog = require('./routes/changelog')
 const courses = require('./routes/courses')
 const students = require('./routes/students')
 const population = require('./routes/population')
@@ -17,14 +17,14 @@ const elementdetails = require('./routes/elementdetails')
 const teachers = require('./routes/teachers')
 const providers = require('./routes/providers')
 const faculties = require('./routes/faculties')
+const university = require('./routes/university')
+const courseExclusions = require('./routes/courseExclusions')
 const semesters = require('./routes/semesters')
-const mandatoryCourses = require('./routes/mandatorycourses')
-const mandatoryCourseLabels = require('./routes/mandatorycourselabels')
 const feedback = require('./routes/feedback')
 const tags = require('./routes/tags')
 const updater = require('./routes/updater')
-const tsaAnalytics = require('./routes/tsaAnalytics')
 const customPopulationSearch = require('./routes/customPopulationSearch')
+const languageCenterData = require('./routes/languageCenterData')
 const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
 const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
@@ -34,7 +34,6 @@ const studyProgrammeCriteria = require('./routes/studyProgrammeCriteria')
 const initializeSentry = require('./util/sentry')
 const completedCoursesSearch = require('./routes/completedCoursesSearch')
 const errorMiddleware = require('./middleware/errorMiddleware')
-
 module.exports = (app, url) => {
  initializeSentry(app)
  app.use(Sentry.Handlers.requestHandler())
@@ -44,7 +43,6 @@ module.exports = (app, url) => {
  app.use(express.json())

  app.use(shibbolethCharsetMiddleware)
-  app.use(url, matomoInit)
  app.use(currentUserMiddleware)
  app.use(accessLogger)
  app.use(url, login)
@@ -57,18 +55,19 @@ module.exports = (app, url) => {
  app.use(url, semesters)
  app.use(url, tags)
  app.use(url, programmeModules)
+  app.use(url, courseExclusions)
  app.use(url, studyProgramme)
  app.use(`${url}/programmecriteria`, studyProgrammeCriteria)
  app.use(`${url}/openunisearch`, auth.roles(['openUniSearch']), customOpenUniSearch)
+  app.use(`${url}/changelog`, changelog)
  app.use(`${url}/completedcoursessearch`, completedCoursesSearch)
-  app.use(`${url}/faculties`, auth.roles(['facultyStatistics']), faculties)
+  app.use(`${url}/languagecenterdata`, languageCenterData)
+  app.use(`${url}/faculties`, auth.roles(['facultyStatistics', 'katselmusViewer']), faculties)
+  app.use(`${url}/university`, university)
  app.use(`${url}/updater`, auth.roles(['admin']), updater)
  app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
  app.use(`${url}/users`, auth.roles(['admin']), users)
  app.use(`${url}/feedback`, feedback)
-  app.use(`${url}/mandatory_courses`, mandatoryCourses)
-  app.use(`${url}/mandatory-course-labels`, mandatoryCourseLabels)
-  app.use(`${url}/tsa`, tsaAnalytics)
  app.use(`${url}/custom-population-search`, customPopulationSearch)
  app.use(`${url}/cool-data-science`, trends)
  app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
@@ -0,0 +1,22 @@
+const router = require('express').Router()
+const axios = require('axios')
+const { isDev } = require('../conf-backend')
+const changelog = {}
+
+router.get('/', async (req, res) => {
+  if (changelog?.data) return res.status(200).send(changelog.data)
+  if (isDev)
+    return res.status(200).json([
+      {
+        description: "### Fake release\nLet's not spam github api in development!",
+        title: 'Fake title for fake release',
+        time: new Date(),
+      },
+    ])
+  const result = await axios.get('https://api.github.com/repos/UniversityOfHelsinkiCS/oodikone/releases')
+  const formattedResult = result.data.map(r => ({ description: r.body, time: r.created_at, title: r.name }))
+  changelog.data = formattedResult
+  res.status(200).send(formattedResult)
+})
+
+module.exports = router
@@ -1,5 +1,7 @@
 const router = require('express').Router()
 const { getCompletedCourses } = require('../services/completedCoursesSearch')
+const { getImporterClient } = require('../util/importerClient')
+const importerClient = getImporterClient()
 const _ = require('lodash')
 const {
  getOpenUniSearches,
@@ -7,16 +9,33 @@ const {
  deleteSearch,
  updateSearch,
 } = require('../services/openUni/openUniManageSearches')
+const logger = require('../util/logger')

 router.get('/', async (req, res) => {
  const studentNumbers = JSON.parse(req.query?.studentlist) || []
  const courseCodes = JSON.parse(req.query?.courselist) || []
  const {
-    user: { isAdmin, studentsUserCanAccess },
+    user: { isAdmin, studentsUserCanAccess, sisPersonId: teacherId },
  } = req
  if (!Array.isArray(studentNumbers)) return res.status(400).json({ error: 'Student numbers must be of type array' })
  if (!Array.isArray(courseCodes)) return res.status(400).json({ error: 'Courses must be of type array' })

+  const answerTimeout = new Promise(resolve => setTimeout(resolve, 6000))
+
+  try {
+    // Teachers also can get rights to students via importer if the students
+    // have enrolled to their courses in last 8 months
+    // (acual logic a bit different, see importer)
+    const teacherRightsToStudents = await Promise.race([
+      importerClient.post(`/teacher-rights/`, { teacherId, studentNumbers }),
+      answerTimeout,
+    ])
+    if (teacherRightsToStudents && Array.isArray(teacherRightsToStudents.data))
+      studentsUserCanAccess.push(...teacherRightsToStudents.data)
+  } catch (e) {
+    logger.error(`Importer teacher-rights request failed with message: ${e?.message}`)
+  }
+
  const filteredStudentNumbers = isAdmin ? studentNumbers : _.intersection(studentNumbers, studentsUserCanAccess)
  const completedCourses = await getCompletedCourses(filteredStudentNumbers, courseCodes)
  const discardedStudentNumbers = _.difference(
@@ -0,0 +1,25 @@
+const router = require('express').Router()
+const { addExcludedCourses, removeExcludedCourses } = require('../services/excludedCourses')
+
+router.delete('/v3/course_exclusions/:code', async (req, res) => {
+  const { programmeCode, curriculumVersion, courseCodes } = req.body
+  const result = await removeExcludedCourses({ programmeCode, curriculumVersion, courseCodes })
+  if (!result) {
+    res.status(400).end()
+    return
+  }
+  res.json(result)
+})
+
+router.post('/v3/course_exclusions/:code', async (req, res) => {
+  const { code } = req.params
+  const { courseCodes, curriculumVersion } = req.body
+  const result = await addExcludedCourses(code, courseCodes, curriculumVersion.join(','))
+  if (!result) {
+    res.status(400).end()
+    return
+  }
+  res.json(result)
+})
+
+module.exports = router
@@ -1,18 +1,34 @@
 const router = require('express').Router()
 const crypto = require('crypto')
-const Course = require('../services/courses')
+const courseService = require('../services/courses')
 const { faculties } = require('../services/organisations')
 const { validateParamLength } = require('../util')

 router.get('/v2/coursesmulti', async (req, res) => {
  let results = { courses: [] }
-  const { name, code } = req.query
+  const { name, code, combineSubstitutions } = req.query

  if (!(validateParamLength(name, 5) || validateParamLength(code, 2))) {
    return res.status(400).json({ error: 'name or code invalid' })
  }

-  results = await Course.byNameAndOrCodeLike(name, code)
+  results = await courseService.byNameAndOrCodeLike(name, code)
+
+  if (combineSubstitutions === 'false') {
+    const courseCodes = results.courses.map(course => course.code)
+
+    const substitutions = [
+      ...new Set(
+        results.courses
+          .flatMap(course => course.substitutions)
+          .filter(value => value !== null && !courseCodes.includes(value))
+      ),
+    ]
+    const substitutionDetails = await courseService.byCodes(substitutions)
+    for (const substitution of substitutionDetails) {
+      results.courses.push(substitution.dataValues)
+    }
+  }
  res.json(results)
 })

@@ -30,6 +46,7 @@ router.get('/v3/courseyearlystats', async (req, res) => {
  }

  const { codes, separate: sep } = req.query
+  const combineSubstitutions = req.query.combineSubstitutions !== 'false'

  const separate = !sep ? false : JSON.parse(sep)
  if (!codes) {
@@ -39,7 +56,7 @@ router.get('/v3/courseyearlystats', async (req, res) => {
    // and users with rights to any specific study programmes
    const anonymize = allowedRoles !== 'admin' && rights.length < 1
    const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
-    const results = await Course.courseYearlyStats(codes, separate, anonymizationSalt)
+    const results = await courseService.courseYearlyStats(codes, separate, anonymizationSalt, combineSubstitutions)
    res.json(results)
  }
 })
@@ -1,5 +1,4 @@
 const router = require('express').Router()
-const MandatoryCourses = require('../services/mandatoryCourses')
 const { getAllProgrammes, getAllElementDetails } = require('../services/studyrights')

 router.get('/elementdetails/all', async (req, res) => {
@@ -12,13 +11,4 @@ router.get('/studyprogrammes', async (req, res) => {
  res.json(studyrights)
 })

-router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
-  if (req.params.id) {
-    const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
-    res.json(codes)
-  } else {
-    res.status(422).end()
-  }
-})
-
 module.exports = router
@@ -1,17 +1,13 @@
 const router = require('express').Router()
-const { faculties } = require('../services/organisations')
 const { combineFacultyBasics } = require('../services/faculty/facultyBasics')
 const { combineFacultyCredits } = require('../services/faculty/facultyCredits')
-const { findFacultyProgrammeCodes } = require('../services/faculty/faculty')
 const { combineFacultyThesisWriters } = require('../services/faculty/facultyThesisWriters')
 const { countGraduationTimes } = require('../services/faculty/facultyGraduationTimes')
 const { updateFacultyOverview, updateFacultyProgressOverview } = require('../services/faculty/facultyUpdates')
 const { combineFacultyStudentProgress } = require('../services/faculty/facultyStudentProgress')
 const { combineFacultyStudents } = require('../services/faculty/facultyStudents')
-
 const {
-  getFacultyProgrammes,
-  setFacultyProgrammes,
+  getProgrammes,
  getBasicStats,
  setBasicStats,
  getCreditStats,
@@ -26,23 +22,13 @@ const {
  setFacultyProgressStats,
 } = require('../services/faculty/facultyService')
 const logger = require('../util/logger')
+const { getFacultyList } = require('../services/faculty/facultyHelpers')

 // Faculty uses a lot of tools designed for Study programme.
 // Some of them have been copied here and slightly edited for faculty purpose.

-const getProgrammes = async (code, programmeFilter) => {
-  const programmes = await getFacultyProgrammes(code, programmeFilter)
-  if (programmes) return programmes
-  let updatedProgrammes = await findFacultyProgrammeCodes(code, programmeFilter)
-  if (updatedProgrammes) updatedProgrammes = await setFacultyProgrammes(code, updatedProgrammes, programmeFilter)
-
-  return updatedProgrammes
-}
-
 router.get('/', async (req, res) => {
-  const ignore = ['Y', 'H99', 'Y01', 'H92', 'H930']
-  const facultyList = (await faculties()).filter(f => !ignore.includes(f.code))
-  facultyList.sort((a, b) => (a.name.fi > b.name.fi ? 1 : -1))
+  const facultyList = await getFacultyList()
  res.json(facultyList)
 })

@@ -140,28 +126,16 @@ router.get('/:id/graduationtimes', async (req, res) => {

 router.get('/:id/progressstats', async (req, res) => {
  const code = req.params.id
-  const programmeFilter = req.query.programme_filter
  const specialGroups = req.query?.special_groups
  const graduated = req.query?.graduated
-  const programmes = await getProgrammes(code, programmeFilter)
+  const programmes = await getProgrammes(code, 'NEW_STUDY_PROGRAMMES')
  if (!programmes) return res.status(422).end()

-  //all programmes are required for correct sorting of transfers
-  let allProgrammeCodes = []
-  const allProgs = await getProgrammes(code, 'ALL_PROGRAMMES')
-  allProgs?.data.forEach(prog => allProgrammeCodes.push(prog.code))
-
  if (!code) return res.status(422).end()
  const data = await getFacultyProgressStats(code, specialGroups, graduated)
  if (data) return res.json(data)

-  let updateStats = await combineFacultyStudentProgress(
-    code,
-    programmes.data,
-    allProgrammeCodes,
-    specialGroups,
-    graduated
-  )
+  let updateStats = await combineFacultyStudentProgress(code, programmes.data, specialGroups, graduated)
  if (updateStats) {
    updateStats = await setFacultyProgressStats(updateStats, specialGroups, graduated)
  }
@@ -170,21 +144,16 @@ router.get('/:id/progressstats', async (req, res) => {

 router.get('/:id/studentstats', async (req, res) => {
  const code = req.params.id
-  const programmeFilter = req.query.programme_filter
  const specialGroups = req.query?.special_groups
  const graduated = req.query?.graduated

  if (!code) return res.status(422).end()
  const data = await getFacultyStudentStats(code, specialGroups, graduated)
  if (data) return res.json(data)
-  const programmes = await getProgrammes(code, programmeFilter)
-  if (!programmes) return res.status(422).end()
-  //all programmes are required for correct sorting of transfers
-  let allProgrammeCodes = []
-  const allProgs = await getProgrammes(code, 'ALL_PROGRAMMES')
-  allProgs?.data.forEach(prog => allProgrammeCodes.push(prog.code))
+  const newProgrammes = await getProgrammes(code, 'NEW_STUDY_PROGRAMMES')
+  if (!newProgrammes) return res.status(422).end()

-  let updateStats = await combineFacultyStudents(code, programmes.data, allProgrammeCodes, specialGroups, graduated)
+  let updateStats = await combineFacultyStudents(code, newProgrammes.data, specialGroups, graduated)
  if (updateStats) {
    updateStats = await setFacultyStudentStats(updateStats, specialGroups, graduated)
  }
@@ -195,12 +164,7 @@ router.get('/:id/update_basicview', async (req, res) => {
  const code = req.params.id
  const statsType = req.query?.stats_type
  if (code) {
-    let result = null
-    try {
-      result = await updateFacultyOverview(code, statsType)
-    } catch (e) {
-      logger.error(`Failed to update faculty ${code} basic tab stats for ${statsType}: ${e}`)
-    }
+    const result = await updateFacultyOverview(code, statsType)
    return res.json(result)
  }
  return res.status(422).end()
@@ -0,0 +1,13 @@
+const router = require('express').Router()
+const { getLanguageCenterData } = require('../services/languageCenterData')
+
+router.get('/', async (req, res) => {
+  const { iamGroups, isAdmin } = req.user
+  if (!isAdmin && !iamGroups.includes('grp-kielikeskus-esihenkilot')) {
+    return res.status(403).json({ error: 'Request failed because of missing rights' })
+  }
+  const result = await getLanguageCenterData()
+  return res.json(result)
+})
+
+module.exports = router
@@ -1,71 +0,0 @@
-const router = require('express').Router()
-const MandatoryCourseLabels = require('../services/mandatoryCourseLabels')
-const logger = require('../util/logger')
-
-router.get('/:programme/labels', async (req, res) => {
-  const { programme } = req.params
-  if (programme) {
-    try {
-      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
-      res.status(200).json(labels)
-    } catch (err) {
-      logger.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.delete('/:programme/labels', async (req, res) => {
-  const { programme } = req.params
-  const { label } = req.body
-  if (programme && label) {
-    try {
-      await MandatoryCourseLabels.destroy(programme, label)
-      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
-      res.status(200).json(labels)
-    } catch (err) {
-      logger.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.post('/:programme/move', async (req, res) => {
-  const { programme } = req.params
-  const { label, direction } = req.body
-  if (programme && label) {
-    try {
-      await MandatoryCourseLabels.move(programme, label, direction)
-      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
-      res.status(200).json(labels)
-    } catch (err) {
-      logger.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.post('/:programme/labels', async (req, res) => {
-  const { programme } = req.params
-  const { label } = req.body
-  if (programme && label) {
-    try {
-      await MandatoryCourseLabels.create(programme, label)
-      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
-      res.status(200).json(labels)
-    } catch (err) {
-      logger.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
-  }
-})
-
-module.exports = router
@@ -1,36 +0,0 @@
-const router = require('express').Router()
-const MandatoryCourses = require('../services/mandatoryCourses')
-
-router.delete('/:programme', async (req, res) => {
-  const { programme } = req.params
-  const { course } = req.body
-  if (!programme || !course) return res.status(400).end()
-  await MandatoryCourses.remove(programme, course)
-  res.status(200).json(course)
-})
-
-router.post('/:programme', async (req, res) => {
-  const { programme } = req.params
-  const { course } = req.body
-  if (!programme || !course) return res.status(400).end()
-  await MandatoryCourses.create(programme, course)
-  const newCourse = await MandatoryCourses.find(programme, course)
-  if (!newCourse) return res.status(400).end()
-  res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
-})
-
-router.post('/:programme/label/:course', async (req, res) => {
-  const { programme, course } = req.params
-  const { label } = req.body
-  if (!programme || !course) return res.status(400).end()
-  await MandatoryCourses.updateLabel(programme, course, label)
-  const updatedCourse = await MandatoryCourses.find(programme, course)
-  if (!updatedCourse) return res.status(400).end()
-  res.status(200).json({
-    name: updatedCourse.course.name,
-    code: updatedCourse.course_code,
-    label: updatedCourse.mandatory_course_label,
-  })
-})
-
-module.exports = router
@@ -1,36 +0,0 @@
-const router = require('express').Router()
-const config = require('../conf-backend')
-
-// Produces the Matomo tracker initialization script for the frontend.
-// Matomo URL and site id are baked here instead of in the frontend
-// so we can fix them on the fly instead of rebuilding the entire frontend.
-
-const acual = `
-var _paq = window._paq || [];
-/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
-_paq.push(["trackPageView"]);
-_paq.push(["enableLinkTracking"]);
-(function() {
-  var u = "${config.MATOMO_URL}";
-  _paq.push(["setTrackerUrl", u + "matomo.php"]);
-  _paq.push(["setSiteId", "${config.MATOMO_SITE_ID}"]);
-  _paq.push(["enableHeartBeatTimer"]);
-  var d = document,
-    g = d.createElement("script"),
-    s = d.getElementsByTagName("script")[0];
-  g.type = "text/javascript";
-  g.async = true;
-  g.defer = true;
-  g.src = u + "matomo.js";
-  s.parentNode.insertBefore(g, s);
-})();
-`
-
-const script = config.MATOMO_URL ? acual : ''
-
-router.get('/matomo-init', (_, res) => {
-  res.contentType('text/javascript; charset=UTF-8')
-  res.send(script)
-})
-
-module.exports = router
@@ -10,7 +10,7 @@ const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
 const { mapToProviders } = require('../util/utils')
 const { encrypt, decrypt } = require('../services/encrypt')
-const { mapCodesToIds } = require('../services/studyprogrammeHelpers')
+const { mapCodesToIds } = require('../services/studyprogramme/studyprogrammeHelpers')

 const { ApplicationError } = require('../util/customErrors')

@@ -86,42 +86,6 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
  }
 })

-router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => {
-  const { from, to, coursecodes } = req.body
-  if (!from || !to || !coursecodes) {
-    return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
-  }
-  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(coursecodes)
-  if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
-    return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
-  }
-
-  const {
-    user: { isAdmin, studentsUserCanAccess },
-  } = req
-
-  const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
-  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
-
-  const result = await Population.bottlenecksOf(
-    {
-      year: 1900,
-      studyRights: [],
-      semesters: ['FALL', 'SPRING'],
-      months: 10000,
-    },
-    studentnumberlist
-  )
-
-  if (result.error) {
-    Sentry.captureException(new Error(result.error))
-    res.status(400).json(result)
-    return
-  }
-
-  res.json(result)
-})
-
 router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
  const { tag } = req.body
  if (!tag) {
@@ -157,7 +121,7 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
 router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
  const {
    user: { isAdmin, studentsUserCanAccess },
-    body: { studentnumberlist: studentnumbersInReq, studentCountLimit, ...query },
+    body: { studentnumberlist: studentnumbersInReq, ...query },
  } = req

  if (!studentnumbersInReq) {
@@ -165,14 +129,12 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
  }

  const studentnumberlist = isAdmin ? studentnumbersInReq : _.intersection(studentnumbersInReq, studentsUserCanAccess)
-
  const result = await Population.bottlenecksOf(
    {
      year: query?.year ?? 1900,
      studyRights: query?.studyRights ?? [],
      semesters: query?.semesters ?? ['FALL', 'SPRING'],
      months: query?.months ?? 10000,
-      studentCountLimit,
    },
    studentnumberlist
  )
@@ -200,7 +162,13 @@ router.get('/v3/populationstatistics', async (req, res) => {
  } = req

  try {
-    if (!isAdmin && !rights.includes(studyRights.programme) && !iamRights.includes(studyRights.programme)) {
+    if (
+      !isAdmin &&
+      !rights.includes(studyRights.programme) &&
+      !rights.includes(studyRights.combinedProgramme) &&
+      !iamRights.includes(studyRights.programme) &&
+      !iamRights.includes(studyRights.combinedProgramme)
+    ) {
      res.status(403).json([])
      return
    }
@@ -209,7 +177,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
    return
  }

-  if (req.query.months == null) {
+  if (req.query.months === null) {
    req.query.months = 12
  }

@@ -218,12 +186,15 @@ router.get('/v3/populationstatistics', async (req, res) => {
    const multipopulationstudentPromises = Promise.all(
      req.query.years.map(year => {
        const newMonths = (upperYearBound - Number(year)) * 12
-        const populationStudents = Population.optimizedStatisticsOf({
-          ...req.query,
-          studyRights,
-          year,
-          months: newMonths,
-        })
+        const populationStudents = Population.optimizedStatisticsOf(
+          {
+            ...req.query,
+            studyRights: { programme: studyRights.programme },
+            year,
+            months: newMonths,
+          },
+          null
+        )
        return populationStudents
      })
    )
@@ -239,8 +210,6 @@ router.get('/v3/populationstatistics', async (req, res) => {

    res.json(filterPersonalTags(result, user.id))
  } else {
-    // If two studyprogrammes are given, query does not work just right
-    // This solution is temporal
    const result = await Population.optimizedStatisticsOf({
      ...req.query,
      studyRights: { programme: studyRights.programme },
@@ -253,7 +222,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
    }

    // Obfuscate if user has only iam rights
-    if (!isAdmin && !rights.includes(studyRights.programme)) {
+    if (!isAdmin && !rights.includes(studyRights.programme) && !rights.includes(studyRights.combinedProgramme)) {
      result.students = result.students.map(student => {
        const { iv, encryptedData } = encrypt(student.studentNumber)
        const obfuscatedBirthDate = new Date(new Date(student.birthdate).setMonth(0, 0)) // only birth year for age distribution
@@ -401,10 +370,14 @@ router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
  } = req
  const filteredStudentNumbers = isAdmin ? studentnumberlist : _.intersection(studentnumberlist, studentsUserCanAccess)

+  const studyProgrammeCode =
+    tags?.studyProgramme && tags?.studyProgramme.includes('+')
+      ? tags?.studyProgramme.split('+')[0]
+      : tags?.studyProgramme
  const result = await Population.optimizedStatisticsOf(
    {
      year: tags?.year || 1900,
-      studyRights: tags?.studyProgramme ? [tags.studyProgramme] : [],
+      studyRights: studyProgrammeCode ? [studyProgrammeCode] : [],
      semesters: ['FALL', 'SPRING'],
      months: 10000,
    },
@@ -414,7 +387,12 @@ router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
    Sentry.captureException(new Error(result.error))
    return
  }
-  res.status(200).json(filterPersonalTags(result, id))
+
+  result.studyProgramme = tags?.studyProgramme
+
+  const discardedStudentNumbers = _.difference(studentnumberlist, filteredStudentNumbers)
+
+  res.status(200).json({ ...filterPersonalTags(result, id), discardedStudentNumbers })
 })

 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
@@ -426,7 +404,11 @@ router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
    mapCodesToIds(studyrights.programmes)
    res.json(studyrights)
  } else {
-    const studyrights = await StudyrightService.getFilteredAssociations(rights.concat(iamRights))
+    const allRights = rights.concat(iamRights)
+    // For combined programme
+    // If more programmes are combined, then a function might be a better idea to add moar rights
+    if (allRights.includes('KH90_001') || allRights.includes('MH90_001')) allRights.push('KH90_001', 'MH90_001')
+    const studyrights = await StudyrightService.getFilteredAssociations(allRights)
    mapCodesToIds(studyrights.programmes)
    res.json(studyrights)
  }
@@ -445,7 +427,7 @@ router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req
  )
  const maxYearsToCreatePopulationFromUni = await CourseService.maxYearsToCreatePopulationFrom(
    JSON.parse(courseCodes),
-    'reqularStats'
+    'regularStats'
  )
  const maxYearsToCreatePopulationFromBoth = await CourseService.maxYearsToCreatePopulationFrom(
    JSON.parse(courseCodes),
@@ -1,43 +1,21 @@
 const router = require('express').Router()
-const {
-  byProgrammeCode,
-  addExcludedCourses,
-  removeExcludedCourses,
-  modulesByProgrammeCode,
-} = require('../services/programmeModules')
+const { getCoursesAndModules, getCurriculumVersions } = require('../services/programmeModules')

-router.get('/v3/programme_modules/:code', async (req, res) => {
+router.get('/v3/get_curriculum_options/:code', async (req, res) => {
  const { code } = req.params
-  const result = await byProgrammeCode(code)
+  const result = await getCurriculumVersions(code)
  res.json(result)
 })

-router.get('/v3/programme_modules/:code/modules', async (req, res) => {
-  const { code } = req.params
-  const result = await modulesByProgrammeCode(code)
-  res.json(result)
-})
-
-router.delete('/v3/programme_modules', async (req, res) => {
-  const { programmecode, ids } = req.body
-  await removeExcludedCourses(ids)
-  const result = await byProgrammeCode(programmecode)
-  if (!result) {
-    res.status(400).end()
-    return
-  }
-  res.json(result)
-})
-
-router.post('/v3/programme_modules/:programmecode/', async (req, res) => {
-  const { programmecode, coursecodes } = req.body
-  await addExcludedCourses(programmecode, coursecodes)
-  const result = await byProgrammeCode(programmecode)
-  if (!result) {
-    res.status(400).end()
-    return
-  }
-  res.json(result)
+router.get('/v3/get_curriculum/:code/:period_ids', async (req, res) => {
+  const { code, period_ids } = req.params
+  const result = await getCoursesAndModules(code, period_ids.replace(' ', '').split(','))
+  res.json({
+    defaultProgrammeCourses: result.defaultProgrammeCourses.courses,
+    defaultProgrammeModules: result.defaultProgrammeCourses.modules,
+    secondProgrammeCourses: result.secondProgrammeCourses?.courses,
+    secondProgrammeModules: result.secondProgrammeCourses?.modules,
+  })
 })

 module.exports = router
@@ -1,21 +0,0 @@
-const express = require('express')
-const logger = require('../util/logger')
-
-const wrapWithErrorHandler = fn => (req, res) =>
-  Promise.resolve(fn(req, res)).catch(error => {
-    logger.error(error)
-    res.status(500).send('Internal server error')
-  })
-
-const routerWithWrapper = () => {
-  const router = express.Router()
-  return {
-    router,
-    wrapper: {
-      get: (url, callback) => router.get(url, wrapWithErrorHandler(callback)),
-      post: (url, callback) => router.post(url, wrapWithErrorHandler(callback)),
-    },
-  }
-}
-
-module.exports = { routerWithWrapper }
@@ -44,9 +44,9 @@ router.get('/students/:id', async (req, res) => {
  if (!isAdmin && !studentsUserCanAccess.includes(studentId)) {
    throw new ApplicationError('Error finding student', 400)
  }
-
  const student = await Student.withId(studentId)
-  res.json(filterStudentTags(student, id))
+  const filteredTags = filterStudentTags(student, id)
+  res.json(filteredTags)
 })

 module.exports = router
@@ -1,9 +1,9 @@
 const router = require('express').Router()
-const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics')
-const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
-const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
-const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
-const { getStudyprogrammeCoursesForStudytrack } = require('../services/studyprogrammeCourses')
+const { getBasicStatsForStudytrack } = require('../services/studyprogramme/studyprogrammeBasics')
+const { getCreditStatsForStudytrack } = require('../services/studyprogramme/studyprogrammeCredits')
+const { getGraduationStatsForStudytrack } = require('../services/studyprogramme/studyprogrammeGraduations')
+const { getStudyprogrammeCoursesForStudytrack } = require('../services/studyprogramme/studyprogrammeCourses')
+const { getStudytrackStatsForStudyprogramme } = require('../services/studyprogramme/studytrackStats')
 const {
  getBasicStats,
  setBasicStats,
@@ -14,22 +14,36 @@ const {
  getStudytrackStats,
  setStudytrackStats,
 } = require('../services/analyticsService')
-const { updateBasicView, updateStudytrackView } = require('../services/studyprogrammeUpdates')
+const { updateBasicView, updateStudytrackView } = require('../services/studyprogramme/studyprogrammeUpdates')
 const { getProgrammeName } = require('../services/studyprogramme')
 const logger = require('../util/logger')
+const { getAssociations } = require('../services/studyrights')
+const { ElementDetail } = require('../models')
+
+// For grafana statistics (idea stolen from Norppa)
+const logInfoForGrafana = async (code, combinedProgramme) => {
+  const programme = await ElementDetail.findOne({ where: { code: code } })
+  const programmeCode = combinedProgramme ? `${programme.code}-${combinedProgramme}` : programme.code
+  logger.info('Study Programme', {
+    studyprogrammeName: combinedProgramme ? `${programme.name.fi} + maisteri` : programme.name.fi,
+    studyprogrammeCode: programmeCode,
+  })
+}

 router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
  const code = req.params.id
  const yearType = req.query?.year_type
  const specialGroups = req.query?.special_groups
-
+  const combinedProgramme = req.query?.combined_programme
  if (!code) return res.status(422).end()

-  const data = await getBasicStats(code, yearType, specialGroups)
+  logInfoForGrafana(code, combinedProgramme)
+  const data = await getBasicStats(code, combinedProgramme, yearType, specialGroups)
  if (data) return res.json(data)

  const updated = await getBasicStatsForStudytrack({
    studyprogramme: req.params.id,
+    combinedProgramme,
    settings: {
      isAcademicYear: yearType === 'ACADEMIC_YEAR',
      includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
@@ -43,13 +57,15 @@ router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
  const code = req.params.id
  const yearType = req.query?.year_type
  const specialGroups = req.query?.special_groups
+  const combinedProgramme = req.query?.combined_programme

  if (!code) return res.status(422).end()

-  const data = await getCreditStats(code, yearType, specialGroups)
+  const data = await getCreditStats(code, combinedProgramme, yearType, specialGroups)
  if (data) return res.json(data)
  const updatedStats = await getCreditStatsForStudytrack({
    studyprogramme: req.params.id,
+    combinedProgramme,
    settings: {
      isAcademicYear: yearType === 'ACADEMIC_YEAR',
      includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
@@ -63,13 +79,15 @@ router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
  const code = req.params.id
  const yearType = req.query?.year_type
  const specialGroups = req.query?.special_groups
+  const combinedProgramme = req.query?.combined_programme

  if (!code) return res.status(422).end()

-  const data = await getGraduationStats(code, yearType, specialGroups)
+  const data = await getGraduationStats(code, combinedProgramme, yearType, specialGroups)
  if (data) return res.json(data)
  const updatedStats = await getGraduationStatsForStudytrack({
    studyprogramme: req.params.id,
+    combinedProgramme,
    settings: {
      isAcademicYear: yearType === 'ACADEMIC_YEAR',
      includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
@@ -82,14 +100,16 @@ router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
 router.get('/v2/studyprogrammes/:id/coursestats', async (req, res) => {
  const code = req.params.id
  const showByYear = req.query.academicyear
+  const combinedProgramme = req.query?.combined_programme
  const date = new Date()
  date.setHours(23, 59, 59, 999)

+  logInfoForGrafana(code, combinedProgramme)
  try {
-    const data = await getStudyprogrammeCoursesForStudytrack(date.getTime(), code, showByYear)
+    const data = await getStudyprogrammeCoursesForStudytrack(date.getTime(), code, showByYear, combinedProgramme)
    return res.json(data)
  } catch (e) {
-    logger.error(`Failed to get code ${code} programme courses stats: ${e}`)
+    logger.error({ message: `Failed to get code ${code} programme courses stats`, meta: `${e}` })
  }
 })

@@ -97,18 +117,23 @@ router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
  const code = req.params.id
  const graduated = req.query?.graduated
  const specialGroups = req.query?.special_groups
+  const combinedProgramme = req.query?.combined_programme

  if (!code) return res.status(422).end()

-  const data = await getStudytrackStats(code, graduated, specialGroups)
+  logInfoForGrafana(code, combinedProgramme)
+  const data = await getStudytrackStats(code, combinedProgramme, graduated, specialGroups)
  if (data) return res.json(data)

+  const associations = await getAssociations()
  const updated = await getStudytrackStatsForStudyprogramme({
    studyprogramme: code,
+    combinedProgramme,
    settings: {
      graduated: graduated === 'GRADUATED_INCLUDED',
      specialGroups: specialGroups === 'SPECIAL_INCLUDED',
    },
+    associations,
  })
  if (updated) await setStudytrackStats(updated, graduated, specialGroups)
  return res.json(updated)
@@ -116,12 +141,13 @@ router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {

 router.get('/v2/studyprogrammes/:id/update_basicview', async (req, res) => {
  const code = req.params.id
+  const combinedProgramme = req.query?.combined_programme
  if (code) {
    let result = null
    try {
-      result = await updateBasicView(code)
+      result = await updateBasicView(code, combinedProgramme)
    } catch (e) {
-      logger.error(`Failed to update code ${code} basic stats: ${e}`)
+      logger.error({ message: `Failed to update code ${code} ${combinedProgramme} basic stats`, meta: `${e}` })
    }
    return res.json(result)
  } else {
@@ -131,12 +157,14 @@ router.get('/v2/studyprogrammes/:id/update_basicview', async (req, res) => {

 router.get('/v2/studyprogrammes/:id/update_studytrackview', async (req, res) => {
  const code = req.params.id
+  const combinedProgramme = req.query?.combined_programme
  if (code) {
    let result = null
    try {
-      result = await updateStudytrackView(code)
+      const associations = await getAssociations()
+      result = await updateStudytrackView(code, combinedProgramme, associations)
    } catch (e) {
-      logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
+      logger.error({ message: `Failed to update code ${code} ${combinedProgramme} studytrack stats`, meta: `${e}` })
    }
    return res.json(result)
  } else {
@@ -153,11 +181,12 @@ router.get('/v2/studyprogrammes/:id/evaluationstats', async (req, res) => {
  if (!code) return res.status(422).end()

  // Statistics for Tilannekuvalomake view
-
-  let gradData = await getGraduationStats(code, yearType, specialGroups)
+  const combinedProgramme = ''
+  let gradData = await getGraduationStats(code, combinedProgramme, yearType, specialGroups)
  if (!gradData) {
    const updatedStats = await getGraduationStatsForStudytrack({
      studyprogramme: req.params.id,
+      combinedProgramme,
      settings: {
        isAcademicYear: yearType === 'ACADEMIC_YEAR',
        includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
@@ -169,14 +198,17 @@ router.get('/v2/studyprogrammes/:id/evaluationstats', async (req, res) => {
    }
  }

-  let progressData = await getStudytrackStats(code, graduated, specialGroups)
+  let progressData = await getStudytrackStats(code, combinedProgramme, graduated, specialGroups)
  if (!progressData) {
+    const associations = await getAssociations()
    const updated = await getStudytrackStatsForStudyprogramme({
      studyprogramme: code,
+      combinedProgramme,
      settings: {
        graduated: graduated === 'GRADUATED_INCLUDED',
        specialGroups: specialGroups === 'SPECIAL_INCLUDED',
      },
+      associations,
    })
    if (updated) {
      await setStudytrackStats(updated, graduated, specialGroups)
@@ -189,27 +221,16 @@ router.get('/v2/studyprogrammes/:id/evaluationstats', async (req, res) => {
  delete gradData.tableStats
  delete gradData.graphStats
  delete gradData.titles
-
  const data = {
    id: code,
    programmeName: programmeName?.name,
    status: gradData?.status,
    lastUpdated: gradData.lastUpdated,
    graduations: gradData,
-    progress: {
-      creditTableStats: {},
-      creditTableTitles: progressData?.creditTableTitles,
-      graphData: {
-        creditGraphStats: {},
-        id: progressData?.id,
-        years: progressData?.years,
-      },
-    },
+    creditCounts: progressData?.creditCounts,
+    years: progressData?.years,
  }

-  data.progress.creditTableStats[code] = progressData?.creditTableStats[code]
-  data.progress.graphData.creditGraphStats[code] = progressData?.creditGraphStats[code]
-
  return res.json(data)
 })

@@ -2,7 +2,7 @@ const {
  getCriteria,
  saveYearlyCourseCriteria,
  saveYearlyCreditCriteria,
-} = require('../services/studyProgrammeCriteria')
+} = require('../services/studyprogramme/studyProgrammeCriteria')

 const router = require('express').Router()

@@ -22,8 +22,11 @@ router.get('/tags/:studytrack', async (req, res) => {
  const {
    user: { rights, roles, id },
  } = req
+  const programmeCodes = studytrack.includes('KH') && studytrack.includes('MH') ? studytrack.split('-') : [studytrack]

-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
+  // Respond with null and 200 instead of 403 if the user isn't authorized to view the tags. This is to avoid unnecessary noise in Sentry
+  if (!rights.includes(programmeCodes[0]) && !rights.includes(programmeCodes[1]) && !roles?.includes('admin'))
+    return res.json(null)

  const tags = await Tags.findTagsByStudytrack(studytrack)
  res.status(200).json(filterRelevantTags(tags, id))
@@ -37,7 +40,9 @@ router.post('/tags', async (req, res) => {
    user: { rights, roles, id },
  } = req

-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
+  const programmeCodes = studytrack.includes('KH') && studytrack.includes('MH') ? studytrack.split('-') : [studytrack]
+  if (!rights.includes(programmeCodes[0]) && !rights.includes(programmeCodes[1]) && !roles?.includes('admin'))
+    return res.status(403).end()

  await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
  const tags = await Tags.findTagsByStudytrack(studytrack)
@@ -50,7 +55,15 @@ router.delete('/tags', async (req, res) => {
    user: { rights, roles, id },
  } = req

-  if (!rights.includes(tag.studytrack) && !roles?.includes('admin') && !(tag.personal_user_id === id))
+  const programmeCodes =
+    tag.studytrack.includes('KH') && tag.studytrack.includes('MH') ? tag.studytrack.split('-') : [tag.studytrack]
+
+  if (
+    !rights.includes(programmeCodes[0]) &&
+    !rights.includes(programmeCodes[1]) &&
+    !roles?.includes('admin') &&
+    !(tag.personal_user_id === id)
+  )
    return res.status(403).end()

  await Tags.deleteTag(tag)
@@ -69,7 +82,11 @@ router.get('/studenttags/:studytrack', async (req, res) => {
    user: { rights, roles, id },
  } = req

-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
+  const programmeCodes = studytrack.includes('KH') && studytrack.includes('MH') ? studytrack.split('-') : [studytrack]
+
+  // Respond with null and 200 instead of 403 if the user isn't authorized to view the tags. This is to avoid unnecessary noise in Sentry
+  if (!rights.includes(programmeCodes[0]) && !rights.includes(programmeCodes[1]) && !roles?.includes('admin'))
+    return res.json(null)

  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
  res.status(200).json(filterRelevantStudentTags(result, id))
@@ -83,33 +100,38 @@ router.get('/studenttags/:studentnumber', async (req, res) => {
 })

 router.post('/studenttags/:studentnumber', async (req, res) => {
-  const { tag, studytrack } = req.body
+  const { tag, studytrack, combinedProgramme } = req.body
  const {
    user: { roles, rights, id },
  } = req

-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
+  if (!rights.includes(studytrack) && !rights.includes(combinedProgramme) && !roles?.includes('admin'))
+    return res.status(403).end()

  await TagStudent.createStudentTag(tag)
-  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  const studytrackCode = combinedProgramme ? `${studytrack}-${combinedProgramme}` : studytrack
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrackCode)
  res.status(200).json(filterRelevantStudentTags(result, id))
 })

 router.post('/studenttags', async (req, res) => {
-  const { tags, studytrack } = req.body
+  const { tags, studytrack, combinedProgramme } = req.body
  const {
    user: { rights, roles, id },
  } = req

-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
+  if (!rights.includes(studytrack) && !rights.includes(combinedProgramme) && !roles?.includes('admin'))
+    return res.status(403).end()

-  const existingtags = await Tags.findTagsByStudytrack(studytrack)
-  const existingTagids = existingtags.map(t => t.tag_id)
+  const studytrackCode = combinedProgramme ? `${studytrack}-${combinedProgramme}` : studytrack
+  const existingTags = await Tags.findTagsByStudytrack(studytrackCode)
+  const existingTagids = existingTags.map(t => t.tag_id)
  const tagids = [...new Set(tags.map(t => t.tag_id))]
  if (!tagids.find(t => existingTagids.includes(t))) return res.status(400).json({ error: 'The tag does not exist' })

  const studentnumbers = tags.map(t => t.studentnumber)
-  const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
+  const studentFromProgrammes = combinedProgramme ? [studytrack, combinedProgramme] : [studytrack]
+  const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, studentFromProgrammes)
  const missingStudents = difference(studentnumbers, students)
  if (missingStudents.length !== 0)
    return res
@@ -117,39 +139,41 @@ router.post('/studenttags', async (req, res) => {
      .json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })

  await TagStudent.createMultipleStudentTags(tags)
-  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrackCode)
  res.status(200).json(filterRelevantStudentTags(result, id))
 })

 router.delete('/studenttags/delete_one', async (req, res) => {
-  const { tag_id, studentnumber, studytrack } = req.body
+  const { tag_id, studentnumber, studytrack, combinedProgramme } = req.body
  const {
    user: { rights, roles, id },
  } = req

-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
-
-  const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
+  if (!rights.includes(studytrack) && !rights.includes(combinedProgramme) && !roles?.includes('admin'))
+    return res.status(403).end()
+  const studytrackCode = combinedProgramme ? `${studytrack}-${combinedProgramme}` : studytrack
+  const tags = await Tags.findTagsFromStudytrackById(studytrackCode, [tag_id])
  if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })

  await TagStudent.deleteStudentTag(studentnumber, tag_id)
-  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrackCode)
  res.status(200).json(filterRelevantStudentTags(result, id))
 })

 router.delete('/studenttags/delete_many', async (req, res) => {
-  const { tagId, studentnumbers, studytrack } = req.body
+  const { tagId, studentnumbers, studytrack, combinedProgramme } = req.body
  const {
    user: { rights, roles, id },
  } = req

-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
-
-  const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
+  if (!rights.includes(studytrack) && !rights.includes(combinedProgramme) && !roles?.includes('admin'))
+    return res.status(403).end()
+  const studytrackCode = combinedProgramme ? `${studytrack}-${combinedProgramme}` : studytrack
+  const tags = await Tags.findTagsFromStudytrackById(studytrackCode, [tagId])
  if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })

  await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
-  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrackCode)
  res.status(200).json(filterRelevantStudentTags(result, id))
 })

@@ -1,31 +0,0 @@
-const Sentry = require('@sentry/node')
-const router = require('express').Router()
-const tsaService = require('../services/tsaService')
-
-router.post('/event', (req, res) => {
-  const { group, name, label, value } = req.body
-
-  if (!group || !name || !value) {
-    return res.status(400).json({ error: 'group, name and value are required' })
-  }
-
-  const {
-    user: { userId },
-  } = req
-
-  // don't await here because frontend doesn't care if it succeeds
-  tsaService.sendTsaEvent(userId, { group, name, label, value }).catch(e => {
-    Sentry.withScope(scope => {
-      // if error was caused by a 400, dig out the JSON response for Sentry
-      if (e.isAxiosError && e.response && e.response.data) {
-        scope.setExtra({ error: e.response.data })
-      }
-
-      Sentry.captureException(e)
-    })
-  })
-
-  res.status(200).end()
-})
-
-module.exports = router
@@ -0,0 +1,192 @@
+const router = require('express').Router()
+const { getFacultyList } = require('../services/faculty/facultyHelpers')
+const { getGraduationStats, getFacultyProgressStats } = require('../services/faculty/facultyService')
+
+const { getMedian } = require('../services/studyprogramme/studyprogrammeHelpers')
+const degreeNames = ['bachelor', 'bachelorMaster', 'master', 'licentiate', 'doctor']
+
+router.get('/allprogressstats', async (req, res) => {
+  const specialGroups = 'SPECIAL_EXCLUDED'
+  const graduated = req.query?.graduated
+  const allFaculties = await getFacultyList()
+  const facultyCodes = allFaculties.map(f => f.code)
+  const codeToData = {}
+
+  for (const facultyCode of facultyCodes) {
+    const data = await getFacultyProgressStats(facultyCode, specialGroups, graduated)
+    if (!data) {
+      return res.status(500).end()
+    }
+    codeToData[facultyCode] = data
+  }
+
+  const universityData = {
+    years: codeToData.H50.years,
+    yearlyBachelorTitles: codeToData.H50.yearlyBachelorTitles,
+    programmeNames: allFaculties.reduce((obj, fac) => {
+      const { name, ...rest } = fac.dataValues
+      obj[fac.code] = { ...rest, ...name }
+      return obj
+    }, {}),
+    bachelorsProgStats: {},
+    creditCounts: {
+      bachelor: {},
+    },
+  }
+
+  const unifyProgressStats = progStats => {
+    return progStats.reduce(
+      (all, prog) => {
+        prog.forEach((yearStats, yearIndex) =>
+          yearStats.forEach((category, categoryIndex) => {
+            all[yearIndex][categoryIndex] += prog[yearIndex][categoryIndex]
+          })
+        )
+        return all
+      },
+      // eslint-disable-next-line no-unused-vars
+      progStats[0].map(year => year.map(_num => 0))
+    )
+  }
+
+  for (const facultyCode of facultyCodes) {
+    const facultyData = codeToData[facultyCode]
+    for (const year of universityData.years.slice(1).reverse()) {
+      for (const fieldName of degreeNames) {
+        if (!facultyData.creditCounts[fieldName] || Object.keys(facultyData.creditCounts[fieldName]).length === 0)
+          continue
+        if (!universityData.creditCounts[fieldName]) universityData.creditCounts[fieldName] = {}
+        if (!universityData.creditCounts[fieldName][year]) universityData.creditCounts[fieldName][year] = []
+        universityData.creditCounts[fieldName][year].push(...facultyData.creditCounts[fieldName][year])
+      }
+      for (const fieldName of [
+        'bachelorsProgStats',
+        'bcMsProgStats',
+        'licentiateProgStats',
+        'mastersProgStats',
+        'doctoralProgStats',
+      ]) {
+        if (Object.keys(facultyData[fieldName]).length === 0) continue
+        if (!universityData[fieldName]) universityData[fieldName] = {}
+        universityData[fieldName][facultyCode] = unifyProgressStats(Object.values(facultyData[fieldName]))
+      }
+    }
+  }
+
+  // Remove ELL bachelor+master progresstats for now, because it is problematic due to different credit limits
+  // when expanding the table rows to show 'programme'(faculty) specific progress bars
+  delete universityData.bcMsProgStats['H90']
+
+  return res.status(200).json(universityData)
+})
+
+router.get('/allgraduationstats', async (req, res) => {
+  const degreeNames = ['bachelor', 'bcMsCombo', 'master', 'licentiate', 'doctor']
+  const allFaculties = await getFacultyList()
+  const facultyCodes = allFaculties.map(f => f.code)
+  const facultyData = {}
+  const timesArrays = [] // keep book of these to null them in the end, large lists not used in frontend
+  for (const facultyCode of facultyCodes) {
+    const data = await getGraduationStats(facultyCode, 'NEW_STUDY_PROGRAMMES', true)
+    if (!data) res.status(500).json({ message: `Did not find data for ${facultyCode}` })
+    facultyData[facultyCode] = data
+  }
+
+  const unifyTotals = (facultyData, universityData, isLast) => {
+    for (const degree of degreeNames) {
+      if (!universityData[degree]) universityData[degree] = []
+      if (!facultyData[degree]) continue
+      for (const yearStats of facultyData[degree]) {
+        const universityStats = universityData[degree]
+        const universityYearStats = universityData[degree].find(stats => stats.name === yearStats.name)
+        if (!universityYearStats) {
+          universityStats.push(yearStats)
+        } else {
+          universityYearStats.times.push(...yearStats.times)
+          timesArrays.push(universityYearStats.times)
+          universityYearStats.amount += yearStats.amount
+          universityYearStats.statistics.onTime += yearStats.statistics.onTime
+          universityYearStats.statistics.yearOver += yearStats.statistics.yearOver
+          universityYearStats.statistics.wayOver += yearStats.statistics.wayOver
+          if (isLast) {
+            universityYearStats.y = getMedian(universityYearStats.times)
+          }
+        }
+      }
+    }
+  }
+
+  const unifyProgrammeStats = (universityData, facultyData, facultyCode) => {
+    for (const degree of degreeNames) {
+      if (!facultyData[degree]) continue
+      if (!universityData[degree]) universityData[degree] = {}
+      for (const yearData of facultyData[degree]) {
+        if (!universityData[degree][yearData.name]) {
+          universityData[degree][yearData.name] = { programmes: [], data: [] }
+        }
+        const uniYearStats = universityData[degree][yearData.name]
+        if (!uniYearStats.programmes.find(prog => prog === facultyCode)) {
+          uniYearStats.programmes.push(facultyCode)
+        }
+        const uniYearFacultyStats = uniYearStats.data.find(item => item.code === facultyCode)
+        const yearDataClone = { ...yearData, times: null, statistics: { ...yearData.statistics } }
+        if (!uniYearFacultyStats) {
+          uniYearStats.data.push({ ...yearDataClone, name: facultyCode, code: facultyCode })
+        } else {
+          uniYearFacultyStats.y = yearData.y
+          uniYearFacultyStats.amount += yearData.amount
+          uniYearFacultyStats.statistics.onTime += yearData.statistics.onTime
+          uniYearFacultyStats.statistics.yearOver += yearData.statistics.yearOver
+          uniYearFacultyStats.statistics.wayOver += yearData.statistics.wayOver
+        }
+      }
+    }
+
+    return universityData
+  }
+
+  const universityData = {
+    goals: {
+      bachelor: 36,
+      bcMsCombo: 60,
+      master: 24,
+      doctor: 48,
+      licentiate: 78,
+    },
+    programmeNames: allFaculties.reduce((obj, fac) => {
+      const { name, ...rest } = fac.dataValues
+      obj[fac.code] = { ...rest, ...name }
+      return obj
+    }, {}),
+    byGradYear: { medians: {}, programmes: { medians: {} } },
+    classSizes: { programmes: {} },
+  }
+
+  for (let i = 0; i < facultyCodes.length; i++) {
+    const facultyCode = facultyCodes[i]
+    const data = facultyData[facultyCode]
+    unifyTotals(data.byGradYear.medians, universityData.byGradYear.medians, i === facultyCodes.length - 1)
+    unifyProgrammeStats(universityData.byGradYear.programmes.medians, data.byGradYear.medians, facultyCode)
+    for (const degree of degreeNames) {
+      if (!universityData.classSizes[degree]) {
+        universityData.classSizes[degree] = data.classSizes[degree]
+      } else {
+        Object.entries(data.classSizes[degree]).forEach(([key, value]) => {
+          universityData.classSizes[degree][key] += value
+        })
+      }
+    }
+    // eslint-disable-next-line no-unused-vars
+    const { programmes, ...rest } = data.classSizes
+    universityData.classSizes.programmes[facultyCode] = rest
+  }
+
+  // Empty "times" arrays because that's not needed anymore.
+  timesArrays.forEach(arr => {
+    arr.length = 0
+  })
+
+  return res.status(200).json(universityData)
+})
+
+module.exports = router
@@ -1,4 +1,5 @@
 const router = require('express').Router()
+const { refreshFaculties, refreshProgrammes } = require('../events')
 const {
  updateSISMetadata,
  updateSISStudents,
@@ -8,7 +9,9 @@ const {
  abort,
  updateCoursesByCourseCode,
 } = require('../services/sisUpdaterService')
-const { refreshStatistics, refreshTrends } = require('../events')
+
+const logger = require('../util/logger')
+const { jobMaker, getJobs } = require('../worker/queue')

 router.get('/update/v2/meta', async (req, res) => {
  const response = await updateSISMetadata(req)
@@ -46,6 +49,7 @@ router.get('/update/v2/programmes', async (req, res) => {
 })

 router.get('/refresh_redis_cache', async (req, res) => {
+  logger.info(`${req.user.userId} requested refresh of redis cache`)
  const response = await updateSISRedisCache()
  if (response) {
    res.status(200).json('Refreshing SIS redis cache scheduled')
@@ -53,13 +57,33 @@ router.get('/refresh_redis_cache', async (req, res) => {
 })

 router.post('/refresh_statistic_v2', async (req, res) => {
-  await refreshStatistics()
-  res.status(200).json('Refreshing sis statistics')
+  logger.info(`${req.user.userId} requested refresh of statistics`)
+  jobMaker.statistics()
+  res.status(200).json('Teacher and study programme statistics refreshed')
+})
+
+router.post('/refresh_study_programmes_v2', async (req, res) => {
+  logger.info(`${req.user.userId} requested refresh of study programmes`)
+  refreshProgrammes()
+  res.status(200).json('Added job for refreshing study programme overviews')
+})
+
+router.post('/refresh_faculties_v2', async (req, res) => {
+  logger.info(`${req.user.userId} requested refresh of faculties`)
+  refreshFaculties()
+  res.status(200).json('Added job for refreshing faculties')
 })

 router.post('/refresh_trends', async (req, res) => {
-  await refreshTrends()
-  res.status(200).json('Refreshing trends')
+  logger.info(`${req.user.userId} requested refresh of trends`)
+  jobMaker.trends()
+  res.status(200).json('Added job for refreshing trends')
+})
+
+router.post('/refresh_language_center_data', async (req, res) => {
+  logger.info(`${req.user.userId} requested refresh of language center data`)
+  jobMaker.languagecenter()
+  res.status(200).json('Added job for refreshing language center data')
 })

 router.get('/abort', async (req, res) => {
@@ -67,4 +91,10 @@ router.get('/abort', async (req, res) => {
  res.status(200).json()
 })

+router.get('/jobs', async (req, res) => {
+  const waiting = await getJobs('waiting')
+  const active = await getJobs('active')
+  res.status(200).json({ waiting, active })
+})
+
 module.exports = router
@@ -10,9 +10,10 @@ const createRedisKeyForGraduationStats = (id, yearType, specialGroups) =>
 const createRedisKeyForStudytrackStats = (id, graduated, specialGroups) =>
  `STUDYTRACK_STATS_${id}_${graduated}_${specialGroups}`

-const getBasicStats = async (id, yearType, specialGroups) => {
+const getBasicStats = async (id, combinedProgramme, yearType, specialGroups) => {
  if (!isUpdatedNewProgramme(id)) return null
-  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
+  let searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
+  const redisKey = createRedisKeyForBasicStats(searchkey, yearType, specialGroups)
  const dataFromRedis = await redisClient.getAsync(redisKey)
  if (!dataFromRedis) return null
  return JSON.parse(dataFromRedis)
@@ -32,9 +33,10 @@ const setBasicStats = async (data, yearType, specialGroups) => {
  return dataToRedis
 }

-const getCreditStats = async (id, yearType, specialGroups) => {
+const getCreditStats = async (id, combinedProgramme, yearType, specialGroups) => {
  if (!isUpdatedNewProgramme(id)) return null
-  const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
+  let searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
+  const redisKey = createRedisKeyForCreditStats(searchkey, yearType, specialGroups)
  const dataFromRedis = await redisClient.getAsync(redisKey)
  if (!dataFromRedis) return null
  return JSON.parse(dataFromRedis)
@@ -54,9 +56,10 @@ const setCreditStats = async (data, yearType, specialGroups) => {
  return dataToRedis
 }

-const getGraduationStats = async (id, yearType, specialGroups) => {
+const getGraduationStats = async (id, combinedProgramme, yearType, specialGroups) => {
  if (!isUpdatedNewProgramme(id)) return null
-  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
+  let searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
+  const redisKey = createRedisKeyForGraduationStats(searchkey, yearType, specialGroups)
  const dataFromRedis = await redisClient.getAsync(redisKey)
  if (!dataFromRedis) return null
  return JSON.parse(dataFromRedis)
@@ -76,9 +79,10 @@ const setGraduationStats = async (data, yearType, specialGroups) => {
  return dataToRedis
 }

-const getStudytrackStats = async (id, graduated, specialGroups) => {
+const getStudytrackStats = async (id, combinedProgramme, graduated, specialGroups) => {
  if (!isUpdatedNewProgramme(id)) return null
-  const redisKey = createRedisKeyForStudytrackStats(id, graduated, specialGroups)
+  let searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
+  const redisKey = createRedisKeyForStudytrackStats(searchkey, graduated, specialGroups)
  const dataFromRedis = await redisClient.getAsync(redisKey)
  if (!dataFromRedis) return null
  return JSON.parse(dataFromRedis)
@@ -1,6 +1,5 @@
 const { Credit, Enrollment, Student, Course } = require('../models')
 const { Op } = require('sequelize')
-const _ = require('lodash')

 const getCompletedCourses = async (studentNumbers, courseCodes) => {
  let courses = await Course.findAll({
@@ -45,7 +44,7 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
  })

  const studentInfo = await Student.findAll({
-    attributes: ['studentnumber', 'firstnames', 'lastname', 'email'],
+    attributes: ['studentnumber', 'firstnames', 'lastname', 'email', 'sis_person_id'],
    where: {
      studentnumber: {
        [Op.in]: studentNumbers,
@@ -88,13 +87,14 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
    if (!studentCredits[s.studentnumber]) {
      studentCredits[s.studentnumber] = { credits: [], enrollments: [] }
    }
-    studentCredits[s.studentnumber].firstNames = s.firstnames
-    studentCredits[s.studentnumber].lastName = s.lastname
+    studentCredits[s.studentnumber].firstnames = s.firstnames
+    studentCredits[s.studentnumber].lastname = s.lastname
    studentCredits[s.studentnumber].email = s.email
+    studentCredits[s.studentnumber].sis_person_id = s.sis_person_id
  })

  credits.forEach(credit => {
-    if (credit.creditType === 9) {
+    if (credit.creditType === 10) {
      return
    }
    const previous = studentCredits[credit.studentNumber].credits?.find(c => credit.courseCode === c.courseCode)
@@ -125,6 +125,7 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
      date: enrollment.date,
      courseCode: enrollment.courseCode,
      state: enrollment.state,
+      substitution: enrollment.substitution,
    })
  })

@@ -133,10 +134,12 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
      ...acc,
      {
        studentNumber: student,
+        sis_person_id: studentCredits[student].sis_person_id,
        credits: studentCredits[student].credits,
-        enrollments: studentCredits[student].enrollments,
-        firstNames: studentCredits[student].firstNames,
-        lastName: studentCredits[student].lastName,
+        enrollments: {},
+        allEnrollments: studentCredits[student].enrollments,
+        firstnames: studentCredits[student].firstnames,
+        lastname: studentCredits[student].lastname,
        email: studentCredits[student].email,
      },
    ],
@@ -145,18 +148,22 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {

  students.forEach(student => {
    courseCodes.forEach(courseCode => {
-      let [hits, rest] = _.partition(student.enrollments, e => e.courseCode === courseCode)
-      if (hits?.length === 0) return
-      let latest = hits[0]
-      for (let hit of hits) {
-        if (hit > latest) {
-          latest = hit
-        }
-      }
-      student.enrollments = [latest, ...rest]
+      student.enrollments[courseCode] = student.allEnrollments
+        .filter(e => e.courseCode === courseCode || e.substitution === courseCode)
+        .sort(e => new Date(e.date).getDate())[0]
    })
  })
-  return { students, courses }
+
+  // Omit allEnrollments, we're only supposed to show the recent, relevant enrollment,
+  // the user does not have rights to see all enrollments.
+  return {
+    students: students.map(student => {
+      // eslint-disable-next-line no-unused-vars
+      const { allEnrollments, ...rest } = student
+      return { ...rest }
+    }),
+    courses,
+  }
 }

 module.exports = { getCompletedCourses }
@@ -1,529 +0,0 @@
-const Sequelize = require('sequelize')
-const {
-  Student,
-  Credit,
-  Course,
-  ElementDetail,
-  StudyrightElement,
-  Studyright,
-  Semester,
-  Organization,
-  Enrollment,
-} = require('../models')
-
-const { parseCredit } = require('./parseCredits')
-const Op = Sequelize.Op
-const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
-const { sortMainCode, getSortRank } = require('../util/utils')
-const { parseEnrollment } = require('./parseEnrollments')
-
-const byNameOrCode = (searchTerm, language) =>
-  Course.findAll({
-    where: {
-      [Op.or]: [
-        {
-          name: {
-            [language]: {
-              [Op.iLike]: searchTerm,
-            },
-          },
-        },
-        {
-          code: {
-            [Op.like]: searchTerm,
-          },
-        },
-      ],
-    },
-  })
-
-const byName = (name, language) =>
-  Course.findAll({
-    where: {
-      name: {
-        [language]: {
-          [Op.eq]: name,
-        },
-      },
-    },
-    order: [['latest_instance_date', 'DESC']],
-    limit: 1,
-  })
-
-const byCode = code => Course.findByPk(code)
-
-const findOneByCode = code => {
-  return Course.findOne({
-    attributes: ['id', 'code', 'name'],
-    where: {
-      code: code,
-    },
-  })
-}
-
-const creditsForCourses = async (codes, anonymizationSalt, unification) => {
-  let is_open = false
-
-  if (unification === 'open') is_open = true
-
-  if (unification === 'unify') {
-    is_open = {
-      [Op.in]: [false, true],
-    }
-  }
-
-  const credits = await Credit.findAll({
-    include: [
-      {
-        model: Student,
-        attributes: ['studentnumber'],
-        include: {
-          model: StudyrightElement,
-          attributes: ['code', 'startdate'],
-          include: [
-            {
-              model: ElementDetail,
-              attributes: ['name', 'type'],
-              where: {
-                type: {
-                  [Op.eq]: 20,
-                },
-              },
-            },
-            {
-              model: Studyright,
-              attributes: ['prioritycode', 'faculty_code'],
-              where: {
-                prioritycode: {
-                  [Op.eq]: 1,
-                },
-              },
-              include: {
-                model: Organization,
-              },
-            },
-          ],
-        },
-      },
-      {
-        model: Semester,
-        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
-        where: {
-          startdate: {
-            [Op.lte]: new Date(),
-          },
-        },
-      },
-    ],
-    where: {
-      course_code: {
-        [Op.in]: codes,
-      },
-      student_studentnumber: {
-        [Op.ne]: null,
-      },
-      [Op.or]: [{ is_open }, { is_open: null }],
-    },
-    order: [['attainment_date', 'ASC']],
-  })
-
-  const parsedCredits = credits.map(credit => parseCredit(credit, anonymizationSalt))
-  return parsedCredits
-}
-
-const enrollmentsForCourses = async (codes, anonymizationSalt, unification) => {
-  let is_open = false
-
-  if (unification === 'open') is_open = true
-
-  if (unification === 'unify') {
-    is_open = {
-      [Op.in]: [false, true],
-    }
-  }
-
-  const enrollments = await Enrollment.findAll({
-    include: [
-      {
-        model: Student,
-        attributes: ['studentnumber'],
-        include: {
-          model: StudyrightElement,
-          attributes: ['code', 'startdate'],
-          include: [
-            {
-              model: ElementDetail,
-              attributes: ['name', 'type'],
-              where: {
-                type: {
-                  [Op.eq]: 20,
-                },
-              },
-            },
-            {
-              model: Studyright,
-              attributes: ['prioritycode', 'faculty_code'],
-              where: {
-                prioritycode: {
-                  [Op.eq]: 1,
-                },
-              },
-              include: {
-                model: Organization,
-              },
-            },
-          ],
-        },
-      },
-      {
-        model: Semester,
-        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
-        where: {
-          startdate: {
-            [Op.lte]: new Date(),
-          },
-        },
-      },
-    ],
-    where: {
-      course_code: {
-        [Op.in]: codes,
-      },
-      studentnumber: {
-        [Op.ne]: null,
-      },
-      enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
-      state: ['ENROLLED', 'CONFIRMED'],
-      [Op.or]: [{ is_open }, { is_open: null }],
-    },
-  })
-  return enrollments.map(enrollment => parseEnrollment(enrollment, anonymizationSalt))
-}
-
-const bySearchTerm = async (term, language) => {
-  const formatCourse = course => ({
-    name: course.name[language],
-    code: course.code,
-    date: course.latest_instance_date,
-  })
-
-  try {
-    const result = await byNameOrCode(`%${term}%`, language)
-    return result.map(formatCourse)
-  } catch (e) {
-    return {
-      error: e,
-    }
-  }
-}
-
-const createCourse = async (code, name, latest_instance_date) =>
-  Course.create({
-    code,
-    name,
-    latest_instance_date,
-  })
-
-const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
-
-const unifyOpenUniversity = code => {
-  const regexresult = isOpenUniCourseCode(code)
-  if (!regexresult) return code
-  return regexresult[1]
-}
-
-const allCodeAlternatives = async code => {
-  const course = await Course.findOne({
-    where: { code: code },
-  })
-
-  return sortMainCode([...course.substitutions, code])
-}
-
-const yearlyStatsOfNew = async (coursecode, separate, unification, anonymizationSalt) => {
-  const courseForSubs = await Course.findOne({
-    where: { code: coursecode },
-  })
-
-  let codes = courseForSubs.substitutions ? sortMainCode([...courseForSubs.substitutions, coursecode]) : [coursecode]
-
-  if (unification === 'reqular') {
-    codes = codes.filter(course => !isOpenUniCourseCode(course))
-  }
-
-  const [credits, enrollments, course] = await Promise.all([
-    creditsForCourses(codes, anonymizationSalt, unification),
-    enrollmentsForCourses(codes, anonymizationSalt, unification),
-    Course.findOne({
-      where: {
-        code: coursecode,
-      },
-    }),
-  ])
-
-  const counter = new CourseYearlyStatsCounter()
-
-  for (let credit of credits) {
-    const {
-      studentnumber,
-      grade,
-      passed,
-      semestercode,
-      semestername,
-      yearcode,
-      yearname,
-      attainment_date,
-      programmes,
-      coursecode,
-      credits,
-    } = credit
-
-    const groupcode = separate ? semestercode : yearcode
-    const groupname = separate ? semestername : yearname
-    const unknownProgramme = [
-      {
-        code: 'OTHER',
-        name: {
-          en: 'Other',
-          fi: 'Muu',
-          sv: 'Andra',
-        },
-        faculty_code: 'OTHER',
-        organization: {
-          name: {
-            en: 'Other',
-            fi: 'Muu',
-            sv: 'Andra',
-          },
-        },
-      },
-    ]
-    counter.markStudyProgrammes(
-      studentnumber,
-      programmes.length === 0 ? unknownProgramme : programmes,
-      yearcode,
-      passed,
-      credits
-    )
-    counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode)
-    counter.markCreditToStudentCategories(studentnumber, passed, attainment_date, groupcode)
-  }
-
-  enrollments.forEach(enrollment => {
-    const { studentnumber, semestercode, semestername, yearcode, yearname, coursecode, state, enrollment_date_time } =
-      enrollment
-
-    const groupcode = separate ? semestercode : yearcode
-    const groupname = separate ? semestername : yearname
-
-    counter.markEnrollmentToGroup(
-      studentnumber,
-      state,
-      enrollment_date_time,
-      groupcode,
-      groupname,
-      coursecode,
-      yearcode
-    )
-  })
-
-  const statistics = counter.getFinalStatistics(anonymizationSalt)
-
-  return {
-    ...statistics,
-    coursecode,
-    alternatives: codes,
-    name: course.name,
-  }
-}
-
-const maxYearsToCreatePopulationFrom = async (coursecodes, unifyCourses) => {
-  let is_open = false
-
-  if (unifyCourses === 'openStats') is_open = true
-
-  if (unifyCourses === 'unifyStats') {
-    is_open = {
-      [Op.in]: [false, true],
-    }
-  }
-
-  const maxAttainmentDate = new Date(
-    Math.max(
-      ...(
-        await Course.findAll({
-          where: {
-            code: {
-              [Op.in]: coursecodes,
-            },
-          },
-          attributes: ['max_attainment_date'],
-        })
-      ).map(c => new Date(c.max_attainment_date).getTime())
-    )
-  )
-
-  const attainmentThreshold = new Date(maxAttainmentDate.getFullYear(), 0, 1)
-  attainmentThreshold.setFullYear(attainmentThreshold.getFullYear() - 6)
-
-  const credits = await Credit.findAll({
-    where: {
-      course_code: {
-        [Op.in]: coursecodes,
-      },
-      attainment_date: {
-        [Op.gt]: attainmentThreshold,
-      },
-      is_open,
-    },
-    order: [['attainment_date', 'ASC']],
-  })
-
-  const yearlyStudents = Object.values(
-    credits.reduce((res, credit) => {
-      const attainmentYear = new Date(credit.attainment_date).getFullYear()
-      if (!res[attainmentYear]) res[attainmentYear] = 0
-      res[attainmentYear]++
-      return res
-    }, {})
-  )
-  const maxYearsToCreatePopulationFrom = Math.max(
-    Math.floor(
-      1200 / // Lower this value to get a smaller result if necessary
-        (yearlyStudents.reduce((acc, curr) => acc + curr, 0) / yearlyStudents.length)
-    ),
-    1
-  )
-
-  return maxYearsToCreatePopulationFrom
-}
-
-const courseYearlyStats = async (coursecodes, separate, anonymizationSalt) => {
-  const statsReqular = await Promise.all(
-    coursecodes.map(async code => {
-      const unifyStats = await yearlyStatsOfNew(code, separate, 'unify', anonymizationSalt)
-      const reqularStats = await yearlyStatsOfNew(code, separate, 'reqular', anonymizationSalt)
-      const openStats = await yearlyStatsOfNew(code, separate, 'open', anonymizationSalt)
-
-      return { unifyStats, reqularStats, openStats }
-    })
-  )
-
-  return statsReqular
-}
-
-const nameLikeTerm = name => {
-  if (!name) {
-    return undefined
-  }
-  const term = `%${name.trim()}%`
-  return {
-    name: {
-      [Op.or]: {
-        fi: {
-          [Op.iLike]: term,
-        },
-        sv: {
-          [Op.iLike]: term,
-        },
-        en: {
-          [Op.iLike]: term,
-        },
-      },
-    },
-  }
-}
-
-const byCodes = codes => {
-  return Course.findAll({
-    where: {
-      code: {
-        [Op.in]: codes,
-      },
-    },
-  })
-}
-
-const codeLikeTerm = code =>
-  !code
-    ? undefined
-    : {
-        code: {
-          // Starts with code or has AY/A in front of the code
-          [Op.iRegexp]: `^(AY|ay|A|a)?${code}`,
-        },
-      }
-
-const byNameAndOrCodeLike = async (name, code) => {
-  let rawCourses = await Course.findAll({
-    where: {
-      ...nameLikeTerm(name),
-      ...codeLikeTerm(code),
-    },
-  })
-
-  const courses = rawCourses
-    .map(course => {
-      return { ...course.dataValues }
-    })
-    .sort((x, y) => getSortRank(y.code) - getSortRank(x.code))
-
-  let substitutionGroupIndex = 0
-  const visited = []
-
-  const organizeSubgroups = course => {
-    if (visited.includes(course.code)) return
-
-    let temp = []
-    if (course.substitutions !== null) {
-      temp = courses.filter(c => course.substitutions.includes(c.code))
-    }
-
-    temp.unshift(course)
-    temp.forEach(cu => {
-      if (visited.includes(course.code)) return
-      visited.push(cu.id)
-      cu.subsId = substitutionGroupIndex
-    })
-  }
-
-  courses.forEach(course => {
-    if (!visited.includes(course.id)) {
-      substitutionGroupIndex++
-      organizeSubgroups(course)
-    }
-  })
-  return { courses }
-}
-
-const getCourseProvidersForCourses = async codes => {
-  return (
-    await Organization.findAll({
-      attributes: ['code'],
-      include: {
-        model: Course,
-        where: {
-          code: {
-            [Op.in]: codes,
-          },
-        },
-      },
-      raw: true,
-    })
-  ).map(({ code }) => code)
-}
-
-module.exports = {
-  byCode,
-  byName,
-  bySearchTerm,
-  createCourse,
-  courseYearlyStats,
-  byNameAndOrCodeLike,
-  byCodes,
-  maxYearsToCreatePopulationFrom,
-  unifyOpenUniversity,
-  allCodeAlternatives,
-  findOneByCode,
-  getCourseProvidersForCourses,
-}
@@ -0,0 +1,158 @@
+const { Op } = require('sequelize')
+const { Course } = require('../../models')
+const { getSortRank } = require('../../util/utils')
+
+const byNameOrCode = (searchTerm, language) =>
+  Course.findAll({
+    where: {
+      [Op.or]: [
+        {
+          name: {
+            [language]: {
+              [Op.iLike]: searchTerm,
+            },
+          },
+        },
+        {
+          code: {
+            [Op.like]: searchTerm,
+          },
+        },
+      ],
+    },
+  })
+
+const byName = (name, language) =>
+  Course.findAll({
+    where: {
+      name: {
+        [language]: {
+          [Op.eq]: name,
+        },
+      },
+    },
+    order: [['latest_instance_date', 'DESC']],
+    limit: 1,
+  })
+
+const byCode = code => Course.findByPk(code)
+
+const findOneByCode = code => {
+  return Course.findOne({
+    attributes: ['id', 'code', 'name'],
+    where: {
+      code: code,
+    },
+  })
+}
+
+const bySearchTerm = async (term, language) => {
+  const formatCourse = course => ({
+    name: course.name[language],
+    code: course.code,
+    date: course.latest_instance_date,
+  })
+
+  try {
+    const result = await byNameOrCode(`%${term}%`, language)
+    return result.map(formatCourse)
+  } catch (e) {
+    return {
+      error: e,
+    }
+  }
+}
+
+const nameLikeTerm = name => {
+  if (!name) {
+    return undefined
+  }
+  const term = `%${name.trim()}%`
+  return {
+    name: {
+      [Op.or]: {
+        fi: {
+          [Op.iLike]: term,
+        },
+        sv: {
+          [Op.iLike]: term,
+        },
+        en: {
+          [Op.iLike]: term,
+        },
+      },
+    },
+  }
+}
+
+const byCodes = codes => {
+  return Course.findAll({
+    where: {
+      code: {
+        [Op.in]: codes,
+      },
+    },
+  })
+}
+
+const codeLikeTerm = code =>
+  !code
+    ? undefined
+    : {
+        code: {
+          // Starts with code or has AY/A in front of the code
+          [Op.iRegexp]: `^(AY|ay|A|a)?${code}`,
+        },
+      }
+
+const byNameAndOrCodeLike = async (name, code) => {
+  const rawCourses = await Course.findAll({
+    where: {
+      ...nameLikeTerm(name),
+      ...codeLikeTerm(code),
+    },
+    order: [['id', 'desc']],
+  })
+
+  const courses = rawCourses
+    .map(course => {
+      return { ...course.dataValues }
+    })
+    .sort((x, y) => getSortRank(y.code) - getSortRank(x.code))
+
+  let substitutionGroupIndex = 0
+  const visited = []
+
+  const organizeSubgroups = course => {
+    if (visited.includes(course.code)) return
+
+    let temp = []
+    if (course.substitutions !== null) {
+      temp = courses.filter(c => course.substitutions.includes(c.code))
+    }
+
+    temp.unshift(course)
+    temp.forEach(cu => {
+      if (visited.includes(course.code)) return
+      visited.push(cu.id)
+      cu.subsId = substitutionGroupIndex
+    })
+  }
+
+  courses.forEach(course => {
+    if (!visited.includes(course.id)) {
+      substitutionGroupIndex++
+      organizeSubgroups(course)
+    }
+  })
+  return { courses }
+}
+
+module.exports = {
+  byCode,
+  byName,
+  bySearchTerm,
+  byNameAndOrCodeLike,
+  byCodes,
+  findOneByCode,
+}
@@ -59,7 +59,6 @@ class CourseStatsCounter {
      passingSemesters[fall[i]] = initialValue
      passingSemesters[spring[i]] = initialValue
    }
-
    return passingSemesters
  }

@@ -214,7 +213,6 @@ class CourseStatsCounter {
    stats.totalStudents = stats.students
    stats.totalEnrolledNoGrade = lengthOf(fileredEnrolledNoGrade)
    stats.percentageWithEnrollments = percentageOf(stats.passed, stats.totalStudents)
-
    return {
      stats,
      students,
@@ -0,0 +1,143 @@
+const { Op } = require('sequelize')
+const {
+  Credit,
+  Student,
+  Semester,
+  Studyright,
+  Organization,
+  StudyrightElement,
+  ElementDetail,
+  Enrollment,
+} = require('../../models')
+
+const creditsForCourses = async (codes, anonymizationSalt, unification) => {
+  let is_open = false
+
+  if (unification === 'open') is_open = true
+
+  if (unification === 'unify') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
+  return await Credit.findAll({
+    include: [
+      {
+        model: Student,
+        attributes: ['studentnumber'],
+      },
+      {
+        model: Semester,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+        where: {
+          startdate: {
+            [Op.lte]: new Date(),
+          },
+        },
+      },
+    ],
+    where: {
+      course_code: {
+        [Op.in]: codes,
+      },
+      student_studentnumber: {
+        [Op.ne]: null,
+      },
+      [Op.or]: [{ is_open }, { is_open: null }],
+    },
+    order: [['attainment_date', 'ASC']],
+  })
+}
+
+const getStudentNumberToSrElementsMap = async studentNumbers => {
+  const studyrights = await Studyright.findAll({
+    attributes: ['prioritycode', 'faculty_code', 'student_studentnumber', 'studyrightid'],
+    where: {
+      prioritycode: {
+        [Op.eq]: 1,
+      },
+      student_studentnumber: { [Op.in]: studentNumbers },
+    },
+    include: {
+      model: Organization,
+    },
+  })
+
+  const studyrightMap = studyrights.reduce((obj, cur) => {
+    obj[cur.studyrightid] = cur
+    return obj
+  }, {})
+
+  const studyrightIds = Object.keys(studyrightMap)
+
+  const studyrightElements = await StudyrightElement.findAll({
+    attributes: ['code', 'startdate', 'studentnumber', 'studyrightid'],
+    include: [
+      {
+        model: ElementDetail,
+        attributes: ['name', 'type'],
+        where: {
+          type: {
+            [Op.eq]: 20,
+          },
+        },
+      },
+    ],
+    where: { studyrightid: { [Op.in]: studyrightIds } },
+  })
+
+  return studyrightElements.reduce((obj, cur) => {
+    if (!obj[cur.studentnumber]) obj[cur.studentnumber] = []
+    cur.studyright = studyrightMap[cur.studyrightid]
+    obj[cur.studentnumber].push(cur)
+    return obj
+  }, {})
+}
+
+const enrollmentsForCourses = async (codes, unification) => {
+  let is_open = false
+
+  if (unification === 'open') is_open = true
+
+  if (unification === 'unify') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
+  return await Enrollment.findAll({
+    include: [
+      {
+        model: Student,
+        attributes: ['studentnumber'],
+      },
+      {
+        model: Semester,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+        where: {
+          startdate: {
+            [Op.lte]: new Date(),
+          },
+        },
+      },
+    ],
+    where: {
+      course_code: {
+        [Op.in]: codes,
+      },
+      studentnumber: {
+        [Op.ne]: null,
+      },
+      enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
+      state: ['ENROLLED', 'CONFIRMED'],
+      [Op.or]: [{ is_open }, { is_open: null }],
+    },
+  })
+}
+
+module.exports = {
+  getStudentNumberToSrElementsMap,
+  enrollmentsForCourses,
+  creditsForCourses,
+}
@@ -0,0 +1,281 @@
+const Sequelize = require('sequelize')
+const { Credit, Course, Organization, Enrollment } = require('../../models')
+
+const { parseCredit } = require('./parseCredits')
+const { parseEnrollment } = require('./parseEnrollments')
+const Op = Sequelize.Op
+const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
+const { sortMainCode } = require('../../util/utils')
+const {
+  enrollmentsForCourses,
+  creditsForCourses,
+  getStudentNumberToSrElementsMap,
+} = require('./creditsAndEnrollmentsOfCourse')
+const { byNameAndOrCodeLike, byCodes } = require('./courseFinders')
+
+const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+
+const unifyOpenUniversity = code => {
+  const regexresult = isOpenUniCourseCode(code)
+  if (!regexresult) return code
+  return regexresult[1]
+}
+
+const allCodeAlternatives = async code => {
+  const course = await Course.findOne({
+    where: { code: code },
+  })
+
+  return sortMainCode([...course.substitutions, code])
+}
+
+const yearlyStatsOfNew = async (
+  coursecode,
+  separate,
+  unification,
+  anonymizationSalt,
+  combineSubstitutions,
+  studentNumberToSrElementsMap
+) => {
+  const courseForSubs = await Course.findOne({
+    where: { code: coursecode },
+  })
+
+  let codes =
+    combineSubstitutions && courseForSubs.substitutions
+      ? sortMainCode([...courseForSubs.substitutions, coursecode])
+      : [coursecode]
+
+  if (unification === 'regular') {
+    codes = codes.filter(course => !isOpenUniCourseCode(course))
+  }
+
+  const [credits, enrollments, course] = await Promise.all([
+    creditsForCourses(codes, anonymizationSalt, unification),
+    enrollmentsForCourses(codes, unification),
+    Course.findOne({
+      where: {
+        code: coursecode,
+      },
+    }),
+  ])
+
+  const counter = new CourseYearlyStatsCounter()
+
+  for (let credit of credits) {
+    const {
+      studentnumber,
+      grade,
+      passed,
+      semestercode,
+      semestername,
+      yearcode,
+      yearname,
+      attainment_date,
+      programmes,
+      coursecode,
+      credits,
+    } = parseCredit(credit, anonymizationSalt, studentNumberToSrElementsMap)
+
+    const groupcode = separate ? semestercode : yearcode
+    const groupname = separate ? semestername : yearname
+    const unknownProgramme = [
+      {
+        code: 'OTHER',
+        name: {
+          en: 'Other',
+          fi: 'Muu',
+          sv: 'Andra',
+        },
+        faculty_code: 'OTHER',
+        organization: {
+          name: {
+            en: 'Other',
+            fi: 'Muu',
+            sv: 'Andra',
+          },
+        },
+      },
+    ]
+    counter.markStudyProgrammes(
+      studentnumber,
+      programmes.length === 0 ? unknownProgramme : programmes,
+      yearcode,
+      passed,
+      credits
+    )
+    counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode)
+    counter.markCreditToStudentCategories(studentnumber, passed, attainment_date, groupcode)
+  }
+
+  enrollments.forEach(enrollment => {
+    const { studentnumber, semestercode, semestername, yearcode, yearname, coursecode, state, enrollment_date_time } =
+      parseEnrollment(enrollment, anonymizationSalt, studentNumberToSrElementsMap)
+
+    const groupcode = separate ? semestercode : yearcode
+    const groupname = separate ? semestername : yearname
+
+    counter.markEnrollmentToGroup(
+      studentnumber,
+      state,
+      enrollment_date_time,
+      groupcode,
+      groupname,
+      coursecode,
+      yearcode
+    )
+  })
+
+  const statistics = counter.getFinalStatistics(anonymizationSalt)
+
+  return {
+    ...statistics,
+    coursecode,
+    alternatives: codes,
+    name: course.name,
+  }
+}
+
+const maxYearsToCreatePopulationFrom = async (coursecodes, unifyCourses) => {
+  let is_open = false
+
+  if (unifyCourses === 'openStats') is_open = true
+
+  if (unifyCourses === 'unifyStats') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
+  const maxAttainmentDate = new Date(
+    Math.max(
+      ...(
+        await Course.findAll({
+          where: {
+            code: {
+              [Op.in]: coursecodes,
+            },
+          },
+          attributes: ['max_attainment_date'],
+        })
+      ).map(c => new Date(c.max_attainment_date).getTime())
+    )
+  )
+
+  const attainmentThreshold = new Date(maxAttainmentDate.getFullYear(), 0, 1)
+  attainmentThreshold.setFullYear(attainmentThreshold.getFullYear() - 6)
+
+  const credits = await Credit.findAll({
+    where: {
+      course_code: {
+        [Op.in]: coursecodes,
+      },
+      attainment_date: {
+        [Op.gt]: attainmentThreshold,
+      },
+      is_open,
+    },
+    order: [['attainment_date', 'ASC']],
+  })
+
+  const yearlyStudents = Object.values(
+    credits.reduce((res, credit) => {
+      const attainmentYear = new Date(credit.attainment_date).getFullYear()
+      if (!res[attainmentYear]) res[attainmentYear] = 0
+      res[attainmentYear]++
+      return res
+    }, {})
+  )
+  const maxYearsToCreatePopulationFrom = Math.max(
+    Math.floor(
+      1200 / // Lower this value to get a smaller result if necessary
+        (yearlyStudents.reduce((acc, curr) => acc + curr, 0) / yearlyStudents.length)
+    ),
+    1
+  )
+
+  return maxYearsToCreatePopulationFrom
+}
+
+const courseYearlyStats = async (coursecodes, separate, anonymizationSalt, combineSubstitutions) => {
+  const credits = await Credit.findAll({
+    attributes: ['student_studentnumber'],
+    where: { course_code: { [Op.in]: coursecodes } },
+  })
+
+  const enrollment = await Enrollment.findAll({
+    attributes: ['studentnumber'],
+    where: { course_code: { [Op.in]: coursecodes } },
+  })
+
+  const studentnumbersObject = {}
+
+  credits.forEach(cr => {
+    studentnumbersObject[cr.student_studentnumber] = true
+  })
+  enrollment.forEach(cr => {
+    studentnumbersObject[cr.studentnumber] = true
+  })
+
+  const studentNumberToSrElementsMap = await getStudentNumberToSrElementsMap(Object.keys(studentnumbersObject))
+
+  const statsRegular = await Promise.all(
+    coursecodes.map(async code => {
+      const unifyStats = await yearlyStatsOfNew(
+        code,
+        separate,
+        'unify',
+        anonymizationSalt,
+        combineSubstitutions,
+        studentNumberToSrElementsMap
+      )
+      const regularStats = await yearlyStatsOfNew(
+        code,
+        separate,
+        'regular',
+        anonymizationSalt,
+        combineSubstitutions,
+        studentNumberToSrElementsMap
+      )
+      const openStats = await yearlyStatsOfNew(
+        code,
+        separate,
+        'open',
+        anonymizationSalt,
+        combineSubstitutions,
+        studentNumberToSrElementsMap
+      )
+
+      return { unifyStats, regularStats, openStats }
+    })
+  )
+
+  return statsRegular
+}
+
+const getCourseProvidersForCourses = async codes => {
+  return (
+    await Organization.findAll({
+      attributes: ['code'],
+      include: {
+        model: Course,
+        where: {
+          code: {
+            [Op.in]: codes,
+          },
+        },
+      },
+      raw: true,
+    })
+  ).map(({ code }) => code)
+}
+
+module.exports = {
+  courseYearlyStats,
+  maxYearsToCreatePopulationFrom,
+  unifyOpenUniversity,
+  allCodeAlternatives,
+  getCourseProvidersForCourses,
+  byCodes,
+  byNameAndOrCodeLike,
+}
@@ -1,5 +1,5 @@
 const crypto = require('crypto')
-const { Credit } = require('../models')
+const { Credit } = require('../../models')

 const formatStudyrightElement = ({ code, element_detail, startdate, studyright: sr }) => {
  const studyright = sr.get({ plain: true })
@@ -17,24 +17,20 @@ const formatStudyrightElement = ({ code, element_detail, startdate, studyright:
  }
 }

-const parseCredit = (credit, anonymizationSalt) => {
-  const { student, semester, grade, course_code, credits, attainment_date } = credit
-  const { studentnumber, studyright_elements } = student
+const parseCredit = (credit, anonymizationSalt, studentNumberToSrElementsMap) => {
+  const { semester, grade, course_code, credits, attainment_date, student_studentnumber: studentnumber } = credit
  const { yearcode, yearname, semestercode, name: semestername } = semester

+  const studyrightElements = studentNumberToSrElementsMap[studentnumber] || []
+
  if (anonymizationSalt) {
    const anonymizedStudentNumber = crypto
      .createHash('sha256')
      .update(`${studentnumber}${anonymizationSalt}`)
      .digest('hex')

-    const anonymizedStudent = {
-      studentnumber: anonymizedStudentNumber,
-    }
-
    return {
      obfuscated: true,
-      student: anonymizedStudent,
      yearcode,
      yearname,
      semestercode,
@@ -44,13 +40,12 @@ const parseCredit = (credit, anonymizationSalt) => {
      grade,
      passed: !Credit.failed(credit) || Credit.passed(credit) || Credit.improved(credit),
      studentnumber: anonymizedStudentNumber,
-      programmes: studyright_elements.map(formatStudyrightElement),
+      programmes: studyrightElements.map(formatStudyrightElement),
      credits,
    }
  }

  return {
-    student,
    yearcode,
    yearname,
    semestercode,
@@ -60,7 +55,7 @@ const parseCredit = (credit, anonymizationSalt) => {
    grade,
    passed: !Credit.failed(credit) || Credit.passed(credit) || Credit.improved(credit),
    studentnumber,
-    programmes: studyright_elements.map(formatStudyrightElement),
+    programmes: studyrightElements.map(formatStudyrightElement),
    credits,
  }
 }
@@ -1,24 +1,20 @@
 const crypto = require('crypto')
 const { formatStudyrightElement } = require('./parseCredits')

-const parseEnrollment = (enrollment, anonymizationSalt) => {
-  const { student, semester, state, enrollment_date_time, course_code } = enrollment
-  const { studentnumber, studyright_elements } = student
+const parseEnrollment = (enrollment, anonymizationSalt, studentNumberToSrElementsMap) => {
+  const { studentnumber, semester, state, enrollment_date_time, course_code } = enrollment
  const { yearcode, yearname, semestercode, name: semestername } = semester

+  const studyrightElements = studentNumberToSrElementsMap[studentnumber] || []
+
  if (anonymizationSalt) {
    const anonymizedStudentNumber = crypto
      .createHash('sha256')
      .update(`${studentnumber}${anonymizationSalt}`)
      .digest('hex')

-    const anonymizedStudent = {
-      studentnumber: anonymizedStudentNumber,
-    }
-
    return {
      obfuscated: true,
-      student: anonymizedStudent,
      yearcode,
      yearname,
      semestercode,
@@ -27,12 +23,11 @@ const parseEnrollment = (enrollment, anonymizationSalt) => {
      state,
      enrollment_date_time,
      studentnumber: anonymizedStudentNumber,
-      programmes: studyright_elements.map(formatStudyrightElement),
+      programmes: studyrightElements.map(formatStudyrightElement),
    }
  }

  return {
-    student,
    yearcode,
    yearname,
    semestercode,
@@ -41,7 +36,7 @@ const parseEnrollment = (enrollment, anonymizationSalt) => {
    state,
    enrollment_date_time,
    studentnumber,
-    programmes: studyright_elements.map(formatStudyrightElement),
+    programmes: studyrightElements.map(formatStudyrightElement),
  }
 }

@@ -1,64 +0,0 @@
-// Returns an array containing all values that are in a but not in b taking account duplicates
-const difference = (a, b) => {
-  return a.filter(
-    function (v) {
-      return !this.get(v) || !this.set(v, this.get(v) - 1)
-    },
-    b.reduce((acc, v) => acc.set(v, (acc.get(v) || 0) + 1), new Map())
-  )
-}
-
-const mapStudentNumbers = (studentnumbers, yearcode, name) =>
-  studentnumbers.map(studentnumber => ({
-    studentnumber,
-    yearcode,
-    year: name,
-  }))
-
-const getDiffForCourse = (oodiCourseStats, sisCourseStats) => {
-  return oodiCourseStats.statistics.reduce(
-    (courseDiff, oodiYearlyStats) => {
-      const sisYearlyStats =
-        sisCourseStats && sisCourseStats.statistics.find(s => s.yearcode === oodiYearlyStats.yearcode)
-
-      if (!sisYearlyStats) {
-        return {
-          missingStudents: mapStudentNumbers(oodiYearlyStats.students.studentnumbers),
-          extraStudents: [],
-        }
-      }
-
-      const missingStudentNumbers = difference(
-        oodiYearlyStats.students.studentnumbers,
-        sisYearlyStats.students.studentnumbers
-      )
-      const extraStudentNumbers = difference(
-        sisYearlyStats.students.studentnumbers,
-        oodiYearlyStats.students.studentnumbers
-      )
-
-      const missingStudents = mapStudentNumbers(missingStudentNumbers, oodiYearlyStats.yearcode, oodiYearlyStats.name)
-      const extraStudents = mapStudentNumbers(extraStudentNumbers, oodiYearlyStats.yearcode, oodiYearlyStats.name)
-
-      const noDifferenceInCourseStats = missingStudents.length === 0 && extraStudents.length === 0
-      if (noDifferenceInCourseStats) {
-        return courseDiff
-      }
-
-      return {
-        missingStudents: courseDiff.missingStudents.concat(missingStudents),
-        extraStudents: courseDiff.extraStudents.concat(extraStudents),
-      }
-    },
-    { missingStudents: [], extraStudents: [] }
-  )
-}
-
-const getCourseYearlyStatsDiff = (sisData, oodiData) => {
-  return oodiData.map(oodiCourseStats => {
-    const sisCourseStats = sisData.find(cs => cs.coursecode === oodiCourseStats.coursecode)
-    return { coursecode: oodiCourseStats.coursecode, ...getDiffForCourse(oodiCourseStats, sisCourseStats) }
-  })
-}
-
-module.exports = { getCourseYearlyStatsDiff }
--- a/Show More
+++ b/Show More