Classic happy birthday to this one guy on the dataset -change

Stop intercepting /superlogin path on user tests
Run migrations against built images in ci pipeline
2026-10-13 03:13:40 +03:00 · 2022-01-04 12:06:26 +02:00 · 2022-01-04 11:53:22 +02:00 · 2022-01-04 11:10:30 +02:00 · 2022-01-04 11:07:22 +02:00 · 2021-12-30 17:41:02 +02:00
312 changed files with 9656 additions and 78178 deletions
@@ -1,4 +1,3 @@
 # Populate and change this to .env
 IMPORTER_DB_API_TOKEN=
-IMPORTER_DB_API_URL=https://importer.cs.helsinki.fi
 PATE_TOKEN=
@@ -1,4 +1,4 @@
-name: "Release oodikone to production"
+name: 'Release oodikone to production'

 on:
  release:
@@ -6,23 +6,23 @@ on:

 jobs:
  build_and_release_backend:
-    name: "Build and release backend"
+    name: 'Build and release backend'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
+      - name: 'Extract release version'
        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
        # This steps extracts release version from GITHUB_REF env var
        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
        id: extract_version
        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
        uses: docker/build-push-action@v2
        with:
          build-args: |
@@ -38,26 +38,26 @@ jobs:
          target: production

  build_and_release_frontend:
-    name: "Build and release frontend"
+    name: 'Build and release frontend'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Get current time"
+      - name: 'Get current time'
        id: get_current_time
        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
-      - name: "Extract release version"
+      - name: 'Extract release version'
        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
        # This steps extracts release version from GITHUB_REF env var
        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
        id: extract_version
        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
        uses: docker/build-push-action@v2
        with:
          build-args: |
@@ -72,11 +72,11 @@ jobs:
          push: true
          tags: toska/oodikone2-frontend:latest
          target: production
-      - name: "Create a container from built image "
+      - name: 'Create a container from built image '
        run: docker create --name frontend toska/oodikone2-frontend:latest
-      - name: "Copy assets from the container"
+      - name: 'Copy assets from the container'
        run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
+      - name: 'Create Sentry release'
        uses: getsentry/action-release@v1
        env:
          SENTRY_URL: ${{ secrets.SENTRY_URL }}
@@ -90,49 +90,16 @@ jobs:
          url_prefix: '~/static/js'
          version: ${{ steps.extract_version.outputs.VERSION }}

-  build_and_release_userservice:
-    name: "Build and release userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
-        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
-        # This steps extracts release version from GITHUB_REF env var
-        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
-        id: extract_version
-        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: true
-          tags: toska/oodikone2-userservice:latest
-          target: production
-
  report_success:
    runs-on: ubuntu-latest
-    name: "Report deployment success to oodikone notifications"
+    name: 'Report deployment success to oodikone notifications'
    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
    if: success()
    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
        with:
          webhook-url: ${{ secrets.WEBHOOK_URL }}
          message-type: deployment
@@ -141,15 +108,14 @@ jobs:

  report_failure:
    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
+    name: 'Report deployment failure to oodikone notifications'
    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
    if: failure()
    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
        with:
          webhook-url: ${{ secrets.WEBHOOK_URL }}
          message-type: deployment-failure
@@ -45,21 +45,7 @@ jobs:
          'eslint --ext .jsx,.js services/frontend' \
          'stylelint services/frontend/src/**/*.css'

-  lint_userservice:
-    name: 'Lint userservice'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: 14
-      - name: 'Install dependencies'
-        run: npm ci && cd services/userservice && npm ci
-      - name: 'Run frontend linter'
-        run: npm run eslint services/userservice
-
  # === Build images ===
-
  build_backend:
    name: 'Build backend'
    runs-on: ubuntu-latest
@@ -116,50 +102,106 @@ jobs:
        with:
          path: /tmp/frontend.tar

-  build_userservice:
-    name: 'Build userservice'
+  build_and_push_anon_sis_db:
+    name: 'Build anon sis-db and push to toska hub if new migrations have been added'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: 'Build image and export it'
-        uses: docker/build-push-action@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: false
-          tags: toska/oodikone2-userservice:staging
-          target: production
-          outputs: type=docker,dest=/tmp/userservice.tar
-      - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v2
+          filters: |
+            migrations:
+              - 'updater/sis-updater-worker/src/db/migrations/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
        with:
-          path: /tmp/userservice.tar
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach sis-updater-worker
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db

-  # === Integration tests ===
-  #
-  run_userservice_integration_tests:
-    name: 'Run userservice integration tests'
+  build_and_push_anon_kone_db:
+    name: 'Build anon kone-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
-      - name: 'Run tests included in docker build'
-        run: docker-compose --file docker-compose.test.yml up --build userservice-test
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_kone/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
+
+  build_and_push_anon_user_db:
+    name: 'Build anon user-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_user/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db

  # === End-to-end tests ===
-
  test_course_statistics:
    name: 'Test course statistics'
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -171,7 +213,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -202,7 +243,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -214,7 +257,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -245,7 +287,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -257,7 +301,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -288,7 +331,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -300,7 +345,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -331,7 +375,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -343,7 +389,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -374,7 +419,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -386,7 +433,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -417,7 +463,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -429,7 +477,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -464,7 +511,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -476,7 +525,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -504,8 +552,6 @@ jobs:
    needs:
      - lint_backend
      - lint_frontend
-      - lint_userservice
-      - run_userservice_integration_tests
      - test_course_statistics
      - test_custom_population
      - test_filters
@@ -524,7 +570,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to DockerHub'
        uses: docker/login-action@v1
        with:
@@ -551,7 +596,6 @@ jobs:
        run: |
          docker push toska/oodikone2-backend:staging
          docker push toska/oodikone2-frontend:staging
-          docker push toska/oodikone2-userservice:staging
      - name: 'Send deployment success notification to slack'
        if: success()
        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
@@ -574,8 +618,6 @@ jobs:
    needs:
      - lint_backend
      - lint_frontend
-      - lint_userservice
-      - run_userservice_integration_tests
      - test_course_statistics
      - test_custom_population
      - test_filters
@@ -7,8 +7,8 @@ module.exports = {
  '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
  '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
  '*.css': files => `stylelint --fix ${files.join(' ')}`,
-  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0-alpine hadolint ${relativeFilePaths(files)}`,
-  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck-alpine:v0.7.2 ${relativeFilePaths(files)} -x`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0 hadolint ${relativeFilePaths(files)}`,
+  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck:v0.7.2 ${relativeFilePaths(files)} -x`,
  '.github/workflows/*': files => `${dockerCmdBase} rhysd/actionlint:1.6.2 ${relativeFilePaths(files)}`,
  'docker-compose*': files => {
    const composeFiles = file => {
@@ -43,8 +43,8 @@ What different CLI options do:
   - Cleans up any previous real data databases
   - Downloads needed database dumps from production servers.
   - Creates real data databases and populates them with downloaded dumps.
-3. Reset sis-importer data
-   - Like option 2, but only for sis-importer database. Useful when developing updater microservice.
+3. Reset single database
+   - Like option 2, but only for single database. Leaves other databases untouched.

 ## ⌨️ Development

@@ -79,7 +79,7 @@ It is recommended to spend some time to become familiar with `docker` and `docke
 - `docker-compose ps`: view the containers in the running environment
 - `docker-compose logs frontend`: print logs for just frontend
 - `docker-compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
- `docker exec -it userservice sh`: open bash terminal inside userservice container
+- `docker exec -it backend sh`: open bash terminal inside backend container
 - `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.

 ### User types for development
@@ -124,12 +124,16 @@ Continuous integration (CI) works with Github actions and is defined in workflow

 ### Modules are missing after updating package.json

-You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container.

 ### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data

 Make sure you have your VPN on

+### I made a change to sis-db models aand created a migration, but tests are failing in CI!
+
+Migrations are handled by sis-importer-worker, since oodikone is using only replica. Since CI is testing only services included in oodikone, your new changes aren't migrated and thus are failing. Open anonyymioodi-folder in this repo to find more about how to to update the anon sis-db.
+
 ### Everything is broken, can't get oodikone running, data is not there etc.

 Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.
@@ -0,0 +1,2 @@
+*.sqz
+
@@ -0,0 +1,65 @@
+# Anonyymioodi - anonymous database setup for oodikone
+
+(name anonyymioodi for :culture: reasons)
+
+This repository contains a guide and scripts to create docker images from database dumps and/or push database images to Toska Hub docker registry. Images are mostly used by oodikone.
+
+## How to update database images
+
+Updating images varies a bit depending on the db you're trying to update. Complete steps as defined below and run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the image you've created.
+
+### Sis-importer-db
+
+The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
+- Move `sis-importer-db.sqz` to this folder
+- Run `./create_container_from_dump.sh sis-importer-db`
+- Enrich data according to what's in `enrich_data.sql` file
+
+### Sis-db:
+
+The following steps are for building a clean sis-importer-db image from scratch using the sis-updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
+- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
+- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
+- Start updater, see oodikone readme for more info.
+- Populate sis-db from sis-importer-db:
+
+```
+curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
+```
+
+(It might be a good idea to follow sis-updater-worker logs while updater is running)
+
+- Clean up any extra stuff you don't want to keep in sis-db.
+- (Bonus: check that cypress tests run okay)
+
+### User-db / kone-db
+
+- Temporarily replace user-db/kone-db -service in oodikone's docker-compose.yml with the same service from this repo's docker-compose.yml
+- Ensure the newest user-db/kone-db image is pulled from the registry with `docker pull`
+- Start oodikone, let it run migrations
+- Modify db to wanted state. Tips:
+  - Easy way to create new users is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
+
+By default, previous user-db/kone-db images are used as a base. If you want to build the image from a clean Postgres image, just comment out toska hub image and uncomment the clean Postgres image.
+
+## How to modify database images
+
+If you need to clean up / modify / do whatever with dbs, either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose.
+
+## How to update Postgres versions
+
+When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
+
+- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
+- Start database with docker-compose
+- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
+- Close container with docker-compose down.
+- Update the Postgres version in docker-compose.yml in this folder.
+- Uncomment row with Postgres image and comment row with Toskahub image.
+- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+infomsg "Committing and pushing to registry"
+database=${args[0]}
+docker commit "$database" "registry.toska.cs.helsinki.fi/$database"
+docker push "registry.toska.cs.helsinki.fi/$database"
+successmsg "Succesfully pushed"
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+# === Create images ===
+
+retry () {
+  sleep 5
+  for i in {1..60}; do
+    "$@" && break || warningmsg "Retry attempt $i failed, waiting..." && sleep 5;
+  done
+}
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+
+infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
+database=${args[0]}
+docker-compose down --remove-orphans
+docker-compose up -d "$database"
+
+database_dump="$database.sqz"
+infomsg "Attempting to restore database $database from dump $database_dump"
+retry docker-compose exec "$database" pg_isready --dbname="$database"
+docker exec -i "$database" /bin/bash -c "pg_restore --username=postgres \
+--format=custom --dbname=$database --no-owner" < "$database_dump"
+successmsg "Database $database succesfully created"
@@ -0,0 +1,50 @@
+x-defaults: &defaults
+  restart: unless-stopped
+
+services:
+  kone-db:
+    <<: *defaults
+    container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/kone-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone
+
+  sis-db:
+    <<: *defaults
+    container_name: sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-db
+    image: postgres:13.2
+    shm_size: 2gb
+
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-importer-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-importer-db
+    image: postgres:13.1
+    profiles:
+      - updater
+
+  user-db:
+    <<: *defaults
+    container_name: user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone
@@ -0,0 +1,133 @@
+-- add these to sis-importer db by running:
+-- docker exec -i sis-importer-db psql -U postgres sis-importer-db < enrich_sampledata.sql
+
+-- if you need to run updater for these students only, see studentnumberlist in the end
+-- this file
+
+-- Add admission types (valintaperusteet) to studyrights for some of the computer
+-- science 2018-2019 population students. See person_ids / student_number below 
+-- (in case you need to update dumps). 
+
+-- TODO: this should be moved into the sampledata creator script in
+-- importer repo, but since this file contains studentnumbers and person ids, it's 
+-- better to keep this here
+
+-- KP (koepisteet)
+-- hy-hlo-1412620   | 013683126
+-- hy-hlo-1485246   | 014340523
+-- hy-hlo-32230770  | 010589388
+-- hy-hlo-35626233  | 010775888
+-- hy-hlo-40150144  | 011065995
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:kp' 
+WHERE person_id IN (
+'hy-hlo-1412620',
+'hy-hlo-1485246',
+'hy-hlo-32230770',
+'hy-hlo-35626233',
+'hy-hlo-40150144'
+);
+
+-- TV (todistusvalinta)
+-- hy-hlo-45783433  | 011321150
+-- hy-hlo-48537946  | 010764653
+-- hy-hlo-49801905  | 011194400
+-- hy-hlo-53321302  | 011338871
+-- hy-hlo-51223300  | 013951984
+-- hy-hlo-53310137  | 012520688
+-- hy-hlo-53621583  | 011196796
+-- hy-hlo-53863064  | 011736884
+-- hy-hlo-57711592  | 010262566
+-- hy-hlo-57732872  | 013670647
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:tv' 
+WHERE person_id IN (
+'hy-hlo-45783433',
+'hy-hlo-48537946',
+'hy-hlo-49801905',
+'hy-hlo-53321302',
+'hy-hlo-51223300',
+'hy-hlo-53310137',
+'hy-hlo-53621583',
+'hy-hlo-53863064',
+'hy-hlo-57711592',
+'hy-hlo-57732872'
+);
+
+-- YP:
+-- hy-hlo-67768022  | 011952325
+-- hy-hlo-73740025  | 011736606
+-- hy-hlo-74780917  | 012775433
+-- hy-hlo-79615371  | 014113941
+-- hy-hlo-78118455  | 014300615
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:yp' 
+WHERE person_id IN (
+'hy-hlo-67768022',
+'hy-hlo-73740025',
+'hy-hlo-74780917',
+'hy-hlo-79615371',
+'hy-hlo-78118455'
+);
+-- AV:
+-- hy-hlo-78777265  | 010779392
+-- hy-hlo-81909550  | 011894153
+-- hy-hlo-84571890  | 013897158
+-- hy-hlo-80330834  | 015085926
+-- hy-hlo-84775722  | 011791272
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:av' 
+WHERE person_id IN (
+'hy-hlo-78777265',
+'hy-hlo-81909550',
+'hy-hlo-84571890',
+'hy-hlo-80330834',
+'hy-hlo-84775722'
+);
+-- M:
+-- hy-hlo-87439827  | 011928456
+-- hy-hlo-96612371  | 014418361
+-- hy-hlo-84840261  | 010975895
+-- hy-hlo-90330681  | 010522688
+-- hy-hlo-96594603  | 013951573
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:m' 
+WHERE person_id IN (
+'hy-hlo-87439827',
+'hy-hlo-96612371',
+'hy-hlo-84840261',
+'hy-hlo-90330681',
+'hy-hlo-96594603'
+);
+
+-- all studentnumbers for running updater:
+013683126
+014340523
+010589388
+010775888
+011065995
+011321150
+010764653
+011194400
+011338871
+013951984
+012520688
+011196796
+011736884
+010262566
+013670647
+011952325
+011736606
+012775433
+014113941
+014300615
+010779392
+011894153
+013897158
+015085926
+011791272
+011928456
+014418361
+010975895
+010522688
+013951573
@@ -44,20 +44,22 @@ will take care of setting up Oodikone for you. See README for more details."
  msg ""
 }

-# Define custom shell prompt for the interactive select loop
-PS3="Please enter your choice: "
+set_custom_select_prompt() {
+  PS3="Please enter your choice: "

-options=(
-  "Set up oodikone from scratch."
-  "Reset all real data."
-  "Reset sis-importer data."
-  "Quit."
-)
+  options=(
+    "Set up oodikone from scratch."
+    "Reset all real data."
+    "Reset single database."
+    "Quit."
+  )
+}

-# Run scripts before showing interactive prompt
+# Run scripts and set the prompt
 show_welcome
 init_dirs
 get_username
+set_custom_select_prompt

 while true; do
  select opt in "${options[@]}"; do
@@ -66,8 +68,10 @@ while true; do
        set_up_oodikone_from_scratch;;
      "Reset all real data.")
        reset_all_real_data;;
-      "Reset sis-importer data.")
-        reset_sis_importer_data;;
+      "Reset single database.")
+        reset_single_database
+        set_custom_select_prompt # Set prompt back to initial values
+        ;;
      "Quit.")
        break 2;;
      *) msg "${RED}Invalid option:${NOFORMAT} $REPLY
@@ -186,6 +186,18 @@ describe('Course Statistics tests', () => {
      cy.contains('td', '010674989').siblings().contains('1')
    })

+    it.skip('Language distribution is correct', () => {
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
+      cy.contains(/^TKT20003$/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
+      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.contains('Language distribution').click()
+      cy.contains('td', 'finnish').siblings().contains('122')
+      cy.contains('td', 'english').siblings().contains('5')
+    })
+
    describe('When searching unified course stats', () => {
      beforeEach(() => {
        cy.url().should('include', '/coursestatistics')
@@ -9,23 +9,24 @@ const checkFilteringResult = studentCount => {
 // Semantic UI doesn't allow injection of data-cy:s for single multiple dropdown selection.
 // This function tries to click "x" on all selections inside dropdown matching given attribute
 const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.delete').click()
+  cy.cs(dataCyAttribute).find('i.delete').click({ force: true })
+  cy.cs(dataCyAttribute).click()
 }

 const clearSingleDropdownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.clear').click()
+  cy.cs(dataCyAttribute).find('i.clear').click({ force: true })
 }

 // Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
 // to every it-function. Reason behind using test function wrapper is that Cypresses internal beforeEach / afterEach
 // functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
 const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
-  return (dataCyAttributeOfHeaderToClick, testStepFunctionToRun) => {
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
+  return (filterName, testStepFunctionToRun) => {
+    cy.get(`[data-cy="${filterName}-header"]`).click()
    checkFilteringResult(amountWithoutFiltering)
    testStepFunctionToRun()
    checkFilteringResult(amountWithoutFiltering)
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
+    cy.get(`[data-cy="${filterName}-header"]`).click({ force: true })
  }
 }

@@ -41,80 +42,85 @@ describe('Population Statistics', () => {

  it('Transfer filter is on not transferred by default', () => {
    cy.contains('By default only students who have not transferred to this study programme are shown.')
-    cy.cs('transferredToProgrammeFilter-havenot').find('input').should('be.checked')
+    const card = cy.cs('TransferredToProgramme-filter-card')
+    card.cs('TransferredToProgramme-header').click()
+    card.get('[data-cy="option-havenot"] input').should('be.checked')
+    card.cs('TransferredToProgramme-header').click()
  })

  it('Graduation filter works', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      const card = cy.cs('GraduatedFromProgramme-filter-card')
+
      const graduated = 11
-      cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+      card.cs('option-graduated-true').click()
      checkFilteringResult(graduated)
-      cy.cs('graduatedFromProgrammeFilter-graduated-false').click()
+      card.cs('option-graduated-false').click()
      checkFilteringResult(defaultAmountOfStudents - graduated)
-      cy.cs('graduatedFromProgrammeFilter-all').click()
+      card.get('[data-cy="GraduatedFromProgramme-filter-card"] [data-cy="option-all"]').click()
    })
  })

  it('Transfer filter works', () => {
-    runTestStepWithPreAndPostParts('transferredToProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
      const transferred = 1
-      cy.cs('transferredToProgrammeFilter-have').click()
+      cy.cs('option-have').click()
      checkFilteringResult(transferred)
-      cy.cs('transferredToProgrammeFilter-all').click()
+      cy.cs('option-all').click()
      checkFilteringResult(defaultAmountOfStudents + transferred)
-      cy.cs('transferredToProgrammeFilter-havenot').click()
+      cy.cs('option-havenot').click()
    })
  })

  it('Enrollment filter works', () => {
-    runTestStepWithPreAndPostParts('enrollmentStatusFilter-header', () => {
-      cy.selectFromDropdown('enrollmentStatusFilter-status', 0)
-      cy.selectFromDropdown('enrollmentStatusFilter-semesters', [0])
+    runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
+      cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
+      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(0)
      checkFilteringResult(137)
      clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
    })
  })

  it('Credit filter works', () => {
-    runTestStepWithPreAndPostParts('credit-filter-header', () => {
+    runTestStepWithPreAndPostParts('CreditsEarned', () => {
      cy.cs('credit-filter-min').type('50')
      cy.cs('credit-filter-max').type('150')
      checkFilteringResult(116)
-      cy.cs('credit-filter-min').find('input').clear()
-      cy.cs('credit-filter-max').find('input').clear()
+      cy.cs('credit-filter-min').clear()
+      cy.cs('credit-filter-max').clear()
    })
  })

  it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
      cy.cs('ageFilter-min').type('20')
      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(43)
+      checkFilteringResult(41)
      cy.cs('ageFilter-min').find('input').clear()
      cy.cs('ageFilter-max').find('input').clear()
    })
  })

  it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
      checkFilteringResult(36)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
      checkFilteringResult(112)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
      checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
      checkFilteringResult(0)
      clearSingleDropdownSelection('genderFilter-dropdown')
    })
  })

  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', 0)
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
      checkFilteringResult(1)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 14)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(14)
      checkFilteringResult(121)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
    })
@@ -122,10 +128,10 @@ describe('Population Statistics', () => {

  // Return when we have +2020 population with admission types in data
  it.skip('Admission type filter works', () => {
-    runTestStepWithPreAndPostParts('admissionTypeFilter-header', () => {
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('AdmissionTypeFilter', () => {
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(0)
      checkFilteringResult(5)
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 2)
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(2)
      checkFilteringResult(10)
      clearSingleDropdownSelection('admissionTypeFilter-dropdown')
    })
@@ -135,31 +141,32 @@ describe('Population Statistics', () => {
  it.skip('Courses filter works', () => {
    // courses takes some time to load, wait for it to complete
    cy.wait(10000)
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
      cy.cs('courseFilter-course-dropdown').click()
      const courses = ['TKT20001', 'MAT11002']
      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
      checkFilteringResult(140)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
      checkFilteringResult(131)
      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
      checkFilteringResult(56)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 2)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
      checkFilteringResult(2)
      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
    })
  })

  it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
-        cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
+        const card = cy.cs('GraduatedFromProgramme-filter-card')
+        card.cs('option-graduated-true').click()
        cy.cs('ageFilter-min').type('20')
        cy.cs('ageFilter-max').type('30')
        checkFilteringResult(11)
        cy.cs('ageFilter-min').find('input').clear()
        cy.cs('ageFilter-max').find('input').clear()
-        cy.cs('graduatedFromProgrammeFilter-all').click()
+        card.cs('option-all').click()
      })
    })
  })
@@ -176,7 +183,7 @@ describe('Course Statistics', () => {
  })

  it('Grade filter works', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
      cy.cs('gradeFilter-5').click()
      checkFilteringResult(20)
      cy.cs('gradeFilter-3').click()
@@ -187,43 +194,43 @@ describe('Course Statistics', () => {
  })

  it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
      cy.cs('ageFilter-min').type('20')
      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(33)
+      checkFilteringResult(32)
      cy.cs('ageFilter-min').find('input').clear()
      cy.cs('ageFilter-max').find('input').clear()
    })
  })

  it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
      checkFilteringResult(27)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
      checkFilteringResult(91)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
      checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
      checkFilteringResult(0)
      clearSingleDropdownSelection('genderFilter-dropdown')
    })
  })

  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
      checkFilteringResult(1)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 12)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(12)
      checkFilteringResult(29)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
    })
  })

  it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
        cy.cs('gradeFilter-3').click()
        cy.cs('ageFilter-min').type('20')
        cy.cs('ageFilter-max').type('30')
@@ -245,13 +252,14 @@ describe('Custom Population Statistics', () => {
    cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
      const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
      runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
-      cy.cs('student-no-input').click().type(students.join('/n'))
+      cy.cs('student-no-input').click().type(students.join('\n'))
+      cy.wait(10000)
    })
    cy.cs('search-button').click()
  })

  it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
      cy.cs('ageFilter-min').type('20')
      cy.cs('ageFilter-max').type('30')
      checkFilteringResult(1)
@@ -261,35 +269,43 @@ describe('Custom Population Statistics', () => {
  })

  it('Gender filter works', () => {
-    cy.cs('genderFilter-header').click()
-    cy.selectFromDropdown('genderFilter-dropdown', 0)
-    checkFilteringResult(3)
-    cy.selectFromDropdown('genderFilter-dropdown', 1)
-    checkFilteringResult(2)
-    cy.selectFromDropdown('genderFilter-dropdown', 2)
-    checkFilteringResult(0)
-    clearSingleDropdownSelection('genderFilter-dropdown')
-    checkFilteringResult(5, true)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+      checkFilteringResult(5, true)
+    })
  })

  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
      checkFilteringResult(1)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.cs('startYearAtUni-dropdown').click()
    })
  })

  it('Courses filter works', () => {
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
      const courses = ['MAT11001', 'TKT20004']
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[0] + ' ')
+        .click()
      checkFilteringResult(5)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
      checkFilteringResult(5)
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[1] + ' ')
+        .click()
      checkFilteringResult(3)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(3)
      checkFilteringResult(1)
      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
    })
@@ -1,5 +1,8 @@
 /// <reference types="Cypress" />

+const moment = require('moment')
+const _ = require('lodash')
+
 const setPopStatsUntil = (until, includeSettings = []) => {
  cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
  includeSettings.forEach(setting => {
@@ -13,6 +16,8 @@ const setPopStatsUntil = (until, includeSettings = []) => {
 describe('Population Statistics tests', () => {
  const pathToCSBach2017 =
    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2017'
+  const pathToCSMaster2019 =
+    '/populations?months=27&semesters=FALL&semesters=SPRING&studyRights=%7B%22programme%22%3A%22MH50_009%22%7D&tag&year=2019'
  describe('when using basic user', () => {
    beforeEach(() => {
      cy.init('/populations')
@@ -59,6 +64,7 @@ describe('Population Statistics tests', () => {

    it('Population statistics is usable on general level', () => {
      cy.visit(pathToCSBach2017)
+      cy.cs('filtered-students')
      setPopStatsUntil('toukokuu 2020')

      cy.get('.card').within(() => {
@@ -69,7 +75,8 @@ describe('Population Statistics tests', () => {
        cy.contains('Excludes students with non-degree study right')
        cy.contains('Excludes students who have transferred out of this programme')
      })
-      cy.contains('Courses of population').click({ force: true })
+      cy.cs('filtered-students')
+      cy.contains('Courses of population').click()

      cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
      // eslint-disable-next-line cypress/no-unnecessary-waiting
@@ -123,18 +130,62 @@ describe('Population Statistics tests', () => {
      cy.contains('Credit accumulation (for 170 students)')
    })

-    it('Credit Statistics, Statistics pane works', () => {
+    it('Credit Statistics, Credits Gained tab works', () => {
      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
      cy.contains('Credit statistics').click()
+      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the population')
+
+      const months = Math.ceil(moment.duration(moment().diff(moment('2017-08-1'))).asMonths())
+
+      const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
+      const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))
+
+      cy.get('.credits-gained-table').should('contain', '(n=149)')
+
+      for (const [start, end] of ranges) {
+        let value = 'credits'
+
+        if (start !== null) {
+          value = `${start} ≤ ${value}`
+        }
+
+        if (end !== null) {
+          value = `${value} < ${end}`
+        }
+
+        cy.get('.credits-gained-table').should('contain', value)
+      }
+
+      cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
+        const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)
+
+        expect(sum).to.equal(149)
+      })
+
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('not.exist')
+    })
+
+    it('Credit Statistics, Credits Gained tab shows stats by admissions', () => {
+      cy.visit(pathToCSMaster2019)
+      cy.cs('filtered-students')
+      cy.contains('Credit statistics')
+      cy.get("[data-cy='credit-statistics']").click()
+      cy.get('.credits-gained-divider').click()
+      cy.get("[data-cy='credits-gained-table-Avoin väylä']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Yhteispisteet']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Muu']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('exist')
+    })
+
+    it('Credit Statistics, Statistics pane works', () => {
+      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Credit statistics').click({ force: true })
      cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()

      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
      cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
      cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
      cy.get("[data-cy='credit-stats-min']").should('contain', '11')
-      cy.get("[data-cy='credit-stats-q1']").should('contain', '70')
-      cy.get("[data-cy='credit-stats-q2']").should('contain', '129')
-      cy.get("[data-cy='credit-stats-q3']").should('contain', '168')
      cy.get("[data-cy='credit-stats-max']").should('contain', '314')
    })
  })
@@ -13,7 +13,7 @@ describe('Student Statistics tests', () => {
    studentnumber: '010654019',
  }

-  it.only('Student statistics search form is usable', () => {
+  it('Student statistics search form is usable', () => {
    cy.contains('Show student names')
    cy.url().should('include', '/students')
    cy.get('.prompt').type(student.lastname)
@@ -80,7 +80,7 @@ describe('Student Statistics tests', () => {
    cy.contains('Student number').should('not.exist')
  })

-  it.only('Can jump to population page', () => {
+  it('Can jump to population page', () => {
    cy.get('.prompt').type(student.studentnumber)
    cy.contains(student.studentnumber).click()
    cy.get('i.level.up.alternate.icon').eq(0).click()
@@ -88,3 +88,13 @@ describe('Student Statistics tests', () => {
    cy.contains('Tietojenkäsittelytieteen kandiohjelma')
  })
 })
+
+// Use admin to see all students
+describe('Testing with admin rights', () => {
+  it('Does not crash if student has no studyright or courses', () => {
+    const studentNumber = '011825096'
+    cy.init(`/students/${studentNumber}`, 'admin') // use admin to see all students
+    cy.contains('Started: Unavailable')
+    cy.contains('Credits: 0')
+  })
+})
@@ -4,16 +4,16 @@ describe('Users tests', () => {
  describe('Using user with just grp-oodikone-user, no other rights', () => {
    it('shows only frontpage, trends feedback', () => {
      cy.init('', 'norights') // login with norights user
-      cy.get('.navBar').contains('Trends').should('exist')
-      cy.get('.navBar').contains('Give feedback').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Give feedback').should('exist')

-      cy.get('.navBar').contains('Study programme').should('not.exist')
-      cy.get('.navBar').contains('Student statistics').should('not.exist')
-      cy.get('.navBar').contains('Course statistics').should('not.exist')
-      cy.get('.navBar').contains('Teachers').should('not.exist')
-      cy.get('.navBar').contains('Users').should('not.exist')
-      cy.get('.navBar').contains('Faculty').should('not.exist')
-      cy.get('.navBar').contains('Updater').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Teachers').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Faculty').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Updater').should('not.exist')
    })
  })

@@ -29,32 +29,27 @@ describe('Users tests', () => {
    })

    it('should see more stuff than others', () => {
-      cy.get('.navBar').contains('Study programme').should('exist')
-      cy.get('.navBar').contains('Student statistics').should('exist')
-      cy.get('.navBar').contains('Course statistics').should('exist')
-      // TODO: admin should probably see this as well, since can access this page: cy.get('.navBar').contains('Teachers').should('exist')
-      cy.get('.navBar').contains('Users').should('exist')
-      cy.get('.navBar').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('exist')
+      // TODO: admin should probably see this as well, since can access this page: cy.get('[data-cy="navBar"]').contains('Teachers').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
    })

    it("mocking normal user shows only the mocked user's programmes", () => {
      cy.contains('mocking').should('not.exist')
-      cy.contains('tr', 'basic').within($row => {
+      cy.contains('tr', 'basic').within(() => {
        cy.contains('.button', 'Edit').click()
      })
-      cy.intercept({
-        method: 'POST',
-        path: '/api/superlogin/basic',
-      }).as('superlogin')
      cy.get('i.spy').click()
-      cy.wait('@superlogin')
      cy.contains('mocking as basic')
      cy.wait(1000)
      cy.contains('Study programme').click().siblings().contains('Search by class').click()
      cy.contains('label', 'Study programme')
      cy.contains('label', 'Study programme')
        .siblings()
-        .within($row => {
+        .within(() => {
          cy.get("div[role='option']").should('have.length', 2).contains('Tietojenkäsittelytieteen kandiohjelma')
        })
    })
@@ -54,8 +54,6 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
    if (!headersToUse) throw Error(`${userId} is not valid user id!`)
    req.headers = headersToUse
  })
-  // Hide SIS warning flag to avoid screen size problems when running headlessly
-  window.localStorage.setItem('SIS_WARNING_FLAG', 'true')
  const baseUrl = Cypress.config().baseUrl
  cy.visit(baseUrl.concat(path))
 })
@@ -63,24 +61,28 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
 /**
 * Shorthand for using "Cypress Selectors" (CS), i.e., `data-cy` attributes.
 */
-Cypress.Commands.add('cs', name => {
-  return cy.get(`[data-cy='${name}']`)
+Cypress.Commands.add('cs', { prevSubject: 'optional' }, (subject, name) => {
+  const p = subject === undefined ? cy : cy.wrap(subject)
+
+  return p.get(`[data-cy='${name}']`)
 })

 /**
 * Select item specified by `index` number from a semantic-ui dropdown with
 * `data-cy` attribute value `name`.
 */
-Cypress.Commands.add('selectFromDropdown', (name, index) => {
+Cypress.Commands.add('selectFromDropdown', { prevSubject: true }, (s, index) => {
+  const subject = cy.wrap(s)
  const indexes = Array.isArray(index) ? index : [index]

  indexes.forEach(i => {
-    cy.cs(name).click().children('.menu').children().eq(i).click({ force: true })
+    subject.click()
+    subject.children('.menu').children().eq(i).click({ force: true })
  })

  // Close multiple selection so it does not block elements underneath it.
  if (Array.isArray(index)) {
-    cy.cs(name).children('.icon').click({ force: true })
+    cy.wrap(s).children('.icon').click({ force: true })
  }
 })

@@ -2,7 +2,6 @@ x-defaults: &defaults
  restart: unless-stopped

 services:
-
  backend:
    <<: *defaults
    container_name: backend
@@ -10,17 +9,16 @@ services:
      - kone-db
      - redis
      - sis-db
+      - user-db
    environment:
-      CI: "true"
+      CI: 'true'
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
      FRONT_URL: http://localhost:1337
      REDIS: redis
      SECRET_TOKEN: dev
      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
    image: toska/oodikone2-backend:staging

  frontend:
@@ -31,11 +29,15 @@ services:
  kone-db:
    <<: *defaults
    container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
    image: registry.toska.cs.helsinki.fi/kone-db

  redis:
    <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
    container_name: redis
    image: redis
    ports:
@@ -45,23 +47,20 @@ services:
    <<: *defaults
    container_name: sis-db
    image: registry.toska.cs.helsinki.fi/sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    shm_size: 2gb

  user-db:
    <<: *defaults
    container_name: user-db
-    image: registry.toska.cs.helsinki.fi/user-db
-
-  userservice:
-    <<: *defaults
-    container_name: userservice
-    depends_on:
-      - user-db
    environment:
-      CI: "true"
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    image: toska/oodikone2-userservice:staging
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db

  nginx:
    image: nginx:latest
@@ -69,8 +68,45 @@ services:
    depends_on:
      - frontend
      - backend
-      - userservice
    ports:
-      -  1337:80
+      - 1337:80
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf
+
+  # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    image: registry.toska.cs.helsinki.fi/sis-importer-db
+    profiles:
+      - updater
+
+  sis-updater-worker:
+    <<: *defaults
+    build:
+      target: development
+      context: updater/sis-updater-worker
+    depends_on:
+      - redis
+      - sis-db
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_URI: //redis:6379
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+    profiles:
+      - updater
+
+  sis-updater-nats:
+    <<: *defaults
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    container_name: sis-updater-nats
+    image: nats-streaming:0.16.2
+    profiles:
+      - updater
@@ -1,7 +1,7 @@
 services:
-
  backend:
    environment:
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db-real
      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db-real
      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db-real

@@ -18,9 +18,5 @@ services:
      DB_URL: postgres://postgres@sis-db:5432/sis-db-real
      SIS_IMPORTER_DATABASE: sis-importer-db-real

-  userservice:
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db-real
-
 volumes:
  redis-data-real:
@@ -1,10 +0,0 @@
-services:
-
-  userservice-test:
-    build:
-      context: services/userservice
-      target: test
-    container_name: userservice-test
-    environment:
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
@@ -26,18 +26,17 @@ services:
      - kone-db
      - redis
      - sis-db
+      - user-db
    env_file:
      - .env
    environment:
      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
      FRONT_URL: http://localhost:3000
      REDIS: redis
      SECRET_TOKEN: dev
      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
    ports:
      - 8080:8080
    profiles:
@@ -165,26 +164,6 @@ services:
    volumes:
      - user-db-data:/data

-  userservice:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/userservice
-    container_name: userservice
-    depends_on:
-      - user-db
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    ports:
-      - 4567:4567
-    profiles:
-      - oodikone
-    volumes:
-      - userservice-nodemod:/usr/src/app/node_modules
-      - ./services/userservice/:/usr/src/app
-
 volumes:
  backend-nodemod:
  frontend-nodemod:
@@ -195,4 +174,3 @@ volumes:
  sis-updater-scheduler-nodemod:
  sis-updater-worker-nodemod:
  user-db-data:
-  userservice-nodemod:
@@ -49,5 +49,9 @@
    "stylelint-config-prettier": "^8.0.2",
    "stylelint-config-standard": "^22.0.0",
    "stylelint-prettier": "^1.2.0"
+  },
+  "dependencies": {
+    "lodash": "^4.17.21",
+    "moment": "^2.29.1"
  }
 }
@@ -110,12 +110,54 @@ reset_all_real_data() {
  reset_databases "${DATABASES[@]}"
 }

-reset_sis_importer_data() {
-  infomsg "Downloading sis-importer-db dump"
-  local database=$SIS_IMPORTER_DB_NAME
-  local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
-  download_real_dump $database $url
-  reset_databases $database
+reset_single_database() {
+  # Define custom shell prompt for the nested interactive select loop
+  PS3="Please enter which database to reset: "
+
+  options=(
+    "kone-db"
+    "sis-db"
+    "sis-importer-db"
+    "user-db"
+    "Return to main menu."
+  )
+
+  local database=""
+  local url=""
+
+  while true; do
+    select opt in "${options[@]}"; do
+      case $opt in
+        "kone-db")
+          local database=$KONE_DB_NAME
+          local url=$KONE_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-db")
+          local database=$SIS_DB_NAME
+          local url=$SIS_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-importer-db")
+          local database=$SIS_IMPORTER_DB_NAME
+          local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
+          break 2;;
+        "user-db")
+          local database=$USER_DB_NAME
+          local url=$USER_DB_REAL_DUMP_URL
+          break 2;;
+        "Return to main menu.")
+          break 2;;
+        *) msg "${RED}Invalid option:${NOFORMAT} $REPLY
+  ";;
+      esac
+      break
+    done
+  done
+
+  if [ -n "$database" ]; then
+    infomsg "Downloading $database dump"
+    download_real_dump $database $url
+    reset_databases $database
+  fi
 }

 set_up_oodikone_from_scratch() {
@@ -123,7 +165,6 @@ set_up_oodikone_from_scratch() {
  folders_to_set_up=(
    "$PROJECT_ROOT"
    "$PROJECT_ROOT/services/frontend"
-    "$PROJECT_ROOT/services/userservice"
    "$PROJECT_ROOT/services/backend"
    "$PROJECT_ROOT/updater/sis-updater-scheduler"
    "$PROJECT_ROOT/updater/sis-updater-worker"
@@ -1,6 +1,3 @@
 .dockerignore
 Dockerfile
 node_modules/
-requests/
-scripts/
-*.log
@@ -1,3 +0,0 @@
-# Student diff output
-/student_diff_*
-/diff_partial_att_data.json
@@ -332,11 +332,6 @@
        "concat-map": "0.0.1"
      }
    },
-    "buffer-equal-constant-time": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
-      "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
-    },
    "buffer-writer": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
@@ -704,14 +699,6 @@
      "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=",
      "dev": true
    },
-    "ecdsa-sig-formatter": {
-      "version": "1.0.11",
-      "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
-      "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
-      "requires": {
-        "safe-buffer": "^5.0.1"
-      }
-    },
    "ee-first": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -1330,49 +1317,6 @@
      "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=",
      "dev": true
    },
-    "jsonwebtoken": {
-      "version": "8.5.1",
-      "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz",
-      "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==",
-      "requires": {
-        "jws": "^3.2.2",
-        "lodash.includes": "^4.3.0",
-        "lodash.isboolean": "^3.0.3",
-        "lodash.isinteger": "^4.0.4",
-        "lodash.isnumber": "^3.0.3",
-        "lodash.isplainobject": "^4.0.6",
-        "lodash.isstring": "^4.0.1",
-        "lodash.once": "^4.0.0",
-        "ms": "^2.1.1",
-        "semver": "^5.6.0"
-      },
-      "dependencies": {
-        "semver": {
-          "version": "5.7.1",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
-        }
-      }
-    },
-    "jwa": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz",
-      "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==",
-      "requires": {
-        "buffer-equal-constant-time": "1.0.1",
-        "ecdsa-sig-formatter": "1.0.11",
-        "safe-buffer": "^5.0.1"
-      }
-    },
-    "jws": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz",
-      "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==",
-      "requires": {
-        "jwa": "^1.4.1",
-        "safe-buffer": "^5.0.1"
-      }
-    },
    "kind-of": {
      "version": "6.0.3",
      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
@@ -1389,41 +1333,6 @@
      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
    },
-    "lodash.includes": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
-      "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8="
-    },
-    "lodash.isboolean": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
-      "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY="
-    },
-    "lodash.isinteger": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
-      "integrity": "sha1-YZwK89A/iwTDH1iChAt3sRzWg0M="
-    },
-    "lodash.isnumber": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
-      "integrity": "sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w="
-    },
-    "lodash.isplainobject": {
-      "version": "4.0.6",
-      "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
-      "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs="
-    },
-    "lodash.isstring": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
-      "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE="
-    },
-    "lodash.once": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
-      "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w="
-    },
    "logform": {
      "version": "2.2.0",
      "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz",
@@ -3353,9 +3262,9 @@
      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A=="
    },
    "validator": {
-      "version": "13.6.0",
-      "resolved": "https://registry.npmjs.org/validator/-/validator-13.6.0.tgz",
-      "integrity": "sha512-gVgKbdbHgtxpRyR8K0O6oFZPhhB5tT1jeEHZR0Znr9Svg03U0+r9DXWMrnRAB+HtCStDQKlaIZm42tVsVjqtjg=="
+      "version": "13.7.0",
+      "resolved": "https://registry.npmjs.org/validator/-/validator-13.7.0.tgz",
+      "integrity": "sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw=="
    },
    "vary": {
      "version": "1.1.2",
@@ -21,7 +21,6 @@
    "cron": "^1.8.2",
    "express": "^4.17.1",
    "express-async-errors": "^3.1.1",
-    "jsonwebtoken": "^8.5.1",
    "lodash": "^4.17.21",
    "lru-cache": "^5.1.1",
    "moment": "^2.29.1",
@@ -1,13 +1,9 @@
 require('express-async-errors')
 const express = require('express')
-const cors = require('cors')
-const Sentry = require('@sentry/node')
-const { baseUrl, backendPort, frontUrl } = require('./conf-backend')
+const { baseUrl, backendPort } = require('./conf-backend')
 const routes = require('./routes')
 const { startCron } = require('./events')
 const { initializeDatabaseConnection, dbConnections } = require('./database/connection')
-const initializeSentry = require('./util/sentry')
-const errorMiddleware = require('./middleware/errormiddleware')
 const logger = require('./util/logger')

 initializeDatabaseConnection()
@@ -21,28 +17,11 @@ initializeDatabaseConnection()
    dbConnections.on('error', () => {
      logger.error('Failed to connect to sis db!')
    })
-
-    const app = express()
-    initializeSentry(app)
-    app.use(Sentry.Handlers.requestHandler())
-    app.use(Sentry.Handlers.tracingHandler())
-
    startCron()

-    app.use(cors({ credentials: true, origin: frontUrl }))
-    app.use(express.json())
-
+    const app = express()
    routes(app, baseUrl)

-    app.get('*', async (_, res) => {
-      const results = { error: 'unknown endpoint' }
-      res.status(404).json(results)
-    })
-
-    app.use(Sentry.Handlers.errorHandler())
-
-    app.use(errorMiddleware)
-
    const server = app.listen(backendPort, () => logger.info(`Backend listening on port ${backendPort}!`))
    process.on('SIGTERM', () => {
      server.close(() => {
@@ -18,7 +18,6 @@ const pateToken = process.env.PATE_API_TOKEN || ''

 // Importer client
 const importerToken = process.env.IMPORTER_DB_API_TOKEN || ''
-const importerUrl = process.env.IMPORTER_DB_API_URL || ''

 // Networking: Urls & ports
 const baseUrl = isDev ? '/api' : ''
@@ -37,6 +36,7 @@ const MATOMO_URL = process.env.MATOMO_URL

 // Other stuff
 const DB_URL_KONE = process.env.DB_URL_KONE
+const DB_URL_USER = process.env.DB_URL_USER

 let DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS, 10)
 if (isNaN(DB_MAX_CONNECTIONS)) {
@@ -48,10 +48,8 @@ if (DB_MAX_CRON_CONNECTIONS < 1) {
 }

 const redis = process.env.REDIS
-const TOKEN_SECRET = process.env.TOKEN_SECRET
 const SECRET_TOKEN = process.env.SECRET_TOKEN
 const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
-const USERSERVICE_URL = process.env.USERSERVICE_URL
 const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL

 const formatURL = url => {
@@ -60,20 +58,14 @@ const formatURL = url => {

 const addSlashToEnd = url => (url.endsWith('/') ? url : url + '/')

-const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
-
-const PWD = process.env.PWD || '/usr/src/app'
-
 module.exports = {
+  DB_URL_USER,
  DB_URL_KONE,
  DB_MAX_CONNECTIONS,
  DB_MAX_CRON_CONNECTIONS,
  redis,
-  TOKEN_SECRET,
  SECRET_TOKEN,
  DB_SCHEMA_KONE,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL),
-  ACCESS_TOKEN_HEADER_KEY,
  requiredGroup,
  SIS_UPDATER_URL,
  ANALYTICS_INFLUXDB_URL: formatURL(ANALYTICS_INFLUXDB_URL),
@@ -91,6 +83,4 @@ module.exports = {
  frontUrl,
  backendPort,
  importerToken,
-  importerUrl,
-  PWD,
 }
@@ -39,54 +39,73 @@ class DbConnection extends EventEmitter {
  }
 }

+// Old-style kone + user db connections
 const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
  schema: conf.DB_SCHEMA_KONE,
  searchPath: conf.DB_SCHEMA_KONE,
  logging: false,
 })
+
 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)

+const sequelizeUser = new Sequelize(conf.DB_URL_USER, {
+  logging: false,
+})
+
 const initializeDatabaseConnection = async () => {
  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-  const waitSeconds = 60
-  for (let i = 1; i <= waitSeconds; i++) {
-    try {
-      await sequelizeKone.authenticate()
-      break
-    } catch (e) {
-      if (i === waitSeconds) {
-        logger.error(`Could not connect to database in ${waitSeconds} seconds`)
-        throw e
+  const rounds = 60
+
+  for (const [seq, dbName] of [
+    [sequelizeKone, 'kone-db'],
+    [sequelizeUser, 'user-db'],
+  ]) {
+    logger.info(`Connecting to ${dbName}...`)
+    for (let round = 1; round <= rounds; round++) {
+      try {
+        await seq.authenticate()
+        break
+      } catch (error) {
+        if (round === rounds) {
+          logger.error(`${dbName} database connection failed!`)
+          throw error
+        }
+        await sleep(1000)
      }
-      await sleep(1000)
    }
-  }
-  try {
+    logger.info(`${dbName} database connection established`)
+
+    const schema = dbName === 'kone-db' ? conf.DB_SCHEMA_KONE : undefined
+    const migrationsFolder = dbName === 'kone-db' ? 'migrations_kone' : 'migrations_user'
    const migrator = new Umzug({
      storage: 'sequelize',
      storageOptions: {
-        sequelize: sequelizeKone,
+        sequelize: seq,
        tableName: 'migrations',
-        schema: conf.DB_SCHEMA_KONE,
+        schema,
      },
      migrations: {
-        params: [sequelizeKone.getQueryInterface(), Sequelize],
-        path: `${process.cwd()}/src/database/migrations_kone`,
+        params: [seq.getQueryInterface(), Sequelize],
+        path: `${process.cwd()}/src/database/${migrationsFolder}`,
        pattern: /\.js$/,
-        schema: conf.DB_SCHEMA_KONE,
+        schema,
      },
    })
-    const migrations = await migrator.up()
-    logger.info({ message: 'Kone Migrations up to date', meta: migrations })
-  } catch (e) {
-    logger.error('Kone Migration error')
-    throw e
+    try {
+      const migrations = await migrator.up()
+      logger.info({ message: `${dbName} migrations up to date: `, meta: migrations.map(m => m.file) })
+    } catch (error) {
+      logger.error({ message: `${dbName} migrations failed`, meta: error })
+      throw error
+    }
  }
 }

 const dbConnections = new DbConnection()
+
 module.exports = {
  dbConnections,
  sequelizeKone,
+  sequelizeUser,
  initializeDatabaseConnection,
 }
@@ -0,0 +1,34 @@
+const { BIGINT, STRING, DATE } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.createTable(
+      'study_guidance_group_tags',
+      {
+        id: {
+          primaryKey: true,
+          type: BIGINT,
+          autoIncrement: true,
+        },
+        study_guidance_group_id: {
+          type: STRING,
+          unique: true,
+        },
+        study_programme: {
+          type: STRING,
+        },
+        year: {
+          type: STRING,
+        },
+        created_at: {
+          type: DATE,
+        },
+        updated_at: {
+          type: DATE,
+        },
+      },
+      { underscored: true, timestamps: true }
+    )
+  },
+  down: async () => {},
+}
@@ -0,0 +1,9 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('user_affiliation')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS user_affilation_id_seq')
+    await queryInterface.dropTable('affiliations')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS affiliations_id_seq')
+  },
+  down: async () => {},
+}
@@ -1,40 +1,3 @@
-const jwt = require('jsonwebtoken')
-const conf = require('../conf-backend')
-const blacklist = require('../services/blacklist')
-const { getUserDataFor } = require('../services/userService')
-const { ACCESS_TOKEN_HEADER_KEY } = require('../conf-backend')
-const { hasRequiredGroup, parseHyGroups } = require('../util/utils')
-
-const TOKEN_VERSION = 1.1 // When token structure changes, increment in userservice, backend and frontend
-
-const checkAuth = async (req, res, next) => {
-  const token = req.headers[ACCESS_TOKEN_HEADER_KEY]
-  const uid = req.headers['uid']
-
-  if (!token) {
-    return res.status(403).json({ error: 'No token in headers' })
-  }
-
-  jwt.verify(token, conf.TOKEN_SECRET, async (err, decoded) => {
-    if (err) {
-      return res.status(403).json(err)
-    }
-
-    if (decoded.version !== TOKEN_VERSION) {
-      return res.status(401).json({ error: 'Token needs to be refreshed - invalid version' })
-    }
-
-    if (decoded.mockedBy ? decoded.mockedBy !== uid : decoded.userId !== uid) {
-      return res.status(403).json({ error: 'User shibboleth id and token id did not match' })
-    }
-
-    const userData = await getUserDataFor(decoded.userId)
-    req.decodedToken = decoded
-    Object.assign(req, userData)
-    next()
-  })
-}
-
 const roles = requiredRoles => async (req, res, next) => {
  if (req.decodedToken) {
    const { roles } = req
@@ -47,37 +10,6 @@ const roles = requiredRoles => async (req, res, next) => {
  res.status(403).json({ error: 'missing required roles' })
 }

-const checkRequiredGroup = async (req, res, next) => {
-  const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-  const enabled = hasRequiredGroup(hyGroups)
-  const tokenOutdated = req.decodedToken.enabled !== enabled
-
-  if (tokenOutdated) {
-    return res.status(401).json({
-      error: 'Token needs to be refreshed - enabled does not match hy-group requirement',
-    })
-  }
-
-  if (!enabled) {
-    return res.status(403).json({ error: 'User is not enabled' })
-  }
-  next()
-}
-
-const checkUserBlacklisting = async (req, res, next) => {
-  const { userId, createdAt } = req.decodedToken
-  const isBlacklisted = await blacklist.isUserBlacklisted(userId, createdAt)
-
-  if (isBlacklisted) {
-    return res.status(401).json({ error: 'Token needs to be refreshed - blacklisted' })
-  }
-
-  next()
-}
-
 module.exports = {
-  checkAuth,
  roles,
-  checkRequiredGroup,
-  checkUserBlacklisting,
 }
@@ -0,0 +1,55 @@
+const Sentry = require('@sentry/node')
+const { ApplicationError } = require('../util/customErrors')
+const { getUser, getUserDataFor, getMockedUser } = require('../services/userService')
+const { requiredGroup } = require('../conf-backend')
+const _ = require('lodash')
+
+const parseIamGroups = iamGroups => iamGroups?.split(';').filter(Boolean) ?? []
+
+const hasRequiredIamGroup = iamGroups => _.intersection(iamGroups, requiredGroup).length > 0
+
+const currentUserMiddleware = async (req, _, next) => {
+  const { uid: username, 'shib-session-id': sessionId, shib_logout_url: logoutUrl } = req.headers
+
+  if (!sessionId) {
+    throw new ApplicationError('No session id found in request headers', 403, { logoutUrl })
+  }
+
+  if (!username) {
+    throw new ApplicationError('No username found in request headers', 403, { logoutUrl })
+  }
+  const { displayname: name, mail: email, hygroupcn: iamGroups, hypersonsisuid: sisId } = req.headers
+  const parsedIamGroups = parseIamGroups(iamGroups)
+
+  req.decodedToken = await getUser({
+    username,
+    name,
+    email,
+    iamGroups: parsedIamGroups,
+    sisId,
+  })
+
+  if (!hasRequiredIamGroup(parsedIamGroups)) {
+    throw new ApplicationError('User is not enabled', 403, { logoutUrl })
+  }
+
+  const showAsUser = req.headers['x-show-as-user']
+
+  if (showAsUser) {
+    const mockedUser = await getMockedUser(username, showAsUser)
+    if (mockedUser) {
+      req.decodedToken = mockedUser
+    }
+  }
+  req.logoutUrl = logoutUrl
+
+  Sentry.setUser({ username: req.decodedToken.mockedBy || req.decodedToken.userId })
+
+  // TODO: remove this garbo
+  const userData = await getUserDataFor(req.decodedToken.userId)
+  Object.assign(req, userData)
+
+  next()
+}
+
+module.exports = currentUserMiddleware
@@ -1,13 +0,0 @@
-const Sentry = require('@sentry/node')
-
-const sentryUserIdMiddleware = (req, res, next) => {
-  Sentry.getCurrentHub().configureScope(scope => {
-    scope.setUser({
-      username: req.decodedToken && req.decodedToken.userId,
-    })
-  })
-
-  next()
-}
-
-module.exports = sentryUserIdMiddleware
@@ -0,0 +1,5 @@
+const headersMiddleware = require('unfuck-utf8-headers-middleware')
+
+const headers = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
+
+module.exports = headersMiddleware(headers)
@@ -47,6 +47,9 @@ Course.init(
    substitutions: {
      type: JSONB,
    },
+    course_unit_type: {
+      type: STRING,
+    },
  },
  {
    underscored: false,
@@ -75,6 +75,9 @@ Credit.init(
    updatedAt: {
      type: DATE,
    },
+    language: {
+      type: STRING,
+    },
  },
  {
    underscored: false,
@@ -135,6 +135,28 @@ const CustomPopulationSearch = sequelizeKone.define('custom_population_searches'
  },
 })

+const StudyGuidanceGroupTag = sequelizeKone.define(
+  'study_guidance_group_tags',
+  {
+    id: {
+      primaryKey: true,
+      type: Sequelize.BIGINT,
+      autoIncrement: true,
+    },
+    studyGuidanceGroupId: {
+      type: Sequelize.STRING,
+      unique: true,
+    },
+    studyProgramme: {
+      type: Sequelize.STRING,
+    },
+    year: {
+      type: Sequelize.STRING,
+    },
+  },
+  { underscored: true, timestamps: true }
+)
+
 MandatoryCourse.belongsTo(MandatoryCourseLabels, {
  foreignKey: 'label',
  sourceKey: 'id',
@@ -157,4 +179,5 @@ module.exports = {
  CustomPopulationSearch,
  sequelizeKone,
  ExcludedCourse,
+  StudyGuidanceGroupTag,
 }
@@ -1,7 +1,7 @@
 const Sequelize = require('sequelize')
-const { sequelize } = require('../database/connection')
+const { sequelizeUser } = require('../database/connection')

-const UserElementDetails = sequelize.define(
+const UserElementDetails = sequelizeUser.define(
  'user_elementdetails',
  {
    userId: {
@@ -18,7 +18,7 @@ const UserElementDetails = sequelize.define(
  }
 )

-const AccessGroup = sequelize.define('access_group', {
+const AccessGroup = sequelizeUser.define('access_group', {
  id: {
    primaryKey: true,
    type: Sequelize.BIGINT,
@@ -39,7 +39,7 @@ const AccessGroup = sequelize.define('access_group', {
  },
 })

-const UserFaculties = sequelize.define('user_faculties', {
+const UserFaculties = sequelizeUser.define('user_faculties', {
  userId: {
    primaryKey: true,
    type: Sequelize.BIGINT,
@@ -60,7 +60,7 @@ const UserFaculties = sequelize.define('user_faculties', {
  },
 })

-const User = sequelize.define(
+const User = sequelizeUser.define(
  'users',
  {
    id: {
@@ -83,25 +83,7 @@ const User = sequelize.define(
    timestamps: false,
  }
 )
-const HyGroup = sequelize.define('hy_group', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true,
-  },
-
-  code: { type: Sequelize.STRING },
-
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})
-
-const Affiliation = sequelize.define('affiliations', {
+const HyGroup = sequelizeUser.define('hy_group', {
  id: {
    primaryKey: true,
    type: Sequelize.BIGINT,
@@ -131,12 +113,6 @@ AccessGroup.belongsToMany(User, { through: 'user_accessgroup' })
 User.belongsToMany(HyGroup, { through: 'user_hy_group', as: 'hy_group' })
 HyGroup.belongsToMany(User, { through: 'user_hy_group' })

-User.belongsToMany(Affiliation, {
-  through: 'user_affiliation',
-  as: 'affiliation',
-})
-Affiliation.belongsToMany(User, { through: 'user_affiliation' })
-
 User.hasMany(UserFaculties, { as: 'faculty', foreignKey: 'userId' })

 module.exports = {
@@ -144,7 +120,6 @@ module.exports = {
  UserElementDetails,
  AccessGroup,
  HyGroup,
-  Affiliation,
  UserFaculties,
-  sequelize,
+  sequelizeUser,
 }
@@ -17,7 +17,6 @@ Student.init(
    dateofuniversityenrollment: { type: DATE },
    email: { type: STRING },
    national_student_number: { type: STRING },
-    home_county_id: { type: INTEGER },
    country_fi: { type: STRING },
    country_sv: { type: STRING },
    country_en: { type: STRING },
@@ -25,6 +24,7 @@ Student.init(
    home_country_sv: { type: STRING },
    home_country_en: { type: STRING },
    gender_code: { type: INTEGER },
+    sis_person_id: { type: STRING },
    createdAt: {
      type: DATE,
    },
@@ -1,50 +1,52 @@
-const shibbolethHeadersFix = require('unfuck-utf8-headers-middleware')
-const accessLogger = require('./middleware/accesslogger')
-const sentryUserId = require('./middleware/sentryUserId')
+const Sentry = require('@sentry/node')
+const express = require('express')
+const cors = require('cors')
+const { frontUrl } = require('./conf-backend')
+const shibbolethCharsetMiddleware = require('./middleware/shibbolethCharsetMiddleware')
+const matomoInit = require('./routes/matomo-init')
+const currentUserMiddleware = require('./middleware/currentUserMiddleware')
+const accessLogger = require('./middleware/accessLogger')
+const auth = require('./middleware/auth')
+
 const courses = require('./routes/courses')
 const students = require('./routes/students')
 const population = require('./routes/population')
 const login = require('./routes/login')
-const superlogin = require('./routes/superlogin')
 const language = require('./routes/language')
 const users = require('./routes/users')
 const elementdetails = require('./routes/elementdetails')
-const auth = require('./middleware/auth')
 const teachers = require('./routes/teachers')
 const providers = require('./routes/providers')
 const faculties = require('./routes/faculties')
 const semesters = require('./routes/semesters')
 const mandatoryCourses = require('./routes/mandatorycourses')
 const mandatoryCourseLabels = require('./routes/mandatorycourselabels')
-const ping = require('./routes/ping')
 const feedback = require('./routes/feedback')
 const tags = require('./routes/tags')
 const updater = require('./routes/updater')
 const tsaAnalytics = require('./routes/tsaAnalytics')
-const matomoInit = require('./routes/matomo-init')
 const customPopulationSearch = require('./routes/customPopulationSearch')
 const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
 const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
+const studyProgramme = require('./routes/studyProgramme')
+const initializeSentry = require('./util/sentry')
+
+const errorMiddleware = require('./middleware/errorMiddleware')

 module.exports = (app, url) => {
-  app.use(url, ping)
-  app.use(
-    shibbolethHeadersFix([
-      'hyGroupCn',
-      'SHIB_LOGOUT_URL',
-      'eduPersonAffiliation',
-      'uid',
-      'displayName',
-      'mail',
-      'hyPersonSisuId',
-      'preferredLanguage',
-    ])
-  )
+  initializeSentry(app)
+  app.use(Sentry.Handlers.requestHandler())
+  app.use(Sentry.Handlers.tracingHandler())
+
+  app.use(cors({ credentials: true, origin: frontUrl }))
+  app.use(express.json())
+
+  app.use(shibbolethCharsetMiddleware)
  app.use(url, matomoInit)
+  app.use(currentUserMiddleware)
+  app.use(accessLogger)
  app.use(url, login)
-  app.use(`${url}/superlogin`, superlogin)
-  app.use(auth.checkAuth, auth.checkRequiredGroup, auth.checkUserBlacklisting, accessLogger, sentryUserId)
  app.use(url, elementdetails)
  app.use(url, courses)
  app.use(url, students)
@@ -54,6 +56,7 @@ module.exports = (app, url) => {
  app.use(url, semesters)
  app.use(url, tags)
  app.use(url, programmeModules)
+  app.use(url, studyProgramme)
  app.use(`${url}/faculties`, faculties)
  app.use(`${url}/updater`, auth.roles(['admin']), updater)
  app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
@@ -65,4 +68,10 @@ module.exports = (app, url) => {
  app.use(`${url}/custom-population-search`, customPopulationSearch)
  app.use(`${url}/cool-data-science`, trends)
  app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
+  app.get('*', async (_, res) => {
+    const results = { error: 'unknown endpoint' }
+    res.status(404).json(results)
+  })
+  app.use(Sentry.Handlers.errorHandler())
+  app.use(errorMiddleware)
 }
@@ -1,47 +1,16 @@
 const router = require('express').Router()
-const userService = require('../services/userService')
-const { parseHyGroups } = require('../util/utils')
-const { sendNotificationAboutNewUser } = require('../services/mailservice')
-const logger = require('../util/logger')
+const { ApplicationError } = require('../util/customErrors')

-router.post('/login', async (req, res) => {
-  const uid = req.headers['uid']
-  if (req.headers['shib-session-id'] && uid) {
-    const full_name = req.headers.displayname || ''
-    const mail = req.headers.mail || ''
-    const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-    const affiliations = parseHyGroups(req.headers['edupersonaffiliation'])
-    const hyPersonSisuId = req.headers.hypersonsisuid || ''
-    logger.info(`${uid} trying to login, referring to userservice`)
-    let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail, hyPersonSisuId)
-    if (isNew) {
-      const result = await sendNotificationAboutNewUser({ userId: uid, userFullName: full_name })
-      if (result.error) {
-        return res.status(400).json(result).end()
-      }
-    }
-    res.status(200).json({ token })
-  } else {
-    res
-      .status(401)
-      .json({
-        message: `Not enough headers login, uid: ${req.headers.uid}
-        session-id ${req.headers['shib-session-id']}`,
-      })
-      .end()
+router.get('/login', async (req, res) => {
+  const { decodedToken, logoutUrl } = req
+
+  if (!decodedToken) {
+    throw new ApplicationError('User not found', 404)
  }
-})

-router.delete('/logout', async (req, res) => {
-  const logoutUrl = req.headers.shib_logout_url
-  const { returnUrl } = req.body
-  if (logoutUrl) {
-    return res
-      .status(200)
-      .send({ logoutUrl: `${logoutUrl}?return=${returnUrl}` })
-      .end()
-  }
-  res.status(200).send({ logoutUrl: returnUrl }).end()
+  res.send({
+    token: decodedToken,
+    logoutUrl,
+  })
 })
-
 module.exports = router
@@ -28,7 +28,7 @@ _paq.push(["enableLinkTracking"]);

 const script = config.MATOMO_URL ? acual : ''

-router.get('/matomo-init', (req, res) => {
+router.get('/matomo-init', (_, res) => {
  res.contentType('text/javascript; charset=UTF-8')
  res.send(script)
 })
@@ -1,85 +0,0 @@
-/* eslint-disable */
-
-const { router, wrapper } = require('./router').routerWithWrapper()
-const axios = require('axios')
-
-wrapper.get('/ping', async (req, res) => {
-  res.json({
-    message: 'pong'
-  })
-})
-
-wrapper.get('/v3/mandatory_courses/:code', async (req, res) => {
-  const courses = []
-  const included = new Set()
-
-  const dumb_flatten = module => {
-    if (module.children) {
-      module.children.forEach(child => {
-        dumb_flatten(child)
-      })
-    } else if (module.code && !module.code.startsWith('KK') && !courses.map(c => c.code).includes(module.code)) {
-      courses.push({
-        name: {
-          fi: module.name,
-          en: module.name
-        },
-        code: module.code,
-        label: { id: `${courses.length}`, label: module.code.replace('-', '').slice(0, 4), orderNumber: courses.length }
-      })
-    }
-  }
-
-  const better_flatten = (module, label) => {
-    if (module.children) {
-      module.children.forEach(child => {
-        better_flatten(child, module.module ? module.module.name : label)
-      })
-    } else if (module.code && !module.code.startsWith('KK')) {
-      courses.push({
-        name: {
-          fi: module.name,
-          en: module.name
-        },
-        code: module.code,
-        label: { id: `${courses.length}`, label, orderNumber: courses.length }
-      })
-    }
-  }
-
-  const superFlattenFlatten = (module, label) => {
-    if (module.children) {
-      module.children.forEach(child => {
-        superFlattenFlatten(child, module.module ? module.module.name : label)
-      })
-    } else if (Array.isArray(module)) {
-      module.forEach(elem => superFlattenFlatten(elem, label))
-    } else if (module.code && !module.code.startsWith('KK')) {
-      if (!included.has(`${label}${module.code}`)) {
-        included.add(`${label}${module.code}`)
-        courses.push({
-          name: {
-            fi: module.name,
-            en: module.name
-          },
-          code: module.code,
-          label: { id: `${courses.length}`, label, orderNumber: courses.length }
-        })
-      }
-    }
-  }
-
-  const code = req.params.code
-  const response = await axios.get(`http://sis-updater-scheduler:8082/v1/courses/${code}?token=dev&superFlatten=true`)
-
-  // dumb_flatten(response.data)
-  // better_flatten(response.data)
-  superFlattenFlatten(response.data)
-
-  const byCode = (c1, c2) => (c1.code < c2.code ? -1 : 1)
-
-  courses.sort(byCode)
-  res.json(courses)
-})
-
-module.exports = router
@@ -2,7 +2,7 @@ const router = require('express').Router()
 const Student = require('../services/students')
 const userService = require('../services/userService')
 const Unit = require('../services/units')
-const { mismatchedStudents } = require('../services/student_credit_total_mismatches')
+const { getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')

 const filterStudentTags = (student, userId) => {
  return {
@@ -11,17 +11,10 @@ const filterStudentTags = (student, userId) => {
  }
 }

-const creditTotalMismatch = student => {
-  return {
-    ...student,
-    mismatch: mismatchedStudents.has(student.studentNumber),
-  }
-}
-
 router.get('/students', async (req, res) => {
  const {
    roles,
-    decodedToken: { userId },
+    decodedToken: { userId, sisPersonId },
    query: { searchTerm },
  } = req

@@ -45,8 +38,14 @@ router.get('/students', async (req, res) => {
  } else {
    const unitsUserCanAccess = await userService.getUnitsFromElementDetails(userId)
    const codes = unitsUserCanAccess.map(unit => unit.id)
+    let extraStudents = []
+    if (roles.includes('studyGuidanceGroups')) {
+      const matchingOnlyByTerm = await Student.bySearchTerm(trimmedSearchTerm)
+      const studentsUserCanAccess = await getAllStudentsUserHasInGroups(sisPersonId)
+      extraStudents.push(...matchingOnlyByTerm.filter(s => studentsUserCanAccess.has(s.studentNumber)))
+    }
    const matchingStudents = await Student.bySearchTermAndElements(trimmedSearchTerm, codes)
-    res.json(matchingStudents)
+    res.json([...matchingStudents, ...extraStudents])
  }
 })

@@ -54,14 +53,11 @@ router.get('/students/:id', async (req, res) => {
  const { id: studentId } = req.params
  const { roles, decodedToken } = req

-  if (roles && roles.includes('admin')) {
+  if (roles?.includes('admin')) {
    const results = await Student.withId(studentId)
    return results.error
      ? res.status(400).json({ error: 'error finding student' }).end()
-      : res
-          .status(200)
-          .json(creditTotalMismatch(filterStudentTags(results, decodedToken.id)))
-          .end()
+      : res.status(200).json(filterStudentTags(results, decodedToken.id)).end()
  }

  const uid = req.decodedToken.userId
@@ -78,6 +74,13 @@ router.get('/students/:id', async (req, res) => {
    })
  )

+  if (roles?.includes('studyGuidanceGroups')) {
+    const studentsUserCanAccess = await getAllStudentsUserHasInGroups(decodedToken.sisPersonId)
+    if (studentsUserCanAccess.has(studentId)) {
+      return res.status(200).json(filterStudentTags(student, decodedToken.id)).end()
+    }
+  }
+
  if (rights.some(right => right !== null)) {
    res.status(200).json(filterStudentTags(student, decodedToken.id)).end()
  } else {
@@ -1,6 +1,6 @@
 const router = require('express').Router()
 const { logger } = require('../util/logger')
-const { getAllGroupsAndStudents } = require('../services/studyGuidanceGroups')
+const { getAllGroupsAndStudents, changeGroupTags } = require('../services/studyGuidanceGroups')

 router.get('/', async (req, res) => {
  const {
@@ -13,4 +13,27 @@ router.get('/', async (req, res) => {
  return res.json(await getAllGroupsAndStudents(sisPersonId))
 })

+router.put('/:id/tags', async (req, res) => {
+  const {
+    decodedToken: { uid, sisPersonId },
+    body: tags,
+    params: { id: groupId },
+  } = req
+
+  if (!sisPersonId) {
+    logger.error(`User ${uid} tried to change study guidance group tags but personId was ${sisPersonId} in header`)
+    return res.status(400).json({ error: 'Not possible to change groups without personId' })
+  }
+
+  const groupsUserHasAccessTo = (await getAllGroupsAndStudents(sisPersonId)).map(group => group.id)
+
+  if (!groupsUserHasAccessTo.includes(groupId)) {
+    logger.error(
+      `User ${uid} tried to change study guidance group tags, but didn't have permission for the group ${groupId}`
+    )
+    return res.status(400).json({ error: 'Access denied' })
+  }
+  return res.status(200).json(await changeGroupTags({ groupId, tags }))
+})
+
 module.exports = router
@@ -0,0 +1,131 @@
+const router = require('express').Router()
+const {
+  getBasicStatsForStudytrack,
+  getCreditStatsForStudytrack,
+  getGraduationStatsForStudytrack,
+} = require('../services/studyprogrammeStats')
+const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
+
+const {
+  getBasicStats,
+  setBasicStats,
+  getCreditStats,
+  setCreditStats,
+  getGraduationStats,
+  setGraduationStats,
+  getStudytrackStats,
+  setStudytrackStats,
+} = require('../services/analyticsService')
+const logger = require('../util/logger')
+
+router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+
+  if (code) {
+    let data = null
+    try {
+      data = await getBasicStats(code, yearType)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} basic stats: ${e}`)
+    }
+    if (!data) {
+      try {
+        const result = await getBasicStatsForStudytrack({
+          studyprogramme: req.params.id,
+          yearType,
+        })
+        data = await setBasicStats(result, yearType)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} basic stats: ${e}`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+
+  if (code) {
+    let data = null
+    try {
+      data = await getCreditStats(code, yearType)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} credit stats`)
+    }
+    if (!data) {
+      try {
+        const result = await getCreditStatsForStudytrack({
+          studyprogramme: req.params.id,
+          yearType,
+        })
+        data = await setCreditStats(result, yearType)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} credit stats`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+
+  if (code) {
+    let data = null
+    try {
+      data = await getGraduationStats(code, yearType)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} graduation stats`)
+    }
+    if (!data) {
+      try {
+        const result = await getGraduationStatsForStudytrack({
+          studyprogramme: req.params.id,
+          yearType,
+        })
+        data = await setGraduationStats(result, yearType)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} graduation stats`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
+  const code = req.params.id
+
+  if (code) {
+    let data = null
+    try {
+      data = await getStudytrackStats(code)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} studytrack stats: ${e}`)
+    }
+    if (!data) {
+      try {
+        const result = await getStudytrackStatsForStudyprogramme({ studyprogramme: req.params.id })
+        data = await setStudytrackStats(result)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.use('*', (req, res, next) => next())
+
+module.exports = router
@@ -1,21 +0,0 @@
-const router = require('express').Router()
-const userService = require('../services/userService')
-
-router.post('/:uid', async (req, res) => {
-  const uid = req.headers['uid']
-  if (req.headers['shib-session-id'] && uid) {
-    const asUser = req.params.uid
-    const token = await userService.superlogin(uid, asUser)
-    res.status(200).json(token)
-  } else {
-    res
-      .status(401)
-      .json({
-        message: `Not enough headers login, uid:
-        ${req.headers.uid} session-id ${req.headers['shib-session-id']}`,
-      })
-      .end()
-  }
-})
-
-module.exports = router
@@ -1,18 +1,8 @@
 const router = require('express').Router()
 const userService = require('../services/userService')
-const blacklist = require('../services/blacklist')
-const { userDataCache } = require('../services/cache')
 const { sendNotificationAboutAccessToUser, previewNotificationAboutAccessToUser } = require('../services/mailservice')
 const logger = require('../util/logger')

-const addUserToBlacklist = async (user, decodedToken) => {
-  if (user) {
-    const { username } = user
-    userDataCache.del(username)
-    if (username != decodedToken.userId) await blacklist.addUserToBlacklist(username)
-  }
-}
-
 router.get('/', async (req, res) => {
  const results = await userService.findAll()
  res.json(results)
@@ -25,12 +15,7 @@ router.get('/access_groups', async (req, res) => {

 router.post('/modifyaccess', async (req, res) => {
  try {
-    const { username } = req.body
    const result = await userService.modifyAccess(req.body)
-    const user = {
-      username,
-    }
-    await addUserToBlacklist(user, req.decodedToken)
    res.status(200).json(result)
  } catch (e) {
    res.status(400).json(e)
@@ -60,7 +45,6 @@ router.post('/:uid/elements', async (req, res) => {
  const { uid } = req.params
  const { codes } = req.body
  const user = await userService.enableElementDetails(uid, codes)
-  await addUserToBlacklist(user, req.decodedToken)
  res.json(user)
 })

@@ -68,7 +52,6 @@ router.delete('/:uid/elements', async (req, res) => {
  const { uid } = req.params
  const { codes } = req.body
  const user = await userService.removeElementDetails(uid, codes)
-  await addUserToBlacklist(user, req.decodedToken)
  res.json(user)
 })

@@ -76,7 +59,6 @@ router.post('/:uid/faculties', async (req, res) => {
  const { uid } = req.params
  const { faculties } = req.body
  const user = await userService.setFaculties(uid, faculties)
-  await addUserToBlacklist(user, req.decodedToken)
  res.json(user)
 })

@@ -3,6 +3,10 @@ const moment = require('moment')

 const createRedisKeyForProductivity = id => `PRODUCTIVITY_${id}`
 const createRedisKeyForThroughput = id => `THROUGHPUT_${id}`
+const createRedisKeyForBasicStats = (id, yearType) => `BASIC_STATS_${id}_${yearType}`
+const createRedisKeyForCreditStats = (id, yearType) => `CREDIT_STATS_${id}_${yearType}`
+const createRedisKeyForGraduationStats = (id, yearType) => `GRADUATION_STATS_${id}_${yearType}`
+const createRedisKeyForStudytrackStats = id => `STUDYTRACK_STATS_${id}`

 const getProductivity = async id => {
  const redisKey = createRedisKeyForProductivity(id)
@@ -87,6 +91,86 @@ const patchThroughput = async data => {
  }
 }

+const getBasicStats = async (id, yearType) => {
+  const redisKey = createRedisKeyForBasicStats(id, yearType)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setBasicStats = async (data, yearType) => {
+  const { id } = data
+  const redisKey = createRedisKeyForBasicStats(id, yearType)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getCreditStats = async (id, yearType) => {
+  const redisKey = createRedisKeyForCreditStats(id, yearType)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setCreditStats = async (data, yearType) => {
+  const { id } = data
+  const redisKey = createRedisKeyForCreditStats(id, yearType)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getGraduationStats = async (id, yearType) => {
+  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setGraduationStats = async (data, yearType) => {
+  const { id } = data
+  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getStudytrackStats = async id => {
+  const redisKey = createRedisKeyForStudytrackStats(id)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setStudytrackStats = async data => {
+  const { id } = data
+  const redisKey = createRedisKeyForStudytrackStats(id)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
 module.exports = {
  getProductivity,
  setProductivity,
@@ -94,4 +178,12 @@ module.exports = {
  getThroughput,
  setThroughput,
  patchThroughput,
+  getBasicStats,
+  setBasicStats,
+  getCreditStats,
+  setCreditStats,
+  getGraduationStats,
+  setGraduationStats,
+  getStudytrackStats,
+  setStudytrackStats,
 }
@@ -1,22 +0,0 @@
-const { redisClient } = require('./redis')
-const moment = require('moment')
-const logger = require('../util/logger')
-
-const MAKE_REDIS_HASH_KEY = id => `BLACKLIST ${id}`
-
-const addUserToBlacklist = async userId => {
-  logger.info(`user ${userId} added to token blacklist`)
-  await redisClient.setAsync(MAKE_REDIS_HASH_KEY(userId), moment().toISOString(false))
-}
-
-const isUserBlacklisted = async (userId, tokenCreatedAt) => {
-  const isBlacklisted = async () => {
-    const blacklistTimestamp = await redisClient.getAsync(MAKE_REDIS_HASH_KEY(userId))
-    return tokenCreatedAt && blacklistTimestamp && moment(tokenCreatedAt) < moment(blacklistTimestamp)
-  }
-
-  const timer = () => new Promise(resolve => setTimeout(() => resolve(true), 5000))
-  return Promise.race([isBlacklisted(), timer()])
-}
-
-module.exports = { addUserToBlacklist, isUserBlacklisted }
@@ -368,39 +368,9 @@ const byNameAndOrCodeLike = async (name, code) => {
  })

  const courses = rawCourses
-    .map(
-      ({
-        id,
-        code,
-        name,
-        latest_instance_date,
-        is_study_module,
-        coursetypecode,
-        startdate,
-        max_attainment_date,
-        min_attainment_date,
-        createdAt,
-        updatedAt,
-        substitutions,
-        organizations,
-      }) => {
-        return {
-          id,
-          code,
-          name,
-          latest_instance_date,
-          is_study_module,
-          coursetypecode,
-          startdate,
-          max_attainment_date,
-          min_attainment_date,
-          createdAt,
-          updatedAt,
-          substitutions,
-          organizations: organizations.map(o => o.id),
-        }
-      }
-    )
+    .map(course => {
+      return { ...course.dataValues, organizations: course.organizations.map(o => o.id) }
+    })
    .sort(a => (a.code.match(/^[A-Za-z]{3}[0-9]{1}/) ? -1 : 1))

  let substitutionGroupIndex = 0
@@ -0,0 +1,338 @@
+const sequelize = require('sequelize')
+const { Op } = sequelize
+const {
+  Credit,
+  Course,
+  Organization,
+  Studyright,
+  StudyrightElement,
+  ElementDetail,
+  Transfer,
+  Student,
+  SemesterEnrollment,
+  Semester,
+} = require('../models')
+const { formatStudyright, formatStudent } = require('./studyprogrammeHelpers')
+
+const whereStudents = studentnumbers => {
+  return studentnumbers ? studentnumbers : { [Op.not]: null }
+}
+
+const studytrackStudents = async studentnumbers =>
+  (
+    await Student.findAll({
+      where: {
+        studentnumber: {
+          [Op.in]: studentnumbers,
+        },
+      },
+    })
+  ).map(formatStudent)
+
+const enrolledStudyrights = async studytrack => {
+  const studyrights = await Studyright.findAll({
+    attributes: ['studyrightid', 'studystartdate', 'enddate', 'prioritycode'],
+    include: [
+      {
+        model: StudyrightElement,
+        required: true,
+        where: { code: studytrack },
+      },
+      {
+        model: Student,
+        attributes: ['studentnumber'],
+        required: true,
+      },
+    ],
+  })
+
+  const studentnumbers = studyrights.map(s => s.student.studentnumber)
+
+  const students = await Student.findAll({
+    attributes: ['studentnumber'],
+    include: [
+      {
+        model: Studyright,
+        include: [
+          {
+            model: StudyrightElement,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        ],
+        attributes: ['studyrightid'],
+      },
+      {
+        model: SemesterEnrollment,
+        attributes: ['semestercode'],
+        include: [
+          {
+            model: Semester,
+            required: true,
+          },
+        ],
+        where: {
+          enrollmenttype: 1,
+        },
+      },
+    ],
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumbers,
+      },
+    },
+  })
+
+  let enrollments = new Set()
+
+  students.forEach(student => {
+    student.semester_enrollments.forEach(e => {
+      const enrollmentStartDate = e.semester.startdate
+      const enrollmentEndDate = e.semester.enddate
+      const correctStudyRight = studyrights.find(
+        s =>
+          student.studentnumber === s.student.studentnumber &&
+          s.studystartdate <= enrollmentStartDate && // Studying in the programme should have started on the same day or earlier than the enrollment
+          s.enddate >= enrollmentEndDate && // Studying in the programme should have ended on the same day or later than the enrollment
+          (!s.canceldate || s.canceldate > enrollmentStartDate) && // Studyright should not be canceled OR the possible canceldate should be after the enrollmentStartDate
+          (s.prioritycode === 1 || s.prioritycode === 30) // Studyright should be primary or the student should have graduated
+      )
+      if (correctStudyRight) {
+        enrollments.add({
+          studentnumber: student.studentnumber,
+          startdate: e.semester.startdate,
+        })
+      }
+    })
+  })
+
+  return enrollments
+}
+
+const startedStudyrights = async (studytrack, since, studentnumbers) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: {
+            model: ElementDetail,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        studystartdate: {
+          [Op.gte]: since,
+        },
+        student_studentnumber: whereStudents(studentnumbers),
+      },
+    })
+  ).map(formatStudyright)
+
+const graduatedStudyRights = async (studytrack, since, studentnumbers) =>
+  await Studyright.findAll({
+    include: {
+      model: StudyrightElement,
+      required: true,
+      include: {
+        model: ElementDetail,
+        required: true,
+        where: {
+          code: studytrack,
+        },
+      },
+    },
+    where: {
+      graduated: 1,
+      enddate: {
+        [Op.gte]: since,
+      },
+      student_studentnumber: whereStudents(studentnumbers),
+    },
+  })
+
+const cancelledStudyRights = async (studytrack, since) => {
+  return await Studyright.findAll({
+    include: {
+      model: StudyrightElement,
+      required: true,
+      where: {
+        code: {
+          [Op.eq]: studytrack,
+        },
+      },
+    },
+    where: {
+      canceldate: {
+        [Op.gte]: since,
+      },
+    },
+  })
+}
+
+const transfersAway = async (studytrack, since) => {
+  return await Transfer.findAll({
+    where: {
+      transferdate: {
+        [Op.gte]: since,
+      },
+      sourcecode: studytrack,
+    },
+    distinct: true,
+    col: 'studentnumber',
+  })
+}
+
+const transfersTo = async (studytrack, since) => {
+  return await Transfer.findAll({
+    where: {
+      transferdate: {
+        [Op.gte]: since,
+      },
+      targetcode: studytrack,
+    },
+    distinct: true,
+    col: 'studentnumber',
+  })
+}
+
+const getProgrammesStudents = async studyprogramme =>
+  (
+    await Studyright.findAll({
+      attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode'],
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          where: {
+            code: {
+              [Op.in]: [studyprogramme],
+            },
+          },
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+    })
+  ).map(formatStudyright)
+
+const getCreditsForStudyProgramme = async (provider, since) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      where: {
+        is_study_module: false,
+      },
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.notIn]: [10, 9, 7],
+      },
+      isStudyModule: {
+        [Op.not]: true,
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+    },
+  })
+
+const getTransferredCredits = async (provider, since) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'credittypecode', 'student_studentnumber'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      where: {
+        is_study_module: false,
+      },
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.eq]: [9],
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+    },
+  })
+
+const getThesisCredits = async (provider, since, thesisType) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      where: {
+        course_unit_type: {
+          [Op.in]: thesisType,
+        },
+      },
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.notIn]: [10, 9, 7],
+      },
+      isStudyModule: {
+        [Op.not]: true,
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+    },
+  })
+
+module.exports = {
+  studytrackStudents,
+  enrolledStudyrights,
+  startedStudyrights,
+  graduatedStudyRights,
+  cancelledStudyRights,
+  transfersAway,
+  transfersTo,
+  getProgrammesStudents,
+  getCreditsForStudyProgramme,
+  getTransferredCredits,
+  getThesisCredits,
+}
@@ -65,7 +65,7 @@ const formatStudentForPopulationStatistics = (
  startDateMoment,
  endDateMoment
 ) => {
-  const toCourse = ({ grade, attainment_date, credits, course_code, credittypecode, isStudyModule }) => {
+  const toCourse = ({ grade, attainment_date, credits, course_code, credittypecode, isStudyModule, language }) => {
    const attainment_date_normailized =
      attainment_date < startDate ? startDateMoment.clone().add(1, 'day').toISOString() : attainment_date
    const passed = Credit.passed({ credittypecode })
@@ -78,6 +78,7 @@ const formatStudentForPopulationStatistics = (
      credits,
      isStudyModuleCredit: isStudyModule,
      credittypecode,
+      language,
    }
  }

@@ -265,6 +266,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
        'isStudyModule',
        'student_studentnumber',
        'course_code',
+        'language',
      ],
      where: creditsOfStudent,
      raw: true,
@@ -494,6 +496,7 @@ const studentnumbersWithAllStudyrightElements = async (
    if (!newestStudytrack) return false
    return studyRights.includes(newestStudytrack.code)
  })
+
  const studentnumberlist = filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers

  // fetch students that have transferred out of the programme and filter out these studentnumbers
@@ -1,28 +1,63 @@
 const { getImporterClient } = require('../util/importerClient')
 const importerClient = getImporterClient()
+const { StudyGuidanceGroupTag } = require('../models/models_kone')
+const logger = require('../util/logger')

 const getAllStudentsUserHasInGroups = async sisPersonId => {
  if (!importerClient) {
    return new Set()
  }
-  const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
-  return new Set(
-    Object.values(data)
-      .map(group => group.members)
-      .flat()
-      .map(member => member.personStudentNumber)
-  )
+  try {
+    const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
+    return new Set(
+      Object.values(data)
+        .map(group => group.members)
+        .flat()
+        .map(member => member.personStudentNumber)
+    )
+  } catch (error) {
+    logger.error("Couldn't fetch users studyguidance groups")
+    return new Set()
+  }
 }

 const getAllGroupsAndStudents = async sisPersonId => {
  if (!importerClient) {
    return []
  }
-  const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
-  return Object.values(data)
+  try {
+    const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
+    const tagsByGroupId = (await StudyGuidanceGroupTag.findAll()).reduce((acc, curr) => {
+      const { studyProgramme, year } = curr
+      return { ...acc, [curr.studyGuidanceGroupId]: { studyProgramme, year } }
+    }, {})
+    return Object.values(data).map(group => ({ ...group, tags: tagsByGroupId[group.id] }))
+  } catch (error) {
+    logger.error("Couldn't fetch users studyguidance groups")
+    return []
+  }
+}
+
+const changeGroupTags = async ({ groupId, tags }) => {
+  const { studyProgramme, year } = tags
+  const tagToUpdate = studyProgramme ? { studyProgramme } : { year }
+
+  const [result] = await StudyGuidanceGroupTag.upsert(
+    {
+      studyGuidanceGroupId: groupId,
+      ...tagToUpdate,
+    },
+    {
+      where: {
+        studyGuidanceGroupId: groupId,
+      },
+    }
+  )
+  return result
 }

 module.exports = {
  getAllStudentsUserHasInGroups,
  getAllGroupsAndStudents,
+  changeGroupTags,
 }
@@ -211,7 +211,7 @@ const findProgrammeThesisCredits = async code => {
    },
    where: {
      credittypecode: {
-        [Op.ne]: 10,
+        [Op.eq]: 4,
      },
    },
  })
@@ -0,0 +1,161 @@
+const { mean } = require('lodash')
+
+// Helper functions
+const formatStudyright = studyright => {
+  const { studyrightid, studystartdate, enddate, graduated, prioritycode, extentcode, student } = studyright
+  return {
+    studyrightid,
+    studystartdate,
+    enddate,
+    graduated,
+    prioritycode,
+    extentcode,
+    studentnumber: student.studentnumber,
+  }
+}
+
+const formatStudent = student => {
+  const { studentnumber, gender_code, home_country_en, creditcount } = student
+  return {
+    studentnumber,
+    gender_code,
+    home_country_en,
+    creditcount,
+  }
+}
+
+const getYearsArray = (since, isAcademicYear) => {
+  const years = []
+  for (let i = since; i <= new Date().getFullYear(); i++) {
+    const year = isAcademicYear ? `${i} - ${i + 1}` : i
+    years.push(year)
+  }
+  return years
+}
+
+const getYearsObject = (years, emptyArrays = false) => {
+  let yearsObject = {}
+  for (const year of years) {
+    yearsObject = { ...yearsObject, [year]: emptyArrays ? [] : 0 }
+  }
+  return yearsObject
+}
+
+const getStatsBasis = years => {
+  return {
+    graphStats: new Array(years.length).fill(0),
+    tableStats: getYearsObject(years),
+  }
+}
+
+const isMajorStudentCredit = (studyright, attainment_date) =>
+  studyright &&
+  (studyright.prioritycode === 1 || studyright.prioritycode === 30) && // Is studyright state = MAIN or state = GRADUATED
+  studyright.studystartdate <= attainment_date && // Has the credit been attained after studying in the programme started
+  studyright.enddate >= attainment_date && // Has the credit been attained before the studyright ended
+  (!studyright.canceldate || studyright.canceldate >= attainment_date) // If the studyright was cancelled, was the credit attained before it was cancelled
+
+const getMedian = values => {
+  if (values.length === 0) return 0
+  values.sort((a, b) => a - b)
+  const half = Math.floor(values.length / 2)
+  if (values.length % 2) return values[half]
+  return (values[half - 1] + values[half]) / 2.0
+}
+
+const getMean = values => {
+  if (values.length === 0) return 0
+  return Math.round(mean(values))
+}
+
+const defineYear = (date, isAcademicYear) => {
+  if (!date) return ''
+  const year = date.getFullYear()
+  if (!isAcademicYear) return year
+  if (date < new Date(`${year}-07-31`)) return `${year - 1} - ${year}`
+  return `${year} - ${year + 1}`
+}
+
+const getStartDate = (studyprogramme, isAcademicYear) => {
+  if ((studyprogramme.includes('KH') || studyprogramme.includes('MH')) && isAcademicYear) return new Date('2017-08-01')
+  if (studyprogramme.includes('KH') || studyprogramme.includes('MH')) return new Date('2017-01-01')
+  if (isAcademicYear) return new Date('2000-08-01')
+  return new Date('2000-01-01')
+}
+
+// There are 9 course_unit_types
+// 1. urn:code:course-unit-type:regular
+// 2. urn:code:course-unit-type:bachelors-thesis
+// 3. urn:code:course-unit-type:masters-thesis
+// 4. urn:code:course-unit-type:doctors-thesis
+// 5. urn:code:course-unit-type:licentiate-thesis
+// 6. urn:code:course-unit-type:bachelors-maturity-examination
+// 7. urn:code:course-unit-type:masters-maturity-examination
+// 8. urn:code:course-unit-type:communication-and-linguistic-studies
+// 9. urn:code:course-unit-type:practical-training-homeland
+// Four of these are thesis types
+
+const getThesisType = studyprogramme => {
+  if (studyprogramme.includes('MH') || studyprogramme.includes('ma'))
+    return ['urn:code:course-unit-type:masters-thesis']
+  if (studyprogramme.includes('KH') || studyprogramme.includes('ba'))
+    return ['urn:code:course-unit-type:bachelors-thesis']
+  if (/^(T)[0-9]{6}$/.test(studyprogramme))
+    return ['urn:code:course-unit-type:doctors-thesis', 'urn:code:course-unit-type:licentiate-thesis']
+  return 'thesis-type-not-defined'
+}
+
+const getPercentage = (value, total) => {
+  if (typeof value !== 'number' || typeof total !== 'number') return 'NA'
+  if (total === 0) return 'NA'
+  if (value === 0) return '0 %'
+  return `${((value / total) * 100).toFixed(1)} %`
+}
+
+const getEmptyArray = length => new Array(length).fill(0)
+
+const getCreditGraphStats = years => ({
+  lte30: {
+    name: 'Less than 30 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte60: {
+    name: '30-59 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte90: {
+    name: '60-89 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte120: {
+    name: '90-119 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte150: {
+    name: '120-149 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte150: {
+    name: 'More than 150 credits',
+    data: getEmptyArray(years.length),
+  },
+})
+
+const creditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'mte150']
+
+module.exports = {
+  formatStudyright,
+  formatStudent,
+  getYearsArray,
+  getYearsObject,
+  getStatsBasis,
+  isMajorStudentCredit,
+  getMedian,
+  getMean,
+  defineYear,
+  getStartDate,
+  getThesisType,
+  getPercentage,
+  getCreditGraphStats,
+  creditThresholds,
+}
@@ -0,0 +1,315 @@
+const moment = require('moment')
+const { indexOf } = require('lodash')
+
+const { mapToProviders } = require('../util/utils')
+const {
+  getStatsBasis,
+  defineYear,
+  isMajorStudentCredit,
+  getYearsObject,
+  getYearsArray,
+  getMedian,
+  getMean,
+  getStartDate,
+  getThesisType,
+} = require('./studyprogrammeHelpers')
+const {
+  graduatedStudyRights,
+  startedStudyrights,
+  cancelledStudyRights,
+  transfersAway,
+  transfersTo,
+  getProgrammesStudents,
+  getCreditsForStudyProgramme,
+  getTransferredCredits,
+  getThesisCredits,
+} = require('./newStudyprogramme')
+
+// const getEnrolledStats = async (studytrack, since, years, isAcademicYear) => {
+//   const students = await enrolledStudyrights(studytrack)
+//   const { graphStats, tableStats } = getStatsBasis(years)
+
+//   students.forEach(({ startdate }) => {
+//     if (startdate.getMonth() === 7) {
+//       const enrolledYear = defineYear(startdate, isAcademicYear)
+
+//       graphStats[indexOf(years, enrolledYear)] += 1
+//       tableStats[enrolledYear] += 1
+//     }
+//   })
+
+//   return { graphStats, tableStats }
+// }
+
+const getGraduatedStats = async (studytrack, since, years, isAcademicYear) => {
+  const studyrights = await graduatedStudyRights(studytrack, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ enddate }) => {
+    const graduationYear = defineYear(enddate, isAcademicYear)
+    graphStats[indexOf(years, graduationYear)] += 1
+    tableStats[graduationYear] += 1
+  })
+  return { graphStats, tableStats }
+}
+
+const getStartedStats = async (studytrack, since, years, isAcademicYear) => {
+  const studyrights = await startedStudyrights(studytrack, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ studystartdate }) => {
+    const startYear = defineYear(studystartdate, isAcademicYear)
+    graphStats[indexOf(years, startYear)] += 1
+    tableStats[startYear] += 1
+  })
+  return { graphStats, tableStats }
+}
+
+const getCancelledStats = async (studytrack, since, years, isAcademicYear) => {
+  const studyrights = await cancelledStudyRights(studytrack, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ canceldate }) => {
+    const cancelYear = defineYear(canceldate, isAcademicYear)
+    graphStats[indexOf(years, cancelYear)] += 1
+    tableStats[cancelYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredAwayStats = async (studytrack, since, years, isAcademicYear) => {
+  const studyrights = await transfersAway(studytrack, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredToStats = async (studytrack, since, years, isAcademicYear) => {
+  const studyrights = await transfersTo(studytrack, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getRegularCreditStats = async (studytrack, since, years, isAcademicYear) => {
+  const providercode = mapToProviders([studytrack])[0]
+  const studyrights = await getProgrammesStudents(studytrack)
+  const credits = await getCreditsForStudyProgramme(providercode, since)
+
+  let majors = getStatsBasis(years)
+  let nonMajors = getStatsBasis(years)
+
+  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
+    const studyright = studyrights.find(studyright => studyright.studentnumber == student_studentnumber)
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+
+    if (isMajorStudentCredit(studyright, attainment_date)) {
+      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      majors.tableStats[attainmentYear] += credits || 0
+    } else {
+      nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      nonMajors.tableStats[attainmentYear] += credits || 0
+    }
+  })
+
+  return { majors, nonMajors }
+}
+
+const getTransferredCreditStats = async (studytrack, since, years, isAcademicYear) => {
+  const providercode = mapToProviders([studytrack])[0]
+  const credits = await getTransferredCredits(providercode, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  credits.forEach(({ attainment_date, credits }) => {
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+    graphStats[indexOf(years, attainmentYear)] += credits || 0
+    tableStats[attainmentYear] += credits || 0
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getThesisStats = async (studytrack, since, years, isAcademicYear) => {
+  const providercode = mapToProviders([studytrack])[0]
+  const thesisType = getThesisType(studytrack)
+  const credits = await getThesisCredits(providercode, since, thesisType)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  credits.forEach(({ attainment_date }) => {
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+    graphStats[indexOf(years, attainmentYear)] += 1
+    tableStats[attainmentYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getGraduationTimeStats = async (studytrack, since, years, isAcademicYear) => {
+  const studyrights = await graduatedStudyRights(studytrack, since)
+  let graduationAmounts = getYearsObject(years)
+  let graduationTimes = getYearsObject(years, true)
+
+  studyrights.forEach(({ enddate, studystartdate }) => {
+    const graduationYear = defineYear(enddate, isAcademicYear)
+    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+    graduationAmounts[graduationYear] += 1
+    graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
+  })
+
+  const medians = getYearsObject(years, true)
+  const means = getYearsObject(years, true)
+
+  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
+  const comparisonValue = studytrack.includes('KH') ? 72 : 48
+
+  // HighCharts graph require the data to have this format (ie. actual value, "empty value")
+  years.forEach(year => {
+    const median = getMedian(graduationTimes[year])
+    const mean = getMean(graduationTimes[year])
+    medians[year] = [
+      ['', median],
+      ['', comparisonValue - median],
+    ]
+    means[year] = [
+      ['', mean],
+      ['', comparisonValue - mean],
+    ]
+  })
+  return { medians, means, graduationAmounts }
+}
+
+const getBasicStatsForStudytrack = async ({ studyprogramme, yearType }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+  const started = await getStartedStats(studyprogramme, since, years, isAcademicYear)
+  const graduated = await getGraduatedStats(studyprogramme, since, years, isAcademicYear)
+  const cancelled = await getCancelledStats(studyprogramme, since, years, isAcademicYear)
+  const transferredAway = await getTransferredAwayStats(studyprogramme, since, years, isAcademicYear)
+  const transferredTo = await getTransferredToStats(studyprogramme, since, years, isAcademicYear)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const tableStats = reversedYears.map(year => [
+    year,
+    started.tableStats[year],
+    graduated.tableStats[year],
+    cancelled.tableStats[year],
+    transferredAway.tableStats[year],
+    transferredTo.tableStats[year],
+  ])
+
+  return {
+    id: studyprogramme,
+    years,
+    graphStats: [
+      {
+        name: 'Started',
+        data: started.graphStats,
+      },
+      {
+        name: 'Graduated',
+        data: graduated.graphStats,
+      },
+      {
+        name: 'Cancelled',
+        data: cancelled.graphStats,
+      },
+      {
+        name: 'Transferred away',
+        data: transferredAway.graphStats,
+      },
+      {
+        name: 'Transferred To',
+        data: transferredTo.graphStats,
+      },
+    ],
+    tableStats,
+  }
+}
+
+const getCreditStatsForStudytrack = async ({ studyprogramme, yearType }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+  const { majors, nonMajors } = await getRegularCreditStats(studyprogramme, since, years, isAcademicYear)
+  const transferred = await getTransferredCreditStats(studyprogramme, since, years, isAcademicYear)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const tableStats = reversedYears.map(year => [
+    year,
+    majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
+    majors.tableStats[year],
+    nonMajors.tableStats[year],
+    transferred.tableStats[year],
+  ])
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    graphStats: [
+      {
+        name: 'Major students credits',
+        data: majors.graphStats,
+      },
+      {
+        name: 'Non-major students credits',
+        data: nonMajors.graphStats,
+      },
+      {
+        name: 'Transferred credits',
+        data: transferred.graphStats,
+      },
+    ],
+  }
+}
+
+const getGraduationStatsForStudytrack = async ({ studyprogramme, yearType }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+  const thesis = await getThesisStats(studyprogramme, since, years, isAcademicYear)
+  const graduated = await getGraduatedStats(studyprogramme, since, years, isAcademicYear)
+  const graduationTimeStats = await getGraduationTimeStats(studyprogramme, since, years, isAcademicYear)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    graphStats: [
+      {
+        name: 'Graduated students',
+        data: graduated.graphStats,
+      },
+      {
+        name: 'Wrote thesis',
+        data: thesis.graphStats,
+      },
+    ],
+    graduationMedianTime: graduationTimeStats.medians,
+    graduationMeanTime: graduationTimeStats.means,
+    graduationAmounts: graduationTimeStats.graduationAmounts,
+  }
+}
+
+module.exports = {
+  getBasicStatsForStudytrack,
+  getCreditStatsForStudytrack,
+  getGraduationStatsForStudytrack,
+}
@@ -0,0 +1,195 @@
+const { indexOf } = require('lodash')
+
+const { getAssociations } = require('./studyrights')
+const { studentnumbersWithAllStudyrightElements } = require('./populations')
+const {
+  getStartDate,
+  getYearsArray,
+  getPercentage,
+  getCreditGraphStats,
+  getYearsObject,
+  creditThresholds,
+} = require('./studyprogrammeHelpers')
+const { studytrackStudents, startedStudyrights, graduatedStudyRights } = require('./newStudyprogramme')
+const { getAcademicYearDates } = require('../util/semester')
+
+const getStudentData = students => {
+  const data = { female: 0, male: 0, finnish: 0, lte30: 0, lte60: 0, lte90: 0, lte120: 0, lte150: 0, mte150: 0 }
+  students.forEach(({ gender_code, home_country_en, creditcount }) => {
+    data.male += gender_code === '1' ? 1 : 0
+    data.female += gender_code === '2' ? 1 : 0
+    data.finnish += home_country_en === 'Finland' ? 1 : 0
+    data.lte30 += creditcount < 30 ? 1 : 0
+    data.lte60 += creditcount >= 30 && creditcount < 60 ? 1 : 0
+    data.lte90 += creditcount >= 60 && creditcount < 90 ? 1 : 0
+    data.lte120 += creditcount >= 90 && creditcount < 120 ? 1 : 0
+    data.lte150 += creditcount >= 120 && creditcount < 150 ? 1 : 0
+    data.mte150 += creditcount >= 150 ? 1 : 0
+  })
+  return data
+}
+
+// Goes through the programme and all its studytracks for the said year and adds the wanted stats to the data objects
+const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studytrackNames, year, years, data }) => {
+  const { mainStatsByYear, mainStatsByTrack, creditGraphStats, creditTableStats, emptyTracks } = data
+  const { startDate, endDate } = getAcademicYearDates(year)
+
+  await Promise.all(
+    studytracks.map(async track => {
+      const codes = studyprogramme === track ? [studyprogramme] : [studyprogramme, track]
+      const studentnumbers = await studentnumbersWithAllStudyrightElements(
+        codes,
+        startDate,
+        endDate,
+        true,
+        true,
+        true,
+        true
+      )
+
+      // All the stats are counted for the students who actually started studying
+      const started = await startedStudyrights(track, startDate, studentnumbers)
+      const startedStudentnumbers = [...new Set(started.map(s => s.studentnumber))]
+      const students = await studytrackStudents(startedStudentnumbers)
+      const studentData = getStudentData(students)
+      const graduated = await graduatedStudyRights(track, startDate, studentnumbers)
+
+      // If the track has no stats for that year, it should be removed from the table and dropdown options
+      if (started.length === 0) {
+        emptyTracks.has(track) ? emptyTracks.set(track, emptyTracks.get(track) + 1) : emptyTracks.set(track, 1)
+        return
+      }
+
+      // Count stats for creditgraph for the year per track
+      creditThresholds.forEach(
+        threshold => (creditGraphStats[track][threshold].data[indexOf(years, year)] = studentData[threshold])
+      )
+
+      // Count stats for the credit progress table for the year per track
+      creditTableStats[track] = [
+        ...creditTableStats[track],
+        [
+          `${year} - ${year + 1}`,
+          started.length,
+          studentData.lte30,
+          studentData.lte60,
+          studentData.lte90,
+          studentData.lte120,
+          studentData.lte150,
+          studentData.mte150,
+        ],
+      ]
+
+      // Count stats for the main studytrack table grouped by tracks
+      mainStatsByTrack[track] = [
+        ...mainStatsByTrack[track],
+        [
+          `${year} - ${year + 1}`,
+          started.length,
+          getPercentage(started.length, started.length),
+          studentData.male,
+          getPercentage(studentData.male, started.length),
+          studentData.female,
+          getPercentage(studentData.female, started.length),
+          studentData.finnish,
+          getPercentage(studentData.finnish, started.length),
+          graduated.length,
+          getPercentage(graduated.length, started.length),
+        ],
+      ]
+
+      // Count stats for the main studytrack table grouped by year
+      mainStatsByYear[year] = [
+        ...mainStatsByYear[year],
+        [
+          studytrackNames[track]?.name['fi'] || `${year} - ${year + 1}`,
+          started.length,
+          getPercentage(started.length, started.length),
+          studentData.male,
+          getPercentage(studentData.male, started.length),
+          studentData.female,
+          getPercentage(studentData.female, started.length),
+          studentData.finnish,
+          getPercentage(studentData.finnish, started.length),
+          graduated.length,
+          getPercentage(graduated.length, started.length),
+        ],
+      ]
+    })
+  )
+}
+
+// Defines the studytrack names for the studytrack selector
+// If the track has no stats for any year, it should be removed the dropdown options
+const getStudytrackOptions = (studyprogramme, studytrackNames, studytracks, emptyTracks, years) => {
+  const names = { [studyprogramme]: 'All students of the programme' }
+  studytracks.forEach(track => {
+    const trackName = studytrackNames[track]?.name['fi']
+    if (trackName && emptyTracks.get(track) !== years.length) {
+      names[track] = trackName
+    }
+  })
+  return names
+}
+
+// Creates empty objects for each statistic type, which are then updated with the studytrack data
+const getEmptyStatsObjects = (years, studytracks) => {
+  const mainStatsByYear = getYearsObject(years, true)
+  const mainStatsByTrack = {}
+  const creditGraphStats = {}
+  const creditTableStats = {}
+  const emptyTracks = new Map()
+
+  studytracks.forEach(async track => {
+    mainStatsByTrack[track] = []
+    creditGraphStats[track] = await getCreditGraphStats(years)
+    creditTableStats[track] = []
+  })
+
+  return { mainStatsByYear, mainStatsByTrack, creditGraphStats, creditTableStats, emptyTracks }
+}
+
+// Combines all the data for the Populations and Studytracks -view
+const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {
+  const isAcademicYear = true
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear())
+
+  const associations = await getAssociations()
+  const studytracks = associations.programmes[studyprogramme]
+    ? [studyprogramme, ...associations.programmes[studyprogramme].studytracks]
+    : [studyprogramme]
+
+  const studytrackNames = associations.studyTracks
+
+  const data = getEmptyStatsObjects(years, studytracks)
+
+  await Promise.all(
+    years.map(async year => {
+      return await getStudytrackDataForTheYear({
+        studyprogramme,
+        studytracks,
+        studytrackNames,
+        year,
+        years,
+        data,
+      })
+    })
+  )
+
+  const studytrackOptions = getStudytrackOptions(studyprogramme, studytrackNames, studytracks, data.emptyTracks, years)
+
+  return {
+    id: studyprogramme,
+    years: getYearsArray(since.getFullYear(), isAcademicYear),
+    mainStatsByTrack: data.mainStatsByTrack,
+    mainStatsByYear: data.mainStatsByYear,
+    creditTableStats: data.creditTableStats,
+    creditGraphStats: data.creditGraphStats,
+    studytrackOptions,
+  }
+}
+
+module.exports = {
+  getStudytrackStatsForStudyprogramme,
+}
@@ -178,7 +178,7 @@ const getTotalCreditsOfCoursesBetween = async (a, b, alias = 'sum', alias2 = 'su
    WHERE
      cr.attainment_date BETWEEN :a AND :b
      AND (cr."isStudyModule" = false OR cr."isStudyModule" IS NULL)
-      AND cr.credittypecode IN (4, 9)
+      AND cr.credittypecode IN (4, 9)      
    GROUP BY co.id, o.code
    -- HAVING SUM(cr.credits) > 0
    `,
@@ -189,13 +189,40 @@ const getTotalCreditsOfCoursesBetween = async (a, b, alias = 'sum', alias2 = 'su
  )
 }

-const makeYearlyCreditsPromises = (currentYear, years, getRange, alias = 'sum', alias2 = 'sum2') => {
+const getTotalOpenUniCreditsOfCoursesBetween = async (a, b, alias = 'sum', alias2 = 'sum2') => {
+  return sequelize.query(
+    `
+    SELECT SUM(cr.credits) AS ${alias}, COUNT(DISTINCT(cr.student_studentnumber)) AS ${alias2},
+    o.code AS organizationcode, co.code, co.id, co.name
+    FROM credit cr
+    INNER JOIN course co ON cr.course_code = co.code
+    INNER JOIN course_providers cp ON cp.coursecode = co.id
+    INNER JOIN organization o ON o.id = cp.organizationcode
+    AND cr.attainment_date BETWEEN :a AND :b
+    AND (cr."isStudyModule" = false OR cr."isStudyModule" IS NULL)
+    AND cr.credittypecode IN (4, 9) AND cp.organizationcode = 'hy-org-48645785'
+    AND cr.student_studentnumber NOT IN
+      (SELECT student_studentnumber FROM studyright
+      WHERE studyright.student_studentnumber = cr.student_studentnumber
+      AND cr.attainment_date BETWEEN studyright.startdate AND studyright.enddate)
+    GROUP BY co.id, o.code -- HAVING SUM(cr.credits) > 0
+    `,
+    {
+      type: sequelize.QueryTypes.SELECT,
+      replacements: { a, b },
+    }
+  )
+}
+
+const makeYearlyCreditsPromises = (currentYear, years, getRange, alias = 'sum', alias2 = 'sum2', avoin) => {
  return years.map(
    year =>
      new Promise(async res => {
        const diff = currentYear - year
        const { from, to } = getRange(diff)
-        const creditsByCourse = await getTotalCreditsOfCoursesBetween(from, to, alias, alias2)
+        const creditsByCourse = avoin
+          ? await getTotalOpenUniCreditsOfCoursesBetween(from, to, alias, alias2)
+          : await getTotalCreditsOfCoursesBetween(from, to, alias, alias2)
        res(
          creditsByCourse.map(c => {
            c['year'] = year
@@ -218,6 +245,27 @@ const getOrgStats = (stats, orgId) =>
    },
  }

+const getOrgsOfOpenUniReqularCounterparts = async codes => {
+  const rows = await sequelize.query(
+    `
+      SELECT co.id, co.code, cp.organizationcode, org.code AS organization
+      FROM course co, course_providers cp, organization org
+      WHERE co.code IN(:codes)
+      AND co.id = cp.coursecode
+      AND org.id = cp.organizationcode;
+    `,
+    {
+      type: sequelize.QueryTypes.SELECT,
+      replacements: { codes },
+    }
+  )
+
+  return rows.reduce((acc, { code, organization }) => {
+    acc[code] = organization
+    return acc
+  }, {})
+}
+
 const calculateStatusStatistics = async (unixMillis, showByYear) => {
  const YEAR_TO_MILLISECONDS = 31556952000
  /* Memoize parses booleans into strings... */
@@ -225,38 +273,83 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
  const startYear = startDate.getFullYear()
  const startTime = startDate.getTime()
  const yearRange = _.range(2017, startYear + 1)
+  const getRangeAcc = diff => ({
+    from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
+    to:
+      showByYear === 'true'
+        ? new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS)
+        : new Date(unixMillis - diff * YEAR_TO_MILLISECONDS),
+  })
+  const getRangeTotal = diff => ({
+    from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
+    to: new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS),
+  })
  const yearlyAccCreditsPromises = makeYearlyCreditsPromises(
    startYear,
    yearRange,
-    diff => ({
-      from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
-      to:
-        showByYear === 'true'
-          ? new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS)
-          : new Date(unixMillis - diff * YEAR_TO_MILLISECONDS),
-    }),
+    getRangeAcc,
    'acc',
-    'students'
+    'students',
+    false
+  )
+  const yearlyOpenUniAccCreditsPromises = makeYearlyCreditsPromises(
+    startYear,
+    yearRange,
+    getRangeAcc,
+    'acc',
+    'students',
+    true
  )
-
  const yearlyTotalCreditsPromises = makeYearlyCreditsPromises(
    startYear,
    yearRange.slice(0, -1),
-    diff => ({
-      from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
-      to: new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS),
-    }),
+    getRangeTotal,
    'total',
-    'students'
+    'students',
+    false
+  )
+
+  const yearlyOpenUniTotalCreditsPromises = makeYearlyCreditsPromises(
+    startYear,
+    yearRange.slice(0, -1),
+    getRangeTotal,
+    'total',
+    'students',
+    true
  )

  /* Gather all required data */
-  const [yearlyAccCredits, yearlyTotalCredits, elementDetails, faculties] = await Promise.all([
+  const [
+    yearlyAccCredits,
+    yearlyTotalCredits,
+    yearlyOpenUniAccCredits,
+    yearlyOpenUniTotalCredits,
+    elementDetails,
+    faculties,
+  ] = await Promise.all([
    Promise.all(yearlyAccCreditsPromises),
    Promise.all(yearlyTotalCreditsPromises),
+    Promise.all(yearlyOpenUniAccCreditsPromises),
+    Promise.all(yearlyOpenUniTotalCreditsPromises),
    ElementDetail.findAll(),
    Organization.findAll(),
  ])
+
+  const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+
+  const unifyOpenUniversity = code => {
+    const regexresult = isOpenUniCourseCode(code)
+    if (!regexresult) return code
+    return regexresult[1]
+  }
+
+  const fromAyToNormalCourses = [
+    ...new Set(_.flatten(yearlyOpenUniAccCredits).map(c => unifyOpenUniversity(c.code))),
+    ...new Set(_.flatten(yearlyOpenUniTotalCredits).map(c => unifyOpenUniversity(c.code))),
+  ]
+
+  const orgsOfOpenUniReqularCounterparts = await getOrgsOfOpenUniReqularCounterparts(fromAyToNormalCourses)
+
  const facultyProgrammes = facultiesAndProgrammesForTrends

  const yearlyOrgStatPromises = yearRange.map(async year => {
@@ -322,6 +415,8 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
    return res
  }, {})

+  programmeToFaculties['H906'] = ['H906']
+
  const providerToProgramme = elementDetails.reduce((res, curr) => {
    const [p] = mapToProviders([curr.code])
    res[p] = {
@@ -331,11 +426,43 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
    return res
  }, {})

+  providerToProgramme['H906'] = {
+    code: 'H906',
+    name: { en: 'Language Centre', fi: 'Kielikeskus', sv: 'Språkcentrum' },
+  }
+
+  providerToProgramme['H930'] = {
+    code: 'H930',
+    name: { en: 'Open University', fi: 'Avoin yliopisto', sv: 'Öppna universitetet' },
+  }
+
  const organizationCodeToOrganization = faculties.reduce((acc, org) => {
    acc[org.code] = org
    return acc
  }, {})

+  const sumMerge = (a, b) => {
+    if (!a) return b
+    if (!b) return a
+    return a + b
+  }
+
+  const mergele = (a, b) => {
+    if (!a) return _.clone(b)
+    if (!b) return _.clone(a)
+    return _.mergeWith(a, b, sumMerge)
+  }
+
+  const refactoredOpenUniCredits = [..._.flatten(yearlyOpenUniAccCredits), ..._.flatten(yearlyOpenUniTotalCredits)].map(
+    ele => {
+      const org = orgsOfOpenUniReqularCounterparts[unifyOpenUniversity(ele.code)]
+      return {
+        ...ele,
+        organizationcode: org ? org : 'H930',
+      }
+    }
+  )
+
  /* Calculate course level stats and group by providers */
  const coursesGroupedByProvider = Object.entries(
    _.groupBy([..._.flatten(yearlyAccCredits), ..._.flatten(yearlyTotalCredits)], 'organizationcode')
@@ -355,8 +482,8 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
          acc[courseCode]['yearly'][year] = {}
          acc[courseCode]['yearly'][year]['acc'] = 0
          acc[courseCode]['yearly'][year]['accStudents'] = 0
-          acc[courseCode]['yearly'][year]['total'] = 0
-          acc[courseCode]['yearly'][year]['totalStudents'] = 0
+          acc[courseCode]['yearly'][year]['total'] = undefined
+          acc[courseCode]['yearly'][year]['totalStudents'] = undefined
        })

        yearlyInstances.forEach(instance => {
@@ -370,6 +497,52 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
            acc[courseCode]['yearly'][instance.year]['totalStudents'] = Number(instance.students)
          }
        })
+
+        acc[courseCode]['current'] = _.get(acc, [courseCode, 'yearly', startYear, 'acc']) || 0
+        acc[courseCode]['previous'] = _.get(acc, [courseCode, 'yearly', startYear - 1, 'acc']) || 0
+        acc[courseCode]['currentStudents'] = _.get(acc, [courseCode, 'yearly', startYear, 'accStudents']) || 0
+        acc[courseCode]['previousStudents'] = _.get(acc, [courseCode, 'yearly', startYear - 1, 'accStudents']) || 0
+        return acc
+      },
+      {}
+    )
+    return acc
+  }, {})
+
+  const openUniCoursesGroupedByProvider = Object.entries(
+    _.groupBy(refactoredOpenUniCredits, 'organizationcode')
+  ).reduce((acc, [organizationcode, courseCredits]) => {
+    acc[organizationcode] = Object.entries(_.groupBy(courseCredits, 'code')).reduce(
+      (acc, [courseCode, yearlyInstances]) => {
+        acc[courseCode] = {
+          yearly: {},
+          name: yearlyInstances[0].name,
+          type: 'course',
+          code: courseCode,
+        }
+
+        const array = _.range(2017, startYear + 1)
+
+        array.forEach(year => {
+          acc[courseCode]['yearly'][year] = {}
+          acc[courseCode]['yearly'][year]['acc'] = 0
+          acc[courseCode]['yearly'][year]['accStudents'] = 0
+          acc[courseCode]['yearly'][year]['total'] = undefined
+          acc[courseCode]['yearly'][year]['totalStudents'] = undefined
+        })
+
+        yearlyInstances.forEach(instance => {
+          if (!acc[courseCode]['yearly'][instance.year]) acc[courseCode]['yearly'][instance.year] = {}
+
+          if (instance.acc !== undefined) {
+            acc[courseCode]['yearly'][instance.year]['acc'] = instance.acc
+            acc[courseCode]['yearly'][instance.year]['accStudents'] = Number(instance.students)
+          } else {
+            acc[courseCode]['yearly'][instance.year]['total'] = instance.total
+            acc[courseCode]['yearly'][instance.year]['totalStudents'] = Number(instance.students)
+          }
+        })
+
        acc[courseCode]['current'] = _.get(acc, [courseCode, 'yearly', startYear, 'acc']) || 0
        acc[courseCode]['previous'] = _.get(acc, [courseCode, 'yearly', startYear - 1, 'acc']) || 0
        acc[courseCode]['currentStudents'] = _.get(acc, [courseCode, 'yearly', startYear, 'accStudents']) || 0
@@ -386,6 +559,25 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
    const programme = providerToProgramme[organizationcode]

    if (programme && programme.code) {
+      if (programme.code === 'H906') {
+        const courseValues = Object.values(courses)
+        const yearlyValues = courseValues.map(c => c.yearly)
+
+        acc[programme.code] = {
+          type: 'programme',
+          code: programme.code,
+          name: programme.name,
+          drill: courses,
+          yearly: _.mergeWith({}, ...yearlyValues, mergele),
+          current: _.sumBy(courseValues, 'current') | undefined,
+          previous: _.sumBy(courseValues, 'previous') | undefined,
+          currentStudents: _.sumBy(courseValues, 'currentStudents') || 'undefined',
+          previousStudents: _.sumBy(courseValues, 'previousStudents') | 'undefined',
+        }
+
+        return acc
+      }
+
      const orgId = organizationCodeToOrganization[organizationcode]?.id

      const { students: currentStudents, credits: current } = getOrgStats(currentOrgStats, orgId).accumulated.aggregated
@@ -401,20 +593,64 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
        yearly,
        current,
        previous,
-        currentStudents,
-        previousStudents,
+        currentStudents: currentStudents,
+        previousStudents: previousStudents,
      }
    }

    return acc
  }, {})

+  const openUniGroupedByProgramme = Object.entries(openUniCoursesGroupedByProvider).reduce(
+    (acc, [organizationcode, courses]) => {
+      const programme = providerToProgramme[organizationcode]
+
+      if (programme && programme.code) {
+        const courseValues = Object.values(courses)
+        const yearlyValues = courseValues.map(c => c.yearly)
+
+        acc[programme.code] = {
+          type: 'programme',
+          code: programme.code,
+          name: programme.name,
+          drill: courses,
+          yearly: _.mergeWith({}, ...yearlyValues, mergele),
+          current: _.sumBy(courseValues, 'current') | undefined,
+          previous: _.sumBy(courseValues, 'previous') | undefined,
+          currentStudents: _.sumBy(courseValues, 'currentStudents') || 'undefined',
+          previousStudents: _.sumBy(courseValues, 'previousStudents') | 'undefined',
+        }
+
+        return acc
+      }
+
+      return acc
+    },
+    {}
+  )
+
  /* Group programmes into faculties and calculate faculty level stats */
  const groupedByFaculty = Object.entries(groupedByProgramme).reduce((acc, [programmeCode, programmeStats]) => {
    const facultyCodes = programmeToFaculties[programmeCode]
    if (!facultyCodes) return acc
    facultyCodes.forEach(facultyCode => {
      if (!facultyCode) return
+      if (facultyCode === 'H906' || facultyCode === 'H930') {
+        if (!acc[facultyCode]) {
+          acc[facultyCode] = {
+            type: 'faculty',
+            code: facultyCode,
+            drill: {},
+            name: facultyCodeToFaculty[facultyCode] ? facultyCodeToFaculty[facultyCode].name : null,
+            yearly: programmeStats.yearly,
+            current: programmeStats.current,
+            previous: programmeStats.previous,
+            currentStudents: programmeStats.currentStudents,
+            previousStudents: programmeStats.previousStudents,
+          }
+        }
+      }
+
      if (!acc[facultyCode]) {
        const orgId = organizationCodeToOrganization[facultyCode].id

@@ -443,7 +679,31 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
    return acc
  }, {})

-  return groupedByFaculty
+  const openUniGroupedByFaculty = Object.entries(openUniGroupedByProgramme).reduce(
+    (acc, [programmeCode, programmeStats]) => {
+      const facultyCode = 'H930'
+
+      if (!acc[facultyCode]) {
+        acc[facultyCode] = {
+          type: 'faculty',
+          code: facultyCode,
+          drill: {},
+          name: facultyCodeToFaculty[facultyCode] ? facultyCodeToFaculty[facultyCode].name : null,
+          yearly: {},
+          current: 0,
+          previous: 0,
+        }
+      }
+
+      acc[facultyCode]['drill'][programmeCode] = programmeStats
+      acc[facultyCode]['yearly'] = _.mergeWith(acc[facultyCode]['yearly'], programmeStats.yearly, mergele)
+      acc[facultyCode]['current'] += programmeStats.current
+      acc[facultyCode]['previous'] += programmeStats.previous
+      return acc
+    },
+    {}
+  )
+  return { ...groupedByFaculty, ...openUniGroupedByFaculty }
 }

 const getStatus = async (unixMillis, showByYear, doRefresh = false) => {
@@ -1,6 +1,3 @@
-const axios = require('axios')
-const _ = require('lodash')
-const { USERSERVICE_URL } = require('../conf-backend')
 const UnitService = require('./units')
 const elementDetailService = require('./elementdetails')
 const { filterStudentnumbersByAccessrights } = require('./students')
@@ -8,8 +5,10 @@ const { userDataCache } = require('./cache')
 const { getImporterClient } = require('../util/importerClient')
 const logger = require('../util/logger')
 const { facultiesAndProgrammesForTrends } = require('../services/organisations')
-
-const client = axios.create({ baseURL: USERSERVICE_URL, headers: { secret: process.env.USERSERVICE_SECRET } })
+const { sendNotificationAboutNewUser } = require('../services/mailservice')
+const User = require('./users/users')
+const AccessGroup = require('./users/accessgroups')
+const _ = require('lodash')

 // Private helpers
 const enrichProgrammesFromFaculties = faculties =>
@@ -44,15 +43,27 @@ const checkStudyGuidanceGroupsAccess = async hyPersonSisuId => {
  }
  const importerClient = getImporterClient()
  if (!importerClient) return false
-  const { data } = await importerClient.get(`/person-groups/person/${hyPersonSisuId}`)
+  const answerTimeout = new Promise(resolve => setTimeout(resolve, 2000))
+  const result = await Promise.race([importerClient.get(`/person-groups/person/${hyPersonSisuId}`), answerTimeout])
+  if (!result) return false
+  const { data } = result
  return data && Object.values(data).length > 0
 }

 const byUsernameData = async username => {
  // OK, enriched
-  const url = `/user/${username}/user_data`
-  const response = await client.get(url)
-  return enrichWithProgrammes(response.data)
+  const user = await User.byUsernameMinified(username)
+  const roles = user.accessgroup.map(({ group_code }) => group_code)
+  const rights = User.getUserProgrammes(user)
+  const faculties = user.faculty.map(({ faculty_code }) => faculty_code)
+  const data = {
+    email: user.email,
+    full_name: user.full_name,
+    roles,
+    rights,
+    faculties,
+  }
+  return enrichWithProgrammes(data)
 }

 // Exported helpers
@@ -95,53 +106,72 @@ const getStudentsUserCanAccess = async (studentnumbers, roles, userId) => {
 }

 // FOR ROUTES 1-to-1 mapping
-const modifyAccess = async body => enrichWithProgrammes((await client.post('/modifyaccess', body)).data)
+const modifyAccess = async body => {
+  const { username, accessgroups } = body
+  await User.modifyRights(username, accessgroups)
+  const user = await User.byUsername(username)
+  return enrichWithProgrammes(User.getUserData(user))
+}

-const getAccessGroups = async () => enrichWithProgrammes((await client.get('/access_groups')).data)
+const getAccessGroups = async () => enrichWithProgrammes(await AccessGroup.findAll())

-const enableElementDetails = async (uid, codes) =>
-  enrichWithProgrammes((await client.post('/add_rights', { uid, codes })).data.user)
+const enableElementDetails = async (uid, codes) => {
+  await User.addProgrammes(uid, codes)
+  const user = await User.byId(uid)
+  return enrichWithProgrammes(User.getUserData(user))
+}

-const findAll = async () => enrichWithProgrammes((await client.get('/findall')).data)
+const findAll = async () => enrichWithProgrammes((await User.findAll()).map(User.getUserData))

-const setFaculties = async (uid, faculties) =>
-  enrichWithProgrammes((await client.post('/set_faculties', { uid, faculties })).data.user)
+const setFaculties = async (uid, faculties) => {
+  await User.setFaculties(uid, faculties)
+  const user = await User.byId(uid)
+  return enrichWithProgrammes(User.getUserData(user))
+}

-const removeElementDetails = async (uid, codes) =>
-  enrichWithProgrammes((await client.post('/remove_rights', { uid, codes })).data.user)
+const removeElementDetails = async (uid, codes) => {
+  await User.removeProgrammes(uid, codes)
+  const user = await User.byId(uid)
+  return enrichWithProgrammes(User.getUserData(user))
+}

 const updateUser = async (username, fields) => {
  userDataCache.del(username)
-  return enrichWithProgrammes((await client.put(`/user/${username}`, fields)).data)
+  const uid = username
+  const user = await User.byUsername(uid)
+  if (!user) {
+    throw new Error(`User ${uid} not found`)
+  }
+  await User.updateUser(user, fields)
+  const returnedUser = await User.byUsername(username)
+  return enrichWithProgrammes(User.getUserData(returnedUser))
 }

-const login = async (uid, full_name, hyGroups, affiliations, email, hyPersonSisuId) => {
+const getLoginDataWithoutToken = async (uid, full_name, hyGroups, email, hyPersonSisuId) => {
  const hasStudyGuidanceGroupAccess = await checkStudyGuidanceGroupsAccess(hyPersonSisuId)
-  return (
-    await client.post('/login', {
-      uid,
-      full_name,
-      hyGroups,
-      affiliations,
-      email,
-      hyPersonSisuId,
-      hasStudyGuidanceGroupAccess,
-    })
-  ).data
+  const result = await User.loginWithoutToken(
+    uid,
+    full_name,
+    hyGroups,
+    email,
+    hyPersonSisuId,
+    hasStudyGuidanceGroupAccess
+  )
+  return result
 }

-const superlogin = async (uid, asUser) =>
-  (
-    await client.post('/superlogin', {
-      uid,
-      asUser,
-    })
-  ).data
+const getMockedUser = async (uid, asUser) => await User.superlogin(uid, asUser)
+
+const getUser = async ({ username, name, email, iamGroups, sisId }) => {
+  const { payload: user, isNew } = await getLoginDataWithoutToken(username, name, iamGroups, email, sisId)
+  if (isNew) {
+    await sendNotificationAboutNewUser({ userId: username, userFullName: name })
+  }
+  return user
+}

 module.exports = {
  updateUser,
-  login,
-  superlogin,
  enableElementDetails,
  removeElementDetails,
  findAll,
@@ -151,4 +181,7 @@ module.exports = {
  setFaculties,
  getUserDataFor,
  getStudentsUserCanAccess,
+  getLoginDataWithoutToken,
+  getUser,
+  getMockedUser,
 }
@@ -1,4 +1,4 @@
-const { AccessGroup } = require('../models/index')
+const { AccessGroup } = require('../../models/models_user')
 const { Op } = require('sequelize')

 const byId = id => AccessGroup.findByPk(id)
@@ -1,4 +1,4 @@
-const { HyGroup } = require('../models/index')
+const { HyGroup } = require('../../models/models_user')
 const { Op } = require('sequelize')

 const byId = id => HyGroup.findByPk(id)
@@ -1,16 +1,28 @@
 const Sequelize = require('sequelize')
-const jwt = require('jsonwebtoken')
 const _ = require('lodash')
 const moment = require('moment')
-const { User, UserElementDetails, AccessGroup, HyGroup, Affiliation, UserFaculties, sequelize } = require('../models')
+const {
+  User,
+  UserElementDetails,
+  AccessGroup,
+  HyGroup,
+  UserFaculties,
+  sequelizeUser,
+} = require('../../models/models_user')
 const AccessService = require('./accessgroups')
-const AffiliationService = require('./affiliations')
 const HyGroupService = require('./hygroups')
-const { requiredGroup, courseStatisticsGroup, TOKEN_SECRET, hyOneGroup } = require('../conf')
 const Op = Sequelize.Op

-const TOKEN_VERSION = 1.1 // When token structure changes, increment in userservice, backend and frontend
-const generateToken = async (uid, mockedBy = null) => {
+// move these to conf
+const courseStatisticsGroup = 'grp-oodikone-basic-users'
+const hyOneGroup = 'hy-one'
+const requiredGroup =
+  process.env.NODE_ENV === 'staging'
+    ? ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
+    : ['grp-oodikone-users', 'grp-oodikone-basic-users']
+
+// help for refactoring away from token
+const generateTokenData = async (uid, mockedBy = null) => {
  const user = await byUsername(uid)
  const userData = getUserData(user)
  const programmes = userData.elementdetails
@@ -24,14 +36,11 @@ const generateToken = async (uid, mockedBy = null) => {
    rights: programmes,
    roles: user.accessgroup,
    createdAt: moment().toISOString(),
-    version: TOKEN_VERSION,
    sisPersonId: user.sisu_person_id,
  }
-  const token = jwt.sign(payload, TOKEN_SECRET)
-
-  // return the information including token as JSON
-  return token
+  return payload
 }
+
 const createMissingGroups = async (group, service) => {
  const savedGroups = await service.findAll()
  group.forEach(async code => {
@@ -41,24 +50,7 @@ const createMissingGroups = async (group, service) => {
  })
 }

-const updateGroups = async (user, affiliations, hyGroups) => {
-  let affiliationsToBeUpdated = (await user.getAffiliation()).map(af => af.code)
-  let affiliationsToAdd = []
-  let affiliationsToDelete = []
-
-  affiliations.forEach(async affilitation => {
-    if (!affiliationsToBeUpdated.includes(affilitation)) {
-      affiliationsToAdd = affiliationsToAdd.concat(affilitation)
-    }
-  })
-  affiliationsToBeUpdated.forEach(async affilitation => {
-    if (!affiliations.includes(affilitation)) {
-      affiliationsToDelete = affiliationsToDelete.concat(affilitation)
-    }
-  })
-  await user.addAffiliation(await AffiliationService.byCodes(affiliationsToAdd))
-  await user.removeAffiliation(await AffiliationService.byCodes(affiliationsToDelete))
-
+const updateGroups = async (user, hyGroups) => {
  let hyGroupsToBeUpdated = (await user.getHy_group()).map(hg => hg.code)
  let hyGroupsToAdd = []
  let hyGroupsToDelete = []
@@ -78,12 +70,11 @@ const updateGroups = async (user, affiliations, hyGroups) => {
  await user.removeHy_group(await HyGroupService.byCodes(hyGroupsToDelete))
 }

-const login = async (uid, full_name, hyGroups, affiliations, mail, hyPersonSisuId, hasStudyGuidanceGroupAccess) => {
+const loginBase = async (uid, full_name, hyGroups, mail, hyPersonSisuId, hasStudyGuidanceGroupAccess) => {
  let user = await byUsername(uid)
  let isNew = false
  const lastLogin = new Date()
  await createMissingGroups(hyGroups, HyGroupService)
-  await createMissingGroups(affiliations, AffiliationService)

  if (!user) {
    user = await createUser(uid, full_name, mail, hyPersonSisuId, lastLogin)
@@ -91,28 +82,45 @@ const login = async (uid, full_name, hyGroups, affiliations, mail, hyPersonSisuI
    const userHyGroups = await HyGroupService.byCodes(hyGroups)
    await user.addHy_group(userHyGroups)

-    const userAffiliations = await AffiliationService.byCodes(affiliations)
-    await user.addAffiliation(userAffiliations)
-
    isNew = true
  } else {
    user = await updateUser(user, { full_name, email: mail, sisu_person_id: hyPersonSisuId, last_login: lastLogin })
-    await updateGroups(user, affiliations, hyGroups)
+    await updateGroups(user, hyGroups)
  }

  await determineAccessToCourseStats(user, hyGroups)
  await determineAccessToTeachersForOne(user, hyGroups)
  await determineAccessToStudyGuidanceGroups(user, hasStudyGuidanceGroupAccess)

-  console.log('Generating token')
-  const token = await generateToken(uid)
-  console.log('Token done')
-  return { token, isNew }
+  return isNew
 }
+
+const loginWithoutToken = async (
+  uid,
+  full_name,
+  hyGroups,
+  affiliations,
+  mail,
+  hyPersonSisuId,
+  hasStudyGuidanceGroupAccess
+) => {
+  const isNew = await loginBase(
+    uid,
+    full_name,
+    hyGroups,
+    affiliations,
+    mail,
+    hyPersonSisuId,
+    hasStudyGuidanceGroupAccess
+  )
+  const payload = await generateTokenData(uid)
+  return { payload, isNew }
+}
+
 const superlogin = async (uid, asUser) => {
  const user = await byUsername(uid)
  if (user && user.accessgroup.map(r => r.group_code).includes('admin')) {
-    const token = await generateToken(asUser, uid)
+    const token = await generateTokenData(asUser, uid)
    return token
  }
  return undefined
@@ -140,10 +148,6 @@ const userIncludes = [
    model: UserFaculties,
    as: 'faculty',
  },
-  {
-    model: Affiliation,
-    as: 'affiliation',
-  },
 ]

 const getUserData = user => {
@@ -252,7 +256,7 @@ const removeProgrammes = async (uid, codes) => {
 }

 const setFaculties = async (uid, faculties) => {
-  await sequelize.transaction(async transaction => {
+  await sequelizeUser.transaction(async transaction => {
    await UserFaculties.destroy({ where: { userId: uid }, transaction })
    for (const faculty of faculties) {
      await UserFaculties.create({ userId: uid, faculty_code: faculty }, { transaction })
@@ -325,11 +329,11 @@ module.exports = {
  getUserProgrammes,
  addProgrammes,
  removeProgrammes,
-  login,
  superlogin,
  modifyRights,
  getUserAccessGroups,
  getRoles,
  getUserData,
  setFaculties,
+  loginWithoutToken,
 }
@@ -15,7 +15,8 @@ class ApplicationError extends Error {

  toJSON() {
    return {
-      error: this.message,
+      message: this.message,
+      ...this.extra,
    }
  }
 }
@@ -1,17 +1,17 @@
 const axios = require('axios')
-const { importerToken, importerUrl } = require('../conf-backend')
+const { importerToken } = require('../conf-backend')
 const logger = require('./logger')

 const importerClient = axios.create({
  headers: {
    token: importerToken,
  },
-  baseURL: importerUrl,
+  baseURL: 'https://importer.cs.helsinki.fi/api/importer',
 })

 const getImporterClient = () => {
-  if (!importerToken || !importerUrl) {
-    logger.error("Importer token or importer url not set, can't return client!")
+  if (!importerToken) {
+    logger.error("Importer token not set, can't return client!")
    return null
  }
  return importerClient
@@ -1,15 +1,10 @@
 const winston = require('winston')
-const path = require('path')
-const { isProduction, PWD } = require('../conf-backend')
-
+const { isDev } = require('../conf-backend')
 const { combine, timestamp, printf, splat } = winston.format

-const transports = []
-// Write logfile to docker containers pwd
-const filename = path.join(PWD, 'debug.log')
-transports.push(new winston.transports.File({ filename, level: 'debug' }))
+let transports = []

-if (!isProduction) {
+if (isDev) {
  const devFormat = printf(
    ({ level, message, timestamp, ...rest }) => `${timestamp} ${level}: ${message} ${JSON.stringify(rest)}`
  )
@@ -20,9 +15,7 @@ if (!isProduction) {
      format: combine(splat(), timestamp(), devFormat),
    })
  )
-}
-
-if (isProduction) {
+} else {
  const levels = {
    error: 0,
    warn: 1,
@@ -40,9 +33,11 @@ if (isProduction) {
    })
  )

-  transports.push(new winston.transports.Console({ format: prodFormat, level: 'info' }))
+  transports.push(new winston.transports.Console({ format: prodFormat }))
 }

 const logger = winston.createLogger({ transports })

+logger.on('error', e => console.error('Logging failed! Reason: ', e)) // eslint-disable-line no-console
+
 module.exports = logger
@@ -29,8 +29,16 @@ const getPassingSemester = (startYear, date) => {
  return yearCount < 6 ? `${yearCount}-${semester}` : 'LATER'
 }

+const getAcademicYearDates = year => {
+  return {
+    startDate: `${year}-${semesterStart['FALL']}`,
+    endDate: `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`,
+  }
+}
+
 module.exports = {
  getPassingSemester,
+  getAcademicYearDates,
  semesterEnd,
  semesterStart,
 }
@@ -1,6 +1,3 @@
-const { requiredGroup } = require('../conf-backend')
-const _ = require('lodash')
-
 const mapToProviders = elementDetails =>
  elementDetails.map(r => {
    const isNumber = str => !Number.isNaN(Number(str))
@@ -14,21 +11,6 @@ const mapToProviders = elementDetails =>
    return r
  })

-const hasRequiredGroup = hyGroups => {
-  const hasGroup = requiredGroup === null || _.intersection(hyGroups, requiredGroup).length > 0
-  return hasGroup
-}
-
-const parseHyGroups = hyGroups => {
-  let parsedHyGroups = []
-  if (!(hyGroups === undefined || hyGroups === '')) {
-    parsedHyGroups = hyGroups.split(';')
-  }
-  return parsedHyGroups
-}
-
 module.exports = {
  mapToProviders,
-  hasRequiredGroup,
-  parseHyGroups,
 }
@@ -1,7 +1,8 @@
 {
  "root": true,
  "env": {
-    "browser": true
+    "browser": true,
+    "node": true
  },
  "parser": "@babel/eslint-parser",
  "parserOptions": {
@@ -30,7 +31,7 @@
    "react/no-unescaped-entities": "off",
    "react/prop-types": "off",
    "react/prefer-stateless-function": "error",
-    "import/no-unresolved": "error",
+    "import/no-unresolved": "off",
    "react/jsx-no-bind": "error",
    "prefer-object-spread/prefer-object-spread": "error",
    "babel/no-invalid-this": "error",
@@ -0,0 +1,6 @@
+{
+  "compilerOptions": {
+    "baseUrl": "src"
+  },
+  "include": ["src"]
+}
@@ -6,14 +6,16 @@
    "build": "DISABLE_ESLINT_PLUGIN=true react-scripts build"
  },
  "dependencies": {
+    "@reduxjs/toolkit": "^1.6.2",
    "@sentry/browser": "^6.9.0",
    "@sentry/tracing": "^6.11.0",
-    "axios": "^0.21.1",
+    "axios": "^0.21.2",
+    "formik": "^2.2.9",
    "highcharts": "^9.1.2",
    "highcharts-custom-events": "^3.0.10",
+    "highcharts-exporting": "^0.1.7",
    "history": "^4.10.1",
-    "jstat": "^1.9.5",
-    "jwt-decode": "^3.1.2",
+    "immer": "^9.0.7",
    "lodash": "^4.17.21",
    "moment": "^2.29.1",
    "path-to-regexp": "^0.1.7",
@@ -22,18 +24,14 @@
    "qs": "^6.10.1",
    "query-string": "^6.14.1",
    "react": "^16.14.0",
-    "react-context-devtool": "^2.0.3",
    "react-datetime": "^3.0.4",
    "react-dom": "^16.14.0",
    "react-highcharts": "^16.1.0",
    "react-markdown": "^6.0.2",
-    "react-redux": "^7.2.4",
+    "react-redux": "^7.2.6",
    "react-router-dom": "^5.2.0",
    "react-scripts": "4.0.3",
    "react-scroll-to-component": "^1.0.2",
-    "redux": "^4.1.0",
-    "redux-thunk": "^2.3.0",
-    "reselect": "^4.0.0",
    "semantic-ui-css": "^2.4.1",
    "semantic-ui-react": "^2.0.3",
    "unfuck-spa-shibboleth-session": "^3.1.3",
@@ -1,16 +1,6 @@
 import React from 'react'
-import PropTypes from 'prop-types'
-import FilterContextProvider from './components/FilterTray/FilterContextProvider'
 import { LanguageProvider } from './components/LanguagePicker/useLanguage'

-const CommonContext = ({ children }) => (
-  <LanguageProvider>
-    <FilterContextProvider>{children}</FilterContextProvider>
-  </LanguageProvider>
-)
-
-CommonContext.propTypes = {
-  children: PropTypes.node.isRequired,
-}
+const CommonContext = ({ children }) => <LanguageProvider>{children}</LanguageProvider>

 export default CommonContext
@@ -1,43 +1,43 @@
 import axios from 'axios'
 import * as Sentry from '@sentry/browser'
-import { getMocked, setMocking } from '../common'
+import { createApi, fetchBaseQuery } from '@reduxjs/toolkit/query/react'
+import { showAsUserKey } from 'common'
 import { apiBasePath, isDev } from '../conf'

-const getHeaders = () => {
+export const getHeaders = () => {
  // Set up dev user for development environment, mimicking production admin user
  const devUserHeaders = {
    uid: 'mluukkai',
    displayName: 'Matti Luukkainen',
    'shib-session-id': 'mock-session',
    hyGroupCn: 'grp-oodikone-users;grp-oodikone-basic-users',
-    eduPersonAffiliation: 'member;employee;faculty',
    mail: 'grp-toska+mockmluukkai@helsinki.fi',
    hyPersonSisuId: 'hy-hlo-1441871',
+    shib_logout_url: 'https://helsinki.fi/shibboleth-sp/Logout',
  }
-  return isDev ? { ...devUserHeaders } : {}
+  const headers = isDev ? { ...devUserHeaders } : {}
+
+  // Set up possible show as user -headers
+  const showAsUser = window.localStorage.getItem(showAsUserKey)
+  if (showAsUser) headers['x-show-as-user'] = showAsUser
+
+  return headers
 }

 export const api = axios.create({ baseURL: apiBasePath, headers: getHeaders() })

-const types = {
-  attempt: prefix => `${prefix}ATTEMPT`,
-  failure: prefix => `${prefix}FAILURE`,
-  success: prefix => `${prefix}SUCCESS`,
+export const actionSuffixes = {
+  attempt: `ATTEMPT`,
+  failure: `FAILURE`,
+  success: `SUCCESS`,
 }

 export const actionTypes = prefix => ({
-  attempt: types.attempt(prefix),
-  failure: types.failure(prefix),
-  success: types.success(prefix),
+  attempt: `${prefix}${actionSuffixes.attempt}`,
+  failure: `${prefix}${actionSuffixes.failure}`,
+  success: `${prefix}${actionSuffixes.success}`,
 })

-export const logout = async () => {
-  setMocking(null)
-  const returnUrl = window.location.origin
-  const response = await api.delete('/logout', { data: { returnUrl } })
-  window.location = response.data.logoutUrl
-}
-
 export const callApi = async (url, method = 'get', data, params, timeout = 0, progressCallback = null) => {
  const options = { headers: getHeaders(), timeout }

@@ -69,7 +69,8 @@ export const callController = (route, prefix, data, method = 'get', query, param
    params,
    onProgress,
  }
-  return { type: `${prefix}ATTEMPT`, requestSettings }
+  const { attempt } = actionTypes(prefix)
+  return { type: attempt, requestSettings }
 }

 const handleError = (err, actionHistory = []) => {
@@ -90,46 +91,30 @@ export const handleRequest = store => next => async action => {
  const { requestSettings } = action
  if (requestSettings) {
    const { route, method, data, prefix, query, params, onProgress } = requestSettings
+    const { success, failure } = actionTypes(prefix)
    try {
      const res = await callApi(route, method, data, params, 0, onProgress)
-      store.dispatch({ type: `${prefix}SUCCESS`, response: res.data, query })
+      store.dispatch({ type: success, response: res.data, query })
    } catch (e) {
-      store.dispatch({ type: `${prefix}FAILURE`, response: e, query })
+      store.dispatch({ type: failure, response: e, query })
      handleError(e, store.getState().actionHistory)
    }
  }
 }

-export const handleAuth = store => next => async action => {
-  next(action)
-  const { type } = action
-  if (type === 'LOGIN_ATTEMPT') {
-    try {
-      let token
-      const mocked = getMocked()
-      if (mocked) {
-        token = (await callApi(`/superlogin/${mocked}`, 'post')).data
-      } else {
-        // eslint-disable-next-line prefer-destructuring
-        token = (await callApi('/login', 'post')).data.token
-      }
-
-      api.defaults.headers.common = {
-        ...api.defaults.headers.common,
-        'x-access-token': token,
-      }
-      store.dispatch({ type: 'LOGIN_SUCCESS', token })
-    } catch (err) {
-      store.dispatch({ type: 'LOGIN_FAILURE' })
-      handleError(err, store.getState().actionHistory)
-    }
-  } else if (type === 'LOGOUT_ATTEMPT') {
-    try {
-      await logout()
-      store.dispatch({ type: 'LOGOUT_SUCCESSFUL' })
-    } catch (err) {
-      store.dispatch({ type: 'LOGOUT_FAILURE' })
-      handleError(err, store.getState().actionHistory)
-    }
-  }
-}
+// Redux-toolkit query based API
+// Api can be extended with enhanceEndpoints method
+// All tags used for invalidating cache must be defined here
+export const RTKApi = createApi({
+  reducerPath: 'api',
+  tagTypes: ['StudyGuidanceGroups'],
+  baseQuery: fetchBaseQuery({
+    baseUrl: apiBasePath,
+    prepareHeaders: headers => {
+      // Add possible default headers
+      Object.entries(getHeaders()).forEach(([key, value]) => headers.set(key, value))
+      return headers
+    },
+  }),
+  endpoints: () => ({}),
+})
@@ -50,41 +50,108 @@ export default {
  **Include attainments attained before the studyright start** laskee mukaan opiskelijan ennen opinto-oikeuden alkua saadut opintopisteet.
  Vakiona opintopisteet ennen opinto-oikeutta suodatetaan pois.
  `,
-  status: `
-  Tilannekuva näyttää "reaaliaikaisesti" eri tiedekuntien, ohjelmien ja kurssien tuottamat opintopisteet nykyisen lukuvuoden aikana 
-  verrattuna edelliseen lukuvuoteen. 
-  
-  Nuoli kuvaa muutosta, paljonko opintopisteitä on kertynyt tämänhetkisen lukuvuoden alusta tarkastelupäivään mennessä 
-  suhteessa edellisen lukuvuoden alusta vastaavaan tarkastelupäivään viime vuonna. Vain suoritetut ja hyväksiluetut opintopisteet huomioidaan.
-  
-  Tilannekuvassa voi porautua kortteja klikkaamalla ensin ohjelmatasolle ja edelleen kurssitasolle. Asetukset-nappi avaa valikon, josta voi valita missä muodossa data halutaan nähdä. 
-  
-  Valinta "Näytä edelliset vuodet" näyttää vuosittaisen opintopiste-kertymän tähän päivään
-  mennessä vuonna X sekä koko lukuvuoden X opintopisteet muodossa "kerääntymä vuonna X / koko lukuvuoden X opintopisteet".
-  
-  Valinta "Näytä kalenterivuosittain" laskee kertymän kalenterivuoden alusta,
-  eikä lukuvuoden alusta.
+  status: {
+    general: `
+      Tilannekuva näyttää "reaaliaikaisesti" eri tiedekuntien, ohjelmien ja kurssien tuottamat opintopisteet nykyisen lukuvuoden aikana 
+      verrattuna edelliseen lukuvuoteen. 
+      
+      Nuoli kuvaa muutosta, paljonko opintopisteitä on kertynyt tämänhetkisen lukuvuoden alusta tarkastelupäivään mennessä 
+      suhteessa edellisen lukuvuoden alusta vastaavaan tarkastelupäivään viime vuonna. Vain suoritetut ja hyväksiluetut opintopisteet huomioidaan.
+      
+      Tilannekuvassa voi porautua kortteja klikkaamalla ensin ohjelmatasolle ja edelleen kurssitasolle. Asetukset-nappi avaa valikon, josta voi valita missä muodossa data halutaan nähdä. Mahdolliset valinnat ovat selitettyinä alla.
+    `,
+    settings: {
+      showByYear: {
+        label: 'Näytä kalenterivuosittain',
+        short: 'Näytä tilastot kalenterivuosittain lukuvuosien sijasta.',
+        long: `
+          Kun tämä valinta on käytössä, vuosittaiset ajanjaksot lasketaan kalenterivuoden alusta sen loppuun.
+          Muulloin vuosittaiset ajanjaksot lasketaan lukukauden alusta seuraavan lukukauden alkuun.
+        `,
+      },
+      showYearlyValues: {
+        label: 'Näytä edelliset vuodet',
+        short: 'Näytä tilastot vuosittain, alkaen vuodesta 2017.',
+        long: `
+          Näyttää tilastot vuodesta 2017 eteenpäin.
+          Jokaiselta vuodelta näytetään kaksi tilastoa: ajanjakson kokonaistilasto ja ns. tähän mennessä "*kertynyt*" tilasto.
+          Kokonaistilasto vastaa nimensä mukaisesti koko ajanjaksoa, kun taas *kertynyt* tilasto kattaa ajanjakson vuoden 
+          tai lukuvuoden alusta "Näytä päivänä"-valintaa vastaavaan päivämäärään tuona vuotena. Luvut näytetään muodossa *<kerynyt>*/*<kokonais>*.
+          Kuluvalta vuodelta näytetään ainoastaan kertynyt tilasto.
+        `,
+      },
+      showRelativeValues: {
+        label: 'Näytä suhteutettuna opiskelijoiden määrään',
+        short:
+          'Näyttää tilastot suhteutettuna opiskelijoiden määrään kyseisellä aikavälillä ja kyseisessä organisaatiossa.',
+        long: `
+          Näyttää tilastot suhteutettuna opiskelijoiden määrään kyseisellä aikavälillä ja kyseisessä organisaatiossa.
+          Opiskelijoiden määrä perustuu ajanjaksolla kyseisen organisaation alaisista kursseista suoritusmerkintöjä saaneiden opiskelijoiden määrään.
+          Luku siis sisältää muutkin kuin kyseiseen ohjelman tai osaston opinto-oikeuden omaavat opiskelijat.
+        `,
+      },
+      showCountingFrom: {
+        label: 'Valitse päivämäärä',
+        short: 'Valitse päivä johon asti kertyneet tilastot näytetään.',
+        long: `
+          Tämä valinta määrittää päivämäärän, jota käyttäen kertyneet tilastot lasketaan.
+          Esimerkiksi "Näytä kalenterivuosittain" valinnan ollessa pois päältä,
+          lasketaan kertyneet tilastot (vrt. lukuvuosien kokonaistilastot) kunkin lukuvuoden alusta
+          tätä päivämäärää vastaavaan päivään kyseisenä lukuvuonna.
+        `,
+      },
+      showStudentCounts: {
+        label: 'Näytä kurssien opiskelijamäärät',
+        short: 'Näyttää suoritettujen opintopisteiden sijasta opiskelijoiden määrät kurssitason näkymässä.',
+        long: `
+          Oletuksena kurssitason näkymässä näytetään organisaatio- ja ohjelmatason näkymien tapaan suoritettujen opintopisteiden
+          kokonaismäärä. Kun tämä valinta on käytössä, tämän sijasta näytettävät luvut vastaavat kurssin suorittaneiden 
+          *yksilöityjen* opiskelijoiden määrää. Kukin opiskelija lasketaan siis vain kerran tähän tilastoon.
+        `,
+      },
+    },
+  },
+  statusGraduated: {
+    general: `
+      Tilannekuva näyttää "reaaliaikaisesti" eri tiedekuntien ja ohjelmien valmistumiset nykyisen lukuvuoden aikana 
+      verrattuna edelliseen lukuvuoteen. 
+      
+      Nuoli kuvaa muutosta, paljonko valmistuneita on kertynyt tämänhetkisen lukuvuoden alusta tarkastelupäivään mennessä 
+      suhteessa edellisen lukuvuoden alusta vastaavaan tarkastelupäivään viime vuonna.
+      
+      Tilannekuvassa voi porautua kortteja klikkaamalla ohjelmatasolle. 
+      Asetukset-nappi avaa valikon, josta voi valita missä muodossa data halutaan nähdä. 
+    `,
+    settings: {
+      showByYear: {
+        label: 'Näytä kalenterivuosittain',
+        short: 'Näytä tilastot kalenterivuosittain lukuvuosien sijasta.',
+        long: `
+          Kun tämä valinta on käytössä, vuosittaiset ajanjaksot lasketaan kalenterivuoden alusta sen loppuun.
+          Muulloin vuosittaiset ajanjaksot lasketaan lukukauden alusta seuraavan lukukauden alkuun.
+        `,
+      },

-  Valinta "Näytä suhteutettuna opiskelijoiden määrään" näyttää opintopistemäärät suhteutettuna kyseisen ajanjakson aikana kyseisen organisaation alaisuuteen kuuluvia kursseja suorittaineiden opiskelijoiden määrään. Tämä luku sisältää myös avoimen yliopiston opiskelijat.
-  
-  Kohdasta "Näytä päivänä" voi valita tarkastelupäivän, joka on oletusarvoisesti tämänhetkinen päivä.
-  
-  Avoimen yliopiston suorituksia ei vielä lasketa mukaan.
-  `,
-  statusGraduated: `
-  Tilannekuva näyttää "reaaliaikaisesti" eri tiedekuntien ja ohjelmien valmistumiset nykyisen lukuvuoden aikana 
-  verrattuna edelliseen lukuvuoteen. 
-  
-  Nuoli kuvaa muutosta, paljonko valmistuneita on kertynyt tämänhetkisen lukuvuoden alusta tarkastelupäivään mennessä 
-  suhteessa edellisen lukuvuoden alusta vastaavaan tarkastelupäivään viime vuonna.
-  
-  Tilannekuvassa voi porautua kortteja klikkaamalla ohjelmatasolle. 
-  Asetukset-nappi avaa valikon, josta voi valita missä muodossa data halutaan nähdä. 
-  
-  Valinta "Näytä edelliset vuodet" näyttää vuosittaisen valmistumiskertymän tähän päivään mennessä vuonna X sekä koko lukuvuoden X valmistuneet muodossa "kerääntymä vuonna X / koko lukuvuoden X valmistuneet". 
-  
-  Valinta "Näytä vuositasolla" laskee kertymän kalenterivuoden alusta, eikä lukuvuoden alusta.
-  
-  Kohdasta "Näytä päivänä" voi valita tarkastelupäivän, joka on oletusarvoisesti tämänhetkinen päivä.
-  `,
+      showYearlyValues: {
+        label: 'Näytä edelliset vuodet',
+        short: 'Näytä tilastot vuosittain, alkaen vuodesta 2017.',
+        long: `
+          Näyttää vuosittaisen valmistumiskertymän tähän päivään mennessä vuonna X
+          sekä koko lukuvuoden X valmistuneet muodossa *"kerääntymä vuonna X / koko
+          lukuvuoden X valmistuneet"*.
+        `,
+      },
+
+      showCountingFrom: {
+        label: 'Näytä päivänä',
+        short: 'Valitse päivä johon asti kertyneet tilastot näytetään.',
+        long: `
+          Tämä valinta määrittää päivämäärän, jota käyttäen kertyneet tilastot lasketaan.
+          Esimerkiksi "Näytä kalenterivuosittain" valinnan ollessa pois päältä,
+          lasketaan kertyneet tilastot (vrt. lukuvuosien kokonaistilastot) kunkin lukuvuoden alusta
+          tätä päivämäärää vastaavaan päivään kyseisenä lukuvuonna.
+        `,
+      },
+    },
+  },
 }
@@ -0,0 +1,15 @@
+export default {
+  enrollmentStatus: {
+    label: 'Filter students present or absent',
+    short:
+      'Suodata opiskelijoita sen mukaan ovatko he ilmoittautuneet läsnä- tai poissaoleviksi kyseiselle lukukaudelle',
+  },
+  transferred: {
+    label: 'Include and exclude students from this program',
+    short: 'Mahdollisuus suodattaa opiskelijat, jotka ovat siirtyneet tutkinto-ohjelmaan',
+  },
+  courseCredits: {
+    label: null,
+    short: 'Rajaa kurssisuorituksia tietyn ajanjakson mukaan. Tämä rajaus ei muuta opiskelijoiden määrää',
+  },
+}
--- a/Show More
+++ b/Show More