Make linter happy

Improve grade filter fault tolerance
Adjust tests
2026-10-12 19:03:39 +03:00 · 2022-01-05 02:06:08 +02:00 · 2022-01-05 01:59:15 +02:00 · 2022-01-05 01:50:43 +02:00 · 2022-01-05 01:49:45 +02:00 · 2022-01-05 01:20:50 +02:00
333 changed files with 10530 additions and 78684 deletions
@@ -1,4 +1,3 @@
 # Populate and change this to .env
 IMPORTER_DB_API_TOKEN=
-IMPORTER_DB_API_URL=https://importer.cs.helsinki.fi
 PATE_TOKEN=
@@ -1,4 +1,4 @@
-name: "Release oodikone to production"
+name: 'Release oodikone to production'

 on:
  release:
@@ -6,23 +6,23 @@ on:

 jobs:
  build_and_release_backend:
-    name: "Build and release backend"
+    name: 'Build and release backend'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
+      - name: 'Extract release version'
        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
        # This steps extracts release version from GITHUB_REF env var
        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
        id: extract_version
        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
        uses: docker/build-push-action@v2
        with:
          build-args: |
@@ -38,26 +38,26 @@ jobs:
          target: production

  build_and_release_frontend:
-    name: "Build and release frontend"
+    name: 'Build and release frontend'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Get current time"
+      - name: 'Get current time'
        id: get_current_time
        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
-      - name: "Extract release version"
+      - name: 'Extract release version'
        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
        # This steps extracts release version from GITHUB_REF env var
        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
        id: extract_version
        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
        uses: docker/build-push-action@v2
        with:
          build-args: |
@@ -72,11 +72,11 @@ jobs:
          push: true
          tags: toska/oodikone2-frontend:latest
          target: production
-      - name: "Create a container from built image "
+      - name: 'Create a container from built image '
        run: docker create --name frontend toska/oodikone2-frontend:latest
-      - name: "Copy assets from the container"
+      - name: 'Copy assets from the container'
        run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
+      - name: 'Create Sentry release'
        uses: getsentry/action-release@v1
        env:
          SENTRY_URL: ${{ secrets.SENTRY_URL }}
@@ -90,49 +90,16 @@ jobs:
          url_prefix: '~/static/js'
          version: ${{ steps.extract_version.outputs.VERSION }}

-  build_and_release_userservice:
-    name: "Build and release userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
-        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
-        # This steps extracts release version from GITHUB_REF env var
-        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
-        id: extract_version
-        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: true
-          tags: toska/oodikone2-userservice:latest
-          target: production
-
  report_success:
    runs-on: ubuntu-latest
-    name: "Report deployment success to oodikone notifications"
+    name: 'Report deployment success to oodikone notifications'
    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
    if: success()
    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
        with:
          webhook-url: ${{ secrets.WEBHOOK_URL }}
          message-type: deployment
@@ -141,15 +108,14 @@ jobs:

  report_failure:
    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
+    name: 'Report deployment failure to oodikone notifications'
    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
    if: failure()
    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
        with:
          webhook-url: ${{ secrets.WEBHOOK_URL }}
          message-type: deployment-failure
@@ -45,21 +45,7 @@ jobs:
          'eslint --ext .jsx,.js services/frontend' \
          'stylelint services/frontend/src/**/*.css'

-  lint_userservice:
-    name: 'Lint userservice'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: 14
-      - name: 'Install dependencies'
-        run: npm ci && cd services/userservice && npm ci
-      - name: 'Run frontend linter'
-        run: npm run eslint services/userservice
-
  # === Build images ===
-
  build_backend:
    name: 'Build backend'
    runs-on: ubuntu-latest
@@ -116,50 +102,106 @@ jobs:
        with:
          path: /tmp/frontend.tar

-  build_userservice:
-    name: 'Build userservice'
+  build_and_push_anon_sis_db:
+    name: 'Build anon sis-db and push to toska hub if new migrations have been added'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: 'Build image and export it'
-        uses: docker/build-push-action@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: false
-          tags: toska/oodikone2-userservice:staging
-          target: production
-          outputs: type=docker,dest=/tmp/userservice.tar
-      - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v2
+          filters: |
+            migrations:
+              - 'updater/sis-updater-worker/src/db/migrations/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
        with:
-          path: /tmp/userservice.tar
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach sis-updater-worker
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db

-  # === Integration tests ===
-  #
-  run_userservice_integration_tests:
-    name: 'Run userservice integration tests'
+  build_and_push_anon_kone_db:
+    name: 'Build anon kone-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
-      - name: 'Run tests included in docker build'
-        run: docker-compose --file docker-compose.test.yml up --build userservice-test
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_kone/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
+
+  build_and_push_anon_user_db:
+    name: 'Build anon user-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_user/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db

  # === End-to-end tests ===
-
  test_course_statistics:
    name: 'Test course statistics'
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -171,7 +213,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -202,7 +243,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -214,7 +257,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -245,7 +287,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -257,7 +301,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -288,7 +331,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -300,7 +345,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -331,7 +375,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -343,7 +389,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -374,7 +419,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -386,7 +433,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -417,7 +463,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -429,7 +477,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -464,7 +511,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -476,7 +525,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -504,8 +552,6 @@ jobs:
    needs:
      - lint_backend
      - lint_frontend
-      - lint_userservice
-      - run_userservice_integration_tests
      - test_course_statistics
      - test_custom_population
      - test_filters
@@ -524,7 +570,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to DockerHub'
        uses: docker/login-action@v1
        with:
@@ -551,7 +596,6 @@ jobs:
        run: |
          docker push toska/oodikone2-backend:staging
          docker push toska/oodikone2-frontend:staging
-          docker push toska/oodikone2-userservice:staging
      - name: 'Send deployment success notification to slack'
        if: success()
        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
@@ -574,8 +618,6 @@ jobs:
    needs:
      - lint_backend
      - lint_frontend
-      - lint_userservice
-      - run_userservice_integration_tests
      - test_course_statistics
      - test_custom_population
      - test_filters
@@ -7,8 +7,8 @@ module.exports = {
  '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
  '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
  '*.css': files => `stylelint --fix ${files.join(' ')}`,
-  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0-alpine hadolint ${relativeFilePaths(files)}`,
-  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck-alpine:v0.7.2 ${relativeFilePaths(files)} -x`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0 hadolint ${relativeFilePaths(files)}`,
+  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck:v0.7.2 ${relativeFilePaths(files)} -x`,
  '.github/workflows/*': files => `${dockerCmdBase} rhysd/actionlint:1.6.2 ${relativeFilePaths(files)}`,
  'docker-compose*': files => {
    const composeFiles = file => {
@@ -43,8 +43,8 @@ What different CLI options do:
   - Cleans up any previous real data databases
   - Downloads needed database dumps from production servers.
   - Creates real data databases and populates them with downloaded dumps.
-3. Reset sis-importer data
-   - Like option 2, but only for sis-importer database. Useful when developing updater microservice.
+3. Reset single database
+   - Like option 2, but only for single database. Leaves other databases untouched.

 ## ⌨️ Development

@@ -79,7 +79,7 @@ It is recommended to spend some time to become familiar with `docker` and `docke
 - `docker-compose ps`: view the containers in the running environment
 - `docker-compose logs frontend`: print logs for just frontend
 - `docker-compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
- `docker exec -it userservice sh`: open bash terminal inside userservice container
+- `docker exec -it backend sh`: open bash terminal inside backend container
 - `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.

 ### User types for development
@@ -124,12 +124,16 @@ Continuous integration (CI) works with Github actions and is defined in workflow

 ### Modules are missing after updating package.json

-You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container.

 ### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data

 Make sure you have your VPN on

+### I made a change to sis-db models aand created a migration, but tests are failing in CI!
+
+Migrations are handled by sis-importer-worker, since oodikone is using only replica. Since CI is testing only services included in oodikone, your new changes aren't migrated and thus are failing. Open anonyymioodi-folder in this repo to find more about how to to update the anon sis-db.
+
 ### Everything is broken, can't get oodikone running, data is not there etc.

 Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.
@@ -0,0 +1,2 @@
+*.sqz
+
@@ -0,0 +1,65 @@
+# Anonyymioodi - anonymous database setup for oodikone
+
+(name anonyymioodi for :culture: reasons)
+
+This folder contains a guide and scripts to create anonymous database images for oodikone, mainly used for testing.
+
+## How to create / modify database images
+
+Creating / modifying images varies a bit depending on the db. Generally, to modify images, you can either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose. 
+
+By default, previous db images from Toska Hub are used as a base. If needed, comment out Toska hub image and uncomment corresponding clean postgres image in docker-compose before running other steps.
+
+See more details on how to create / modify databases below.
+
+### Sis-importer-db
+
+The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
+- Move `sis-importer-db.sqz` to this folder
+- Run `./create_container_from_dump.sh sis-importer-db`
+- Enrich db with command in file `enrich_data.sql` either by piping command to container or by running command inside container.
+
+### Sis-db:
+
+The following steps are for building a clean sis-db image from scratch using oodikone's updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
+- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
+- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
+- Start updater, see oodikone readme for more info.
+- Populate sis-db from sis-importer-db:
+
+```
+curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
+```
+(or run these from frontend. You might also want to follow sis-updater-workers logs to ensure things are running smoothly.)
+- Clean up any extra stuff you don't want to keep in sis-db.
+- Bonus: check that cypress tests run okay
+
+### User-db / kone-db
+
+- Temporarily replace the db service in oodikone's docker-compose.yml with the same db service from this folder's docker-compose.yml
+- Ensure the newest user-db/kone-db image is pulled from the registry with `docker-compose pull`
+- Modify db to wanted state. Tips:
+  - Run oodikone if you want to upgrade something from frontend (e.g. tags in kone-db, courses in sis-db etc.).
+  - Easy way to create new users to user-db is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
+
+## Publishing the image
+
+After completing the steps above and making sure your local version has the correct data, run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the db image to Toska Hub.
+
+## How to update Postgres versions
+
+When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
+
+- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
+- Start database with docker-compose
+- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
+- Close container with docker-compose down.
+- Update the Postgres version in docker-compose.yml in this folder.
+- Uncomment row with Postgres image and comment row with Toskahub image.
+- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+infomsg "Committing and pushing to registry"
+database=${args[0]}
+docker commit "$database" "registry.toska.cs.helsinki.fi/$database"
+docker push "registry.toska.cs.helsinki.fi/$database"
+successmsg "Succesfully pushed"
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+# === Create images ===
+
+retry () {
+  sleep 5
+  for i in {1..60}; do
+    "$@" && break || warningmsg "Retry attempt $i failed, waiting..." && sleep 5;
+  done
+}
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+
+infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
+database=${args[0]}
+docker-compose down --remove-orphans
+docker-compose up -d "$database"
+
+database_dump="$database.sqz"
+infomsg "Attempting to restore database $database from dump $database_dump"
+retry docker-compose exec "$database" pg_isready --dbname="$database"
+docker exec -i "$database" /bin/bash -c "pg_restore --username=postgres \
+--format=custom --dbname=$database --no-owner" < "$database_dump"
+successmsg "Database $database succesfully created"
@@ -0,0 +1,50 @@
+x-defaults: &defaults
+  restart: unless-stopped
+
+services:
+  kone-db:
+    <<: *defaults
+    container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/kone-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone
+
+  sis-db:
+    <<: *defaults
+    container_name: sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-db
+    image: postgres:13.2
+    shm_size: 2gb
+
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-importer-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-importer-db
+    image: postgres:13.1
+    profiles:
+      - updater
+
+  user-db:
+    <<: *defaults
+    container_name: user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone
@@ -0,0 +1,133 @@
+-- add these to sis-importer db by running:
+-- docker exec -i sis-importer-db psql -U postgres sis-importer-db < enrich_sampledata.sql
+
+-- if you need to run updater for these students only, see studentnumberlist in the end
+-- this file
+
+-- Add admission types (valintaperusteet) to studyrights for some of the computer
+-- science 2018-2019 population students. See person_ids / student_number below 
+-- (in case you need to update dumps). 
+
+-- TODO: this should be moved into the sampledata creator script in
+-- importer repo, but since this file contains studentnumbers and person ids, it's 
+-- better to keep this here
+
+-- KP (koepisteet)
+-- hy-hlo-1412620   | 013683126
+-- hy-hlo-1485246   | 014340523
+-- hy-hlo-32230770  | 010589388
+-- hy-hlo-35626233  | 010775888
+-- hy-hlo-40150144  | 011065995
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:kp' 
+WHERE person_id IN (
+'hy-hlo-1412620',
+'hy-hlo-1485246',
+'hy-hlo-32230770',
+'hy-hlo-35626233',
+'hy-hlo-40150144'
+);
+
+-- TV (todistusvalinta)
+-- hy-hlo-45783433  | 011321150
+-- hy-hlo-48537946  | 010764653
+-- hy-hlo-49801905  | 011194400
+-- hy-hlo-53321302  | 011338871
+-- hy-hlo-51223300  | 013951984
+-- hy-hlo-53310137  | 012520688
+-- hy-hlo-53621583  | 011196796
+-- hy-hlo-53863064  | 011736884
+-- hy-hlo-57711592  | 010262566
+-- hy-hlo-57732872  | 013670647
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:tv' 
+WHERE person_id IN (
+'hy-hlo-45783433',
+'hy-hlo-48537946',
+'hy-hlo-49801905',
+'hy-hlo-53321302',
+'hy-hlo-51223300',
+'hy-hlo-53310137',
+'hy-hlo-53621583',
+'hy-hlo-53863064',
+'hy-hlo-57711592',
+'hy-hlo-57732872'
+);
+
+-- YP:
+-- hy-hlo-67768022  | 011952325
+-- hy-hlo-73740025  | 011736606
+-- hy-hlo-74780917  | 012775433
+-- hy-hlo-79615371  | 014113941
+-- hy-hlo-78118455  | 014300615
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:yp' 
+WHERE person_id IN (
+'hy-hlo-67768022',
+'hy-hlo-73740025',
+'hy-hlo-74780917',
+'hy-hlo-79615371',
+'hy-hlo-78118455'
+);
+-- AV:
+-- hy-hlo-78777265  | 010779392
+-- hy-hlo-81909550  | 011894153
+-- hy-hlo-84571890  | 013897158
+-- hy-hlo-80330834  | 015085926
+-- hy-hlo-84775722  | 011791272
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:av' 
+WHERE person_id IN (
+'hy-hlo-78777265',
+'hy-hlo-81909550',
+'hy-hlo-84571890',
+'hy-hlo-80330834',
+'hy-hlo-84775722'
+);
+-- M:
+-- hy-hlo-87439827  | 011928456
+-- hy-hlo-96612371  | 014418361
+-- hy-hlo-84840261  | 010975895
+-- hy-hlo-90330681  | 010522688
+-- hy-hlo-96594603  | 013951573
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:m' 
+WHERE person_id IN (
+'hy-hlo-87439827',
+'hy-hlo-96612371',
+'hy-hlo-84840261',
+'hy-hlo-90330681',
+'hy-hlo-96594603'
+);
+
+-- all studentnumbers for running updater:
+013683126
+014340523
+010589388
+010775888
+011065995
+011321150
+010764653
+011194400
+011338871
+013951984
+012520688
+011196796
+011736884
+010262566
+013670647
+011952325
+011736606
+012775433
+014113941
+014300615
+010779392
+011894153
+013897158
+015085926
+011791272
+011928456
+014418361
+010975895
+010522688
+013951573
@@ -44,20 +44,22 @@ will take care of setting up Oodikone for you. See README for more details."
  msg ""
 }

-# Define custom shell prompt for the interactive select loop
-PS3="Please enter your choice: "
+set_custom_select_prompt() {
+  PS3="Please enter your choice: "

-options=(
-  "Set up oodikone from scratch."
-  "Reset all real data."
-  "Reset sis-importer data."
-  "Quit."
-)
+  options=(
+    "Set up oodikone from scratch."
+    "Reset all real data."
+    "Reset single database."
+    "Quit."
+  )
+}

-# Run scripts before showing interactive prompt
+# Run scripts and set the prompt
 show_welcome
 init_dirs
 get_username
+set_custom_select_prompt

 while true; do
  select opt in "${options[@]}"; do
@@ -66,8 +68,10 @@ while true; do
        set_up_oodikone_from_scratch;;
      "Reset all real data.")
        reset_all_real_data;;
-      "Reset sis-importer data.")
-        reset_sis_importer_data;;
+      "Reset single database.")
+        reset_single_database
+        set_custom_select_prompt # Set prompt back to initial values
+        ;;
      "Quit.")
        break 2;;
      *) msg "${RED}Invalid option:${NOFORMAT} $REPLY
@@ -186,6 +186,18 @@ describe('Course Statistics tests', () => {
      cy.contains('td', '010674989').siblings().contains('1')
    })

+    it.skip('Language distribution is correct', () => {
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
+      cy.contains(/^TKT20003$/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
+      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.contains('Language distribution').click()
+      cy.contains('td', 'finnish').siblings().contains('122')
+      cy.contains('td', 'english').siblings().contains('5')
+    })
+
    describe('When searching unified course stats', () => {
      beforeEach(() => {
        cy.url().should('include', '/coursestatistics')
@@ -9,23 +9,24 @@ const checkFilteringResult = studentCount => {
 // Semantic UI doesn't allow injection of data-cy:s for single multiple dropdown selection.
 // This function tries to click "x" on all selections inside dropdown matching given attribute
 const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.delete').click()
+  cy.cs(dataCyAttribute).find('i.delete').click({ force: true })
+  cy.cs(dataCyAttribute).click()
 }

 const clearSingleDropdownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.clear').click()
+  cy.cs(dataCyAttribute).find('i.clear').click({ force: true })
 }

 // Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
 // to every it-function. Reason behind using test function wrapper is that Cypresses internal beforeEach / afterEach
 // functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
 const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
-  return (dataCyAttributeOfHeaderToClick, testStepFunctionToRun) => {
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
+  return (filterName, testStepFunctionToRun) => {
+    cy.get(`[data-cy="${filterName}-header"]`).click()
    checkFilteringResult(amountWithoutFiltering)
    testStepFunctionToRun()
    checkFilteringResult(amountWithoutFiltering)
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
+    cy.get(`[data-cy="${filterName}-header"]`).click({ force: true })
  }
 }

@@ -41,80 +42,109 @@ describe('Population Statistics', () => {

  it('Transfer filter is on not transferred by default', () => {
    cy.contains('By default only students who have not transferred to this study programme are shown.')
-    cy.cs('transferredToProgrammeFilter-havenot').find('input').should('be.checked')
+    const card = cy.cs('TransferredToProgramme-filter-card')
+    card.cs('TransferredToProgramme-header').click()
+    card.get('[data-cy="option-havenot"] input').should('be.checked')
+    card.cs('TransferredToProgramme-header').click()
+  })
+
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(5)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
  })

  it('Graduation filter works', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      const card = cy.cs('GraduatedFromProgramme-filter-card')
+
      const graduated = 11
-      cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+      card.cs('option-graduated-true').click()
      checkFilteringResult(graduated)
-      cy.cs('graduatedFromProgrammeFilter-graduated-false').click()
+      card.cs('option-graduated-false').click()
      checkFilteringResult(defaultAmountOfStudents - graduated)
-      cy.cs('graduatedFromProgrammeFilter-all').click()
+      card.get('[data-cy="GraduatedFromProgramme-filter-card"] [data-cy="option-all"]').click()
    })
  })

  it('Transfer filter works', () => {
-    runTestStepWithPreAndPostParts('transferredToProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
      const transferred = 1
-      cy.cs('transferredToProgrammeFilter-have').click()
+      cy.cs('option-have').click()
      checkFilteringResult(transferred)
-      cy.cs('transferredToProgrammeFilter-all').click()
+      cy.cs('option-all').click()
      checkFilteringResult(defaultAmountOfStudents + transferred)
-      cy.cs('transferredToProgrammeFilter-havenot').click()
+      cy.cs('option-havenot').click()
    })
  })

  it('Enrollment filter works', () => {
-    runTestStepWithPreAndPostParts('enrollmentStatusFilter-header', () => {
-      cy.selectFromDropdown('enrollmentStatusFilter-status', 0)
-      cy.selectFromDropdown('enrollmentStatusFilter-semesters', [0])
+    runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
+      cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
+      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(0)
      checkFilteringResult(137)
      clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
    })
  })

  it('Credit filter works', () => {
-    runTestStepWithPreAndPostParts('credit-filter-header', () => {
+    runTestStepWithPreAndPostParts('CreditsEarned', () => {
      cy.cs('credit-filter-min').type('50')
      cy.cs('credit-filter-max').type('150')
      checkFilteringResult(116)
-      cy.cs('credit-filter-min').find('input').clear()
-      cy.cs('credit-filter-max').find('input').clear()
+      cy.cs('credit-filter-min').clear()
+      cy.cs('credit-filter-max').clear()
    })
  })

  it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
      cy.cs('ageFilter-min').type('20')
      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(43)
+      checkFilteringResult(41)
      cy.cs('ageFilter-min').find('input').clear()
      cy.cs('ageFilter-max').find('input').clear()
    })
  })

  it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
      checkFilteringResult(36)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
      checkFilteringResult(112)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
      checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
      checkFilteringResult(0)
      clearSingleDropdownSelection('genderFilter-dropdown')
    })
  })

  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', 0)
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
      checkFilteringResult(1)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 14)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(14)
      checkFilteringResult(121)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
    })
@@ -122,10 +152,10 @@ describe('Population Statistics', () => {

  // Return when we have +2020 population with admission types in data
  it.skip('Admission type filter works', () => {
-    runTestStepWithPreAndPostParts('admissionTypeFilter-header', () => {
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('AdmissionTypeFilter', () => {
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(0)
      checkFilteringResult(5)
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 2)
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(2)
      checkFilteringResult(10)
      clearSingleDropdownSelection('admissionTypeFilter-dropdown')
    })
@@ -135,31 +165,32 @@ describe('Population Statistics', () => {
  it.skip('Courses filter works', () => {
    // courses takes some time to load, wait for it to complete
    cy.wait(10000)
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
      cy.cs('courseFilter-course-dropdown').click()
      const courses = ['TKT20001', 'MAT11002']
      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
      checkFilteringResult(140)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
      checkFilteringResult(131)
      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
      checkFilteringResult(56)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 2)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
      checkFilteringResult(2)
      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
    })
  })

  it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
-        cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
+        const card = cy.cs('GraduatedFromProgramme-filter-card')
+        card.cs('option-graduated-true').click()
        cy.cs('ageFilter-min').type('20')
        cy.cs('ageFilter-max').type('30')
        checkFilteringResult(11)
        cy.cs('ageFilter-min').find('input').clear()
        cy.cs('ageFilter-max').find('input').clear()
-        cy.cs('graduatedFromProgrammeFilter-all').click()
+        card.cs('option-all').click()
      })
    })
  })
@@ -176,7 +207,7 @@ describe('Course Statistics', () => {
  })

  it('Grade filter works', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
      cy.cs('gradeFilter-5').click()
      checkFilteringResult(20)
      cy.cs('gradeFilter-3').click()
@@ -186,44 +217,68 @@ describe('Course Statistics', () => {
    })
  })

+  it('Programme defaults to "Attainment" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Attainment')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(116)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
  it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
      cy.cs('ageFilter-min').type('20')
      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(33)
+      checkFilteringResult(32)
      cy.cs('ageFilter-min').find('input').clear()
      cy.cs('ageFilter-max').find('input').clear()
    })
  })

  it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
      checkFilteringResult(27)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
      checkFilteringResult(91)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
      checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
      checkFilteringResult(0)
      clearSingleDropdownSelection('genderFilter-dropdown')
    })
  })

  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
      checkFilteringResult(1)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 12)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(12)
      checkFilteringResult(29)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
    })
  })

  it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
        cy.cs('gradeFilter-3').click()
        cy.cs('ageFilter-min').type('20')
        cy.cs('ageFilter-max').type('30')
@@ -245,13 +300,14 @@ describe('Custom Population Statistics', () => {
    cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
      const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
      runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
-      cy.cs('student-no-input').click().type(students.join('/n'))
+      cy.cs('student-no-input').click().type(students.join('\n'))
+      cy.wait(10000)
    })
    cy.cs('search-button').click()
  })

  it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
      cy.cs('ageFilter-min').type('20')
      cy.cs('ageFilter-max').type('30')
      checkFilteringResult(1)
@@ -260,36 +316,59 @@ describe('Custom Population Statistics', () => {
    })
  })

-  it('Gender filter works', () => {
-    cy.cs('genderFilter-header').click()
-    cy.selectFromDropdown('genderFilter-dropdown', 0)
-    checkFilteringResult(3)
-    cy.selectFromDropdown('genderFilter-dropdown', 1)
-    checkFilteringResult(2)
-    cy.selectFromDropdown('genderFilter-dropdown', 2)
-    checkFilteringResult(0)
-    clearSingleDropdownSelection('genderFilter-dropdown')
-    checkFilteringResult(5, true)
-  })
-
-  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
-      checkFilteringResult(1)
-      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
    })
  })

-  it('Courses filter works', () => {
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
-      const courses = ['MAT11001', 'TKT20004']
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
      checkFilteringResult(5)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
-      checkFilteringResult(5)
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
      checkFilteringResult(3)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+      checkFilteringResult(5, true)
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.cs('startYearAtUni-dropdown').click()
+    })
+  })
+
+  it.skip('Courses filter works', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
+      const courses = ['MAT11001', 'TKT20004']
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[0] + ' ')
+        .click()
+      checkFilteringResult(5)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
+      checkFilteringResult(5)
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[1] + ' ')
+        .click()
+      checkFilteringResult(3)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(3)
      checkFilteringResult(1)
      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
    })
@@ -1,5 +1,8 @@
 /// <reference types="Cypress" />

+const moment = require('moment')
+const _ = require('lodash')
+
 const setPopStatsUntil = (until, includeSettings = []) => {
  cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
  includeSettings.forEach(setting => {
@@ -13,6 +16,8 @@ const setPopStatsUntil = (until, includeSettings = []) => {
 describe('Population Statistics tests', () => {
  const pathToCSBach2017 =
    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2017'
+  const pathToCSMaster2019 =
+    '/populations?months=27&semesters=FALL&semesters=SPRING&studyRights=%7B%22programme%22%3A%22MH50_009%22%7D&tag&year=2019'
  describe('when using basic user', () => {
    beforeEach(() => {
      cy.init('/populations')
@@ -59,6 +64,7 @@ describe('Population Statistics tests', () => {

    it('Population statistics is usable on general level', () => {
      cy.visit(pathToCSBach2017)
+      cy.cs('filtered-students')
      setPopStatsUntil('toukokuu 2020')

      cy.get('.card').within(() => {
@@ -69,7 +75,8 @@ describe('Population Statistics tests', () => {
        cy.contains('Excludes students with non-degree study right')
        cy.contains('Excludes students who have transferred out of this programme')
      })
-      cy.contains('Courses of population').click({ force: true })
+      cy.cs('filtered-students')
+      cy.contains('Courses of population').click()

      cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
      // eslint-disable-next-line cypress/no-unnecessary-waiting
@@ -123,18 +130,62 @@ describe('Population Statistics tests', () => {
      cy.contains('Credit accumulation (for 170 students)')
    })

-    it('Credit Statistics, Statistics pane works', () => {
+    it('Credit Statistics, Credits Gained tab works', () => {
      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
      cy.contains('Credit statistics').click()
+      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the population')
+
+      const months = Math.ceil(moment.duration(moment().diff(moment('2017-08-1'))).asMonths())
+
+      const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
+      const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))
+
+      cy.get('.credits-gained-table').should('contain', '(n=149)')
+
+      for (const [start, end] of ranges) {
+        let value = 'credits'
+
+        if (start !== null) {
+          value = `${start} ≤ ${value}`
+        }
+
+        if (end !== null) {
+          value = `${value} < ${end}`
+        }
+
+        cy.get('.credits-gained-table').should('contain', value)
+      }
+
+      cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
+        const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)
+
+        expect(sum).to.equal(149)
+      })
+
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('not.exist')
+    })
+
+    it('Credit Statistics, Credits Gained tab shows stats by admissions', () => {
+      cy.visit(pathToCSMaster2019)
+      cy.cs('filtered-students')
+      cy.contains('Credit statistics')
+      cy.get("[data-cy='credit-statistics']").click()
+      cy.get('.credits-gained-divider').click()
+      cy.get("[data-cy='credits-gained-table-Avoin väylä']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Yhteispisteet']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Muu']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('exist')
+    })
+
+    it('Credit Statistics, Statistics pane works', () => {
+      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Credit statistics').click({ force: true })
      cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()

      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
      cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
      cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
      cy.get("[data-cy='credit-stats-min']").should('contain', '11')
-      cy.get("[data-cy='credit-stats-q1']").should('contain', '70')
-      cy.get("[data-cy='credit-stats-q2']").should('contain', '129')
-      cy.get("[data-cy='credit-stats-q3']").should('contain', '168')
      cy.get("[data-cy='credit-stats-max']").should('contain', '314')
    })
  })
@@ -13,7 +13,7 @@ describe('Student Statistics tests', () => {
    studentnumber: '010654019',
  }

-  it.only('Student statistics search form is usable', () => {
+  it('Student statistics search form is usable', () => {
    cy.contains('Show student names')
    cy.url().should('include', '/students')
    cy.get('.prompt').type(student.lastname)
@@ -80,7 +80,7 @@ describe('Student Statistics tests', () => {
    cy.contains('Student number').should('not.exist')
  })

-  it.only('Can jump to population page', () => {
+  it('Can jump to population page', () => {
    cy.get('.prompt').type(student.studentnumber)
    cy.contains(student.studentnumber).click()
    cy.get('i.level.up.alternate.icon').eq(0).click()
@@ -88,3 +88,13 @@ describe('Student Statistics tests', () => {
    cy.contains('Tietojenkäsittelytieteen kandiohjelma')
  })
 })
+
+// Use admin to see all students
+describe('Testing with admin rights', () => {
+  it('Does not crash if student has no studyright or courses', () => {
+    const studentNumber = '011825096'
+    cy.init(`/students/${studentNumber}`, 'admin') // use admin to see all students
+    cy.contains('Started: Unavailable')
+    cy.contains('Credits: 0')
+  })
+})
@@ -4,16 +4,16 @@ describe('Users tests', () => {
  describe('Using user with just grp-oodikone-user, no other rights', () => {
    it('shows only frontpage, trends feedback', () => {
      cy.init('', 'norights') // login with norights user
-      cy.get('.navBar').contains('Trends').should('exist')
-      cy.get('.navBar').contains('Give feedback').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Give feedback').should('exist')

-      cy.get('.navBar').contains('Study programme').should('not.exist')
-      cy.get('.navBar').contains('Student statistics').should('not.exist')
-      cy.get('.navBar').contains('Course statistics').should('not.exist')
-      cy.get('.navBar').contains('Teachers').should('not.exist')
-      cy.get('.navBar').contains('Users').should('not.exist')
-      cy.get('.navBar').contains('Faculty').should('not.exist')
-      cy.get('.navBar').contains('Updater').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Teachers').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Faculty').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Updater').should('not.exist')
    })
  })

@@ -29,32 +29,27 @@ describe('Users tests', () => {
    })

    it('should see more stuff than others', () => {
-      cy.get('.navBar').contains('Study programme').should('exist')
-      cy.get('.navBar').contains('Student statistics').should('exist')
-      cy.get('.navBar').contains('Course statistics').should('exist')
-      // TODO: admin should probably see this as well, since can access this page: cy.get('.navBar').contains('Teachers').should('exist')
-      cy.get('.navBar').contains('Users').should('exist')
-      cy.get('.navBar').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('exist')
+      // TODO: admin should probably see this as well, since can access this page: cy.get('[data-cy="navBar"]').contains('Teachers').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
    })

    it("mocking normal user shows only the mocked user's programmes", () => {
      cy.contains('mocking').should('not.exist')
-      cy.contains('tr', 'basic').within($row => {
+      cy.contains('tr', 'basic').within(() => {
        cy.contains('.button', 'Edit').click()
      })
-      cy.intercept({
-        method: 'POST',
-        path: '/api/superlogin/basic',
-      }).as('superlogin')
      cy.get('i.spy').click()
-      cy.wait('@superlogin')
      cy.contains('mocking as basic')
      cy.wait(1000)
      cy.contains('Study programme').click().siblings().contains('Search by class').click()
      cy.contains('label', 'Study programme')
      cy.contains('label', 'Study programme')
        .siblings()
-        .within($row => {
+        .within(() => {
          cy.get("div[role='option']").should('have.length', 2).contains('Tietojenkäsittelytieteen kandiohjelma')
        })
    })
@@ -54,8 +54,6 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
    if (!headersToUse) throw Error(`${userId} is not valid user id!`)
    req.headers = headersToUse
  })
-  // Hide SIS warning flag to avoid screen size problems when running headlessly
-  window.localStorage.setItem('SIS_WARNING_FLAG', 'true')
  const baseUrl = Cypress.config().baseUrl
  cy.visit(baseUrl.concat(path))
 })
@@ -63,24 +61,33 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
 /**
 * Shorthand for using "Cypress Selectors" (CS), i.e., `data-cy` attributes.
 */
-Cypress.Commands.add('cs', name => {
-  return cy.get(`[data-cy='${name}']`)
+Cypress.Commands.add('cs', { prevSubject: 'optional' }, (subject, name) => {
+  const p = subject === undefined ? cy : cy.wrap(subject)
+
+  return p.get(`[data-cy='${name}']`)
 })

 /**
 * Select item specified by `index` number from a semantic-ui dropdown with
 * `data-cy` attribute value `name`.
 */
-Cypress.Commands.add('selectFromDropdown', (name, index) => {
+Cypress.Commands.add('selectFromDropdown', { prevSubject: true }, (s, index) => {
+  const subject = cy.wrap(s)
  const indexes = Array.isArray(index) ? index : [index]

  indexes.forEach(i => {
-    cy.cs(name).click().children('.menu').children().eq(i).click({ force: true })
+    subject.click()
+
+    if (typeof i === 'number') {
+      subject.children('.menu').children().eq(i).click({ force: true })
+    } else {
+      subject.children('.menu').containing(i).eq(0).click({ force: true })
+    }
  })

  // Close multiple selection so it does not block elements underneath it.
  if (Array.isArray(index)) {
-    cy.cs(name).children('.icon').click({ force: true })
+    cy.wrap(s).children('.icon').click({ force: true })
  }
 })

@@ -2,7 +2,6 @@ x-defaults: &defaults
  restart: unless-stopped

 services:
-
  backend:
    <<: *defaults
    container_name: backend
@@ -10,17 +9,16 @@ services:
      - kone-db
      - redis
      - sis-db
+      - user-db
    environment:
-      CI: "true"
+      CI: 'true'
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
      FRONT_URL: http://localhost:1337
      REDIS: redis
      SECRET_TOKEN: dev
      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
    image: toska/oodikone2-backend:staging

  frontend:
@@ -31,11 +29,15 @@ services:
  kone-db:
    <<: *defaults
    container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
    image: registry.toska.cs.helsinki.fi/kone-db

  redis:
    <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
    container_name: redis
    image: redis
    ports:
@@ -45,23 +47,20 @@ services:
    <<: *defaults
    container_name: sis-db
    image: registry.toska.cs.helsinki.fi/sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    shm_size: 2gb

  user-db:
    <<: *defaults
    container_name: user-db
-    image: registry.toska.cs.helsinki.fi/user-db
-
-  userservice:
-    <<: *defaults
-    container_name: userservice
-    depends_on:
-      - user-db
    environment:
-      CI: "true"
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    image: toska/oodikone2-userservice:staging
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db

  nginx:
    image: nginx:latest
@@ -69,8 +68,45 @@ services:
    depends_on:
      - frontend
      - backend
-      - userservice
    ports:
-      -  1337:80
+      - 1337:80
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf
+
+  # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    image: registry.toska.cs.helsinki.fi/sis-importer-db
+    profiles:
+      - updater
+
+  sis-updater-worker:
+    <<: *defaults
+    build:
+      target: development
+      context: updater/sis-updater-worker
+    depends_on:
+      - redis
+      - sis-db
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_URI: //redis:6379
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+    profiles:
+      - updater
+
+  sis-updater-nats:
+    <<: *defaults
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    container_name: sis-updater-nats
+    image: nats-streaming:0.16.2
+    profiles:
+      - updater
@@ -1,7 +1,7 @@
 services:
-
  backend:
    environment:
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db-real
      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db-real
      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db-real

@@ -18,9 +18,5 @@ services:
      DB_URL: postgres://postgres@sis-db:5432/sis-db-real
      SIS_IMPORTER_DATABASE: sis-importer-db-real

-  userservice:
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db-real
-
 volumes:
  redis-data-real:
@@ -1,10 +0,0 @@
-services:
-
-  userservice-test:
-    build:
-      context: services/userservice
-      target: test
-    container_name: userservice-test
-    environment:
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
@@ -26,18 +26,17 @@ services:
      - kone-db
      - redis
      - sis-db
+      - user-db
    env_file:
      - .env
    environment:
      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
      FRONT_URL: http://localhost:3000
      REDIS: redis
      SECRET_TOKEN: dev
      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
    ports:
      - 8080:8080
    profiles:
@@ -165,26 +164,6 @@ services:
    volumes:
      - user-db-data:/data

-  userservice:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/userservice
-    container_name: userservice
-    depends_on:
-      - user-db
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    ports:
-      - 4567:4567
-    profiles:
-      - oodikone
-    volumes:
-      - userservice-nodemod:/usr/src/app/node_modules
-      - ./services/userservice/:/usr/src/app
-
 volumes:
  backend-nodemod:
  frontend-nodemod:
@@ -195,4 +174,3 @@ volumes:
  sis-updater-scheduler-nodemod:
  sis-updater-worker-nodemod:
  user-db-data:
-  userservice-nodemod:
@@ -49,5 +49,9 @@
    "stylelint-config-prettier": "^8.0.2",
    "stylelint-config-standard": "^22.0.0",
    "stylelint-prettier": "^1.2.0"
+  },
+  "dependencies": {
+    "lodash": "^4.17.21",
+    "moment": "^2.29.1"
  }
 }
@@ -110,12 +110,54 @@ reset_all_real_data() {
  reset_databases "${DATABASES[@]}"
 }

-reset_sis_importer_data() {
-  infomsg "Downloading sis-importer-db dump"
-  local database=$SIS_IMPORTER_DB_NAME
-  local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
-  download_real_dump $database $url
-  reset_databases $database
+reset_single_database() {
+  # Define custom shell prompt for the nested interactive select loop
+  PS3="Please enter which database to reset: "
+
+  options=(
+    "kone-db"
+    "sis-db"
+    "sis-importer-db"
+    "user-db"
+    "Return to main menu."
+  )
+
+  local database=""
+  local url=""
+
+  while true; do
+    select opt in "${options[@]}"; do
+      case $opt in
+        "kone-db")
+          local database=$KONE_DB_NAME
+          local url=$KONE_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-db")
+          local database=$SIS_DB_NAME
+          local url=$SIS_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-importer-db")
+          local database=$SIS_IMPORTER_DB_NAME
+          local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
+          break 2;;
+        "user-db")
+          local database=$USER_DB_NAME
+          local url=$USER_DB_REAL_DUMP_URL
+          break 2;;
+        "Return to main menu.")
+          break 2;;
+        *) msg "${RED}Invalid option:${NOFORMAT} $REPLY
+  ";;
+      esac
+      break
+    done
+  done
+
+  if [ -n "$database" ]; then
+    infomsg "Downloading $database dump"
+    download_real_dump $database $url
+    reset_databases $database
+  fi
 }

 set_up_oodikone_from_scratch() {
@@ -123,7 +165,6 @@ set_up_oodikone_from_scratch() {
  folders_to_set_up=(
    "$PROJECT_ROOT"
    "$PROJECT_ROOT/services/frontend"
-    "$PROJECT_ROOT/services/userservice"
    "$PROJECT_ROOT/services/backend"
    "$PROJECT_ROOT/updater/sis-updater-scheduler"
    "$PROJECT_ROOT/updater/sis-updater-worker"
@@ -1,6 +1,3 @@
 .dockerignore
 Dockerfile
 node_modules/
-requests/
-scripts/
-*.log
@@ -1,3 +0,0 @@
-# Student diff output
-/student_diff_*
-/diff_partial_att_data.json
@@ -332,11 +332,6 @@
        "concat-map": "0.0.1"
      }
    },
-    "buffer-equal-constant-time": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
-      "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
-    },
    "buffer-writer": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
@@ -704,14 +699,6 @@
      "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=",
      "dev": true
    },
-    "ecdsa-sig-formatter": {
-      "version": "1.0.11",
-      "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
-      "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
-      "requires": {
-        "safe-buffer": "^5.0.1"
-      }
-    },
    "ee-first": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -1330,49 +1317,6 @@
      "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=",
      "dev": true
    },
-    "jsonwebtoken": {
-      "version": "8.5.1",
-      "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz",
-      "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==",
-      "requires": {
-        "jws": "^3.2.2",
-        "lodash.includes": "^4.3.0",
-        "lodash.isboolean": "^3.0.3",
-        "lodash.isinteger": "^4.0.4",
-        "lodash.isnumber": "^3.0.3",
-        "lodash.isplainobject": "^4.0.6",
-        "lodash.isstring": "^4.0.1",
-        "lodash.once": "^4.0.0",
-        "ms": "^2.1.1",
-        "semver": "^5.6.0"
-      },
-      "dependencies": {
-        "semver": {
-          "version": "5.7.1",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
-        }
-      }
-    },
-    "jwa": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz",
-      "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==",
-      "requires": {
-        "buffer-equal-constant-time": "1.0.1",
-        "ecdsa-sig-formatter": "1.0.11",
-        "safe-buffer": "^5.0.1"
-      }
-    },
-    "jws": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz",
-      "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==",
-      "requires": {
-        "jwa": "^1.4.1",
-        "safe-buffer": "^5.0.1"
-      }
-    },
    "kind-of": {
      "version": "6.0.3",
      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
@@ -1389,41 +1333,6 @@
      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
    },
-    "lodash.includes": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
-      "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8="
-    },
-    "lodash.isboolean": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
-      "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY="
-    },
-    "lodash.isinteger": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
-      "integrity": "sha1-YZwK89A/iwTDH1iChAt3sRzWg0M="
-    },
-    "lodash.isnumber": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
-      "integrity": "sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w="
-    },
-    "lodash.isplainobject": {
-      "version": "4.0.6",
-      "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
-      "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs="
-    },
-    "lodash.isstring": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
-      "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE="
-    },
-    "lodash.once": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
-      "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w="
-    },
    "logform": {
      "version": "2.2.0",
      "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz",
@@ -3353,9 +3262,9 @@
      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A=="
    },
    "validator": {
-      "version": "13.6.0",
-      "resolved": "https://registry.npmjs.org/validator/-/validator-13.6.0.tgz",
-      "integrity": "sha512-gVgKbdbHgtxpRyR8K0O6oFZPhhB5tT1jeEHZR0Znr9Svg03U0+r9DXWMrnRAB+HtCStDQKlaIZm42tVsVjqtjg=="
+      "version": "13.7.0",
+      "resolved": "https://registry.npmjs.org/validator/-/validator-13.7.0.tgz",
+      "integrity": "sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw=="
    },
    "vary": {
      "version": "1.1.2",
@@ -21,7 +21,6 @@
    "cron": "^1.8.2",
    "express": "^4.17.1",
    "express-async-errors": "^3.1.1",
-    "jsonwebtoken": "^8.5.1",
    "lodash": "^4.17.21",
    "lru-cache": "^5.1.1",
    "moment": "^2.29.1",
@@ -1,13 +1,9 @@
 require('express-async-errors')
 const express = require('express')
-const cors = require('cors')
-const Sentry = require('@sentry/node')
-const { baseUrl, backendPort, frontUrl } = require('./conf-backend')
+const { baseUrl, backendPort } = require('./conf-backend')
 const routes = require('./routes')
 const { startCron } = require('./events')
 const { initializeDatabaseConnection, dbConnections } = require('./database/connection')
-const initializeSentry = require('./util/sentry')
-const errorMiddleware = require('./middleware/errormiddleware')
 const logger = require('./util/logger')

 initializeDatabaseConnection()
@@ -21,28 +17,11 @@ initializeDatabaseConnection()
    dbConnections.on('error', () => {
      logger.error('Failed to connect to sis db!')
    })
-
-    const app = express()
-    initializeSentry(app)
-    app.use(Sentry.Handlers.requestHandler())
-    app.use(Sentry.Handlers.tracingHandler())
-
    startCron()

-    app.use(cors({ credentials: true, origin: frontUrl }))
-    app.use(express.json())
-
+    const app = express()
    routes(app, baseUrl)

-    app.get('*', async (_, res) => {
-      const results = { error: 'unknown endpoint' }
-      res.status(404).json(results)
-    })
-
-    app.use(Sentry.Handlers.errorHandler())
-
-    app.use(errorMiddleware)
-
    const server = app.listen(backendPort, () => logger.info(`Backend listening on port ${backendPort}!`))
    process.on('SIGTERM', () => {
      server.close(() => {
@@ -18,7 +18,6 @@ const pateToken = process.env.PATE_API_TOKEN || ''

 // Importer client
 const importerToken = process.env.IMPORTER_DB_API_TOKEN || ''
-const importerUrl = process.env.IMPORTER_DB_API_URL || ''

 // Networking: Urls & ports
 const baseUrl = isDev ? '/api' : ''
@@ -37,6 +36,7 @@ const MATOMO_URL = process.env.MATOMO_URL

 // Other stuff
 const DB_URL_KONE = process.env.DB_URL_KONE
+const DB_URL_USER = process.env.DB_URL_USER

 let DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS, 10)
 if (isNaN(DB_MAX_CONNECTIONS)) {
@@ -48,10 +48,8 @@ if (DB_MAX_CRON_CONNECTIONS < 1) {
 }

 const redis = process.env.REDIS
-const TOKEN_SECRET = process.env.TOKEN_SECRET
 const SECRET_TOKEN = process.env.SECRET_TOKEN
 const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
-const USERSERVICE_URL = process.env.USERSERVICE_URL
 const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL

 const formatURL = url => {
@@ -60,20 +58,14 @@ const formatURL = url => {

 const addSlashToEnd = url => (url.endsWith('/') ? url : url + '/')

-const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
-
-const PWD = process.env.PWD || '/usr/src/app'
-
 module.exports = {
+  DB_URL_USER,
  DB_URL_KONE,
  DB_MAX_CONNECTIONS,
  DB_MAX_CRON_CONNECTIONS,
  redis,
-  TOKEN_SECRET,
  SECRET_TOKEN,
  DB_SCHEMA_KONE,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL),
-  ACCESS_TOKEN_HEADER_KEY,
  requiredGroup,
  SIS_UPDATER_URL,
  ANALYTICS_INFLUXDB_URL: formatURL(ANALYTICS_INFLUXDB_URL),
@@ -91,6 +83,4 @@ module.exports = {
  frontUrl,
  backendPort,
  importerToken,
-  importerUrl,
-  PWD,
 }
@@ -39,54 +39,73 @@ class DbConnection extends EventEmitter {
  }
 }

+// Old-style kone + user db connections
 const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
  schema: conf.DB_SCHEMA_KONE,
  searchPath: conf.DB_SCHEMA_KONE,
  logging: false,
 })
+
 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)

+const sequelizeUser = new Sequelize(conf.DB_URL_USER, {
+  logging: false,
+})
+
 const initializeDatabaseConnection = async () => {
  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-  const waitSeconds = 60
-  for (let i = 1; i <= waitSeconds; i++) {
-    try {
-      await sequelizeKone.authenticate()
-      break
-    } catch (e) {
-      if (i === waitSeconds) {
-        logger.error(`Could not connect to database in ${waitSeconds} seconds`)
-        throw e
+  const rounds = 60
+
+  for (const [seq, dbName] of [
+    [sequelizeKone, 'kone-db'],
+    [sequelizeUser, 'user-db'],
+  ]) {
+    logger.info(`Connecting to ${dbName}...`)
+    for (let round = 1; round <= rounds; round++) {
+      try {
+        await seq.authenticate()
+        break
+      } catch (error) {
+        if (round === rounds) {
+          logger.error(`${dbName} database connection failed!`)
+          throw error
+        }
+        await sleep(1000)
      }
-      await sleep(1000)
    }
-  }
-  try {
+    logger.info(`${dbName} database connection established`)
+
+    const schema = dbName === 'kone-db' ? conf.DB_SCHEMA_KONE : undefined
+    const migrationsFolder = dbName === 'kone-db' ? 'migrations_kone' : 'migrations_user'
    const migrator = new Umzug({
      storage: 'sequelize',
      storageOptions: {
-        sequelize: sequelizeKone,
+        sequelize: seq,
        tableName: 'migrations',
-        schema: conf.DB_SCHEMA_KONE,
+        schema,
      },
      migrations: {
-        params: [sequelizeKone.getQueryInterface(), Sequelize],
-        path: `${process.cwd()}/src/database/migrations_kone`,
+        params: [seq.getQueryInterface(), Sequelize],
+        path: `${process.cwd()}/src/database/${migrationsFolder}`,
        pattern: /\.js$/,
-        schema: conf.DB_SCHEMA_KONE,
+        schema,
      },
    })
-    const migrations = await migrator.up()
-    logger.info({ message: 'Kone Migrations up to date', meta: migrations })
-  } catch (e) {
-    logger.error('Kone Migration error')
-    throw e
+    try {
+      const migrations = await migrator.up()
+      logger.info({ message: `${dbName} migrations up to date: `, meta: migrations.map(m => m.file) })
+    } catch (error) {
+      logger.error({ message: `${dbName} migrations failed`, meta: error })
+      throw error
+    }
  }
 }

 const dbConnections = new DbConnection()
+
 module.exports = {
  dbConnections,
  sequelizeKone,
+  sequelizeUser,
  initializeDatabaseConnection,
 }
@@ -0,0 +1,34 @@
+const { BIGINT, STRING, DATE } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.createTable(
+      'study_guidance_group_tags',
+      {
+        id: {
+          primaryKey: true,
+          type: BIGINT,
+          autoIncrement: true,
+        },
+        study_guidance_group_id: {
+          type: STRING,
+          unique: true,
+        },
+        study_programme: {
+          type: STRING,
+        },
+        year: {
+          type: STRING,
+        },
+        created_at: {
+          type: DATE,
+        },
+        updated_at: {
+          type: DATE,
+        },
+      },
+      { underscored: true, timestamps: true }
+    )
+  },
+  down: async () => {},
+}
@@ -0,0 +1,6 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('faculty_programmes')
+  },
+  down: async () => {},
+}
@@ -0,0 +1,9 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('user_affiliation')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS user_affilation_id_seq')
+    await queryInterface.dropTable('affiliations')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS affiliations_id_seq')
+  },
+  down: async () => {},
+}
@@ -7,7 +7,8 @@ const logger = require('../util/logger')
 // so don't e.g. make it into an async function or it'll print [object Promise] on every request
 const accessLogger = morgan((tokens, req, res) => {
  const fields = ['method', 'url', 'status', 'response-time', 'remote-addr', 'remote-user', 'user-agent', 'referrer']
-  const meta = { ...req.decodedToken }
+  const { user } = req
+  const meta = { ...user }
  fields.forEach(field => (meta[field] = tokens[field](req, res)))
  meta['time'] = tokens['date'](req, res, 'iso')
  if (req.route) {
@@ -15,7 +16,7 @@ const accessLogger = morgan((tokens, req, res) => {
  }

  const message = [
-    req.decodedToken.name,
+    user.name,
    ':',
    tokens['method'](req, res),
    tokens['url'](req, res),
@@ -1,42 +1,5 @@
-const jwt = require('jsonwebtoken')
-const conf = require('../conf-backend')
-const blacklist = require('../services/blacklist')
-const { getUserDataFor } = require('../services/userService')
-const { ACCESS_TOKEN_HEADER_KEY } = require('../conf-backend')
-const { hasRequiredGroup, parseHyGroups } = require('../util/utils')
-
-const TOKEN_VERSION = 1.1 // When token structure changes, increment in userservice, backend and frontend
-
-const checkAuth = async (req, res, next) => {
-  const token = req.headers[ACCESS_TOKEN_HEADER_KEY]
-  const uid = req.headers['uid']
-
-  if (!token) {
-    return res.status(403).json({ error: 'No token in headers' })
-  }
-
-  jwt.verify(token, conf.TOKEN_SECRET, async (err, decoded) => {
-    if (err) {
-      return res.status(403).json(err)
-    }
-
-    if (decoded.version !== TOKEN_VERSION) {
-      return res.status(401).json({ error: 'Token needs to be refreshed - invalid version' })
-    }
-
-    if (decoded.mockedBy ? decoded.mockedBy !== uid : decoded.userId !== uid) {
-      return res.status(403).json({ error: 'User shibboleth id and token id did not match' })
-    }
-
-    const userData = await getUserDataFor(decoded.userId)
-    req.decodedToken = decoded
-    Object.assign(req, userData)
-    next()
-  })
-}
-
 const roles = requiredRoles => async (req, res, next) => {
-  if (req.decodedToken) {
+  if (req.user) {
    const { roles } = req

    if (requiredRoles.every(r => roles.indexOf(r) >= 0) || roles.includes('admin')) {
@@ -47,37 +10,6 @@ const roles = requiredRoles => async (req, res, next) => {
  res.status(403).json({ error: 'missing required roles' })
 }

-const checkRequiredGroup = async (req, res, next) => {
-  const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-  const enabled = hasRequiredGroup(hyGroups)
-  const tokenOutdated = req.decodedToken.enabled !== enabled
-
-  if (tokenOutdated) {
-    return res.status(401).json({
-      error: 'Token needs to be refreshed - enabled does not match hy-group requirement',
-    })
-  }
-
-  if (!enabled) {
-    return res.status(403).json({ error: 'User is not enabled' })
-  }
-  next()
-}
-
-const checkUserBlacklisting = async (req, res, next) => {
-  const { userId, createdAt } = req.decodedToken
-  const isBlacklisted = await blacklist.isUserBlacklisted(userId, createdAt)
-
-  if (isBlacklisted) {
-    return res.status(401).json({ error: 'Token needs to be refreshed - blacklisted' })
-  }
-
-  next()
-}
-
 module.exports = {
-  checkAuth,
  roles,
-  checkRequiredGroup,
-  checkUserBlacklisting,
 }
@@ -0,0 +1,57 @@
+const Sentry = require('@sentry/node')
+const { ApplicationError } = require('../util/customErrors')
+const { getUser, getUserDataFor, getMockedUser } = require('../services/userService')
+const { requiredGroup } = require('../conf-backend')
+const _ = require('lodash')
+
+const parseIamGroups = iamGroups => iamGroups?.split(';').filter(Boolean) ?? []
+
+const hasRequiredIamGroup = iamGroups => _.intersection(iamGroups, requiredGroup).length > 0
+
+const currentUserMiddleware = async (req, _, next) => {
+  const { uid: username, 'shib-session-id': sessionId, shib_logout_url: logoutUrl } = req.headers
+
+  if (!sessionId) {
+    throw new ApplicationError('No session id found in request headers', 403, { logoutUrl })
+  }
+
+  if (!username) {
+    throw new ApplicationError('No username found in request headers', 403, { logoutUrl })
+  }
+  const { displayname: name, mail: email, hygroupcn: iamGroups, hypersonsisuid: sisId } = req.headers
+  const parsedIamGroups = parseIamGroups(iamGroups)
+
+  req.user = await getUser({
+    username,
+    name,
+    email,
+    iamGroups: parsedIamGroups,
+    sisId,
+  })
+
+  if (!hasRequiredIamGroup(parsedIamGroups)) {
+    throw new ApplicationError('User is not enabled', 403, { logoutUrl })
+  }
+
+  const showAsUser = req.headers['x-show-as-user']
+
+  if (showAsUser) {
+    const mockedUser = await getMockedUser(username, showAsUser)
+    if (mockedUser) {
+      req.user = mockedUser
+    }
+  }
+  req.logoutUrl = logoutUrl
+
+  const { userId, mockedBy } = req.user
+
+  Sentry.setUser({ username: mockedBy ?? userId })
+
+  // TODO: remove this garbo
+  const userData = await getUserDataFor(userId)
+  Object.assign(req, userData)
+
+  next()
+}
+
+module.exports = currentUserMiddleware
@@ -1,13 +0,0 @@
-const Sentry = require('@sentry/node')
-
-const sentryUserIdMiddleware = (req, res, next) => {
-  Sentry.getCurrentHub().configureScope(scope => {
-    scope.setUser({
-      username: req.decodedToken && req.decodedToken.userId,
-    })
-  })
-
-  next()
-}
-
-module.exports = sentryUserIdMiddleware
@@ -0,0 +1,5 @@
+const headersMiddleware = require('unfuck-utf8-headers-middleware')
+
+const headers = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
+
+module.exports = headersMiddleware(headers)
@@ -47,6 +47,9 @@ Course.init(
    substitutions: {
      type: JSONB,
    },
+    course_unit_type: {
+      type: STRING,
+    },
  },
  {
    underscored: false,
@@ -75,6 +75,12 @@ Credit.init(
    updatedAt: {
      type: DATE,
    },
+    language: {
+      type: STRING,
+    },
+    is_open: {
+      type: BOOLEAN,
+    },
  },
  {
    underscored: false,
@@ -135,6 +135,28 @@ const CustomPopulationSearch = sequelizeKone.define('custom_population_searches'
  },
 })

+const StudyGuidanceGroupTag = sequelizeKone.define(
+  'study_guidance_group_tags',
+  {
+    id: {
+      primaryKey: true,
+      type: Sequelize.BIGINT,
+      autoIncrement: true,
+    },
+    studyGuidanceGroupId: {
+      type: Sequelize.STRING,
+      unique: true,
+    },
+    studyProgramme: {
+      type: Sequelize.STRING,
+    },
+    year: {
+      type: Sequelize.STRING,
+    },
+  },
+  { underscored: true, timestamps: true }
+)
+
 MandatoryCourse.belongsTo(MandatoryCourseLabels, {
  foreignKey: 'label',
  sourceKey: 'id',
@@ -157,4 +179,5 @@ module.exports = {
  CustomPopulationSearch,
  sequelizeKone,
  ExcludedCourse,
+  StudyGuidanceGroupTag,
 }
@@ -1,7 +1,7 @@
 const Sequelize = require('sequelize')
-const { sequelize } = require('../database/connection')
+const { sequelizeUser } = require('../database/connection')

-const UserElementDetails = sequelize.define(
+const UserElementDetails = sequelizeUser.define(
  'user_elementdetails',
  {
    userId: {
@@ -18,7 +18,7 @@ const UserElementDetails = sequelize.define(
  }
 )

-const AccessGroup = sequelize.define('access_group', {
+const AccessGroup = sequelizeUser.define('access_group', {
  id: {
    primaryKey: true,
    type: Sequelize.BIGINT,
@@ -39,7 +39,7 @@ const AccessGroup = sequelize.define('access_group', {
  },
 })

-const UserFaculties = sequelize.define('user_faculties', {
+const UserFaculties = sequelizeUser.define('user_faculties', {
  userId: {
    primaryKey: true,
    type: Sequelize.BIGINT,
@@ -59,24 +59,8 @@ const UserFaculties = sequelize.define('user_faculties', {
    type: Sequelize.DATE,
  },
 })
-const FacultyProgrammes = sequelize.define('faculty_programmes', {
-  faculty_code: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  programme_code: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})

-const User = sequelize.define(
+const User = sequelizeUser.define(
  'users',
  {
    id: {
@@ -99,25 +83,7 @@ const User = sequelize.define(
    timestamps: false,
  }
 )
-const HyGroup = sequelize.define('hy_group', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true,
-  },
-
-  code: { type: Sequelize.STRING },
-
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})
-
-const Affiliation = sequelize.define('affiliations', {
+const HyGroup = sequelizeUser.define('hy_group', {
  id: {
    primaryKey: true,
    type: Sequelize.BIGINT,
@@ -147,26 +113,13 @@ AccessGroup.belongsToMany(User, { through: 'user_accessgroup' })
 User.belongsToMany(HyGroup, { through: 'user_hy_group', as: 'hy_group' })
 HyGroup.belongsToMany(User, { through: 'user_hy_group' })

-User.belongsToMany(Affiliation, {
-  through: 'user_affiliation',
-  as: 'affiliation',
-})
-Affiliation.belongsToMany(User, { through: 'user_affiliation' })
-
 User.hasMany(UserFaculties, { as: 'faculty', foreignKey: 'userId' })
-UserFaculties.hasMany(FacultyProgrammes, {
-  as: 'programme',
-  foreignKey: 'faculty_code',
-  sourceKey: 'faculty_code',
-})

 module.exports = {
  User,
  UserElementDetails,
  AccessGroup,
  HyGroup,
-  Affiliation,
  UserFaculties,
-  FacultyProgrammes,
-  sequelize,
+  sequelizeUser,
 }
@@ -17,7 +17,6 @@ Student.init(
    dateofuniversityenrollment: { type: DATE },
    email: { type: STRING },
    national_student_number: { type: STRING },
-    home_county_id: { type: INTEGER },
    country_fi: { type: STRING },
    country_sv: { type: STRING },
    country_en: { type: STRING },
@@ -25,6 +24,7 @@ Student.init(
    home_country_sv: { type: STRING },
    home_country_en: { type: STRING },
    gender_code: { type: INTEGER },
+    sis_person_id: { type: STRING },
    createdAt: {
      type: DATE,
    },
@@ -1,50 +1,52 @@
-const shibbolethHeadersFix = require('unfuck-utf8-headers-middleware')
-const accessLogger = require('./middleware/accesslogger')
-const sentryUserId = require('./middleware/sentryUserId')
+const Sentry = require('@sentry/node')
+const express = require('express')
+const cors = require('cors')
+const { frontUrl } = require('./conf-backend')
+const shibbolethCharsetMiddleware = require('./middleware/shibbolethCharsetMiddleware')
+const matomoInit = require('./routes/matomo-init')
+const currentUserMiddleware = require('./middleware/currentUserMiddleware')
+const accessLogger = require('./middleware/accessLogger')
+const auth = require('./middleware/auth')
+
 const courses = require('./routes/courses')
 const students = require('./routes/students')
 const population = require('./routes/population')
 const login = require('./routes/login')
-const superlogin = require('./routes/superlogin')
 const language = require('./routes/language')
 const users = require('./routes/users')
 const elementdetails = require('./routes/elementdetails')
-const auth = require('./middleware/auth')
 const teachers = require('./routes/teachers')
 const providers = require('./routes/providers')
 const faculties = require('./routes/faculties')
 const semesters = require('./routes/semesters')
 const mandatoryCourses = require('./routes/mandatorycourses')
 const mandatoryCourseLabels = require('./routes/mandatorycourselabels')
-const ping = require('./routes/ping')
 const feedback = require('./routes/feedback')
 const tags = require('./routes/tags')
 const updater = require('./routes/updater')
 const tsaAnalytics = require('./routes/tsaAnalytics')
-const matomoInit = require('./routes/matomo-init')
 const customPopulationSearch = require('./routes/customPopulationSearch')
 const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
 const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
+const studyProgramme = require('./routes/studyProgramme')
+const initializeSentry = require('./util/sentry')
+
+const errorMiddleware = require('./middleware/errorMiddleware')

 module.exports = (app, url) => {
-  app.use(url, ping)
-  app.use(
-    shibbolethHeadersFix([
-      'hyGroupCn',
-      'SHIB_LOGOUT_URL',
-      'eduPersonAffiliation',
-      'uid',
-      'displayName',
-      'mail',
-      'hyPersonSisuId',
-      'preferredLanguage',
-    ])
-  )
+  initializeSentry(app)
+  app.use(Sentry.Handlers.requestHandler())
+  app.use(Sentry.Handlers.tracingHandler())
+
+  app.use(cors({ credentials: true, origin: frontUrl }))
+  app.use(express.json())
+
+  app.use(shibbolethCharsetMiddleware)
  app.use(url, matomoInit)
+  app.use(currentUserMiddleware)
+  app.use(accessLogger)
  app.use(url, login)
-  app.use(`${url}/superlogin`, superlogin)
-  app.use(auth.checkAuth, auth.checkRequiredGroup, auth.checkUserBlacklisting, accessLogger, sentryUserId)
  app.use(url, elementdetails)
  app.use(url, courses)
  app.use(url, students)
@@ -54,6 +56,7 @@ module.exports = (app, url) => {
  app.use(url, semesters)
  app.use(url, tags)
  app.use(url, programmeModules)
+  app.use(url, studyProgramme)
  app.use(`${url}/faculties`, faculties)
  app.use(`${url}/updater`, auth.roles(['admin']), updater)
  app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
@@ -65,4 +68,10 @@ module.exports = (app, url) => {
  app.use(`${url}/custom-population-search`, customPopulationSearch)
  app.use(`${url}/cool-data-science`, trends)
  app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
+  app.get('*', async (_, res) => {
+    const results = { error: 'unknown endpoint' }
+    res.status(404).json(results)
+  })
+  app.use(Sentry.Handlers.errorHandler())
+  app.use(errorMiddleware)
 }
@@ -42,6 +42,4 @@ router.get('/v3/courseyearlystats', async (req, res) => {
  }
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -8,7 +8,7 @@ const {

 router.get('/', async (req, res) => {
  const {
-    decodedToken: { id },
+    user: { id },
  } = req
  const customPopulationSearches = await getCustomPopulationSearchesByUser(id)
  res.json(customPopulationSearches)
@@ -16,7 +16,7 @@ router.get('/', async (req, res) => {

 router.post('/', async (req, res) => {
  const {
-    decodedToken: { id },
+    user: { id },
    body: { name, students },
  } = req

@@ -29,7 +29,7 @@ router.post('/', async (req, res) => {

 router.put('/:id', async (req, res) => {
  const {
-    decodedToken: { id: userId },
+    user: { id: userId },
    body: { students },
  } = req
  const { id } = req.params
@@ -46,7 +46,7 @@ router.put('/:id', async (req, res) => {

 router.delete('/:id', async (req, res) => {
  const {
-    decodedToken: { id: userId },
+    user: { id: userId },
  } = req
  const { id } = req.params

@@ -188,6 +188,4 @@ router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
  }
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -1,14 +1,15 @@
 const router = require('express').Router()
-const userService = require('../services/userService')
+const { getUserDataFor } = require('../services/userService')
 const { sendFeedbackToToska } = require('../services/mailservice')
 const logger = require('../util/logger')

 router.post('/email', async (req, res) => {
  const { content } = req.body
  const {
-    decodedToken: { userId },
+    user: { userId },
  } = req
-  const { email, full_name } = await userService.byUsername(userId)
+
+  const { email, full_name } = await getUserDataFor(userId)

  const result = await sendFeedbackToToska({
    feedbackContent: content,
@@ -2,8 +2,10 @@ const router = require('express').Router()
 const userService = require('../services/userService')

 router.post('/language', async (req, res) => {
-  const { language } = req.body
-  const { userId } = req.decodedToken
+  const {
+    body: { language },
+    user: { userId },
+  } = req
  if (!['fi', 'sv', 'en'].includes(language)) {
    return res.status(400).json('invalid language')
  }
@@ -1,47 +1,22 @@
 const router = require('express').Router()
-const userService = require('../services/userService')
-const { parseHyGroups } = require('../util/utils')
-const { sendNotificationAboutNewUser } = require('../services/mailservice')
-const logger = require('../util/logger')
+const { ApplicationError } = require('../util/customErrors')

-router.post('/login', async (req, res) => {
-  const uid = req.headers['uid']
-  if (req.headers['shib-session-id'] && uid) {
-    const full_name = req.headers.displayname || ''
-    const mail = req.headers.mail || ''
-    const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-    const affiliations = parseHyGroups(req.headers['edupersonaffiliation'])
-    const hyPersonSisuId = req.headers.hypersonsisuid || ''
-    logger.info(`${uid} trying to login, referring to userservice`)
-    let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail, hyPersonSisuId)
-    if (isNew) {
-      const result = await sendNotificationAboutNewUser({ userId: uid, userFullName: full_name })
-      if (result.error) {
-        return res.status(400).json(result).end()
-      }
-    }
-    res.status(200).json({ token })
-  } else {
-    res
-      .status(401)
-      .json({
-        message: `Not enough headers login, uid: ${req.headers.uid}
-        session-id ${req.headers['shib-session-id']}`,
-      })
-      .end()
+router.get('/login', async (req, res) => {
+  const { user: userFromReq, logoutUrl } = req
+
+  if (!userFromReq) {
+    throw new ApplicationError('User not found', 404)
  }
-})
-
-router.delete('/logout', async (req, res) => {
-  const logoutUrl = req.headers.shib_logout_url
-  const { returnUrl } = req.body
-  if (logoutUrl) {
-    return res
-      .status(200)
-      .send({ logoutUrl: `${logoutUrl}?return=${returnUrl}` })
-      .end()
+  const roles = userFromReq.roles.map(role => role.group_code)
+  const user = {
+    ...userFromReq,
+    roles,
+    isAdmin: roles.includes('admin'),
  }
-  res.status(200).send({ logoutUrl: returnUrl }).end()
-})

+  res.send({
+    user,
+    logoutUrl,
+  })
+})
 module.exports = router
@@ -28,7 +28,7 @@ _paq.push(["enableLinkTracking"]);

 const script = config.MATOMO_URL ? acual : ''

-router.get('/matomo-init', (req, res) => {
+router.get('/matomo-init', (_, res) => {
  res.contentType('text/javascript; charset=UTF-8')
  res.send(script)
 })
@@ -1,85 +0,0 @@
-/* eslint-disable */
-
-const { router, wrapper } = require('./router').routerWithWrapper()
-const axios = require('axios')
-
-wrapper.get('/ping', async (req, res) => {
-  res.json({
-    message: 'pong'
-  })
-})
-
-wrapper.get('/v3/mandatory_courses/:code', async (req, res) => {
-  const courses = []
-  const included = new Set()
-
-  const dumb_flatten = module => {
-    if (module.children) {
-      module.children.forEach(child => {
-        dumb_flatten(child)
-      })
-    } else if (module.code && !module.code.startsWith('KK') && !courses.map(c => c.code).includes(module.code)) {
-      courses.push({
-        name: {
-          fi: module.name,
-          en: module.name
-        },
-        code: module.code,
-        label: { id: `${courses.length}`, label: module.code.replace('-', '').slice(0, 4), orderNumber: courses.length }
-      })
-    }
-  }
-
-  const better_flatten = (module, label) => {
-    if (module.children) {
-      module.children.forEach(child => {
-        better_flatten(child, module.module ? module.module.name : label)
-      })
-    } else if (module.code && !module.code.startsWith('KK')) {
-      courses.push({
-        name: {
-          fi: module.name,
-          en: module.name
-        },
-        code: module.code,
-        label: { id: `${courses.length}`, label, orderNumber: courses.length }
-      })
-    }
-  }
-
-  const superFlattenFlatten = (module, label) => {
-    if (module.children) {
-      module.children.forEach(child => {
-        superFlattenFlatten(child, module.module ? module.module.name : label)
-      })
-    } else if (Array.isArray(module)) {
-      module.forEach(elem => superFlattenFlatten(elem, label))
-    } else if (module.code && !module.code.startsWith('KK')) {
-      if (!included.has(`${label}${module.code}`)) {
-        included.add(`${label}${module.code}`)
-        courses.push({
-          name: {
-            fi: module.name,
-            en: module.name
-          },
-          code: module.code,
-          label: { id: `${courses.length}`, label, orderNumber: courses.length }
-        })
-      }
-    }
-  }
-
-  const code = req.params.code
-  const response = await axios.get(`http://sis-updater-scheduler:8082/v1/courses/${code}?token=dev&superFlatten=true`)
-
-  // dumb_flatten(response.data)
-  // better_flatten(response.data)
-  superFlattenFlatten(response.data)
-
-  const byCode = (c1, c2) => (c1.code < c2.code ? -1 : 1)
-
-  courses.sort(byCode)
-  res.json(courses)
-})
-
-module.exports = router
@@ -83,7 +83,7 @@ router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) =>
  let studentnumberlist
  const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
  const {
-    decodedToken: { userId },
+    user: { userId },
    roles,
  } = req

@@ -123,7 +123,7 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
  let studentnumberlist
  const studentnumbers = await Student.findByTag(tag)
  const {
-    decodedToken: { userId },
+    user: { userId },
    roles,
  } = req
  if (roles && roles.includes('admin')) {
@@ -160,7 +160,7 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
  }
  let studentnumberlist
  const {
-    decodedToken: { userId, sisPersonId },
+    user: { userId, sisPersonId },
    roles,
  } = req

@@ -195,7 +195,7 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r

 router.get('/v3/populationstatistics', async (req, res) => {
  const { year, semesters, studyRights: studyRightsJSON } = req.query
-  const { decodedToken } = req
+  const { user } = req

  if (!year || !semesters || !studyRightsJSON) {
    res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
@@ -245,7 +245,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
      return
    }

-    res.json(filterPersonalTags(result, decodedToken.id))
+    res.json(filterPersonalTags(result, user.id))
  } else {
    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
    if (result.error) {
@@ -254,13 +254,13 @@ router.get('/v3/populationstatistics', async (req, res) => {
      return
    }

-    res.json(filterPersonalTags(result, decodedToken.id))
+    res.json(filterPersonalTags(result, user.id))
  }
 })

 router.get('/v3/populationstatisticsbytag', async (req, res) => {
  const { tag, studyRights: studyRightsJSON, months, year } = req.query
-  const { decodedToken } = req
+  const { user } = req

  if (!tag) return res.status(400).json({ error: 'The query should have a tag defined' })
  const foundTag = await TagService.findTagById(tag)
@@ -270,7 +270,7 @@ router.get('/v3/populationstatisticsbytag', async (req, res) => {
  let studentnumberlist
  const studentnumbers = await Student.findByTag(tag)
  const {
-    decodedToken: { userId },
+    user: { userId },
    roles,
  } = req
  if (roles && roles.includes('admin')) {
@@ -301,12 +301,12 @@ router.get('/v3/populationstatisticsbytag', async (req, res) => {
    return
  }

-  res.json(filterPersonalTags(result, decodedToken.id))
+  res.json(filterPersonalTags(result, user.id))
 })

 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
  const { coursecodes, from, to, separate: sep } = req.query
-  const { decodedToken, roles } = req
+  const { user, roles } = req
  const separate = sep ? JSON.parse(sep) : false

  if (!coursecodes || !from || !to) {
@@ -339,7 +339,7 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
  if (roles && roles.includes('admin')) {
    studentsUserCanAccess = new Set(studentnumbers)
  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(decodedToken.userId)
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(user.userId)
    const codes = unitsUserCanAccess.map(unit => unit.id)
    studentsUserCanAccess = new Set(await Student.filterStudentnumbersByAccessrights(studentnumbers, codes))
  }
@@ -368,25 +368,23 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
    return
  }

-  res.json(filterPersonalTags(result, decodedToken.id))
+  res.json(filterPersonalTags(result, user.id))
 })

 router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
  const { studentnumberlist, usingStudyGuidanceGroups } = req.body
-  const { roles, decodedToken } = req
+  const { roles, user } = req

  if (usingStudyGuidanceGroups) {
-    if (!decodedToken.sisPersonId) {
-      logger.error(
-        `User ${decodedToken.userId} tried to get person groups but personId was ${decodedToken.sisPersonId} in header`
-      )
+    if (!user.sisPersonId) {
+      logger.error(`User ${user.userId} tried to get person groups but personId was ${user.sisPersonId} in header`)
      return res.status(400).json({ error: 'Not possible to get groups without personId' })
    }
  }

  const studentsUserCanAccess = usingStudyGuidanceGroups
-    ? await getAllStudentsUserHasInGroups(decodedToken.sisPersonId)
-    : await UserService.getStudentsUserCanAccess(studentnumberlist, roles, decodedToken.userId)
+    ? await getAllStudentsUserHasInGroups(user.sisPersonId)
+    : await UserService.getStudentsUserCanAccess(studentnumberlist, roles, user.userId)
  const filteredStudentNumbers = studentnumberlist.filter(s => studentsUserCanAccess.has(s))

  const result = await Population.optimizedStatisticsOf(
@@ -402,7 +400,7 @@ router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
    Sentry.captureException(new Error(result.error))
    return
  }
-  res.status(200).json(filterPersonalTags(result, decodedToken.id))
+  res.status(200).json(filterPersonalTags(result, user.id))
 })

 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
@@ -427,6 +425,4 @@ router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req
  return res.json(maxYearsToCreatePopulationFrom)
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -6,6 +6,4 @@ router.get('/semesters/codes', async (req, res) => {
  res.json(providers)
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -2,7 +2,7 @@ const router = require('express').Router()
 const Student = require('../services/students')
 const userService = require('../services/userService')
 const Unit = require('../services/units')
-const { mismatchedStudents } = require('../services/student_credit_total_mismatches')
+const { getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')

 const filterStudentTags = (student, userId) => {
  return {
@@ -11,17 +11,10 @@ const filterStudentTags = (student, userId) => {
  }
 }

-const creditTotalMismatch = student => {
-  return {
-    ...student,
-    mismatch: mismatchedStudents.has(student.studentNumber),
-  }
-}
-
 router.get('/students', async (req, res) => {
  const {
    roles,
-    decodedToken: { userId },
+    user: { userId, sisPersonId },
    query: { searchTerm },
  } = req

@@ -45,26 +38,29 @@ router.get('/students', async (req, res) => {
  } else {
    const unitsUserCanAccess = await userService.getUnitsFromElementDetails(userId)
    const codes = unitsUserCanAccess.map(unit => unit.id)
+    let extraStudents = []
+    if (roles.includes('studyGuidanceGroups')) {
+      const matchingOnlyByTerm = await Student.bySearchTerm(trimmedSearchTerm)
+      const studentsUserCanAccess = await getAllStudentsUserHasInGroups(sisPersonId)
+      extraStudents.push(...matchingOnlyByTerm.filter(s => studentsUserCanAccess.has(s.studentNumber)))
+    }
    const matchingStudents = await Student.bySearchTermAndElements(trimmedSearchTerm, codes)
-    res.json(matchingStudents)
+    res.json([...matchingStudents, ...extraStudents])
  }
 })

 router.get('/students/:id', async (req, res) => {
  const { id: studentId } = req.params
-  const { roles, decodedToken } = req
+  const { roles, user } = req

-  if (roles && roles.includes('admin')) {
+  if (roles?.includes('admin')) {
    const results = await Student.withId(studentId)
    return results.error
      ? res.status(400).json({ error: 'error finding student' }).end()
-      : res
-          .status(200)
-          .json(creditTotalMismatch(filterStudentTags(results, decodedToken.id)))
-          .end()
+      : res.status(200).json(filterStudentTags(results, user.id)).end()
  }

-  const uid = req.decodedToken.userId
+  const uid = req.user.userId
  const student = await Student.withId(studentId)
  if (student.error) {
    return res.status(400).json({ error: 'error finding student' }).end()
@@ -78,13 +74,18 @@ router.get('/students/:id', async (req, res) => {
    })
  )

+  if (roles?.includes('studyGuidanceGroups')) {
+    const studentsUserCanAccess = await getAllStudentsUserHasInGroups(user.sisPersonId)
+    if (studentsUserCanAccess.has(studentId)) {
+      return res.status(200).json(filterStudentTags(student, user.id)).end()
+    }
+  }
+
  if (rights.some(right => right !== null)) {
-    res.status(200).json(filterStudentTags(student, decodedToken.id)).end()
+    res.status(200).json(filterStudentTags(student, user.id)).end()
  } else {
    res.status(400).json({ error: 'error finding student' }).end()
  }
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -1,10 +1,10 @@
 const router = require('express').Router()
 const { logger } = require('../util/logger')
-const { getAllGroupsAndStudents } = require('../services/studyGuidanceGroups')
+const { getAllGroupsAndStudents, changeGroupTags } = require('../services/studyGuidanceGroups')

 router.get('/', async (req, res) => {
  const {
-    decodedToken: { uid, sisPersonId },
+    user: { uid, sisPersonId },
  } = req
  if (!sisPersonId) {
    logger.error(`User ${uid} tried to get person groups but personId was ${sisPersonId} in header`)
@@ -13,4 +13,27 @@ router.get('/', async (req, res) => {
  return res.json(await getAllGroupsAndStudents(sisPersonId))
 })

+router.put('/:id/tags', async (req, res) => {
+  const {
+    user: { uid, sisPersonId },
+    body: tags,
+    params: { id: groupId },
+  } = req
+
+  if (!sisPersonId) {
+    logger.error(`User ${uid} tried to change study guidance group tags but personId was ${sisPersonId} in header`)
+    return res.status(400).json({ error: 'Not possible to change groups without personId' })
+  }
+
+  const groupsUserHasAccessTo = (await getAllGroupsAndStudents(sisPersonId)).map(group => group.id)
+
+  if (!groupsUserHasAccessTo.includes(groupId)) {
+    logger.error(
+      `User ${uid} tried to change study guidance group tags, but didn't have permission for the group ${groupId}`
+    )
+    return res.status(400).json({ error: 'Access denied' })
+  }
+  return res.status(200).json(await changeGroupTags({ groupId, tags }))
+})
+
 module.exports = router
@@ -0,0 +1,129 @@
+const router = require('express').Router()
+const {
+  getBasicStatsForStudytrack,
+  getCreditStatsForStudytrack,
+  getGraduationStatsForStudytrack,
+} = require('../services/studyprogrammeStats')
+const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
+
+const {
+  getBasicStats,
+  setBasicStats,
+  getCreditStats,
+  setCreditStats,
+  getGraduationStats,
+  setGraduationStats,
+  getStudytrackStats,
+  setStudytrackStats,
+} = require('../services/analyticsService')
+const logger = require('../util/logger')
+
+router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+
+  if (code) {
+    let data = null
+    try {
+      data = await getBasicStats(code, yearType)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} basic stats: ${e}`)
+    }
+    if (!data) {
+      try {
+        const result = await getBasicStatsForStudytrack({
+          studyprogramme: req.params.id,
+          yearType,
+        })
+        data = await setBasicStats(result, yearType)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} basic stats: ${e}`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+
+  if (code) {
+    let data = null
+    try {
+      data = await getCreditStats(code, yearType)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} credit stats`)
+    }
+    if (!data) {
+      try {
+        const result = await getCreditStatsForStudytrack({
+          studyprogramme: req.params.id,
+          yearType,
+        })
+        data = await setCreditStats(result, yearType)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} credit stats`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+
+  if (code) {
+    let data = null
+    try {
+      data = await getGraduationStats(code, yearType)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} graduation stats`)
+    }
+    if (!data) {
+      try {
+        const result = await getGraduationStatsForStudytrack({
+          studyprogramme: req.params.id,
+          yearType,
+        })
+        data = await setGraduationStats(result, yearType)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} graduation stats`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
+  const code = req.params.id
+
+  if (code) {
+    let data = null
+    try {
+      data = await getStudytrackStats(code)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} studytrack stats: ${e}`)
+    }
+    if (!data) {
+      try {
+        const result = await getStudytrackStatsForStudyprogramme({ studyprogramme: req.params.id })
+        data = await setStudytrackStats(result)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+module.exports = router
@@ -1,21 +0,0 @@
-const router = require('express').Router()
-const userService = require('../services/userService')
-
-router.post('/:uid', async (req, res) => {
-  const uid = req.headers['uid']
-  if (req.headers['shib-session-id'] && uid) {
-    const asUser = req.params.uid
-    const token = await userService.superlogin(uid, asUser)
-    res.status(200).json(token)
-  } else {
-    res
-      .status(401)
-      .json({
-        message: `Not enough headers login, uid:
-        ${req.headers.uid} session-id ${req.headers['shib-session-id']}`,
-      })
-      .end()
-  }
-})
-
-module.exports = router
@@ -19,41 +19,37 @@ router.get('/tags', async (req, res) => {

 router.get('/tags/:studytrack', async (req, res) => {
  const { studytrack } = req.params
-  const { rights, roles, decodedToken } = req
+  const { rights, roles, user } = req

  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()

  const tags = await Tags.findTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantTags(tags, decodedToken.id))
+  res.status(200).json(filterRelevantTags(tags, user.id))
 })

 router.post('/tags', async (req, res) => {
  const {
    tag: { studytrack, tagname, year, personal_user_id },
  } = req.body
-  const { rights, roles, decodedToken } = req
+  const { rights, roles, user } = req

  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()

  await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
  const tags = await Tags.findTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantTags(tags, decodedToken.id))
+  res.status(200).json(filterRelevantTags(tags, user.id))
 })

 router.delete('/tags', async (req, res) => {
  const { tag } = req.body
-  const { rights, roles, decodedToken } = req
+  const { rights, roles, user } = req

-  if (
-    !rights.includes(tag.studytrack) &&
-    !(roles && roles.includes('admin')) &&
-    !(tag.personal_user_id === decodedToken.id)
-  )
+  if (!rights.includes(tag.studytrack) && !(roles && roles.includes('admin')) && !(tag.personal_user_id === user.id))
    return res.status(403).end()

  await Tags.deleteTag(tag)
  const t = await Tags.findTagsByStudytrack(tag.studytrack)
-  res.status(200).json(filterRelevantTags(t, decodedToken.id))
+  res.status(200).json(filterRelevantTags(t, user.id))
 })

 router.get('/studenttags', async (req, res) => {
@@ -63,35 +59,35 @@ router.get('/studenttags', async (req, res) => {

 router.get('/studenttags/:studytrack', async (req, res) => {
  const { studytrack } = req.params
-  const { rights, roles, decodedToken } = req
+  const { rights, roles, user } = req

  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()

  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, user.id))
 })

 router.get('/studenttags/:studentnumber', async (req, res) => {
  const { studentnumber } = req.params
-  const { decodedToken } = req
+  const { user } = req
  const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, user.id))
 })

 router.post('/studenttags/:studentnumber', async (req, res) => {
  const { tag, studytrack } = req.body
-  const { rights, roles, decodedToken } = req
+  const { rights, roles, user } = req

  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()

  await TagStudent.createStudentTag(tag)
  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, user.id))
 })

 router.post('/studenttags', async (req, res) => {
  const { tags, studytrack } = req.body
-  const { rights, roles, decodedToken } = req
+  const { rights, roles, user } = req

  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()

@@ -110,12 +106,12 @@ router.post('/studenttags', async (req, res) => {

  await TagStudent.createMultipleStudentTags(tags)
  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, user.id))
 })

 router.delete('/studenttags/delete_one', async (req, res) => {
  const { tag_id, studentnumber, studytrack } = req.body
-  const { rights, roles, decodedToken } = req
+  const { rights, roles, user } = req

  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()

@@ -124,12 +120,12 @@ router.delete('/studenttags/delete_one', async (req, res) => {

  await TagStudent.deleteStudentTag(studentnumber, tag_id)
  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, user.id))
 })

 router.delete('/studenttags/delete_many', async (req, res) => {
  const { tagId, studentnumbers, studytrack } = req.body
-  const { rights, roles, decodedToken } = req
+  const { rights, roles, user } = req

  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()

@@ -138,9 +134,7 @@ router.delete('/studenttags/delete_many', async (req, res) => {

  await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, user.id))
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -78,6 +78,4 @@ router.get('/:id', async (req, res) => {
  res.json(result)
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -9,8 +9,12 @@ router.post('/event', (req, res) => {
    return res.status(400).json({ error: 'group, name and value are required' })
  }

+  const {
+    user: { userId },
+  } = req
+
  // don't await here because frontend doesn't care if it succeeds
-  tsaService.sendTsaEvent(req.decodedToken.userId, { group, name, label, value }).catch(e => {
+  tsaService.sendTsaEvent(userId, { group, name, label, value }).catch(e => {
    Sentry.withScope(scope => {
      // if error was caused by a 400, dig out the JSON response for Sentry
      if (e.isAxiosError && e.response && e.response.data) {
@@ -1,18 +1,8 @@
 const router = require('express').Router()
 const userService = require('../services/userService')
-const blacklist = require('../services/blacklist')
-const { userDataCache } = require('../services/cache')
 const { sendNotificationAboutAccessToUser, previewNotificationAboutAccessToUser } = require('../services/mailservice')
 const logger = require('../util/logger')

-const addUserToBlacklist = async (user, decodedToken) => {
-  if (user) {
-    const { username } = user
-    userDataCache.del(username)
-    if (username != decodedToken.userId) await blacklist.addUserToBlacklist(username)
-  }
-}
-
 router.get('/', async (req, res) => {
  const results = await userService.findAll()
  res.json(results)
@@ -25,10 +15,7 @@ router.get('/access_groups', async (req, res) => {

 router.post('/modifyaccess', async (req, res) => {
  try {
-    const { uid } = req.body
    const result = await userService.modifyAccess(req.body)
-    const user = await userService.byId(uid)
-    await addUserToBlacklist(user, req.decodedToken)
    res.status(200).json(result)
  } catch (e) {
    res.status(400).json(e)
@@ -58,7 +45,6 @@ router.post('/:uid/elements', async (req, res) => {
  const { uid } = req.params
  const { codes } = req.body
  const user = await userService.enableElementDetails(uid, codes)
-  await addUserToBlacklist(user, req.decodedToken)
  res.json(user)
 })

@@ -66,7 +52,6 @@ router.delete('/:uid/elements', async (req, res) => {
  const { uid } = req.params
  const { codes } = req.body
  const user = await userService.removeElementDetails(uid, codes)
-  await addUserToBlacklist(user, req.decodedToken)
  res.json(user)
 })

@@ -74,7 +59,6 @@ router.post('/:uid/faculties', async (req, res) => {
  const { uid } = req.params
  const { faculties } = req.body
  const user = await userService.setFaculties(uid, faculties)
-  await addUserToBlacklist(user, req.decodedToken)
  res.json(user)
 })

@@ -3,6 +3,10 @@ const moment = require('moment')

 const createRedisKeyForProductivity = id => `PRODUCTIVITY_${id}`
 const createRedisKeyForThroughput = id => `THROUGHPUT_${id}`
+const createRedisKeyForBasicStats = (id, yearType) => `BASIC_STATS_${id}_${yearType}`
+const createRedisKeyForCreditStats = (id, yearType) => `CREDIT_STATS_${id}_${yearType}`
+const createRedisKeyForGraduationStats = (id, yearType) => `GRADUATION_STATS_${id}_${yearType}`
+const createRedisKeyForStudytrackStats = id => `STUDYTRACK_STATS_${id}`

 const getProductivity = async id => {
  const redisKey = createRedisKeyForProductivity(id)
@@ -87,6 +91,86 @@ const patchThroughput = async data => {
  }
 }

+const getBasicStats = async (id, yearType) => {
+  const redisKey = createRedisKeyForBasicStats(id, yearType)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setBasicStats = async (data, yearType) => {
+  const { id } = data
+  const redisKey = createRedisKeyForBasicStats(id, yearType)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getCreditStats = async (id, yearType) => {
+  const redisKey = createRedisKeyForCreditStats(id, yearType)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setCreditStats = async (data, yearType) => {
+  const { id } = data
+  const redisKey = createRedisKeyForCreditStats(id, yearType)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getGraduationStats = async (id, yearType) => {
+  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setGraduationStats = async (data, yearType) => {
+  const { id } = data
+  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getStudytrackStats = async id => {
+  const redisKey = createRedisKeyForStudytrackStats(id)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setStudytrackStats = async data => {
+  const { id } = data
+  const redisKey = createRedisKeyForStudytrackStats(id)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
 module.exports = {
  getProductivity,
  setProductivity,
@@ -94,4 +178,12 @@ module.exports = {
  getThroughput,
  setThroughput,
  patchThroughput,
+  getBasicStats,
+  setBasicStats,
+  getCreditStats,
+  setCreditStats,
+  getGraduationStats,
+  setGraduationStats,
+  getStudytrackStats,
+  setStudytrackStats,
 }
@@ -1,22 +0,0 @@
-const { redisClient } = require('./redis')
-const moment = require('moment')
-const logger = require('../util/logger')
-
-const MAKE_REDIS_HASH_KEY = id => `BLACKLIST ${id}`
-
-const addUserToBlacklist = async userId => {
-  logger.info(`user ${userId} added to token blacklist`)
-  await redisClient.setAsync(MAKE_REDIS_HASH_KEY(userId), moment().toISOString(false))
-}
-
-const isUserBlacklisted = async (userId, tokenCreatedAt) => {
-  const isBlacklisted = async () => {
-    const blacklistTimestamp = await redisClient.getAsync(MAKE_REDIS_HASH_KEY(userId))
-    return tokenCreatedAt && blacklistTimestamp && moment(tokenCreatedAt) < moment(blacklistTimestamp)
-  }
-
-  const timer = () => new Promise(resolve => setTimeout(() => resolve(true), 5000))
-  return Promise.race([isBlacklisted(), timer()])
-}
-
-module.exports = { addUserToBlacklist, isUserBlacklisted }
@@ -13,7 +13,7 @@ const {
 const { parseCredit } = require('./parseCredits')
 const Op = Sequelize.Op
 const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
-const _ = require('lodash')
+const { sortMainCode, getSortRank } = require('../util/utils')

 const byNameOrCode = (searchTerm, language) =>
  Course.findAll({
@@ -59,7 +59,15 @@ const findOneByCode = code => {
  })
 }

-const creditsForCourses = async (codes, anonymizationSalt) => {
+const creditsForCourses = async (codes, anonymizationSalt, unifyOpenUniCourses) => {
+  let is_open = false
+
+  if (unifyOpenUniCourses) {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
  const credits = await Credit.findAll({
    include: [
      {
@@ -105,9 +113,11 @@ const creditsForCourses = async (codes, anonymizationSalt) => {
      student_studentnumber: {
        [Op.ne]: null,
      },
+      [Op.or]: [{ is_open }, { is_open: null }],
    },
    order: [['attainment_date', 'ASC']],
  })
+
  const parsedCredits = credits.map(credit => parseCredit(credit, anonymizationSalt))
  return parsedCredits
 }
@@ -144,47 +154,20 @@ const unifyOpenUniversity = code => {
  return regexresult[1]
 }

-const allCodeAltenatives = async code => {
-  let course = await Course.findAll({
-    raw: true,
-    attributes: ['id', 'code', 'substitutions'],
-    where: {
-      code: code,
-    },
+const allCodeAlternatives = async code => {
+  const course = await Course.findOne({
+    where: { code: code },
  })

-  const allSubstitutions = _.flatten(course.map(c => c.substitutions))
-
-  let subcodes = []
-  if (allSubstitutions) {
-    subcodes = await Course.findAll({
-      raw: true,
-      attributes: ['code'],
-      where: {
-        id: {
-          [Op.in]: allSubstitutions,
-        },
-      },
-    })
-  }
-
-  const courses = [...course, ...subcodes]
-  const temp = courses
-    .map(c => c.code)
-    .map(c => {
-      if (c.match(/^AY?/)) return [c, 4] // open university codes come last
-      if (c.match(/^\d/)) return [c, 2] // old numeric codes come second
-      if (c.match(/^[A-Za-z]/)) return [c, 1] // new letter based codes come first
-      return [c, 3] // unknown, comes before open uni?
-    })
-    .sort((a, b) => a[1] - b[1])
-    .map(c => c[0])
-
-  return allSubstitutions ? _.uniq(temp) : [code]
+  return sortMainCode([...course.substitutions, code])
 }

 const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anonymizationSalt) => {
-  let codes = await allCodeAltenatives(coursecode)
+  const courseForSubs = await Course.findOne({
+    where: { code: coursecode },
+  })
+
+  let codes = courseForSubs.substitutions ? sortMainCode([...courseForSubs.substitutions, coursecode]) : [coursecode]

  if (isOpenUniCourseCode(coursecode) && !unifyOpenUniCourses) {
    codes = [coursecode]
@@ -195,7 +178,7 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
  }

  const [credits, course] = await Promise.all([
-    creditsForCourses(codes, anonymizationSalt),
+    creditsForCourses(codes, anonymizationSalt, unifyOpenUniCourses),
    Course.findOne({
      where: {
        code: coursecode,
@@ -204,6 +187,7 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
  ])

  const counter = new CourseYearlyStatsCounter()
+
  for (let credit of credits) {
    const {
      studentnumber,
@@ -357,10 +341,6 @@ const codeLikeTerm = code =>

 const byNameAndOrCodeLike = async (name, code) => {
  let rawCourses = await Course.findAll({
-    include: {
-      model: Organization,
-      required: true,
-    },
    where: {
      ...nameLikeTerm(name),
      ...codeLikeTerm(code),
@@ -368,40 +348,10 @@ const byNameAndOrCodeLike = async (name, code) => {
  })

  const courses = rawCourses
-    .map(
-      ({
-        id,
-        code,
-        name,
-        latest_instance_date,
-        is_study_module,
-        coursetypecode,
-        startdate,
-        max_attainment_date,
-        min_attainment_date,
-        createdAt,
-        updatedAt,
-        substitutions,
-        organizations,
-      }) => {
-        return {
-          id,
-          code,
-          name,
-          latest_instance_date,
-          is_study_module,
-          coursetypecode,
-          startdate,
-          max_attainment_date,
-          min_attainment_date,
-          createdAt,
-          updatedAt,
-          substitutions,
-          organizations: organizations.map(o => o.id),
-        }
-      }
-    )
-    .sort(a => (a.code.match(/^[A-Za-z]{3}[0-9]{1}/) ? -1 : 1))
+    .map(course => {
+      return { ...course.dataValues }
+    })
+    .sort((x, y) => getSortRank(y.code) - getSortRank(x.code))

  let substitutionGroupIndex = 0
  const visited = []
@@ -409,7 +359,11 @@ const byNameAndOrCodeLike = async (name, code) => {
  const organizeSubgroups = course => {
    if (visited.includes(course.code)) return

-    let temp = courses.filter(c => (course.substitutions ? course.substitutions.includes(c.id) : false))
+    let temp = []
+    if (course.substitutions !== null) {
+      temp = courses.filter(c => course.substitutions.includes(c.code))
+    }
+
    temp.unshift(course)
    temp.forEach(cu => {
      if (visited.includes(course.code)) return
@@ -437,6 +391,6 @@ module.exports = {
  byCodes,
  maxYearsToCreatePopulationFrom,
  unifyOpenUniversity,
-  allCodeAltenatives,
+  allCodeAlternatives,
  findOneByCode,
 }
@@ -0,0 +1,338 @@
+const sequelize = require('sequelize')
+const { Op } = sequelize
+const {
+  Credit,
+  Course,
+  Organization,
+  Studyright,
+  StudyrightElement,
+  ElementDetail,
+  Transfer,
+  Student,
+  SemesterEnrollment,
+  Semester,
+} = require('../models')
+const { formatStudyright, formatStudent } = require('./studyprogrammeHelpers')
+
+const whereStudents = studentnumbers => {
+  return studentnumbers ? studentnumbers : { [Op.not]: null }
+}
+
+const studytrackStudents = async studentnumbers =>
+  (
+    await Student.findAll({
+      where: {
+        studentnumber: {
+          [Op.in]: studentnumbers,
+        },
+      },
+    })
+  ).map(formatStudent)
+
+const enrolledStudyrights = async studytrack => {
+  const studyrights = await Studyright.findAll({
+    attributes: ['studyrightid', 'studystartdate', 'enddate', 'prioritycode'],
+    include: [
+      {
+        model: StudyrightElement,
+        required: true,
+        where: { code: studytrack },
+      },
+      {
+        model: Student,
+        attributes: ['studentnumber'],
+        required: true,
+      },
+    ],
+  })
+
+  const studentnumbers = studyrights.map(s => s.student.studentnumber)
+
+  const students = await Student.findAll({
+    attributes: ['studentnumber'],
+    include: [
+      {
+        model: Studyright,
+        include: [
+          {
+            model: StudyrightElement,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        ],
+        attributes: ['studyrightid'],
+      },
+      {
+        model: SemesterEnrollment,
+        attributes: ['semestercode'],
+        include: [
+          {
+            model: Semester,
+            required: true,
+          },
+        ],
+        where: {
+          enrollmenttype: 1,
+        },
+      },
+    ],
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumbers,
+      },
+    },
+  })
+
+  let enrollments = new Set()
+
+  students.forEach(student => {
+    student.semester_enrollments.forEach(e => {
+      const enrollmentStartDate = e.semester.startdate
+      const enrollmentEndDate = e.semester.enddate
+      const correctStudyRight = studyrights.find(
+        s =>
+          student.studentnumber === s.student.studentnumber &&
+          s.studystartdate <= enrollmentStartDate && // Studying in the programme should have started on the same day or earlier than the enrollment
+          s.enddate >= enrollmentEndDate && // Studying in the programme should have ended on the same day or later than the enrollment
+          (!s.canceldate || s.canceldate > enrollmentStartDate) && // Studyright should not be canceled OR the possible canceldate should be after the enrollmentStartDate
+          (s.prioritycode === 1 || s.prioritycode === 30) // Studyright should be primary or the student should have graduated
+      )
+      if (correctStudyRight) {
+        enrollments.add({
+          studentnumber: student.studentnumber,
+          startdate: e.semester.startdate,
+        })
+      }
+    })
+  })
+
+  return enrollments
+}
+
+const startedStudyrights = async (studytrack, since, studentnumbers) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: {
+            model: ElementDetail,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        studystartdate: {
+          [Op.gte]: since,
+        },
+        student_studentnumber: whereStudents(studentnumbers),
+      },
+    })
+  ).map(formatStudyright)
+
+const graduatedStudyRights = async (studytrack, since, studentnumbers) =>
+  await Studyright.findAll({
+    include: {
+      model: StudyrightElement,
+      required: true,
+      include: {
+        model: ElementDetail,
+        required: true,
+        where: {
+          code: studytrack,
+        },
+      },
+    },
+    where: {
+      graduated: 1,
+      enddate: {
+        [Op.gte]: since,
+      },
+      student_studentnumber: whereStudents(studentnumbers),
+    },
+  })
+
+const cancelledStudyRights = async (studytrack, since) => {
+  return await Studyright.findAll({
+    include: {
+      model: StudyrightElement,
+      required: true,
+      where: {
+        code: {
+          [Op.eq]: studytrack,
+        },
+      },
+    },
+    where: {
+      canceldate: {
+        [Op.gte]: since,
+      },
+    },
+  })
+}
+
+const transfersAway = async (studytrack, since) => {
+  return await Transfer.findAll({
+    where: {
+      transferdate: {
+        [Op.gte]: since,
+      },
+      sourcecode: studytrack,
+    },
+    distinct: true,
+    col: 'studentnumber',
+  })
+}
+
+const transfersTo = async (studytrack, since) => {
+  return await Transfer.findAll({
+    where: {
+      transferdate: {
+        [Op.gte]: since,
+      },
+      targetcode: studytrack,
+    },
+    distinct: true,
+    col: 'studentnumber',
+  })
+}
+
+const getProgrammesStudents = async studyprogramme =>
+  (
+    await Studyright.findAll({
+      attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode'],
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          where: {
+            code: {
+              [Op.in]: [studyprogramme],
+            },
+          },
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+    })
+  ).map(formatStudyright)
+
+const getCreditsForStudyProgramme = async (provider, since) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      where: {
+        is_study_module: false,
+      },
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.notIn]: [10, 9, 7],
+      },
+      isStudyModule: {
+        [Op.not]: true,
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+    },
+  })
+
+const getTransferredCredits = async (provider, since) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'credittypecode', 'student_studentnumber'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      where: {
+        is_study_module: false,
+      },
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.eq]: [9],
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+    },
+  })
+
+const getThesisCredits = async (provider, since, thesisType) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      where: {
+        course_unit_type: {
+          [Op.in]: thesisType,
+        },
+      },
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.notIn]: [10, 9, 7],
+      },
+      isStudyModule: {
+        [Op.not]: true,
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+    },
+  })
+
+module.exports = {
+  studytrackStudents,
+  enrolledStudyrights,
+  startedStudyrights,
+  graduatedStudyRights,
+  cancelledStudyRights,
+  transfersAway,
+  transfersTo,
+  getProgrammesStudents,
+  getCreditsForStudyProgramme,
+  getTransferredCredits,
+  getThesisCredits,
+}
@@ -3,6 +3,154 @@ const { Organization, ProgrammeModule } = require('../models')
 const Op = Sequelize.Op
 const { dbConnections } = require('../database/connection')

+// temp refactoring faculty <-> userservice mappings away from userservice
+const facultiesAndProgrammesInUserService = [
+  ['H10', 'MH10_001'],
+  ['H10', 'MH40_011'],
+  ['H10', 'KH10_001'],
+  ['H20', 'KH20_001'],
+  ['H20', 'MH20_001'],
+  ['H20', 'MH20_002'],
+  ['H30', 'KH30_001'],
+  ['H30', 'KH30_002'],
+  ['H30', 'MH30_002'],
+  ['H30', 'MH30_003'],
+  ['H30', 'MH30_004'],
+  ['H30', 'MH30_005'],
+  ['H30', 'MH30_001'],
+  ['H40', 'KH40_001'],
+  ['H40', 'KH40_002'],
+  ['H40', 'KH40_003'],
+  ['H40', 'KH40_004'],
+  ['H40', 'KH40_005'],
+  ['H40', 'KH40_006'],
+  ['H40', 'MH40_001'],
+  ['H40', 'MH40_002'],
+  ['H40', 'MH40_003'],
+  ['H40', 'MH40_004'],
+  ['H40', 'MH40_005'],
+  ['H40', 'MH40_006'],
+  ['H40', 'MH40_007'],
+  ['H40', 'MH40_008'],
+  ['H40', 'MH40_009'],
+  ['H40', 'MH40_010'],
+  ['H40', 'MH40_011'],
+  ['H40', 'MH40_012'],
+  ['H40', 'MH40_013'],
+  ['H40', 'MH40_014'],
+  ['H40', 'MH40_015'],
+  ['H40', 'MH70_001'],
+  ['H40', 'MH70_006'],
+  ['H50', 'KH50_001'],
+  ['H50', 'KH50_002'],
+  ['H50', 'KH50_003'],
+  ['H50', 'KH50_004'],
+  ['H50', 'KH50_005'],
+  ['H50', 'KH50_006'],
+  ['H50', 'KH50_007'],
+  ['H50', 'MH50_001'],
+  ['H50', 'MH50_002'],
+  ['H50', 'MH50_003'],
+  ['H50', 'MH50_004'],
+  ['H50', 'MH50_005'],
+  ['H50', 'MH50_006'],
+  ['H50', 'MH50_007'],
+  ['H50', 'MH50_008'],
+  ['H50', 'MH50_009'],
+  ['H50', 'MH50_010'],
+  ['H50', 'MH50_011'],
+  ['H50', 'MH50_012'],
+  ['H50', 'MH50_013'],
+  ['H55', 'KH55_001'],
+  ['H55', 'MH55_001'],
+  ['H57', 'KH57_001'],
+  ['H57', 'KH57_002'],
+  ['H57', 'KH57_003'],
+  ['H57', 'MH50_002'],
+  ['H57', 'MH50_013'],
+  ['H57', 'MH57_001'],
+  ['H57', 'MH57_002'],
+  ['H57', 'MH57_003'],
+  ['H57', 'MH57_004'],
+  ['H57', 'MH57_005'],
+  ['H57', 'MH80_007'],
+  ['H60', 'KH60_001'],
+  ['H60', 'MH60_001'],
+  ['H70', 'KH40_001'],
+  ['H70', 'KH50_001'],
+  ['H70', 'KH70_001'],
+  ['H70', 'KH70_002'],
+  ['H70', 'KH70_003'],
+  ['H70', 'KH70_004'],
+  ['H70', 'MH50_001'],
+  ['H70', 'MH57_005'],
+  ['H70', 'MH70_001'],
+  ['H70', 'MH70_002'],
+  ['H70', 'MH70_003'],
+  ['H70', 'MH70_004'],
+  ['H70', 'MH70_005'],
+  ['H70', 'MH70_006'],
+  ['H70', 'MH70_007'],
+  ['H70', 'MH70_008'],
+  ['H70', 'MH70_009'],
+  ['H70', 'MH70_010'],
+  ['H74', 'KH74_001'],
+  ['H80', 'KH57_002'],
+  ['H80', 'KH80_001'],
+  ['H80', 'KH80_002'],
+  ['H80', 'KH80_003'],
+  ['H80', 'KH80_004'],
+  ['H80', 'MH57_002'],
+  ['H80', 'MH57_003'],
+  ['H80', 'MH57_005'],
+  ['H80', 'MH80_001'],
+  ['H80', 'MH80_002'],
+  ['H80', 'MH80_003'],
+  ['H80', 'MH80_004'],
+  ['H80', 'MH80_005'],
+  ['H80', 'MH80_006'],
+  ['H80', 'MH80_007'],
+  ['H90', 'KH90_001'],
+  ['H30', '447000'],
+  ['H30', '447001'],
+  ['H30', '447002'],
+  ['H60', 'EDUK730'],
+  ['H60', 'K-MUUT-ERIL'],
+  ['H60', 'ED400'],
+  ['H60', 'ED200'],
+  ['H60', 'ED700'],
+  ['H60', 'ED100'],
+  ['H60', 'ED300'],
+  ['H60', 'ED600'],
+  ['H60', '8009'],
+  ['H60', '8118'],
+  ['H60', '8600'],
+  ['H60', 'DUK-PED'],
+  ['H60', '65'],
+  ['H60', '1102'],
+  ['H60', '20018'],
+  ['H60', '8601'],
+  ['H60', '9290'],
+  ['H60', '8172'],
+  ['H60', '83600'],
+  ['H60', 'DUM-PED'],
+  ['H60', 'ED500'],
+  ['H60', '83601'],
+  ['H60', '0130'],
+  ['H60', '20005-e'],
+  ['H60', '1040'],
+  ['H60', '8703'],
+  ['H60', '0005'],
+  ['H60', '7900'],
+  ['H60', '0152 '],
+  ['H90', 'MH90_001'],
+]
+
+const facultiesAndProgrammesForTrends = facultiesAndProgrammesInUserService.map(f => ({
+  faculty_code: f[0],
+  programme_code: f[1],
+}))
+
 // Have facultyfetching to work like it worked during oodi-db time
 const facultiesInOodi = [
  'H10',
@@ -66,4 +214,5 @@ module.exports = {
  degreeProgrammeCodesOfFaculty,
  isFaculty,
  providersOfFaculty,
+  facultiesAndProgrammesForTrends,
 }
@@ -1,6 +1,7 @@
 const { Op } = require('sequelize')
 const moment = require('moment')
 const { sortBy } = require('lodash')
+const { sortMainCode } = require('../util/utils')

 const {
  Student,
@@ -19,7 +20,6 @@ const {
  dbConnections: { sequelize },
 } = require('../database/connection')
 const { Tag, TagStudent } = require('../models/models_kone')
-const { /*allCodeAltenatives, */ findOneByCode } = require('./courses')
 const { CourseStatsCounter } = require('./course_stats_counter')
 const { getPassingSemester, semesterEnd, semesterStart } = require('../util/semester')
 const { getAllProgrammes } = require('./studyrights')
@@ -65,7 +65,7 @@ const formatStudentForPopulationStatistics = (
  startDateMoment,
  endDateMoment
 ) => {
-  const toCourse = ({ grade, attainment_date, credits, course_code, credittypecode, isStudyModule }) => {
+  const toCourse = ({ grade, attainment_date, credits, course_code, credittypecode, isStudyModule, language }) => {
    const attainment_date_normailized =
      attainment_date < startDate ? startDateMoment.clone().add(1, 'day').toISOString() : attainment_date
    const passed = Credit.passed({ credittypecode })
@@ -78,6 +78,7 @@ const formatStudentForPopulationStatistics = (
      credits,
      isStudyModuleCredit: isStudyModule,
      credittypecode,
+      language,
    }
  }

@@ -265,6 +266,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
        'isStudyModule',
        'student_studentnumber',
        'course_code',
+        'language',
      ],
      where: creditsOfStudent,
      raw: true,
@@ -494,6 +496,7 @@ const studentnumbersWithAllStudyrightElements = async (
    if (!newestStudytrack) return false
    return studyRights.includes(newestStudytrack.code)
  })
+
  const studentnumberlist = filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers

  // fetch students that have transferred out of the programme and filter out these studentnumbers
@@ -816,10 +819,6 @@ const findCourses = async (studentnumbers, beforeDate) => {
          },
        },
      },
-      /* {
-        attributes: ['discipline_id', 'name'],
-        model: Discipline
-      }, */
      {
        attributes: ['coursetypecode', 'name'],
        model: CourseType,
@@ -842,13 +841,26 @@ const parseCreditInfo = credit => ({
  date: credit.attainment_date,
 })

-const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+// const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)

-const unifyOpenUniversity = code => {
+/* const unifyOpenUniversity = code => {
  const regexresult = isOpenUniCourseCode(code)
  if (!regexresult) return code
  return regexresult[1]
 }
+ */
+const findMainCourse = async code => {
+  const course = await Course.findOne({
+    where: { code: code },
+  })
+  const sortedCourses = sortMainCode([...course.substitutions, code])
+  const result = await Course.findOne({
+    attributes: ['code', 'name'],
+    where: { code: sortedCourses[0] },
+  })
+
+  return result
+}

 const bottlenecksOf = async (query, studentnumberlist) => {
  const isValidRequest = async (query, params) => {
@@ -918,6 +930,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
          transferredStudents,
          tag
        ))
+
      const allstudents = studentnumbers.reduce((numbers, num) => {
        numbers[num] = true
        return numbers
@@ -954,14 +967,14 @@ const bottlenecksOf = async (query, studentnumberlist) => {
  for (const course of courses) {
    let { course_type } = course
    let maincourse = course
-    if (isOpenUniCourseCode) {
+    const sortedMaincourse = await findMainCourse(course.code)
+    if (sortedMaincourse) maincourse = sortedMaincourse
+
+    /*  if (isOpenUniCourseCode) {
      const nonOpen = unifyOpenUniversity(course.code)
      const response = await findOneByCode(nonOpen)
+ */

-      if (response) {
-        maincourse = response
-      }
-    }
    if (!stats[maincourse.code]) {
      stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudentslength)
    }
@@ -982,6 +995,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {

  bottlenecks.coursestatistics = Object.values(stats).map(coursestatistics => coursestatistics.getFinalStats())
  bottlenecks.allStudents = allstudentslength
+
  return bottlenecks
 }

@@ -6,7 +6,6 @@ const logger = require('../util/logger')
 const { dbConnections } = require('../database/connection')

 const recursivelyGetModuleAndChildren = async (code, type) => {
-  // TODO use only sis connection once sis is stable
  const connection = dbConnections.sequelize
  try {
    const [result] = await connection.query(
@@ -1,28 +1,63 @@
 const { getImporterClient } = require('../util/importerClient')
 const importerClient = getImporterClient()
+const { StudyGuidanceGroupTag } = require('../models/models_kone')
+const logger = require('../util/logger')

 const getAllStudentsUserHasInGroups = async sisPersonId => {
  if (!importerClient) {
    return new Set()
  }
-  const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
-  return new Set(
-    Object.values(data)
-      .map(group => group.members)
-      .flat()
-      .map(member => member.personStudentNumber)
-  )
+  try {
+    const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
+    return new Set(
+      Object.values(data)
+        .map(group => group.members)
+        .flat()
+        .map(member => member.personStudentNumber)
+    )
+  } catch (error) {
+    logger.error("Couldn't fetch users studyguidance groups")
+    return new Set()
+  }
 }

 const getAllGroupsAndStudents = async sisPersonId => {
  if (!importerClient) {
    return []
  }
-  const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
-  return Object.values(data)
+  try {
+    const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
+    const tagsByGroupId = (await StudyGuidanceGroupTag.findAll()).reduce((acc, curr) => {
+      const { studyProgramme, year } = curr
+      return { ...acc, [curr.studyGuidanceGroupId]: { studyProgramme, year } }
+    }, {})
+    return Object.values(data).map(group => ({ ...group, tags: tagsByGroupId[group.id] }))
+  } catch (error) {
+    logger.error("Couldn't fetch users studyguidance groups")
+    return []
+  }
+}
+
+const changeGroupTags = async ({ groupId, tags }) => {
+  const { studyProgramme, year } = tags
+  const tagToUpdate = studyProgramme ? { studyProgramme } : { year }
+
+  const [result] = await StudyGuidanceGroupTag.upsert(
+    {
+      studyGuidanceGroupId: groupId,
+      ...tagToUpdate,
+    },
+    {
+      where: {
+        studyGuidanceGroupId: groupId,
+      },
+    }
+  )
+  return result
 }

 module.exports = {
  getAllStudentsUserHasInGroups,
  getAllGroupsAndStudents,
+  changeGroupTags,
 }
@@ -211,7 +211,7 @@ const findProgrammeThesisCredits = async code => {
    },
    where: {
      credittypecode: {
-        [Op.ne]: 10,
+        [Op.eq]: 4,
      },
    },
  })
@@ -0,0 +1,161 @@
+const { mean } = require('lodash')
+
+// Helper functions
+const formatStudyright = studyright => {
+  const { studyrightid, studystartdate, enddate, graduated, prioritycode, extentcode, student } = studyright
+  return {
+    studyrightid,
+    studystartdate,
+    enddate,
+    graduated,
+    prioritycode,
+    extentcode,
+    studentnumber: student.studentnumber,
+  }
+}
+
+const formatStudent = student => {
+  const { studentnumber, gender_code, home_country_en, creditcount } = student
+  return {
+    studentnumber,
+    gender_code,
+    home_country_en,
+    creditcount,
+  }
+}
+
+const getYearsArray = (since, isAcademicYear) => {
+  const years = []
+  for (let i = since; i <= new Date().getFullYear(); i++) {
+    const year = isAcademicYear ? `${i} - ${i + 1}` : i
+    years.push(year)
+  }
+  return years
+}
+
+const getYearsObject = (years, emptyArrays = false) => {
+  let yearsObject = {}
+  for (const year of years) {
+    yearsObject = { ...yearsObject, [year]: emptyArrays ? [] : 0 }
+  }
+  return yearsObject
+}
+
+const getStatsBasis = years => {
+  return {
+    graphStats: new Array(years.length).fill(0),
+    tableStats: getYearsObject(years),
+  }
+}
+
+const isMajorStudentCredit = (studyright, attainment_date) =>
+  studyright &&
+  (studyright.prioritycode === 1 || studyright.prioritycode === 30) && // Is studyright state = MAIN or state = GRADUATED
+  studyright.studystartdate <= attainment_date && // Has the credit been attained after studying in the programme started
+  studyright.enddate >= attainment_date && // Has the credit been attained before the studyright ended
+  (!studyright.canceldate || studyright.canceldate >= attainment_date) // If the studyright was cancelled, was the credit attained before it was cancelled
+
+const getMedian = values => {
+  if (values.length === 0) return 0
+  values.sort((a, b) => a - b)
+  const half = Math.floor(values.length / 2)
+  if (values.length % 2) return values[half]
+  return (values[half - 1] + values[half]) / 2.0
+}
+
+const getMean = values => {
+  if (values.length === 0) return 0
+  return Math.round(mean(values))
+}
+
+const defineYear = (date, isAcademicYear) => {
+  if (!date) return ''
+  const year = date.getFullYear()
+  if (!isAcademicYear) return year
+  if (date < new Date(`${year}-07-31`)) return `${year - 1} - ${year}`
+  return `${year} - ${year + 1}`
+}
+
+const getStartDate = (studyprogramme, isAcademicYear) => {
+  if ((studyprogramme.includes('KH') || studyprogramme.includes('MH')) && isAcademicYear) return new Date('2017-08-01')
+  if (studyprogramme.includes('KH') || studyprogramme.includes('MH')) return new Date('2017-01-01')
+  if (isAcademicYear) return new Date('2000-08-01')
+  return new Date('2000-01-01')
+}
+
+// There are 9 course_unit_types
+// 1. urn:code:course-unit-type:regular
+// 2. urn:code:course-unit-type:bachelors-thesis
+// 3. urn:code:course-unit-type:masters-thesis
+// 4. urn:code:course-unit-type:doctors-thesis
+// 5. urn:code:course-unit-type:licentiate-thesis
+// 6. urn:code:course-unit-type:bachelors-maturity-examination
+// 7. urn:code:course-unit-type:masters-maturity-examination
+// 8. urn:code:course-unit-type:communication-and-linguistic-studies
+// 9. urn:code:course-unit-type:practical-training-homeland
+// Four of these are thesis types
+
+const getThesisType = studyprogramme => {
+  if (studyprogramme.includes('MH') || studyprogramme.includes('ma'))
+    return ['urn:code:course-unit-type:masters-thesis']
+  if (studyprogramme.includes('KH') || studyprogramme.includes('ba'))
+    return ['urn:code:course-unit-type:bachelors-thesis']
+  if (/^(T)[0-9]{6}$/.test(studyprogramme))
+    return ['urn:code:course-unit-type:doctors-thesis', 'urn:code:course-unit-type:licentiate-thesis']
+  return 'thesis-type-not-defined'
+}
+
+const getPercentage = (value, total) => {
+  if (typeof value !== 'number' || typeof total !== 'number') return 'NA'
+  if (total === 0) return 'NA'
+  if (value === 0) return '0 %'
+  return `${((value / total) * 100).toFixed(1)} %`
+}
+
+const getEmptyArray = length => new Array(length).fill(0)
+
+const getCreditGraphStats = years => ({
+  lte30: {
+    name: 'Less than 30 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte60: {
+    name: '30-59 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte90: {
+    name: '60-89 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte120: {
+    name: '90-119 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte150: {
+    name: '120-149 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte150: {
+    name: 'More than 150 credits',
+    data: getEmptyArray(years.length),
+  },
+})
+
+const creditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'mte150']
+
+module.exports = {
+  formatStudyright,
+  formatStudent,
+  getYearsArray,
+  getYearsObject,
+  getStatsBasis,
+  isMajorStudentCredit,
+  getMedian,
+  getMean,
+  defineYear,
+  getStartDate,
+  getThesisType,
+  getPercentage,
+  getCreditGraphStats,
+  creditThresholds,
+}
@@ -0,0 +1,315 @@
+const moment = require('moment')
+const { indexOf } = require('lodash')
+
+const { mapToProviders } = require('../util/utils')
+const {
+  getStatsBasis,
+  defineYear,
+  isMajorStudentCredit,
+  getYearsObject,
+  getYearsArray,
+  getMedian,
+  getMean,
+  getStartDate,
+  getThesisType,
+} = require('./studyprogrammeHelpers')
+const {
+  graduatedStudyRights,
+  startedStudyrights,
+  cancelledStudyRights,
+  transfersAway,
+  transfersTo,
+  getProgrammesStudents,
+  getCreditsForStudyProgramme,
+  getTransferredCredits,
+  getThesisCredits,
+} = require('./newStudyprogramme')
+
+// const getEnrolledStats = async (studytrack, since, years, isAcademicYear) => {
+//   const students = await enrolledStudyrights(studytrack)
+//   const { graphStats, tableStats } = getStatsBasis(years)
+
+//   students.forEach(({ startdate }) => {
+//     if (startdate.getMonth() === 7) {
+//       const enrolledYear = defineYear(startdate, isAcademicYear)
+
+//       graphStats[indexOf(years, enrolledYear)] += 1
+//       tableStats[enrolledYear] += 1
+//     }
+//   })
+
+//   return { graphStats, tableStats }
+// }
+
+const getGraduatedStats = async (studytrack, since, years, isAcademicYear) => {
+  const studyrights = await graduatedStudyRights(studytrack, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ enddate }) => {
+    const graduationYear = defineYear(enddate, isAcademicYear)
+    graphStats[indexOf(years, graduationYear)] += 1
+    tableStats[graduationYear] += 1
+  })
+  return { graphStats, tableStats }
+}
+
+const getStartedStats = async (studytrack, since, years, isAcademicYear) => {
+  const studyrights = await startedStudyrights(studytrack, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ studystartdate }) => {
+    const startYear = defineYear(studystartdate, isAcademicYear)
+    graphStats[indexOf(years, startYear)] += 1
+    tableStats[startYear] += 1
+  })
+  return { graphStats, tableStats }
+}
+
+const getCancelledStats = async (studytrack, since, years, isAcademicYear) => {
+  const studyrights = await cancelledStudyRights(studytrack, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ canceldate }) => {
+    const cancelYear = defineYear(canceldate, isAcademicYear)
+    graphStats[indexOf(years, cancelYear)] += 1
+    tableStats[cancelYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredAwayStats = async (studytrack, since, years, isAcademicYear) => {
+  const studyrights = await transfersAway(studytrack, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredToStats = async (studytrack, since, years, isAcademicYear) => {
+  const studyrights = await transfersTo(studytrack, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getRegularCreditStats = async (studytrack, since, years, isAcademicYear) => {
+  const providercode = mapToProviders([studytrack])[0]
+  const studyrights = await getProgrammesStudents(studytrack)
+  const credits = await getCreditsForStudyProgramme(providercode, since)
+
+  let majors = getStatsBasis(years)
+  let nonMajors = getStatsBasis(years)
+
+  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
+    const studyright = studyrights.find(studyright => studyright.studentnumber == student_studentnumber)
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+
+    if (isMajorStudentCredit(studyright, attainment_date)) {
+      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      majors.tableStats[attainmentYear] += credits || 0
+    } else {
+      nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      nonMajors.tableStats[attainmentYear] += credits || 0
+    }
+  })
+
+  return { majors, nonMajors }
+}
+
+const getTransferredCreditStats = async (studytrack, since, years, isAcademicYear) => {
+  const providercode = mapToProviders([studytrack])[0]
+  const credits = await getTransferredCredits(providercode, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  credits.forEach(({ attainment_date, credits }) => {
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+    graphStats[indexOf(years, attainmentYear)] += credits || 0
+    tableStats[attainmentYear] += credits || 0
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getThesisStats = async (studytrack, since, years, isAcademicYear) => {
+  const providercode = mapToProviders([studytrack])[0]
+  const thesisType = getThesisType(studytrack)
+  const credits = await getThesisCredits(providercode, since, thesisType)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  credits.forEach(({ attainment_date }) => {
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+    graphStats[indexOf(years, attainmentYear)] += 1
+    tableStats[attainmentYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getGraduationTimeStats = async (studytrack, since, years, isAcademicYear) => {
+  const studyrights = await graduatedStudyRights(studytrack, since)
+  let graduationAmounts = getYearsObject(years)
+  let graduationTimes = getYearsObject(years, true)
+
+  studyrights.forEach(({ enddate, studystartdate }) => {
+    const graduationYear = defineYear(enddate, isAcademicYear)
+    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+    graduationAmounts[graduationYear] += 1
+    graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
+  })
+
+  const medians = getYearsObject(years, true)
+  const means = getYearsObject(years, true)
+
+  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
+  const comparisonValue = studytrack.includes('KH') ? 72 : 48
+
+  // HighCharts graph require the data to have this format (ie. actual value, "empty value")
+  years.forEach(year => {
+    const median = getMedian(graduationTimes[year])
+    const mean = getMean(graduationTimes[year])
+    medians[year] = [
+      ['', median],
+      ['', comparisonValue - median],
+    ]
+    means[year] = [
+      ['', mean],
+      ['', comparisonValue - mean],
+    ]
+  })
+  return { medians, means, graduationAmounts }
+}
+
+const getBasicStatsForStudytrack = async ({ studyprogramme, yearType }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+  const started = await getStartedStats(studyprogramme, since, years, isAcademicYear)
+  const graduated = await getGraduatedStats(studyprogramme, since, years, isAcademicYear)
+  const cancelled = await getCancelledStats(studyprogramme, since, years, isAcademicYear)
+  const transferredAway = await getTransferredAwayStats(studyprogramme, since, years, isAcademicYear)
+  const transferredTo = await getTransferredToStats(studyprogramme, since, years, isAcademicYear)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const tableStats = reversedYears.map(year => [
+    year,
+    started.tableStats[year],
+    graduated.tableStats[year],
+    cancelled.tableStats[year],
+    transferredAway.tableStats[year],
+    transferredTo.tableStats[year],
+  ])
+
+  return {
+    id: studyprogramme,
+    years,
+    graphStats: [
+      {
+        name: 'Started',
+        data: started.graphStats,
+      },
+      {
+        name: 'Graduated',
+        data: graduated.graphStats,
+      },
+      {
+        name: 'Cancelled',
+        data: cancelled.graphStats,
+      },
+      {
+        name: 'Transferred away',
+        data: transferredAway.graphStats,
+      },
+      {
+        name: 'Transferred To',
+        data: transferredTo.graphStats,
+      },
+    ],
+    tableStats,
+  }
+}
+
+const getCreditStatsForStudytrack = async ({ studyprogramme, yearType }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+  const { majors, nonMajors } = await getRegularCreditStats(studyprogramme, since, years, isAcademicYear)
+  const transferred = await getTransferredCreditStats(studyprogramme, since, years, isAcademicYear)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const tableStats = reversedYears.map(year => [
+    year,
+    majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
+    majors.tableStats[year],
+    nonMajors.tableStats[year],
+    transferred.tableStats[year],
+  ])
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    graphStats: [
+      {
+        name: 'Major students credits',
+        data: majors.graphStats,
+      },
+      {
+        name: 'Non-major students credits',
+        data: nonMajors.graphStats,
+      },
+      {
+        name: 'Transferred credits',
+        data: transferred.graphStats,
+      },
+    ],
+  }
+}
+
+const getGraduationStatsForStudytrack = async ({ studyprogramme, yearType }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+  const thesis = await getThesisStats(studyprogramme, since, years, isAcademicYear)
+  const graduated = await getGraduatedStats(studyprogramme, since, years, isAcademicYear)
+  const graduationTimeStats = await getGraduationTimeStats(studyprogramme, since, years, isAcademicYear)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    graphStats: [
+      {
+        name: 'Graduated students',
+        data: graduated.graphStats,
+      },
+      {
+        name: 'Wrote thesis',
+        data: thesis.graphStats,
+      },
+    ],
+    graduationMedianTime: graduationTimeStats.medians,
+    graduationMeanTime: graduationTimeStats.means,
+    graduationAmounts: graduationTimeStats.graduationAmounts,
+  }
+}
+
+module.exports = {
+  getBasicStatsForStudytrack,
+  getCreditStatsForStudytrack,
+  getGraduationStatsForStudytrack,
+}
@@ -2,7 +2,6 @@ const {
  dbConnections: { sequelize },
 } = require('../database/connection')
 const { Studyright, StudyrightElement, ElementDetail } = require('../models')
-const { getUserElementDetails } = require('./userService')
 const moment = require('moment')
 const { redisClient } = require('../services/redis')
 const _ = require('lodash')
@@ -140,15 +139,6 @@ const getAllStudyrightElementsAndAssociations = async () => {
  return JSON.parse(studyright_elements)
 }

-const getStudyrightElementsAndAssociationsForUser = async username => {
-  const studyrightelementcodes = await getUserElementDetails(username)
-  if (studyrightelementcodes.length === 0) {
-    return []
-  }
-  const associations = await getAssociatedStudyrights(studyrightelementcodes)
-  return formatStudyrightElements(studyrightelementcodes, associations)
-}
-
 const getAllProgrammes = async () => {
  const elementDetails = ElementDetail.findAll({
    where: {
@@ -324,21 +314,13 @@ const getFilteredAssociations = async codes => {
  return associations
 }

-const getUserAssociations = async userid => {
-  const codes = await getUserElementDetails(userid)
-  const associations = await getFilteredAssociations(codes)
-  return associations
-}
-
 module.exports = {
  byStudent,
  studentNumbersWithAllStudyRightElements,
  getAssociatedStudyrights,
  getAllStudyrightElementsAndAssociations,
-  getStudyrightElementsAndAssociationsForUser,
  getAssociations,
  getFilteredAssociations,
-  getUserAssociations,
  refreshAssociationsInRedis,
  getAllProgrammes,
  getAllElementDetails,
@@ -0,0 +1,195 @@
+const { indexOf } = require('lodash')
+
+const { getAssociations } = require('./studyrights')
+const { studentnumbersWithAllStudyrightElements } = require('./populations')
+const {
+  getStartDate,
+  getYearsArray,
+  getPercentage,
+  getCreditGraphStats,
+  getYearsObject,
+  creditThresholds,
+} = require('./studyprogrammeHelpers')
+const { studytrackStudents, startedStudyrights, graduatedStudyRights } = require('./newStudyprogramme')
+const { getAcademicYearDates } = require('../util/semester')
+
+const getStudentData = students => {
+  const data = { female: 0, male: 0, finnish: 0, lte30: 0, lte60: 0, lte90: 0, lte120: 0, lte150: 0, mte150: 0 }
+  students.forEach(({ gender_code, home_country_en, creditcount }) => {
+    data.male += gender_code === '1' ? 1 : 0
+    data.female += gender_code === '2' ? 1 : 0
+    data.finnish += home_country_en === 'Finland' ? 1 : 0
+    data.lte30 += creditcount < 30 ? 1 : 0
+    data.lte60 += creditcount >= 30 && creditcount < 60 ? 1 : 0
+    data.lte90 += creditcount >= 60 && creditcount < 90 ? 1 : 0
+    data.lte120 += creditcount >= 90 && creditcount < 120 ? 1 : 0
+    data.lte150 += creditcount >= 120 && creditcount < 150 ? 1 : 0
+    data.mte150 += creditcount >= 150 ? 1 : 0
+  })
+  return data
+}
+
+// Goes through the programme and all its studytracks for the said year and adds the wanted stats to the data objects
+const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studytrackNames, year, years, data }) => {
+  const { mainStatsByYear, mainStatsByTrack, creditGraphStats, creditTableStats, emptyTracks } = data
+  const { startDate, endDate } = getAcademicYearDates(year)
+
+  await Promise.all(
+    studytracks.map(async track => {
+      const codes = studyprogramme === track ? [studyprogramme] : [studyprogramme, track]
+      const studentnumbers = await studentnumbersWithAllStudyrightElements(
+        codes,
+        startDate,
+        endDate,
+        true,
+        true,
+        true,
+        true
+      )
+
+      // All the stats are counted for the students who actually started studying
+      const started = await startedStudyrights(track, startDate, studentnumbers)
+      const startedStudentnumbers = [...new Set(started.map(s => s.studentnumber))]
+      const students = await studytrackStudents(startedStudentnumbers)
+      const studentData = getStudentData(students)
+      const graduated = await graduatedStudyRights(track, startDate, studentnumbers)
+
+      // If the track has no stats for that year, it should be removed from the table and dropdown options
+      if (started.length === 0) {
+        emptyTracks.has(track) ? emptyTracks.set(track, emptyTracks.get(track) + 1) : emptyTracks.set(track, 1)
+        return
+      }
+
+      // Count stats for creditgraph for the year per track
+      creditThresholds.forEach(
+        threshold => (creditGraphStats[track][threshold].data[indexOf(years, year)] = studentData[threshold])
+      )
+
+      // Count stats for the credit progress table for the year per track
+      creditTableStats[track] = [
+        ...creditTableStats[track],
+        [
+          `${year} - ${year + 1}`,
+          started.length,
+          studentData.lte30,
+          studentData.lte60,
+          studentData.lte90,
+          studentData.lte120,
+          studentData.lte150,
+          studentData.mte150,
+        ],
+      ]
+
+      // Count stats for the main studytrack table grouped by tracks
+      mainStatsByTrack[track] = [
+        ...mainStatsByTrack[track],
+        [
+          `${year} - ${year + 1}`,
+          started.length,
+          getPercentage(started.length, started.length),
+          studentData.male,
+          getPercentage(studentData.male, started.length),
+          studentData.female,
+          getPercentage(studentData.female, started.length),
+          studentData.finnish,
+          getPercentage(studentData.finnish, started.length),
+          graduated.length,
+          getPercentage(graduated.length, started.length),
+        ],
+      ]
+
+      // Count stats for the main studytrack table grouped by year
+      mainStatsByYear[year] = [
+        ...mainStatsByYear[year],
+        [
+          studytrackNames[track]?.name['fi'] || `${year} - ${year + 1}`,
+          started.length,
+          getPercentage(started.length, started.length),
+          studentData.male,
+          getPercentage(studentData.male, started.length),
+          studentData.female,
+          getPercentage(studentData.female, started.length),
+          studentData.finnish,
+          getPercentage(studentData.finnish, started.length),
+          graduated.length,
+          getPercentage(graduated.length, started.length),
+        ],
+      ]
+    })
+  )
+}
+
+// Defines the studytrack names for the studytrack selector
+// If the track has no stats for any year, it should be removed the dropdown options
+const getStudytrackOptions = (studyprogramme, studytrackNames, studytracks, emptyTracks, years) => {
+  const names = { [studyprogramme]: 'All students of the programme' }
+  studytracks.forEach(track => {
+    const trackName = studytrackNames[track]?.name['fi']
+    if (trackName && emptyTracks.get(track) !== years.length) {
+      names[track] = trackName
+    }
+  })
+  return names
+}
+
+// Creates empty objects for each statistic type, which are then updated with the studytrack data
+const getEmptyStatsObjects = (years, studytracks) => {
+  const mainStatsByYear = getYearsObject(years, true)
+  const mainStatsByTrack = {}
+  const creditGraphStats = {}
+  const creditTableStats = {}
+  const emptyTracks = new Map()
+
+  studytracks.forEach(async track => {
+    mainStatsByTrack[track] = []
+    creditGraphStats[track] = await getCreditGraphStats(years)
+    creditTableStats[track] = []
+  })
+
+  return { mainStatsByYear, mainStatsByTrack, creditGraphStats, creditTableStats, emptyTracks }
+}
+
+// Combines all the data for the Populations and Studytracks -view
+const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {
+  const isAcademicYear = true
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear())
+
+  const associations = await getAssociations()
+  const studytracks = associations.programmes[studyprogramme]
+    ? [studyprogramme, ...associations.programmes[studyprogramme].studytracks]
+    : [studyprogramme]
+
+  const studytrackNames = associations.studyTracks
+
+  const data = getEmptyStatsObjects(years, studytracks)
+
+  await Promise.all(
+    years.map(async year => {
+      return await getStudytrackDataForTheYear({
+        studyprogramme,
+        studytracks,
+        studytrackNames,
+        year,
+        years,
+        data,
+      })
+    })
+  )
+
+  const studytrackOptions = getStudytrackOptions(studyprogramme, studytrackNames, studytracks, data.emptyTracks, years)
+
+  return {
+    id: studyprogramme,
+    years: getYearsArray(since.getFullYear(), isAcademicYear),
+    mainStatsByTrack: data.mainStatsByTrack,
+    mainStatsByYear: data.mainStatsByYear,
+    creditTableStats: data.creditTableStats,
+    creditGraphStats: data.creditGraphStats,
+    studytrackOptions,
+  }
+}
+
+module.exports = {
+  getStudytrackStatsForStudyprogramme,
+}
@@ -1,15 +1,8 @@
-const axios = require('axios')
 const { redisClient } = require('../../services/redis')
-const { USERSERVICE_URL } = require('../../conf-backend')
 const {
  dbConnections: { sequelize },
 } = require('../../database/connection')

-const userServiceClient = axios.create({
-  baseURL: USERSERVICE_URL,
-  headers: { secret: process.env.USERSERVICE_SECRET },
-})
-
 const getRedisCDS = async REDIS_KEY => {
  const raw = await redisClient.getAsync(REDIS_KEY)
  return raw && JSON.parse(raw)
@@ -177,6 +170,5 @@ const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNon
 module.exports = {
  getRedisCDS,
  saveToRedis,
-  userServiceClient,
  getTargetStudentCounts,
 }
@@ -4,9 +4,10 @@ const {
 } = require('../../database/connection')
 const { mapToProviders } = require('../../util/utils')
 const { ElementDetail, Organization } = require('../../models')
-const { getRedisCDS, saveToRedis, userServiceClient } = require('./shared')
+const { getRedisCDS, saveToRedis } = require('./shared')
+const { facultiesAndProgrammesForTrends } = require('../../services/organisations')

-const REDIS_KEY_STATUS = 'STATUS_DATA_V3'
+const REDIS_KEY_STATUS = 'STATUS_DATA_V5'

 const getCurrentStudyYearStartDate = _.memoize(
  async unixMillis =>
@@ -117,13 +118,13 @@ const getOrganizationStatistics = async (organizations, start, end) => {
    if (organizationStats[orgId] === undefined) {
      organizationStats[orgId] = {
        direct: { credits: 0, students: new Set() },
-        accumulated: { credits: 0, students: new Set() },
+        aggregated: { credits: 0, students: new Set() },
      }
    }

    const orgStats = organizationStats[orgId]

-    const statsRefs = isDirect ? [orgStats.direct, orgStats.accumulated] : [orgStats.accumulated]
+    const statsRefs = isDirect ? [orgStats.direct, orgStats.aggregated] : [orgStats.aggregated]

    statsRefs.forEach(statsRef => {
      statsRef.credits += stats.credits
@@ -142,20 +143,20 @@ const getOrganizationStatistics = async (organizations, start, end) => {
    updateOrgStats(orgId, { credits, students }, true)

    if (parentOrgId) {
-      updateOrgStats(parentOrgId, organizationStats[orgId].accumulated, false)
+      updateOrgStats(parentOrgId, organizationStats[orgId].aggregated, false)
    }
  }

-  return Object.entries(organizationStats).reduce((acc, [id, { direct, accumulated }]) => {
+  return Object.entries(organizationStats).reduce((acc, [id, { direct, aggregated }]) => {
    acc[id] = {
      id,
      direct: {
        students: direct.students.size,
        credits: direct.credits,
      },
-      accumulated: {
-        students: accumulated.students.size,
-        credits: accumulated.credits,
+      aggregated: {
+        students: aggregated.students.size,
+        credits: aggregated.credits,
      },
    }

@@ -177,7 +178,7 @@ const getTotalCreditsOfCoursesBetween = async (a, b, alias = 'sum', alias2 = 'su
    WHERE
      cr.attainment_date BETWEEN :a AND :b
      AND (cr."isStudyModule" = false OR cr."isStudyModule" IS NULL)
-      AND cr.credittypecode IN (4, 9)
+      AND cr.credittypecode IN (4, 9)      
    GROUP BY co.id, o.code
    -- HAVING SUM(cr.credits) > 0
    `,
@@ -188,13 +189,40 @@ const getTotalCreditsOfCoursesBetween = async (a, b, alias = 'sum', alias2 = 'su
  )
 }

-const makeYearlyCreditsPromises = (currentYear, years, getRange, alias = 'sum', alias2 = 'sum2') => {
+const getTotalOpenUniCreditsOfCoursesBetween = async (a, b, alias = 'sum', alias2 = 'sum2') => {
+  return sequelize.query(
+    `
+    SELECT SUM(cr.credits) AS ${alias}, COUNT(DISTINCT(cr.student_studentnumber)) AS ${alias2},
+    o.code AS organizationcode, co.code, co.id, co.name
+    FROM credit cr
+    INNER JOIN course co ON cr.course_code = co.code
+    INNER JOIN course_providers cp ON cp.coursecode = co.id
+    INNER JOIN organization o ON o.id = cp.organizationcode
+    AND cr.attainment_date BETWEEN :a AND :b
+    AND (cr."isStudyModule" = false OR cr."isStudyModule" IS NULL)
+    AND cr.credittypecode IN (4, 9) AND cp.organizationcode = 'hy-org-48645785'
+    AND cr.student_studentnumber NOT IN
+      (SELECT student_studentnumber FROM studyright
+      WHERE studyright.student_studentnumber = cr.student_studentnumber
+      AND cr.attainment_date BETWEEN studyright.startdate AND studyright.enddate)
+    GROUP BY co.id, o.code -- HAVING SUM(cr.credits) > 0
+    `,
+    {
+      type: sequelize.QueryTypes.SELECT,
+      replacements: { a, b },
+    }
+  )
+}
+
+const makeYearlyCreditsPromises = (currentYear, years, getRange, alias = 'sum', alias2 = 'sum2', avoin) => {
  return years.map(
    year =>
      new Promise(async res => {
        const diff = currentYear - year
        const { from, to } = getRange(diff)
-        const creditsByCourse = await getTotalCreditsOfCoursesBetween(from, to, alias, alias2)
+        const creditsByCourse = avoin
+          ? await getTotalOpenUniCreditsOfCoursesBetween(from, to, alias, alias2)
+          : await getTotalCreditsOfCoursesBetween(from, to, alias, alias2)
        res(
          creditsByCourse.map(c => {
            c['year'] = year
@@ -207,10 +235,37 @@ const makeYearlyCreditsPromises = (currentYear, years, getRange, alias = 'sum',

 const getOrgStats = (stats, orgId) =>
  stats[orgId] ?? {
-    accumulated: { students: 0, credits: 0 },
-    direct: { students: 0, credits: 0 },
+    accumulated: {
+      aggregated: { students: 0, credits: 0 },
+      direct: { students: 0, credits: 0 },
+    },
+    total: {
+      aggregated: { students: 0, credits: 0 },
+      direct: { students: 0, credits: 0 },
+    },
  }

+const getOrgsOfOpenUniReqularCounterparts = async codes => {
+  const rows = await sequelize.query(
+    `
+      SELECT co.id, co.code, cp.organizationcode, org.code AS organization
+      FROM course co, course_providers cp, organization org
+      WHERE co.code IN(:codes)
+      AND co.id = cp.coursecode
+      AND org.id = cp.organizationcode;
+    `,
+    {
+      type: sequelize.QueryTypes.SELECT,
+      replacements: { codes },
+    }
+  )
+
+  return rows.reduce((acc, { code, organization }) => {
+    acc[code] = organization
+    return acc
+  }, {})
+}
+
 const calculateStatusStatistics = async (unixMillis, showByYear) => {
  const YEAR_TO_MILLISECONDS = 31556952000
  /* Memoize parses booleans into strings... */
@@ -218,67 +273,135 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
  const startYear = startDate.getFullYear()
  const startTime = startDate.getTime()
  const yearRange = _.range(2017, startYear + 1)
+  const getRangeAcc = diff => ({
+    from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
+    to:
+      showByYear === 'true'
+        ? new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS)
+        : new Date(unixMillis - diff * YEAR_TO_MILLISECONDS),
+  })
+  const getRangeTotal = diff => ({
+    from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
+    to: new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS),
+  })
  const yearlyAccCreditsPromises = makeYearlyCreditsPromises(
    startYear,
    yearRange,
-    diff => ({
-      from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
-      to:
-        showByYear === 'true'
-          ? new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS)
-          : new Date(unixMillis - diff * YEAR_TO_MILLISECONDS),
-    }),
+    getRangeAcc,
    'acc',
-    'students'
+    'students',
+    false
+  )
+  const yearlyOpenUniAccCreditsPromises = makeYearlyCreditsPromises(
+    startYear,
+    yearRange,
+    getRangeAcc,
+    'acc',
+    'students',
+    true
  )
-
  const yearlyTotalCreditsPromises = makeYearlyCreditsPromises(
    startYear,
    yearRange.slice(0, -1),
-    diff => ({
-      from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
-      to: new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS),
-    }),
+    getRangeTotal,
    'total',
-    'students'
+    'students',
+    false
+  )
+
+  const yearlyOpenUniTotalCreditsPromises = makeYearlyCreditsPromises(
+    startYear,
+    yearRange.slice(0, -1),
+    getRangeTotal,
+    'total',
+    'students',
+    true
  )

  /* Gather all required data */
-  const [yearlyAccCredits, yearlyTotalCredits, elementDetails, faculties, { data: facultyProgrammes }] =
-    await Promise.all([
-      Promise.all(yearlyAccCreditsPromises),
-      Promise.all(yearlyTotalCreditsPromises),
-      ElementDetail.findAll(),
-      Organization.findAll(),
-      userServiceClient.get('/faculty_programmes'),
-    ])
-
-  const yearlyOrgStatPromises = yearRange.map(async year => [
-    year,
-    await getOrganizationStatistics(
-      faculties,
-      new Date(startTime - (startYear - year) * YEAR_TO_MILLISECONDS),
-      new Date(startTime - (startYear - year - 1) * YEAR_TO_MILLISECONDS)
-    ),
+  const [
+    yearlyAccCredits,
+    yearlyTotalCredits,
+    yearlyOpenUniAccCredits,
+    yearlyOpenUniTotalCredits,
+    elementDetails,
+    faculties,
+  ] = await Promise.all([
+    Promise.all(yearlyAccCreditsPromises),
+    Promise.all(yearlyTotalCreditsPromises),
+    Promise.all(yearlyOpenUniAccCreditsPromises),
+    Promise.all(yearlyOpenUniTotalCreditsPromises),
+    ElementDetail.findAll(),
+    Organization.findAll(),
  ])

+  const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+
+  const unifyOpenUniversity = code => {
+    const regexresult = isOpenUniCourseCode(code)
+    if (!regexresult) return code
+    return regexresult[1]
+  }
+
+  const fromAyToNormalCourses = [
+    ...new Set(_.flatten(yearlyOpenUniAccCredits).map(c => unifyOpenUniversity(c.code))),
+    ...new Set(_.flatten(yearlyOpenUniTotalCredits).map(c => unifyOpenUniversity(c.code))),
+  ]
+
+  const orgsOfOpenUniReqularCounterparts = await getOrgsOfOpenUniReqularCounterparts(fromAyToNormalCourses)
+
+  const facultyProgrammes = facultiesAndProgrammesForTrends
+
+  const yearlyOrgStatPromises = yearRange.map(async year => {
+    const accumulatedStats = await getOrganizationStatistics(
+      faculties,
+      new Date(startTime - (startYear - year) * YEAR_TO_MILLISECONDS),
+      showByYear === 'true'
+        ? new Date(startTime - (startYear - year - 1) * YEAR_TO_MILLISECONDS)
+        : new Date(unixMillis - (startYear - year) * YEAR_TO_MILLISECONDS)
+    )
+
+    const totalStats =
+      year !== startYear &&
+      (await getOrganizationStatistics(
+        faculties,
+        new Date(startTime - (startYear - year) * YEAR_TO_MILLISECONDS),
+        new Date(startTime - (startYear - year - 1) * YEAR_TO_MILLISECONDS)
+      ))
+
+    return [
+      year,
+      _.merge(
+        _.mapValues(totalStats, total => ({ total })),
+        _.mapValues(accumulatedStats, accumulated => ({ accumulated }))
+      ),
+    ]
+  })
+
  const orgStats = await Promise.all(yearlyOrgStatPromises)
+
  const yearlyOrgStats = Object.fromEntries(orgStats)
  const [[, currentOrgStats], [, prevOrgStats]] = orgStats.reverse()

+  const defaultYearStats = _.chain(_.range(2017, startYear + 1))
+    .map(year => [year, { acc: 0, accStudents: 0 }])
+    .fromPairs()
+    .value()
+
  const getOrgYearlyStats = orgId =>
-    Object.entries(yearlyOrgStats).reduce((acc, [year, stats]) => {
-      const { credits, students } = getOrgStats(stats, orgId).accumulated
+    _.chain(yearlyOrgStats)
+      .mapValues(stats => {
+        const orgStats = getOrgStats(stats, orgId)

-      acc[year] = {
-        acc: credits,
-        accStudents: students,
-        total: credits,
-        totalStudents: students,
-      }
-
-      return acc
-    }, {})
+        return {
+          acc: orgStats.accumulated.aggregated.credits,
+          accStudents: orgStats.accumulated.aggregated.students,
+          total: orgStats.total?.aggregated?.credits,
+          totalStudents: orgStats.total?.aggregated?.students,
+        }
+      })
+      .defaults(defaultYearStats)
+      .value()

  /* Construct some helper maps */
  const facultyCodeToFaculty = faculties.reduce((res, curr) => {
@@ -292,6 +415,8 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
    return res
  }, {})

+  programmeToFaculties['H906'] = ['H906']
+
  const providerToProgramme = elementDetails.reduce((res, curr) => {
    const [p] = mapToProviders([curr.code])
    res[p] = {
@@ -301,26 +426,66 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
    return res
  }, {})

+  providerToProgramme['H906'] = {
+    code: 'H906',
+    name: { en: 'Language Centre', fi: 'Kielikeskus', sv: 'Språkcentrum' },
+  }
+
+  providerToProgramme['H930'] = {
+    code: 'H930',
+    name: { en: 'Open University', fi: 'Avoin yliopisto', sv: 'Öppna universitetet' },
+  }
+
  const organizationCodeToOrganization = faculties.reduce((acc, org) => {
    acc[org.code] = org
    return acc
  }, {})

+  const sumMerge = (a, b) => {
+    if (!a) return b
+    if (!b) return a
+    return a + b
+  }
+
+  const mergele = (a, b) => {
+    if (!a) return _.clone(b)
+    if (!b) return _.clone(a)
+    return _.mergeWith(a, b, sumMerge)
+  }
+
+  const refactoredOpenUniCredits = [..._.flatten(yearlyOpenUniAccCredits), ..._.flatten(yearlyOpenUniTotalCredits)].map(
+    ele => {
+      const org = orgsOfOpenUniReqularCounterparts[unifyOpenUniversity(ele.code)]
+      return {
+        ...ele,
+        organizationcode: org ? org : 'H930',
+      }
+    }
+  )
+
  /* Calculate course level stats and group by providers */
  const coursesGroupedByProvider = Object.entries(
    _.groupBy([..._.flatten(yearlyAccCredits), ..._.flatten(yearlyTotalCredits)], 'organizationcode')
  ).reduce((acc, [organizationcode, courseCredits]) => {
    acc[organizationcode] = Object.entries(_.groupBy(courseCredits, 'code')).reduce(
      (acc, [courseCode, yearlyInstances]) => {
-        acc[courseCode] = { yearly: {}, name: yearlyInstances[0].name }
-        const array = [2020, 2019, 2018, 2017]
+        acc[courseCode] = {
+          yearly: {},
+          name: yearlyInstances[0].name,
+          type: 'course',
+          code: courseCode,
+        }
+
+        const array = _.range(2017, startYear + 1)
+
        array.forEach(year => {
          acc[courseCode]['yearly'][year] = {}
          acc[courseCode]['yearly'][year]['acc'] = 0
          acc[courseCode]['yearly'][year]['accStudents'] = 0
-          acc[courseCode]['yearly'][year]['total'] = 0
-          acc[courseCode]['yearly'][year]['totalStudents'] = 0
+          acc[courseCode]['yearly'][year]['total'] = undefined
+          acc[courseCode]['yearly'][year]['totalStudents'] = undefined
        })
+
        yearlyInstances.forEach(instance => {
          if (!acc[courseCode]['yearly'][instance.year]) acc[courseCode]['yearly'][instance.year] = {}

@@ -332,6 +497,52 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
            acc[courseCode]['yearly'][instance.year]['totalStudents'] = Number(instance.students)
          }
        })
+
+        acc[courseCode]['current'] = _.get(acc, [courseCode, 'yearly', startYear, 'acc']) || 0
+        acc[courseCode]['previous'] = _.get(acc, [courseCode, 'yearly', startYear - 1, 'acc']) || 0
+        acc[courseCode]['currentStudents'] = _.get(acc, [courseCode, 'yearly', startYear, 'accStudents']) || 0
+        acc[courseCode]['previousStudents'] = _.get(acc, [courseCode, 'yearly', startYear - 1, 'accStudents']) || 0
+        return acc
+      },
+      {}
+    )
+    return acc
+  }, {})
+
+  const openUniCoursesGroupedByProvider = Object.entries(
+    _.groupBy(refactoredOpenUniCredits, 'organizationcode')
+  ).reduce((acc, [organizationcode, courseCredits]) => {
+    acc[organizationcode] = Object.entries(_.groupBy(courseCredits, 'code')).reduce(
+      (acc, [courseCode, yearlyInstances]) => {
+        acc[courseCode] = {
+          yearly: {},
+          name: yearlyInstances[0].name,
+          type: 'course',
+          code: courseCode,
+        }
+
+        const array = _.range(2017, startYear + 1)
+
+        array.forEach(year => {
+          acc[courseCode]['yearly'][year] = {}
+          acc[courseCode]['yearly'][year]['acc'] = 0
+          acc[courseCode]['yearly'][year]['accStudents'] = 0
+          acc[courseCode]['yearly'][year]['total'] = undefined
+          acc[courseCode]['yearly'][year]['totalStudents'] = undefined
+        })
+
+        yearlyInstances.forEach(instance => {
+          if (!acc[courseCode]['yearly'][instance.year]) acc[courseCode]['yearly'][instance.year] = {}
+
+          if (instance.acc !== undefined) {
+            acc[courseCode]['yearly'][instance.year]['acc'] = instance.acc
+            acc[courseCode]['yearly'][instance.year]['accStudents'] = Number(instance.students)
+          } else {
+            acc[courseCode]['yearly'][instance.year]['total'] = instance.total
+            acc[courseCode]['yearly'][instance.year]['totalStudents'] = Number(instance.students)
+          }
+        })
+
        acc[courseCode]['current'] = _.get(acc, [courseCode, 'yearly', startYear, 'acc']) || 0
        acc[courseCode]['previous'] = _.get(acc, [courseCode, 'yearly', startYear - 1, 'acc']) || 0
        acc[courseCode]['currentStudents'] = _.get(acc, [courseCode, 'yearly', startYear, 'accStudents']) || 0
@@ -348,44 +559,111 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
    const programme = providerToProgramme[organizationcode]

    if (programme && programme.code) {
+      if (programme.code === 'H906') {
+        const courseValues = Object.values(courses)
+        const yearlyValues = courseValues.map(c => c.yearly)
+
+        acc[programme.code] = {
+          type: 'programme',
+          code: programme.code,
+          name: programme.name,
+          drill: courses,
+          yearly: _.mergeWith({}, ...yearlyValues, mergele),
+          current: _.sumBy(courseValues, 'current') | undefined,
+          previous: _.sumBy(courseValues, 'previous') | undefined,
+          currentStudents: _.sumBy(courseValues, 'currentStudents') || 'undefined',
+          previousStudents: _.sumBy(courseValues, 'previousStudents') | 'undefined',
+        }
+
+        return acc
+      }
+
      const orgId = organizationCodeToOrganization[organizationcode]?.id

-      const { students: currentStudents, credits: current } = getOrgStats(currentOrgStats, orgId).accumulated
-      const { students: previousStudents, credits: previous } = getOrgStats(prevOrgStats, orgId).accumulated
+      const { students: currentStudents, credits: current } = getOrgStats(currentOrgStats, orgId).accumulated.aggregated
+      const { students: previousStudents, credits: previous } = getOrgStats(prevOrgStats, orgId).accumulated.aggregated

      const yearly = getOrgYearlyStats(orgId)

      acc[programme.code] = {
        type: 'programme',
+        code: programme.code,
        name: programme.name,
        drill: courses,
        yearly,
        current,
        previous,
-        currentStudents,
-        previousStudents,
+        currentStudents: currentStudents,
+        previousStudents: previousStudents,
      }
    }

    return acc
  }, {})

+  const openUniGroupedByProgramme = Object.entries(openUniCoursesGroupedByProvider).reduce(
+    (acc, [organizationcode, courses]) => {
+      const programme = providerToProgramme[organizationcode]
+
+      if (programme && programme.code) {
+        const courseValues = Object.values(courses)
+        const yearlyValues = courseValues.map(c => c.yearly)
+
+        acc[programme.code] = {
+          type: 'programme',
+          code: programme.code,
+          name: programme.name,
+          drill: courses,
+          yearly: _.mergeWith({}, ...yearlyValues, mergele),
+          current: _.sumBy(courseValues, 'current') | undefined,
+          previous: _.sumBy(courseValues, 'previous') | undefined,
+          currentStudents: _.sumBy(courseValues, 'currentStudents') || 'undefined',
+          previousStudents: _.sumBy(courseValues, 'previousStudents') | 'undefined',
+        }
+
+        return acc
+      }
+
+      return acc
+    },
+    {}
+  )
+
  /* Group programmes into faculties and calculate faculty level stats */
  const groupedByFaculty = Object.entries(groupedByProgramme).reduce((acc, [programmeCode, programmeStats]) => {
    const facultyCodes = programmeToFaculties[programmeCode]
    if (!facultyCodes) return acc
    facultyCodes.forEach(facultyCode => {
      if (!facultyCode) return
+      if (facultyCode === 'H906' || facultyCode === 'H930') {
+        if (!acc[facultyCode]) {
+          acc[facultyCode] = {
+            type: 'faculty',
+            code: facultyCode,
+            drill: {},
+            name: facultyCodeToFaculty[facultyCode] ? facultyCodeToFaculty[facultyCode].name : null,
+            yearly: programmeStats.yearly,
+            current: programmeStats.current,
+            previous: programmeStats.previous,
+            currentStudents: programmeStats.currentStudents,
+            previousStudents: programmeStats.previousStudents,
+          }
+        }
+      }
+
      if (!acc[facultyCode]) {
        const orgId = organizationCodeToOrganization[facultyCode].id

        const { students: currentStudents, credits: current } = getOrgStats(currentOrgStats, orgId).accumulated
+          .aggregated
        const { students: previousStudents, credits: previous } = getOrgStats(prevOrgStats, orgId).accumulated
+          .aggregated

        const yearly = getOrgYearlyStats(orgId)

        acc[facultyCode] = {
          type: 'faculty',
+          code: facultyCode,
          drill: {},
          name: facultyCodeToFaculty[facultyCode] ? facultyCodeToFaculty[facultyCode].name : null,
          yearly,
@@ -401,7 +679,31 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
    return acc
  }, {})

-  return groupedByFaculty
+  const openUniGroupedByFaculty = Object.entries(openUniGroupedByProgramme).reduce(
+    (acc, [programmeCode, programmeStats]) => {
+      const facultyCode = 'H930'
+
+      if (!acc[facultyCode]) {
+        acc[facultyCode] = {
+          type: 'faculty',
+          code: facultyCode,
+          drill: {},
+          name: facultyCodeToFaculty[facultyCode] ? facultyCodeToFaculty[facultyCode].name : null,
+          yearly: {},
+          current: 0,
+          previous: 0,
+        }
+      }
+
+      acc[facultyCode]['drill'][programmeCode] = programmeStats
+      acc[facultyCode]['yearly'] = _.mergeWith(acc[facultyCode]['yearly'], programmeStats.yearly, mergele)
+      acc[facultyCode]['current'] += programmeStats.current
+      acc[facultyCode]['previous'] += programmeStats.previous
+      return acc
+    },
+    {}
+  )
+  return { ...groupedByFaculty, ...openUniGroupedByFaculty }
 }

 const getStatus = async (unixMillis, showByYear, doRefresh = false) => {
@@ -1,24 +1,41 @@
-const axios = require('axios')
-const { USERSERVICE_URL } = require('../conf-backend')
 const UnitService = require('./units')
 const elementDetailService = require('./elementdetails')
 const { filterStudentnumbersByAccessrights } = require('./students')
 const { userDataCache } = require('./cache')
-const client = axios.create({ baseURL: USERSERVICE_URL, headers: { secret: process.env.USERSERVICE_SECRET } })
 const { getImporterClient } = require('../util/importerClient')
 const logger = require('../util/logger')
+const { facultiesAndProgrammesForTrends } = require('../services/organisations')
+const { sendNotificationAboutNewUser } = require('../services/mailservice')
+const User = require('./users/users')
+const AccessGroup = require('./users/accessgroups')
+const _ = require('lodash')

-const ping = async () => {
-  const url = '/ping'
-  const response = await axios.get(url)
-  return response.data
+// Private helpers
+const enrichProgrammesFromFaculties = faculties =>
+  facultiesAndProgrammesForTrends.filter(f => faculties.includes(f.faculty_code)).map(f => f.programme_code)
+
+const enrichWithProgrammes = toEnrich => {
+  if (Array.isArray(toEnrich)) {
+    return toEnrich.map(obj => enrichWithProgrammes(obj))
+  }
+  if (toEnrich.rights) {
+    return {
+      ...toEnrich,
+      rights: _.union(toEnrich.rights, enrichProgrammesFromFaculties(toEnrich.faculties)),
+    }
+  }
+
+  if (toEnrich.elementdetails) {
+    return {
+      ...toEnrich,
+      elementdetails: _.union(
+        toEnrich.elementdetails,
+        enrichProgrammesFromFaculties(toEnrich.faculty.map(f => f.faculty_code))
+      ),
+    }
+  }
+  return toEnrich
 }
-
-const findAll = async () => {
-  const response = await client.get('/findall')
-  return response.data
-}
-
 const checkStudyGuidanceGroupsAccess = async hyPersonSisuId => {
  if (!hyPersonSisuId) {
    logger.error('Not possible to get groups without personId header')
@@ -26,105 +43,36 @@ const checkStudyGuidanceGroupsAccess = async hyPersonSisuId => {
  }
  const importerClient = getImporterClient()
  if (!importerClient) return false
-  const { data } = await importerClient.get(`/person-groups/person/${hyPersonSisuId}`)
+  const answerTimeout = new Promise(resolve => setTimeout(resolve, 2000))
+  const result = await Promise.race([importerClient.get(`/person-groups/person/${hyPersonSisuId}`), answerTimeout])
+  if (!result) return false
+  const { data } = result
  return data && Object.values(data).length > 0
 }

-const login = async (uid, full_name, hyGroups, affiliations, email, hyPersonSisuId) => {
-  const hasStudyGuidanceGroupAccess = await checkStudyGuidanceGroupsAccess(hyPersonSisuId)
-  const response = await client.post('/login', {
-    uid,
-    full_name,
-    hyGroups,
-    affiliations,
-    email,
-    hyPersonSisuId,
-    hasStudyGuidanceGroupAccess,
-  })
-  return response.data
+const byUsernameData = async username => {
+  // OK, enriched
+  const user = await User.byUsernameMinified(username)
+  const roles = user.accessgroup.map(({ group_code }) => group_code)
+  const rights = User.getUserProgrammes(user)
+  const faculties = user.faculty.map(({ faculty_code }) => faculty_code)
+  const data = {
+    email: user.email,
+    full_name: user.full_name,
+    roles,
+    rights,
+    faculties,
+  }
+  return enrichWithProgrammes(data)
 }

-const superlogin = async (uid, asUser) => {
-  const response = await client.post('/superlogin', {
-    uid,
-    asUser,
-  })
-  return response.data
-}
-
-const byUsername = async uid => {
-  const url = `/user/${uid}`
-  const response = await client.get(url)
-  return response.data
-}
-
-const byUsernameData = async uid => {
-  const url = `/user/${uid}/user_data`
-  const response = await client.get(url)
-  return response.data
-}
-
-const getUserElementDetails = async username => {
-  const url = `/user/elementdetails/${username}`
-  const response = await client.get(url)
-  const elementdetailcodes = response.data
-  return elementDetailService.byCodes(elementdetailcodes)
-}
-
-const byId = async id => {
-  const url = `/user/id/${id}`
-  const response = await client.get(url)
-  return response.data
-}
-
-const updateUser = async (uid, fields) => {
-  userDataCache.del(uid)
-  const url = `/user/${uid}`
-  const response = await client.put(url, fields)
-  return response.data
-}
-
-const getRolesFor = async user => {
-  const url = `/get_roles/${user}`
-  const response = await client.get(url)
-  return response.data
-}
-
-const enableElementDetails = async (uid, codes) => {
-  const response = await client.post('/add_rights', { uid, codes })
-  return response.data.user
-}
-
-const removeElementDetails = async (uid, codes) => {
-  const response = await client.post('/remove_rights', { uid, codes })
-  return response.data.user
-}
-
-const setFaculties = async (uid, faculties) => {
-  const response = await client.post('/set_faculties', { uid, faculties })
-  return response.data.user
-}
-
-const getUnitsFromElementDetails = async username => {
-  const elementDetails = await getUserElementDetails(username)
-  return elementDetails.map(element => UnitService.parseUnitFromElement(element))
-}
-
-const modifyAccess = async body => {
-  const response = await client.post('/modifyaccess', body)
-  return response.data
-}
-
-const getAccessGroups = async () => {
-  const response = await client.get('/access_groups')
-  return response.data
-}
-
-const getUserDataFor = async uid => {
-  let userData = userDataCache.get(uid)
+// Exported helpers
+const getUserDataFor = async username => {
+  // no need to enrich
+  let userData = userDataCache.get(username)
  if (!userData) {
-    userData = await byUsernameData(uid)
-    userDataCache.set(uid, userData)
+    userData = await byUsernameData(username)
+    userDataCache.set(username, userData)
  }

  return {
@@ -133,9 +81,21 @@ const getUserDataFor = async uid => {
  }
 }

+const getUnitsFromElementDetails = async username => {
+  // No need to enrich
+  let userData = userDataCache.get(username)
+  if (!userData) {
+    userData = await byUsernameData(username)
+    userDataCache.set(username, userData)
+  }
+  const elementDetails = await elementDetailService.byCodes(userData.rights)
+  return elementDetails.map(element => UnitService.parseUnitFromElement(element))
+}
+
 const getStudentsUserCanAccess = async (studentnumbers, roles, userId) => {
+  // No need to enrich
  let studentsUserCanAccess
-  if (roles && roles.includes('admin')) {
+  if (roles?.includes('admin')) {
    studentsUserCanAccess = new Set(studentnumbers)
  } else {
    const unitsUserCanAccess = await getUnitsFromElementDetails(userId)
@@ -145,22 +105,83 @@ const getStudentsUserCanAccess = async (studentnumbers, roles, userId) => {
  return studentsUserCanAccess
 }

+// FOR ROUTES 1-to-1 mapping
+const modifyAccess = async body => {
+  const { username, accessgroups } = body
+  await User.modifyRights(username, accessgroups)
+  const user = await User.byUsername(username)
+  return enrichWithProgrammes(User.getUserData(user))
+}
+
+const getAccessGroups = async () => enrichWithProgrammes(await AccessGroup.findAll())
+
+const enableElementDetails = async (uid, codes) => {
+  await User.addProgrammes(uid, codes)
+  const user = await User.byId(uid)
+  return enrichWithProgrammes(User.getUserData(user))
+}
+
+const findAll = async () => enrichWithProgrammes((await User.findAll()).map(User.getUserData))
+
+const setFaculties = async (uid, faculties) => {
+  await User.setFaculties(uid, faculties)
+  const user = await User.byId(uid)
+  return enrichWithProgrammes(User.getUserData(user))
+}
+
+const removeElementDetails = async (uid, codes) => {
+  await User.removeProgrammes(uid, codes)
+  const user = await User.byId(uid)
+  return enrichWithProgrammes(User.getUserData(user))
+}
+
+const updateUser = async (username, fields) => {
+  userDataCache.del(username)
+  const uid = username
+  const user = await User.byUsername(uid)
+  if (!user) {
+    throw new Error(`User ${uid} not found`)
+  }
+  await User.updateUser(user, fields)
+  const returnedUser = await User.byUsername(username)
+  return enrichWithProgrammes(User.getUserData(returnedUser))
+}
+
+const getLoginDataWithoutToken = async (uid, full_name, hyGroups, email, hyPersonSisuId) => {
+  const hasStudyGuidanceGroupAccess = await checkStudyGuidanceGroupsAccess(hyPersonSisuId)
+  const result = await User.loginWithoutToken(
+    uid,
+    full_name,
+    hyGroups,
+    email,
+    hyPersonSisuId,
+    hasStudyGuidanceGroupAccess
+  )
+  return result
+}
+
+const getMockedUser = async (uid, asUser) => await User.superlogin(uid, asUser)
+
+const getUser = async ({ username, name, email, iamGroups, sisId }) => {
+  const { payload: user, isNew } = await getLoginDataWithoutToken(username, name, iamGroups, email, sisId)
+  if (isNew) {
+    await sendNotificationAboutNewUser({ userId: username, userFullName: name })
+  }
+  return user
+}
+
 module.exports = {
-  ping,
-  byUsername,
  updateUser,
-  byId,
-  getUserElementDetails,
-  login,
-  superlogin,
  enableElementDetails,
  removeElementDetails,
  findAll,
  modifyAccess,
  getAccessGroups,
  getUnitsFromElementDetails,
-  getRolesFor,
  setFaculties,
  getUserDataFor,
  getStudentsUserCanAccess,
+  getLoginDataWithoutToken,
+  getUser,
+  getMockedUser,
 }
@@ -1,4 +1,4 @@
-const { AccessGroup } = require('../models/index')
+const { AccessGroup } = require('../../models/models_user')
 const { Op } = require('sequelize')

 const byId = id => AccessGroup.findByPk(id)
@@ -1,4 +1,4 @@
-const { HyGroup } = require('../models/index')
+const { HyGroup } = require('../../models/models_user')
 const { Op } = require('sequelize')

 const byId = id => HyGroup.findByPk(id)
--- a/Show More
+++ b/Show More