Add admin user to always have admin rights

Fix populations failing because of invalid admin role check
Promote custom populations to navbar & fix normal populations being show to studyguidancegroup users
2026-10-13 03:13:40 +03:00 · 2022-01-13 08:56:22 +02:00 · 2022-01-13 08:53:55 +02:00 · 2022-01-13 08:06:53 +02:00 · 2022-01-13 07:55:46 +02:00 · 2022-01-13 07:35:27 +02:00
394 changed files with 13106 additions and 63305 deletions
@@ -1,4 +1,3 @@
 # Populate and change this to .env
 IMPORTER_DB_API_TOKEN=
-IMPORTER_DB_API_URL=
 PATE_TOKEN=
@@ -8,17 +8,10 @@
  "parserOptions": {
    "requireConfigFile": false
  },
-  "extends": [
-    "eslint:recommended",
-    "plugin:prettier/recommended",
-    "plugin:cypress/recommended"
-  ],
-  "plugins": [
-    "prettier",
-    "cypress"
-  ],
+  "extends": ["eslint:recommended", "plugin:prettier/recommended", "plugin:cypress/recommended"],
+  "plugins": ["prettier", "cypress"],
  "rules": {
-    "no-console": "off",
+    "no-console": "error",
    "react/no-unescaped-entities": "off",
    "no-async-promise-executor": "off",
    "prettier/prettier": "error"
@@ -1,4 +1,4 @@
-name: "Release oodikone to production"
+name: 'Release oodikone to production'

 on:
  release:
@@ -6,23 +6,23 @@ on:

 jobs:
  build_and_release_backend:
-    name: "Build and release backend"
+    name: 'Build and release backend'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
+      - name: 'Extract release version'
        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
        # This steps extracts release version from GITHUB_REF env var
        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
        id: extract_version
        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
        uses: docker/build-push-action@v2
        with:
          build-args: |
@@ -38,26 +38,26 @@ jobs:
          target: production

  build_and_release_frontend:
-    name: "Build and release frontend"
+    name: 'Build and release frontend'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Get current time"
+      - name: 'Get current time'
        id: get_current_time
        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
-      - name: "Extract release version"
+      - name: 'Extract release version'
        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
        # This steps extracts release version from GITHUB_REF env var
        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
        id: extract_version
        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
        uses: docker/build-push-action@v2
        with:
          build-args: |
@@ -72,11 +72,11 @@ jobs:
          push: true
          tags: toska/oodikone2-frontend:latest
          target: production
-      - name: "Create a container from built image "
+      - name: 'Create a container from built image '
        run: docker create --name frontend toska/oodikone2-frontend:latest
-      - name: "Copy assets from the container"
+      - name: 'Copy assets from the container'
        run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
+      - name: 'Create Sentry release'
        uses: getsentry/action-release@v1
        env:
          SENTRY_URL: ${{ secrets.SENTRY_URL }}
@@ -87,52 +87,19 @@ jobs:
          environment: production
          set_commits: 'skip'
          sourcemaps: './build/static/js'
-          url_prefix: '~/oodikone/static/js'
+          url_prefix: '~/static/js'
          version: ${{ steps.extract_version.outputs.VERSION }}

-  build_and_release_userservice:
-    name: "Build and release userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
-        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
-        # This steps extracts release version from GITHUB_REF env var
-        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
-        id: extract_version
-        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: true
-          tags: toska/oodikone2-userservice:latest
-          target: production
-
  report_success:
    runs-on: ubuntu-latest
-    name: "Report deployment success to oodikone notifications"
+    name: 'Report deployment success to oodikone notifications'
    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
    if: success()
    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
        with:
          webhook-url: ${{ secrets.WEBHOOK_URL }}
          message-type: deployment
@@ -141,15 +108,14 @@ jobs:

  report_failure:
    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
+    name: 'Report deployment failure to oodikone notifications'
    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
    if: failure()
    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
        with:
          webhook-url: ${{ secrets.WEBHOOK_URL }}
          message-type: deployment-failure
@@ -45,21 +45,7 @@ jobs:
          'eslint --ext .jsx,.js services/frontend' \
          'stylelint services/frontend/src/**/*.css'

-  lint_userservice:
-    name: 'Lint userservice'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: 14
-      - name: 'Install dependencies'
-        run: npm ci && cd services/userservice && npm ci
-      - name: 'Run frontend linter'
-        run: npm run eslint services/userservice
-
  # === Build images ===
-
  build_backend:
    name: 'Build backend'
    runs-on: ubuntu-latest
@@ -116,50 +102,106 @@ jobs:
        with:
          path: /tmp/frontend.tar

-  build_userservice:
-    name: 'Build userservice'
+  build_and_push_anon_sis_db:
+    name: 'Build anon sis-db and push to toska hub if new migrations have been added'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: 'Build image and export it'
-        uses: docker/build-push-action@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: false
-          tags: toska/oodikone2-userservice:staging
-          target: production
-          outputs: type=docker,dest=/tmp/userservice.tar
-      - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v2
+          filters: |
+            migrations:
+              - 'updater/sis-updater-worker/src/db/migrations/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
        with:
-          path: /tmp/userservice.tar
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach sis-updater-worker
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db

-  # === Integration tests ===
-  #
-  run_userservice_integration_tests:
-    name: 'Run userservice integration tests'
+  build_and_push_anon_kone_db:
+    name: 'Build anon kone-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
-      - name: 'Run tests included in docker build'
-        run: docker-compose --file docker-compose.test.yml up --build userservice-test
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_kone/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
+
+  build_and_push_anon_user_db:
+    name: 'Build anon user-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_user/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db

  # === End-to-end tests ===
-
  test_course_statistics:
    name: 'Test course statistics'
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -171,7 +213,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -202,7 +243,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -214,7 +257,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -245,7 +287,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -257,7 +301,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -288,7 +331,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -300,7 +345,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -331,7 +375,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -343,7 +389,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -374,7 +419,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -386,7 +433,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -417,7 +463,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -429,7 +477,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -464,7 +511,9 @@ jobs:
    needs:
      - build_backend
      - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -476,7 +525,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to toska docker registry'
        uses: docker/login-action@v1
        with:
@@ -504,8 +552,6 @@ jobs:
    needs:
      - lint_backend
      - lint_frontend
-      - lint_userservice
-      - run_userservice_integration_tests
      - test_course_statistics
      - test_custom_population
      - test_filters
@@ -524,7 +570,6 @@ jobs:
        run: |
          docker load --input /tmp/artifact/backend.tar
          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
      - name: 'Login to DockerHub'
        uses: docker/login-action@v1
        with:
@@ -545,13 +590,12 @@ jobs:
          environment: staging
          set_commits: 'skip'
          sourcemaps: './build/static/js'
-          url_prefix: '~/oodikone/static/js'
+          url_prefix: '~/static/js'
          version: ${{ github.sha }}
      - name: 'Push to docker hub'
        run: |
          docker push toska/oodikone2-backend:staging
          docker push toska/oodikone2-frontend:staging
-          docker push toska/oodikone2-userservice:staging
      - name: 'Send deployment success notification to slack'
        if: success()
        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
@@ -574,8 +618,6 @@ jobs:
    needs:
      - lint_backend
      - lint_frontend
-      - lint_userservice
-      - run_userservice_integration_tests
      - test_course_statistics
      - test_custom_population
      - test_filters
@@ -7,8 +7,8 @@ module.exports = {
  '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
  '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
  '*.css': files => `stylelint --fix ${files.join(' ')}`,
-  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0-alpine hadolint ${relativeFilePaths(files)}`,
-  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck-alpine:v0.7.2 ${relativeFilePaths(files)} -x`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0 hadolint ${relativeFilePaths(files)}`,
+  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck:v0.7.2 ${relativeFilePaths(files)} -x`,
  '.github/workflows/*': files => `${dockerCmdBase} rhysd/actionlint:1.6.2 ${relativeFilePaths(files)}`,
  'docker-compose*': files => {
    const composeFiles = file => {
@@ -43,8 +43,8 @@ What different CLI options do:
   - Cleans up any previous real data databases
   - Downloads needed database dumps from production servers.
   - Creates real data databases and populates them with downloaded dumps.
-3. Reset sis-importer data
-   - Like option 2, but only for sis-importer database. Useful when developing updater microservice.
+3. Reset single database
+   - Like option 2, but only for single database. Leaves other databases untouched.

 ## ⌨️ Development

@@ -79,7 +79,7 @@ It is recommended to spend some time to become familiar with `docker` and `docke
 - `docker-compose ps`: view the containers in the running environment
 - `docker-compose logs frontend`: print logs for just frontend
 - `docker-compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
- `docker exec -it userservice sh`: open bash terminal inside userservice container
+- `docker exec -it backend sh`: open bash terminal inside backend container
 - `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.

 ### User types for development
@@ -124,7 +124,15 @@ Continuous integration (CI) works with Github actions and is defined in workflow

 ### Modules are missing after updating package.json

-You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container.
+
+### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data
+
+Make sure you have your VPN on
+
+### I made a change to sis-db models aand created a migration, but tests are failing in CI!
+
+Migrations are handled by sis-importer-worker, since oodikone is using only replica. Since CI is testing only services included in oodikone, your new changes aren't migrated and thus are failing. Open anonyymioodi-folder in this repo to find more about how to to update the anon sis-db.

 ### Everything is broken, can't get oodikone running, data is not there etc.

@@ -135,3 +143,7 @@ Just do clean install by launching cli with `npm cli` and running option 1: _Set
 Toska of course.

 University of Helsinki
+
+## Tietosuoja / data protection
+
+See [tietosuoja.md](tietosuoja.md).
@@ -0,0 +1,2 @@
+*.sqz
+
@@ -0,0 +1,65 @@
+# Anonyymioodi - anonymous database setup for oodikone
+
+(name anonyymioodi for :culture: reasons)
+
+This folder contains a guide and scripts to create anonymous database images for oodikone, mainly used for testing.
+
+## How to create / modify database images
+
+Creating / modifying images varies a bit depending on the db. Generally, to modify images, you can either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose. 
+
+By default, previous db images from Toska Hub are used as a base. If needed, comment out Toska hub image and uncomment corresponding clean postgres image in docker-compose before running other steps.
+
+See more details on how to create / modify databases below.
+
+### Sis-importer-db
+
+The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
+- Move `sis-importer-db.sqz` to this folder
+- Run `./create_container_from_dump.sh sis-importer-db`
+- Enrich db with command in file `enrich_data.sql` either by piping command to container or by running command inside container.
+
+### Sis-db:
+
+The following steps are for building a clean sis-db image from scratch using oodikone's updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
+- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
+- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
+- Start updater, see oodikone readme for more info.
+- Populate sis-db from sis-importer-db:
+
+```
+curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
+```
+(or run these from frontend. You might also want to follow sis-updater-workers logs to ensure things are running smoothly.)
+- Clean up any extra stuff you don't want to keep in sis-db.
+- Bonus: check that cypress tests run okay
+
+### User-db / kone-db
+
+- Temporarily replace the db service in oodikone's docker-compose.yml with the same db service from this folder's docker-compose.yml
+- Ensure the newest user-db/kone-db image is pulled from the registry with `docker-compose pull`
+- Modify db to wanted state. Tips:
+  - Run oodikone if you want to upgrade something from frontend (e.g. tags in kone-db, courses in sis-db etc.).
+  - Easy way to create new users to user-db is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
+
+## Publishing the image
+
+After completing the steps above and making sure your local version has the correct data, run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the db image to Toska Hub.
+
+## How to update Postgres versions
+
+When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
+
+- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
+- Start database with docker-compose
+- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
+- Close container with docker-compose down.
+- Update the Postgres version in docker-compose.yml in this folder.
+- Uncomment row with Postgres image and comment row with Toskahub image.
+- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+infomsg "Committing and pushing to registry"
+database=${args[0]}
+docker commit "$database" "registry.toska.cs.helsinki.fi/$database"
+docker push "registry.toska.cs.helsinki.fi/$database"
+successmsg "Succesfully pushed"
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+# === Create images ===
+
+retry () {
+  sleep 5
+  for i in {1..60}; do
+    "$@" && break || warningmsg "Retry attempt $i failed, waiting..." && sleep 5;
+  done
+}
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+
+infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
+database=${args[0]}
+docker-compose down --remove-orphans
+docker-compose up -d "$database"
+
+database_dump="$database.sqz"
+infomsg "Attempting to restore database $database from dump $database_dump"
+retry docker-compose exec "$database" pg_isready --dbname="$database"
+docker exec -i "$database" /bin/bash -c "pg_restore --username=postgres \
+--format=custom --dbname=$database --no-owner" < "$database_dump"
+successmsg "Database $database succesfully created"
@@ -0,0 +1,50 @@
+x-defaults: &defaults
+  restart: unless-stopped
+
+services:
+  kone-db:
+    <<: *defaults
+    container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/kone-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone
+
+  sis-db:
+    <<: *defaults
+    container_name: sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-db
+    image: postgres:13.2
+    shm_size: 2gb
+
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-importer-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-importer-db
+    image: postgres:13.1
+    profiles:
+      - updater
+
+  user-db:
+    <<: *defaults
+    container_name: user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone
@@ -0,0 +1,133 @@
+-- add these to sis-importer db by running:
+-- docker exec -i sis-importer-db psql -U postgres sis-importer-db < enrich_sampledata.sql
+
+-- if you need to run updater for these students only, see studentnumberlist in the end
+-- this file
+
+-- Add admission types (valintaperusteet) to studyrights for some of the computer
+-- science 2018-2019 population students. See person_ids / student_number below 
+-- (in case you need to update dumps). 
+
+-- TODO: this should be moved into the sampledata creator script in
+-- importer repo, but since this file contains studentnumbers and person ids, it's 
+-- better to keep this here
+
+-- KP (koepisteet)
+-- hy-hlo-1412620   | 013683126
+-- hy-hlo-1485246   | 014340523
+-- hy-hlo-32230770  | 010589388
+-- hy-hlo-35626233  | 010775888
+-- hy-hlo-40150144  | 011065995
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:kp' 
+WHERE person_id IN (
+'hy-hlo-1412620',
+'hy-hlo-1485246',
+'hy-hlo-32230770',
+'hy-hlo-35626233',
+'hy-hlo-40150144'
+);
+
+-- TV (todistusvalinta)
+-- hy-hlo-45783433  | 011321150
+-- hy-hlo-48537946  | 010764653
+-- hy-hlo-49801905  | 011194400
+-- hy-hlo-53321302  | 011338871
+-- hy-hlo-51223300  | 013951984
+-- hy-hlo-53310137  | 012520688
+-- hy-hlo-53621583  | 011196796
+-- hy-hlo-53863064  | 011736884
+-- hy-hlo-57711592  | 010262566
+-- hy-hlo-57732872  | 013670647
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:tv' 
+WHERE person_id IN (
+'hy-hlo-45783433',
+'hy-hlo-48537946',
+'hy-hlo-49801905',
+'hy-hlo-53321302',
+'hy-hlo-51223300',
+'hy-hlo-53310137',
+'hy-hlo-53621583',
+'hy-hlo-53863064',
+'hy-hlo-57711592',
+'hy-hlo-57732872'
+);
+
+-- YP:
+-- hy-hlo-67768022  | 011952325
+-- hy-hlo-73740025  | 011736606
+-- hy-hlo-74780917  | 012775433
+-- hy-hlo-79615371  | 014113941
+-- hy-hlo-78118455  | 014300615
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:yp' 
+WHERE person_id IN (
+'hy-hlo-67768022',
+'hy-hlo-73740025',
+'hy-hlo-74780917',
+'hy-hlo-79615371',
+'hy-hlo-78118455'
+);
+-- AV:
+-- hy-hlo-78777265  | 010779392
+-- hy-hlo-81909550  | 011894153
+-- hy-hlo-84571890  | 013897158
+-- hy-hlo-80330834  | 015085926
+-- hy-hlo-84775722  | 011791272
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:av' 
+WHERE person_id IN (
+'hy-hlo-78777265',
+'hy-hlo-81909550',
+'hy-hlo-84571890',
+'hy-hlo-80330834',
+'hy-hlo-84775722'
+);
+-- M:
+-- hy-hlo-87439827  | 011928456
+-- hy-hlo-96612371  | 014418361
+-- hy-hlo-84840261  | 010975895
+-- hy-hlo-90330681  | 010522688
+-- hy-hlo-96594603  | 013951573
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:m' 
+WHERE person_id IN (
+'hy-hlo-87439827',
+'hy-hlo-96612371',
+'hy-hlo-84840261',
+'hy-hlo-90330681',
+'hy-hlo-96594603'
+);
+
+-- all studentnumbers for running updater:
+013683126
+014340523
+010589388
+010775888
+011065995
+011321150
+010764653
+011194400
+011338871
+013951984
+012520688
+011196796
+011736884
+010262566
+013670647
+011952325
+011736606
+012775433
+014113941
+014300615
+010779392
+011894153
+013897158
+015085926
+011791272
+011928456
+014418361
+010975895
+010522688
+013951573
@@ -44,20 +44,22 @@ will take care of setting up Oodikone for you. See README for more details."
  msg ""
 }

-# Define custom shell prompt for the interactive select loop
-PS3="Please enter your choice: "
+set_custom_select_prompt() {
+  PS3="Please enter your choice: "

-options=(
-  "Set up oodikone from scratch."
-  "Reset all real data."
-  "Reset sis-importer data."
-  "Quit."
-)
+  options=(
+    "Set up oodikone from scratch."
+    "Reset all real data."
+    "Reset single database."
+    "Quit."
+  )
+}

-# Run scripts before showing interactive prompt
+# Run scripts and set the prompt
 show_welcome
 init_dirs
 get_username
+set_custom_select_prompt

 while true; do
  select opt in "${options[@]}"; do
@@ -66,8 +68,10 @@ while true; do
        set_up_oodikone_from_scratch;;
      "Reset all real data.")
        reset_all_real_data;;
-      "Reset sis-importer data.")
-        reset_sis_importer_data;;
+      "Reset single database.")
+        reset_single_database
+        set_custom_select_prompt # Set prompt back to initial values
+        ;;
      "Quit.")
        break 2;;
      *) msg "${RED}Invalid option:${NOFORMAT} $REPLY
@@ -186,6 +186,18 @@ describe('Course Statistics tests', () => {
      cy.contains('td', '010674989').siblings().contains('1')
    })

+    it.skip('Language distribution is correct', () => {
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
+      cy.contains(/^TKT20003$/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
+      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.contains('Language distribution').click()
+      cy.contains('td', 'finnish').siblings().contains('122')
+      cy.contains('td', 'english').siblings().contains('5')
+    })
+
    describe('When searching unified course stats', () => {
      beforeEach(() => {
        cy.url().should('include', '/coursestatistics')
@@ -9,23 +9,38 @@ const checkFilteringResult = studentCount => {
 // Semantic UI doesn't allow injection of data-cy:s for single multiple dropdown selection.
 // This function tries to click "x" on all selections inside dropdown matching given attribute
 const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.delete').click()
+  cy.cs(dataCyAttribute).find('i.delete').click({ force: true })
+  cy.cs(dataCyAttribute).click()
 }

 const clearSingleDropdownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.clear').click()
+  cy.cs(dataCyAttribute).find('i.clear').click({ force: true })
 }

 // Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
 // to every it-function. Reason behind using test function wrapper is that Cypresses internal beforeEach / afterEach
 // functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
 const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
-  return (dataCyAttributeOfHeaderToClick, testStepFunctionToRun) => {
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
-    checkFilteringResult(amountWithoutFiltering)
-    testStepFunctionToRun()
-    checkFilteringResult(amountWithoutFiltering)
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
+  return (filterName, testStepFunctionToRun) => {
+    const card = cy.cs(`${filterName}-filter-card`)
+
+    card.invoke('attr', 'data-open').then(open => {
+      console.log('open', open)
+
+      const getHeader = () => cy.cs(`${filterName}-header`)
+
+      if (open === 'false') {
+        getHeader().click({ force: true })
+      }
+
+      checkFilteringResult(amountWithoutFiltering)
+      testStepFunctionToRun()
+      checkFilteringResult(amountWithoutFiltering)
+
+      if (open === 'false') {
+        getHeader().click({ force: true })
+      }
+    })
  }
 }

@@ -41,90 +56,121 @@ describe('Population Statistics', () => {

  it('Transfer filter is on not transferred by default', () => {
    cy.contains('By default only students who have not transferred to this study programme are shown.')
-    cy.cs('transferredToProgrammeFilter-havenot').find('input').should('be.checked')
+
+    runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
+      const card = cy.cs('TransferredToProgramme-filter-card')
+      card.get('[data-cy="option-havenot"] input').should('be.checked')
+    })
+  })
+
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(5)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
  })

  it('Graduation filter works', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
+
      const graduated = 11
-      cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+      getCard().cs('option-graduated-true').click()
      checkFilteringResult(graduated)
-      cy.cs('graduatedFromProgrammeFilter-graduated-false').click()
+      getCard().cs('option-graduated-false').click()
      checkFilteringResult(defaultAmountOfStudents - graduated)
-      cy.cs('graduatedFromProgrammeFilter-all').click()
+      getCard().cs('option-all').click()
    })
  })

  it('Transfer filter works', () => {
-    runTestStepWithPreAndPostParts('transferredToProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
      const transferred = 1
-      cy.cs('transferredToProgrammeFilter-have').click()
+      cy.cs('TransferredToProgramme-filter-card').cs('option-have').click()
      checkFilteringResult(transferred)
-      cy.cs('transferredToProgrammeFilter-all').click()
+      cy.cs('TransferredToProgramme-filter-card').cs('option-all').click()
      checkFilteringResult(defaultAmountOfStudents + transferred)
-      cy.cs('transferredToProgrammeFilter-havenot').click()
+      cy.cs('TransferredToProgramme-filter-card').cs('option-havenot').click()
    })
  })

  it('Enrollment filter works', () => {
-    runTestStepWithPreAndPostParts('enrollmentStatusFilter-header', () => {
-      cy.selectFromDropdown('enrollmentStatusFilter-status', 0)
-      cy.selectFromDropdown('enrollmentStatusFilter-semesters', [0])
+    runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
+      cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
+      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(0)
      checkFilteringResult(137)
      clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
    })
  })

  it('Credit filter works', () => {
-    runTestStepWithPreAndPostParts('credit-filter-header', () => {
+    runTestStepWithPreAndPostParts('CreditsEarned', () => {
      cy.cs('credit-filter-min').type('50')
      cy.cs('credit-filter-max').type('150')
      checkFilteringResult(116)
-      cy.cs('credit-filter-min').find('input').clear()
-      cy.cs('credit-filter-max').find('input').clear()
+      cy.cs('credit-filter-min').clear()
+      cy.cs('credit-filter-max').clear()
    })
  })

  it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
      cy.cs('ageFilter-min').type('20')
      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(43)
+      checkFilteringResult(40)
      cy.cs('ageFilter-min').find('input').clear()
      cy.cs('ageFilter-max').find('input').clear()
    })
  })

  it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
      checkFilteringResult(36)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
      checkFilteringResult(112)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
      checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
      checkFilteringResult(0)
      clearSingleDropdownSelection('genderFilter-dropdown')
    })
  })

  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', 0)
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
      checkFilteringResult(1)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 14)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(14)
      checkFilteringResult(121)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
    })
  })

-  it('Admission type filter works', () => {
-    runTestStepWithPreAndPostParts('admissionTypeFilter-header', () => {
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 0)
+  // Return when we have +2020 population with admission types in data
+  it.skip('Admission type filter works', () => {
+    runTestStepWithPreAndPostParts('AdmissionTypeFilter', () => {
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(0)
      checkFilteringResult(5)
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 2)
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(2)
      checkFilteringResult(10)
      clearSingleDropdownSelection('admissionTypeFilter-dropdown')
    })
@@ -134,31 +180,32 @@ describe('Population Statistics', () => {
  it.skip('Courses filter works', () => {
    // courses takes some time to load, wait for it to complete
    cy.wait(10000)
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
      cy.cs('courseFilter-course-dropdown').click()
      const courses = ['TKT20001', 'MAT11002']
      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
      checkFilteringResult(140)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
      checkFilteringResult(131)
      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
      checkFilteringResult(56)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 2)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
      checkFilteringResult(2)
      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
    })
  })

  it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
-        cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
+        const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
+        getCard().cs('option-graduated-true').click()
        cy.cs('ageFilter-min').type('20')
        cy.cs('ageFilter-max').type('30')
        checkFilteringResult(11)
        cy.cs('ageFilter-min').find('input').clear()
        cy.cs('ageFilter-max').find('input').clear()
-        cy.cs('graduatedFromProgrammeFilter-all').click()
+        getCard().cs('option-all').click()
      })
    })
  })
@@ -175,7 +222,7 @@ describe('Course Statistics', () => {
  })

  it('Grade filter works', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
      cy.cs('gradeFilter-5').click()
      checkFilteringResult(20)
      cy.cs('gradeFilter-3').click()
@@ -185,44 +232,68 @@ describe('Course Statistics', () => {
    })
  })

+  it('Programme defaults to "Attainment" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Attainment')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(116)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
  it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
      cy.cs('ageFilter-min').type('20')
      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(33)
+      checkFilteringResult(31)
      cy.cs('ageFilter-min').find('input').clear()
      cy.cs('ageFilter-max').find('input').clear()
    })
  })

  it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
      checkFilteringResult(27)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
      checkFilteringResult(91)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
      checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
      checkFilteringResult(0)
      clearSingleDropdownSelection('genderFilter-dropdown')
    })
  })

  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
      checkFilteringResult(1)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 12)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(12)
      checkFilteringResult(29)
      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
    })
  })

  it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
        cy.cs('gradeFilter-3').click()
        cy.cs('ageFilter-min').type('20')
        cy.cs('ageFilter-max').type('30')
@@ -244,13 +315,14 @@ describe('Custom Population Statistics', () => {
    cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
      const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
      runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
-      cy.cs('student-no-input').click().type(students.join('/n'))
+      cy.cs('student-no-input').click().type(students.join('\n'))
+      cy.wait(10000)
    })
    cy.cs('search-button').click()
  })

  it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
      cy.cs('ageFilter-min').type('20')
      cy.cs('ageFilter-max').type('30')
      checkFilteringResult(1)
@@ -259,36 +331,59 @@ describe('Custom Population Statistics', () => {
    })
  })

-  it('Gender filter works', () => {
-    cy.cs('genderFilter-header').click()
-    cy.selectFromDropdown('genderFilter-dropdown', 0)
-    checkFilteringResult(3)
-    cy.selectFromDropdown('genderFilter-dropdown', 1)
-    checkFilteringResult(2)
-    cy.selectFromDropdown('genderFilter-dropdown', 2)
-    checkFilteringResult(0)
-    clearSingleDropdownSelection('genderFilter-dropdown')
-    checkFilteringResult(5, true)
-  })
-
-  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
-      checkFilteringResult(1)
-      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
    })
  })

-  it('Courses filter works', () => {
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
-      const courses = ['MAT11001', 'TKT20004']
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
      checkFilteringResult(5)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
-      checkFilteringResult(5)
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
      checkFilteringResult(3)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+      checkFilteringResult(5, true)
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.cs('startYearAtUni-dropdown').click()
+    })
+  })
+
+  it.skip('Courses filter works', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
+      const courses = ['MAT11001', 'TKT20004']
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[0] + ' ')
+        .click()
+      checkFilteringResult(5)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
+      checkFilteringResult(5)
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[1] + ' ')
+        .click()
+      checkFilteringResult(3)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(3)
      checkFilteringResult(1)
      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
    })
@@ -1,5 +1,8 @@
 /// <reference types="Cypress" />

+const moment = require('moment')
+const _ = require('lodash')
+
 const setPopStatsUntil = (until, includeSettings = []) => {
  cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
  includeSettings.forEach(setting => {
@@ -13,6 +16,8 @@ const setPopStatsUntil = (until, includeSettings = []) => {
 describe('Population Statistics tests', () => {
  const pathToCSBach2017 =
    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2017'
+  const pathToCSMaster2019 =
+    '/populations?months=27&semesters=FALL&semesters=SPRING&studyRights=%7B%22programme%22%3A%22MH50_009%22%7D&tag&year=2019'
  describe('when using basic user', () => {
    beforeEach(() => {
      cy.init('/populations')
@@ -59,6 +64,7 @@ describe('Population Statistics tests', () => {

    it('Population statistics is usable on general level', () => {
      cy.visit(pathToCSBach2017)
+      cy.cs('filtered-students')
      setPopStatsUntil('toukokuu 2020')

      cy.get('.card').within(() => {
@@ -69,7 +75,8 @@ describe('Population Statistics tests', () => {
        cy.contains('Excludes students with non-degree study right')
        cy.contains('Excludes students who have transferred out of this programme')
      })
-      cy.contains('Courses of population').click({ force: true })
+      cy.cs('filtered-students')
+      cy.contains('Courses of population').click()

      cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
      // eslint-disable-next-line cypress/no-unnecessary-waiting
@@ -123,18 +130,62 @@ describe('Population Statistics tests', () => {
      cy.contains('Credit accumulation (for 170 students)')
    })

-    it('Credit Statistics, Statistics pane works', () => {
+    it('Credit Statistics, Credits Gained tab works', () => {
      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
      cy.contains('Credit statistics').click()
+      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the population')
+
+      const months = Math.ceil(moment.duration(moment().diff(moment('2017-08-1'))).asMonths())
+
+      const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
+      const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))
+
+      cy.get('.credits-gained-table').should('contain', '(n=149)')
+
+      for (const [start, end] of ranges) {
+        let value = 'credits'
+
+        if (start !== null) {
+          value = `${start} ≤ ${value}`
+        }
+
+        if (end !== null) {
+          value = `${value} < ${end}`
+        }
+
+        cy.get('.credits-gained-table').should('contain', value)
+      }
+
+      cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
+        const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)
+
+        expect(sum).to.equal(149)
+      })
+
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('not.exist')
+    })
+
+    it('Credit Statistics, Credits Gained tab shows stats by admissions', () => {
+      cy.visit(pathToCSMaster2019)
+      cy.cs('filtered-students')
+      cy.contains('Credit statistics')
+      cy.get("[data-cy='credit-statistics']").click()
+      cy.get('.credits-gained-divider').click()
+      cy.get("[data-cy='credits-gained-table-Avoin väylä']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Yhteispisteet']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Muu']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('exist')
+    })
+
+    it('Credit Statistics, Statistics pane works', () => {
+      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Credit statistics').click({ force: true })
      cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()

      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
      cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
      cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
      cy.get("[data-cy='credit-stats-min']").should('contain', '11')
-      cy.get("[data-cy='credit-stats-q1']").should('contain', '70')
-      cy.get("[data-cy='credit-stats-q2']").should('contain', '129')
-      cy.get("[data-cy='credit-stats-q3']").should('contain', '168')
      cy.get("[data-cy='credit-stats-max']").should('contain', '314')
    })
  })
@@ -13,7 +13,7 @@ describe('Student Statistics tests', () => {
    studentnumber: '010654019',
  }

-  it.only('Student statistics search form is usable', () => {
+  it('Student statistics search form is usable', () => {
    cy.contains('Show student names')
    cy.url().should('include', '/students')
    cy.get('.prompt').type(student.lastname)
@@ -80,7 +80,7 @@ describe('Student Statistics tests', () => {
    cy.contains('Student number').should('not.exist')
  })

-  it.only('Can jump to population page', () => {
+  it('Can jump to population page', () => {
    cy.get('.prompt').type(student.studentnumber)
    cy.contains(student.studentnumber).click()
    cy.get('i.level.up.alternate.icon').eq(0).click()
@@ -88,3 +88,13 @@ describe('Student Statistics tests', () => {
    cy.contains('Tietojenkäsittelytieteen kandiohjelma')
  })
 })
+
+// Use admin to see all students
+describe('Testing with admin rights', () => {
+  it('Does not crash if student has no studyright or courses', () => {
+    const studentNumber = '011825096'
+    cy.init(`/students/${studentNumber}`, 'admin') // use admin to see all students
+    cy.contains('Started: Unavailable')
+    cy.contains('Credits: 0')
+  })
+})
@@ -79,7 +79,7 @@ describe('Studyprogramme overview', () => {
    let originalProductivityCalculatedText
    const doRecalculation = () => {
      // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
-      cy.wait(2000)
+      cy.wait(5000)
      cy.get('.attached').contains('Admin').click()
      cy.contains('recalculate productivity').click()
      cy.contains('recalculate throughput').click()
@@ -4,16 +4,16 @@ describe('Users tests', () => {
  describe('Using user with just grp-oodikone-user, no other rights', () => {
    it('shows only frontpage, trends feedback', () => {
      cy.init('', 'norights') // login with norights user
-      cy.get('.navBar').contains('Trends').should('exist')
-      cy.get('.navBar').contains('Give feedback').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Give feedback').should('exist')

-      cy.get('.navBar').contains('Study programme').should('not.exist')
-      cy.get('.navBar').contains('Student statistics').should('not.exist')
-      cy.get('.navBar').contains('Course statistics').should('not.exist')
-      cy.get('.navBar').contains('Teachers').should('not.exist')
-      cy.get('.navBar').contains('Users').should('not.exist')
-      cy.get('.navBar').contains('Faculty').should('not.exist')
-      cy.get('.navBar').contains('Updater').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Teachers').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Faculty').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Updater').should('not.exist')
    })
  })

@@ -29,32 +29,27 @@ describe('Users tests', () => {
    })

    it('should see more stuff than others', () => {
-      cy.get('.navBar').contains('Study programme').should('exist')
-      cy.get('.navBar').contains('Student statistics').should('exist')
-      cy.get('.navBar').contains('Course statistics').should('exist')
-      // TODO: admin should probably see this as well, since can access this page: cy.get('.navBar').contains('Teachers').should('exist')
-      cy.get('.navBar').contains('Users').should('exist')
-      cy.get('.navBar').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('exist')
+      // TODO: admin should probably see this as well, since can access this page: cy.get('[data-cy="navBar"]').contains('Teachers').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
    })

    it("mocking normal user shows only the mocked user's programmes", () => {
      cy.contains('mocking').should('not.exist')
-      cy.contains('tr', 'basic').within($row => {
+      cy.contains('tr', 'basic').within(() => {
        cy.contains('.button', 'Edit').click()
      })
-      cy.intercept({
-        method: 'POST',
-        path: '/api/superlogin/basic',
-      }).as('superlogin')
      cy.get('i.spy').click()
-      cy.wait('@superlogin')
      cy.contains('mocking as basic')
      cy.wait(1000)
      cy.contains('Study programme').click().siblings().contains('Search by class').click()
      cy.contains('label', 'Study programme')
      cy.contains('label', 'Study programme')
        .siblings()
-        .within($row => {
+        .within(() => {
          cy.get("div[role='option']").should('have.length', 2).contains('Tietojenkäsittelytieteen kandiohjelma')
        })
    })
@@ -7,7 +7,7 @@ const adminUserHeaders = {
  uid: 'admin',
  displayname: 'Admin User',
  'shib-session-id': 'mock-cypress-session',
-  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users;grp-toska',
  edupersonaffiliation: 'member;employee;faculty',
  mail: 'grp-toska+mockadminuser@helsinki.fi',
  hypersonsisuid: 'hy-hlo-6666666',
@@ -54,8 +54,6 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
    if (!headersToUse) throw Error(`${userId} is not valid user id!`)
    req.headers = headersToUse
  })
-  // Hide SIS warning flag to avoid screen size problems when running headlessly
-  window.localStorage.setItem('SIS_WARNING_FLAG', 'true')
  const baseUrl = Cypress.config().baseUrl
  cy.visit(baseUrl.concat(path))
 })
@@ -63,24 +61,37 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
 /**
 * Shorthand for using "Cypress Selectors" (CS), i.e., `data-cy` attributes.
 */
-Cypress.Commands.add('cs', name => {
-  return cy.get(`[data-cy='${name}']`)
+Cypress.Commands.add('cs', { prevSubject: 'optional' }, (subject, name) => {
+  const selector = `[data-cy="${name}"]`
+
+  if (subject === undefined) {
+    return cy.get(selector)
+  } else {
+    return cy.wrap(subject).find(selector)
+  }
 })

 /**
 * Select item specified by `index` number from a semantic-ui dropdown with
 * `data-cy` attribute value `name`.
 */
-Cypress.Commands.add('selectFromDropdown', (name, index) => {
+Cypress.Commands.add('selectFromDropdown', { prevSubject: true }, (s, index) => {
+  const subject = cy.wrap(s)
  const indexes = Array.isArray(index) ? index : [index]

  indexes.forEach(i => {
-    cy.cs(name).click().children('.menu').children().eq(i).click({ force: true })
+    subject.click()
+
+    if (typeof i === 'number') {
+      subject.children('.menu').children().eq(i).click({ force: true })
+    } else {
+      subject.children('.menu').containing(i).eq(0).click({ force: true })
+    }
  })

  // Close multiple selection so it does not block elements underneath it.
  if (Array.isArray(index)) {
-    cy.cs(name).children('.icon').click({ force: true })
+    cy.wrap(s).children('.icon').click({ force: true })
  }
 })

@@ -2,7 +2,6 @@ x-defaults: &defaults
  restart: unless-stopped

 services:
-
  backend:
    <<: *defaults
    container_name: backend
@@ -10,17 +9,16 @@ services:
      - kone-db
      - redis
      - sis-db
+      - user-db
    environment:
-      CI: "true"
+      CI: 'true'
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
      FRONT_URL: http://localhost:1337
      REDIS: redis
      SECRET_TOKEN: dev
      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
    image: toska/oodikone2-backend:staging

  frontend:
@@ -31,11 +29,15 @@ services:
  kone-db:
    <<: *defaults
    container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
    image: registry.toska.cs.helsinki.fi/kone-db

  redis:
    <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
    container_name: redis
    image: redis
    ports:
@@ -45,23 +47,20 @@ services:
    <<: *defaults
    container_name: sis-db
    image: registry.toska.cs.helsinki.fi/sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    shm_size: 2gb

  user-db:
    <<: *defaults
    container_name: user-db
-    image: registry.toska.cs.helsinki.fi/user-db
-
-  userservice:
-    <<: *defaults
-    container_name: userservice
-    depends_on:
-      - user-db
    environment:
-      CI: "true"
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    image: toska/oodikone2-userservice:staging
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db

  nginx:
    image: nginx:latest
@@ -69,8 +68,45 @@ services:
    depends_on:
      - frontend
      - backend
-      - userservice
    ports:
-      -  1337:80
+      - 1337:80
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf
+
+  # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    image: registry.toska.cs.helsinki.fi/sis-importer-db
+    profiles:
+      - updater
+
+  sis-updater-worker:
+    <<: *defaults
+    build:
+      target: development
+      context: updater/sis-updater-worker
+    depends_on:
+      - redis
+      - sis-db
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_URI: //redis:6379
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+    profiles:
+      - updater
+
+  sis-updater-nats:
+    <<: *defaults
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    container_name: sis-updater-nats
+    image: nats-streaming:0.16.2
+    profiles:
+      - updater
@@ -1,7 +1,7 @@
 services:
-
  backend:
    environment:
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db-real
      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db-real
      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db-real

@@ -18,9 +18,5 @@ services:
      DB_URL: postgres://postgres@sis-db:5432/sis-db-real
      SIS_IMPORTER_DATABASE: sis-importer-db-real

-  userservice:
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db-real
-
 volumes:
  redis-data-real:
@@ -1,10 +0,0 @@
-services:
-
-  userservice-test:
-    build:
-      context: services/userservice
-      target: test
-    container_name: userservice-test
-    environment:
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
@@ -26,18 +26,17 @@ services:
      - kone-db
      - redis
      - sis-db
+      - user-db
    env_file:
      - .env
    environment:
      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
      FRONT_URL: http://localhost:3000
      REDIS: redis
      SECRET_TOKEN: dev
      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
    ports:
      - 8080:8080
    profiles:
@@ -165,26 +164,6 @@ services:
    volumes:
      - user-db-data:/data

-  userservice:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/userservice
-    container_name: userservice
-    depends_on:
-      - user-db
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    ports:
-      - 4567:4567
-    profiles:
-      - oodikone
-    volumes:
-      - userservice-nodemod:/usr/src/app/node_modules
-      - ./services/userservice/:/usr/src/app
-
 volumes:
  backend-nodemod:
  frontend-nodemod:
@@ -195,4 +174,3 @@ volumes:
  sis-updater-scheduler-nodemod:
  sis-updater-worker-nodemod:
  user-db-data:
-  userservice-nodemod:
@@ -6,6 +6,7 @@
    "concurrently": "concurrently",
    "eslint": "eslint",
    "stylelint": "stylelint",
+    "prettier": "prettier",
    "cli": "./cli.sh",
    "oodikone": "./run.sh oodikone anon up --detach",
    "oodikone:real": "./run.sh oodikone real up --detach",
@@ -48,5 +49,9 @@
    "stylelint-config-prettier": "^8.0.2",
    "stylelint-config-standard": "^22.0.0",
    "stylelint-prettier": "^1.2.0"
+  },
+  "dependencies": {
+    "lodash": "^4.17.21",
+    "moment": "^2.29.1"
  }
 }
@@ -64,8 +64,7 @@ parse_profiles() {
  return 0
 }

-# Set which docker-compose files to use based on version. In ci, profiles are also
-# emptied, since they're passed from github actions.
+# Set which docker-compose files to use based on version.
 parse_env() {
  env=""
  if [[ "$version" == "real" ]]; then
@@ -20,10 +20,10 @@ USER_DB_NAME="user-db"
 DATABASES=("$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")

 ## Urls should be in same order as databases as both are iterated through by indexes.
-KONE_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-kone-pg.sqz"
+KONE_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-kone-db.sqz"
 SIS_DB_REAL_DUMP_URL="svm-96.cs.helsinki.fi:/home/updater_user/backups/latest-sis.sqz"
-SIS_IMPORTER_DB_REAL_DUMP_URL="importer:/home/importer_user/importer-db/backup/importer-db.sqz"
-USER_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-user-pg.sqz"
+SIS_IMPORTER_DB_REAL_DUMP_URL="svm-92.cs.helsinki.fi:/home/importer_user/importer-db/backup/importer-db.sqz"
+USER_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-user-db.sqz"
 REAL_DUMP_URLS=("$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")

 # Source utility functions
@@ -110,12 +110,54 @@ reset_all_real_data() {
  reset_databases "${DATABASES[@]}"
 }

-reset_sis_importer_data() {
-  infomsg "Downloading sis-importer-db dump"
-  local database=$SIS_IMPORTER_DB_NAME
-  local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
-  download_real_dump $database $url
-  reset_databases $database
+reset_single_database() {
+  # Define custom shell prompt for the nested interactive select loop
+  PS3="Please enter which database to reset: "
+
+  options=(
+    "kone-db"
+    "sis-db"
+    "sis-importer-db"
+    "user-db"
+    "Return to main menu."
+  )
+
+  local database=""
+  local url=""
+
+  while true; do
+    select opt in "${options[@]}"; do
+      case $opt in
+        "kone-db")
+          local database=$KONE_DB_NAME
+          local url=$KONE_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-db")
+          local database=$SIS_DB_NAME
+          local url=$SIS_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-importer-db")
+          local database=$SIS_IMPORTER_DB_NAME
+          local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
+          break 2;;
+        "user-db")
+          local database=$USER_DB_NAME
+          local url=$USER_DB_REAL_DUMP_URL
+          break 2;;
+        "Return to main menu.")
+          break 2;;
+        *) msg "${RED}Invalid option:${NOFORMAT} $REPLY
+  ";;
+      esac
+      break
+    done
+  done
+
+  if [ -n "$database" ]; then
+    infomsg "Downloading $database dump"
+    download_real_dump $database $url
+    reset_databases $database
+  fi
 }

 set_up_oodikone_from_scratch() {
@@ -123,7 +165,6 @@ set_up_oodikone_from_scratch() {
  folders_to_set_up=(
    "$PROJECT_ROOT"
    "$PROJECT_ROOT/services/frontend"
-    "$PROJECT_ROOT/services/userservice"
    "$PROJECT_ROOT/services/backend"
    "$PROJECT_ROOT/updater/sis-updater-scheduler"
    "$PROJECT_ROOT/updater/sis-updater-worker"
@@ -152,10 +193,10 @@ linting and formatting"
  npm run prepare

  infomsg "Pulling images"
-  "$PROJECT_ROOT"/run.sh oodikone anon pull
+  "$PROJECT_ROOT"/run.sh both anon pull

  infomsg "Building images"
-  "$PROJECT_ROOT"/run.sh oodikone anon build
+  "$PROJECT_ROOT"/run.sh both anon build

  successmsg "Setup ready, oodikone can be started! See README for more info."
 }
@@ -1,6 +1,3 @@
 .dockerignore
 Dockerfile
 node_modules/
-requests/
-scripts/
-*.log
@@ -1,3 +0,0 @@
-# Student diff output
-/student_diff_*
-/diff_partial_att_data.json
@@ -1,5 +1,6 @@
 require('./src/app')
+const logger = require('./src/util/logger')

 process.on('unhandledRejection', reason => {
-  console.log(reason)
+  logger.error({ message: 'unhandledRejection: ', meta: reason })
 })
@@ -15,12 +15,12 @@
    "@sentry/tracing": "^6.11.0",
    "async": "^3.2.1",
    "async-lock": "^1.3.0",
-    "axios": "^0.21.1",
+    "axios": "^0.21.2",
    "bluebird": "^3.7.2",
    "cors": "^2.8.5",
    "cron": "^1.8.2",
    "express": "^4.17.1",
-    "jsonwebtoken": "^8.5.1",
+    "express-async-errors": "^3.1.1",
    "lodash": "^4.17.21",
    "lru-cache": "^5.1.1",
    "moment": "^2.29.1",
@@ -32,9 +32,7 @@
    "umzug": "^2.3.0",
    "unfuck-utf8-headers-middleware": "^1.0.1",
    "uuid": "^3.4.0",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0",
-    "winston-transport": "^4.4.0"
+    "winston": "^3.3.3"
  },
  "devDependencies": {
    "nodemon": "^1.19.4"
@@ -1,51 +1,35 @@
+require('express-async-errors')
 const express = require('express')
-const cors = require('cors')
-const Sentry = require('@sentry/node')
-const { baseUrl, backendPort, frontUrl } = require('./conf-backend')
+const { baseUrl, backendPort } = require('./conf-backend')
 const routes = require('./routes')
 const { startCron } = require('./events')
 const { initializeDatabaseConnection, dbConnections } = require('./database/connection')
-const initializeSentry = require('./util/sentry')
+const logger = require('./util/logger')

 initializeDatabaseConnection()
  .then(() => {
    dbConnections.connect()

    dbConnections.on('connect', () => {
-      console.log('Connected to sis db successfully')
+      logger.info('Connected to sis db successfully')
    })

    dbConnections.on('error', () => {
-      console.log('Failed connecting to sis db')
+      logger.error('Failed to connect to sis db!')
    })
-
-    const app = express()
-    initializeSentry(app)
-    app.use(Sentry.Handlers.requestHandler())
-    app.use(Sentry.Handlers.tracingHandler())
-
    startCron()

-    app.use(cors({ credentials: true, origin: frontUrl }))
-    app.use(express.json())
-
+    const app = express()
    routes(app, baseUrl)

-    app.get('*', async (req, res) => {
-      const results = { error: 'unknown endpoint' }
-      res.status(404).json(results)
-    })
-
-    app.use(Sentry.Handlers.errorHandler())
-
-    const server = app.listen(backendPort, () => console.log(`Backend listening on port ${backendPort}!`))
+    const server = app.listen(backendPort, () => logger.info(`Backend listening on port ${backendPort}!`))
    process.on('SIGTERM', () => {
      server.close(() => {
-        console.log('Process terminated')
+        logger.info('Process terminated')
      })
    })
  })
  .catch(e => {
    process.exitCode = 1
-    console.log(e)
+    logger.error(e)
  })
@@ -18,7 +18,6 @@ const pateToken = process.env.PATE_API_TOKEN || ''

 // Importer client
 const importerToken = process.env.IMPORTER_DB_API_TOKEN || ''
-const importerUrl = process.env.IMPORTER_DB_API_URL || ''

 // Networking: Urls & ports
 const baseUrl = isDev ? '/api' : ''
@@ -37,6 +36,7 @@ const MATOMO_URL = process.env.MATOMO_URL

 // Other stuff
 const DB_URL_KONE = process.env.DB_URL_KONE
+const DB_URL_USER = process.env.DB_URL_USER

 let DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS, 10)
 if (isNaN(DB_MAX_CONNECTIONS)) {
@@ -48,10 +48,8 @@ if (DB_MAX_CRON_CONNECTIONS < 1) {
 }

 const redis = process.env.REDIS
-const TOKEN_SECRET = process.env.TOKEN_SECRET
 const SECRET_TOKEN = process.env.SECRET_TOKEN
 const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
-const USERSERVICE_URL = process.env.USERSERVICE_URL
 const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL

 const formatURL = url => {
@@ -60,18 +58,14 @@ const formatURL = url => {

 const addSlashToEnd = url => (url.endsWith('/') ? url : url + '/')

-const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
-
 module.exports = {
+  DB_URL_USER,
  DB_URL_KONE,
  DB_MAX_CONNECTIONS,
  DB_MAX_CRON_CONNECTIONS,
  redis,
-  TOKEN_SECRET,
  SECRET_TOKEN,
  DB_SCHEMA_KONE,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL),
-  ACCESS_TOKEN_HEADER_KEY,
  requiredGroup,
  SIS_UPDATER_URL,
  ANALYTICS_INFLUXDB_URL: formatURL(ANALYTICS_INFLUXDB_URL),
@@ -89,5 +83,4 @@ module.exports = {
  frontUrl,
  backendPort,
  importerToken,
-  importerUrl,
 }
@@ -3,6 +3,8 @@ const EventEmitter = require('events')
 const Umzug = require('umzug')
 const conf = require('../conf-backend')
 const { SIS_DB_URL } = process.env
+const logger = require('../util/logger')
+
 class DbConnection extends EventEmitter {
  constructor() {
    super()
@@ -31,74 +33,79 @@ class DbConnection extends EventEmitter {
        this.emit('error', e)
        return
      }
-      console.log(`Sis database connection failed! Attempt ${attempt}/${this.RETRY_ATTEMPTS}`)
-      console.log(e)
+      logger.error({ message: `Sis database connection failed! Attempt ${attempt}/${this.RETRY_ATTEMPTS}`, meta: e })
      setTimeout(() => this.connect(attempt + 1), 1000 * attempt)
    }
  }
 }

+// Old-style kone + user db connections
 const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
  schema: conf.DB_SCHEMA_KONE,
  searchPath: conf.DB_SCHEMA_KONE,
  logging: false,
 })
+
 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)

+const sequelizeUser = new Sequelize(conf.DB_URL_USER, {
+  logging: false,
+})
+
 const initializeDatabaseConnection = async () => {
  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-  const waitSeconds = 60
-  for (let i = 1; i <= waitSeconds; i++) {
-    try {
-      await sequelizeKone.authenticate()
-      break
-    } catch (e) {
-      if (i === waitSeconds) {
-        console.log(`Could not connect to database in ${waitSeconds} seconds`)
-        throw e
+  const rounds = 60
+
+  for (const [seq, dbName] of [
+    [sequelizeKone, 'kone-db'],
+    [sequelizeUser, 'user-db'],
+  ]) {
+    logger.info(`Connecting to ${dbName}...`)
+    for (let round = 1; round <= rounds; round++) {
+      try {
+        await seq.authenticate()
+        break
+      } catch (error) {
+        if (round === rounds) {
+          logger.error(`${dbName} database connection failed!`)
+          throw error
+        }
+        await sleep(1000)
      }
-      console.log('.')
-      await sleep(1000)
    }
-  }
-  try {
+    logger.info(`${dbName} database connection established`)
+
+    const schema = dbName === 'kone-db' ? conf.DB_SCHEMA_KONE : undefined
+    const migrationsFolder = dbName === 'kone-db' ? 'migrations_kone' : 'migrations_user'
    const migrator = new Umzug({
      storage: 'sequelize',
      storageOptions: {
-        sequelize: sequelizeKone,
+        sequelize: seq,
        tableName: 'migrations',
-        schema: conf.DB_SCHEMA_KONE,
+        schema,
      },
-      logging: console.log,
      migrations: {
-        params: [sequelizeKone.getQueryInterface(), Sequelize],
-        path: `${process.cwd()}/src/database/migrations_kone`,
+        params: [seq.getQueryInterface(), Sequelize],
+        path: `${process.cwd()}/src/database/${migrationsFolder}`,
        pattern: /\.js$/,
-        schema: conf.DB_SCHEMA_KONE,
+        schema,
      },
    })
-    const migrations = await migrator.up()
-
-    console.log('Kone Migrations up to date', migrations)
-  } catch (e) {
-    console.log('Kone Migration error')
-    throw e
+    try {
+      const migrations = await migrator.up()
+      logger.info({ message: `${dbName} migrations up to date: `, meta: migrations.map(m => m.file) })
+    } catch (error) {
+      logger.error({ message: `${dbName} migrations failed`, meta: error })
+      throw error
+    }
  }
 }

-const forceSyncDatabase = async () => {
-  try {
-    await sequelizeKone.getQueryInterface().createSchema(conf.DB_SCHEMA_KONE)
-  } catch (e) {
-    //console.log(e)
-  }
-  await sequelizeKone.sync({ force: true })
-}
-
 const dbConnections = new DbConnection()
+
 module.exports = {
  dbConnections,
  sequelizeKone,
+  sequelizeUser,
  initializeDatabaseConnection,
-  forceSyncDatabase,
 }
@@ -0,0 +1,34 @@
+const { BIGINT, STRING, DATE } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.createTable(
+      'study_guidance_group_tags',
+      {
+        id: {
+          primaryKey: true,
+          type: BIGINT,
+          autoIncrement: true,
+        },
+        study_guidance_group_id: {
+          type: STRING,
+          unique: true,
+        },
+        study_programme: {
+          type: STRING,
+        },
+        year: {
+          type: STRING,
+        },
+        created_at: {
+          type: DATE,
+        },
+        updated_at: {
+          type: DATE,
+        },
+      },
+      { underscored: true, timestamps: true }
+    )
+  },
+  down: async () => {},
+}
@@ -0,0 +1,10 @@
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const Op = Sequelize.Op
+    return queryInterface.bulkDelete('access_groups', [
+      {
+        group_code: { [Op.in]: ['dev', 'users', 'cooldata', 'studyprogramme'] },
+      },
+    ])
+  },
+}
@@ -0,0 +1,10 @@
+const { DATE } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.addColumn('users', 'last_login', DATE)
+  },
+  down: async queryInterface => {
+    await queryInterface.removeColumn('users', 'last_login')
+  },
+}
@@ -0,0 +1,8 @@
+module.exports = {
+  up: async queryInterface =>
+    queryInterface.sequelize.query(`
+      --- add lisensiaatti to lääkis faculty
+      INSERT INTO faculty_programmes VALUES ('H90', 'MH90_001', 'NOW()', 'NOW()');
+    `),
+  down: async () => {},
+}
@@ -0,0 +1,6 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('faculty_programmes')
+  },
+  down: async () => {},
+}
@@ -0,0 +1,9 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('user_affiliation')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS user_affilation_id_seq')
+    await queryInterface.dropTable('affiliations')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS affiliations_id_seq')
+  },
+  down: async () => {},
+}
@@ -0,0 +1,9 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('user_hy_group')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS user_hy_group_id_seq')
+    await queryInterface.dropTable('hy_groups')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS hy_groups_id_seq')
+  },
+  down: async () => {},
+}
@@ -9,209 +9,168 @@ const {
  getStartYears,
 } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
-const { getAllProgrammes, nonGraduatedStudentsOfElementDetail } = require('./services/studyrights')
+const { getAllProgrammes } = require('./services/studyrights')
 const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studyprogramme')
 const { findAndSaveTeachers } = require('./services/topteachers')
-const {
-  setProductivity,
-  setThroughput,
-  patchProductivity,
-  patchThroughput,
-  patchNonGraduatedStudents,
-} = require('./services/analyticsService')
-const { isNewHYStudyProgramme } = require('./util')
+const { setProductivity, setThroughput, patchProductivity, patchThroughput } = require('./services/analyticsService')
 const { isProduction } = require('./conf-backend')
+const { getCurrentSemester } = require('./services/semesters')
+const logger = require('./util/logger')

 const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })

 const refreshStudyrightAssociations = async () => {
-  try {
-    console.log('Refreshing studyright associations...')
-    await refreshAssociationsInRedis()
-  } catch (e) {
-    console.error(e)
-  }
+  await refreshAssociationsInRedis()
+  logger.info('Studyright associations refreshed!')
 }

 const refreshOverview = async () => {
-  try {
-    console.log('Refreshing throughput and productivity for programmes...')
-    const codes = (await getAllProgrammes()).map(p => p.code)
-    let ready = 0
-    for (const code of codes) {
-      let programmeStatsSince = new Date('2017-07-31')
-      if (code.includes('MH') || code.includes('KH')) {
-        programmeStatsSince = new Date('2017-07-31')
-      } else {
-        programmeStatsSince = new Date('2000-07-31')
-      }
-      try {
-        await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-        const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
-        await setThroughput(data)
-      } catch (e) {
-        try {
-          await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-        } catch (e) {
-          console.error(e)
-        }
-        console.error(e)
-        console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
-      }
-      try {
-        await patchProductivity({ [code]: { status: 'RECALCULATING' } })
-        const data = await productivityStatsForStudytrack(code, programmeStatsSince)
-        await setProductivity(data)
-      } catch (e) {
-        try {
-          await patchProductivity({
-            [code]: { status: 'RECALCULATION ERRORED' },
-          })
-        } catch (e) {
-          console.error(e)
-        }
-        console.error(e)
-        console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-      }
-      ready += 1
-      console.log(`${ready}/${codes.length} programmes done`)
+  logger.info('Refreshing throughput and productivity for programmes...')
+  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH') || code.includes('MH'))
+  let ready = 0
+  for (const code of codes) {
+    let programmeStatsSince = new Date('2017-07-31')
+    if (code.includes('MH') || code.includes('KH')) {
+      programmeStatsSince = new Date('2017-07-31')
+    } else {
+      programmeStatsSince = new Date('2000-07-31')
    }
-  } catch (e) {
-    console.error(e)
+    try {
+      await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+      const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
+      await setThroughput(data)
+    } catch (e) {
+      logger.error({ message: `failed to recalculate throughput for ${code}`, meta: e })
+      try {
+        await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
+      } catch (e) {
+        logger.error({ message: 'failed to update throughtput status to error', meta: e })
+      }
+    }
+    try {
+      await patchProductivity({ [code]: { status: 'RECALCULATING' } })
+      const data = await productivityStatsForStudytrack(code, programmeStatsSince)
+      await setProductivity(data)
+    } catch (e) {
+      logger.error({ message: `failed to recalculate productivity for ${code}`, meta: e })
+      try {
+        await patchProductivity({
+          [code]: { status: 'RECALCULATION ERRORED' },
+        })
+      } catch (e) {
+        logger.error({ message: 'failed to update productivity status to error', meta: e })
+      }
+    }
+    ready += 1
+    logger.info(`${ready}/${codes.length} programmes done`)
  }
+  logger.info('Throughput and productivity for programmes refreshed')
 }

 const refreshTeacherLeaderboard = async () => {
-  try {
-    const startyearcode = new Date().getFullYear() - 1950
-    const endyearcode = startyearcode + 1
-    console.log('Refreshing teacher leaderboard...')
-    await findAndSaveTeachers(startyearcode, endyearcode)
-  } catch (e) {
-    console.log(e)
-  }
-}
-
-const refreshNonGraduatedStudentsOfOldProgrammes = async () => {
-  try {
-    const oldProgrammeCodes = (await getAllProgrammes()).map(p => p.code).filter(c => !isNewHYStudyProgramme(c))
-    let i = 0
-    console.log('Refreshing non-graduated students of old programmes...')
-    await Promise.all(
-      oldProgrammeCodes.map(
-        c =>
-          new Promise(async res => {
-            try {
-              const [nonGraduatedStudents, studentnumbers] = await nonGraduatedStudentsOfElementDetail(c)
-              await patchNonGraduatedStudents({ [c]: { formattedData: nonGraduatedStudents, studentnumbers } })
-              console.log(`${++i}/${oldProgrammeCodes.length}`)
-            } catch (e) {
-              console.log(`Failed refreshing non-graduated students of programme ${c}!`)
-            }
-            res()
-          })
-      )
-    )
-  } catch (e) {
-    console.log(e)
-  }
+  // refresh this and previous year
+  const currentSemestersYearCode = (await getCurrentSemester()).getDataValue('yearcode')
+  await findAndSaveTeachers(currentSemestersYearCode - 1, currentSemestersYearCode)
 }

 const refreshProtoCtoRedis = async () => {
-  try {
-    const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false' }
-    const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false' }
-    const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true' }
-    const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true' }
-    console.log('Refreshing CDS ProtoC')
-    await refreshProtoC(defaultQuery)
-    await refreshProtoC(onlyOld)
-    await refreshProtoC(onlyEnr)
-    await refreshProtoC(bothToggles)
-  } catch (e) {
-    console.log(e)
-  }
+  logger.info('Refreshing CDS ProtoC')
+  const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false' }
+  const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false' }
+  const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true' }
+  const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true' }
+  await refreshProtoC(defaultQuery)
+  await refreshProtoC(onlyOld)
+  await refreshProtoC(onlyEnr)
+  await refreshProtoC(bothToggles)
+  logger.info('Refreshing CDS ProtoC doned')
 }

 const refreshStatusToRedis = async () => {
-  try {
-    const unixMillis = moment().valueOf()
-    const date = new Date(Number(unixMillis))
+  const unixMillis = moment().valueOf()
+  const date = new Date(Number(unixMillis))

-    date.setHours(23, 59, 59, 999)
-    const showByYearOff = 'false'
-    const showByYear = 'true'
-    console.log('Refreshing CDS Status')
-    await refreshStatus(date.getTime(), showByYearOff)
-    await refreshStatus(date.getTime(), showByYear)
+  date.setHours(23, 59, 59, 999)
+  const showByYearOff = 'false'
+  const showByYear = 'true'
+  logger.info('Refreshing CDS Status')
+  await refreshStatus(date.getTime(), showByYearOff)
+  await refreshStatus(date.getTime(), showByYear)
+  logger.info('Refreshing CDS Status doned')

-    console.log('Refreshing CDS Graduated')
-    await refreshStatusGraduated(date.getTime(), showByYearOff)
-    await refreshStatusGraduated(date.getTime(), showByYear)
-  } catch (e) {
-    console.log(e)
-  }
+  logger.info('Refreshing CDS Graduated')
+  await refreshStatusGraduated(date.getTime(), showByYearOff)
+  await refreshStatusGraduated(date.getTime(), showByYear)
+  logger.info('Refreshing CDS graduated doned')
 }

 const refreshUberToRedis = async () => {
-  try {
-    const years = await getStartYears()
-    const mappedYears = years.map(({ studystartdate }) => studystartdate)
-    mappedYears.forEach(async year => {
-      console.log('Refreshing CDS Uber data for date', year)
-      const defaultQuery = { include_old_attainments: 'false', start_date: year }
-      const oldAttainmentsQuery = { include_old_attainments: 'true', start_date: year }
+  const years = (await getStartYears()).map(({ studystartdate }) => studystartdate)
+  for (const year of years) {
+    const formattedYear = new Date(year).getFullYear()
+    logger.info(`Refreshing CDS uber year ${formattedYear}`)
+    const defaultQuery = { include_old_attainments: 'false', start_date: year }
+    const oldAttainmentsQuery = { include_old_attainments: 'true', start_date: year }
+    try {
      await refreshUber(defaultQuery)
      await refreshUber(oldAttainmentsQuery)
-    })
-  } catch (e) {
-    console.log(e)
+      logger.info(`Refreshing CDS uber year ${formattedYear} doned`)
+    } catch (e) {
+      logger.error({ message: `Error when refreshing CDS uber year ${formattedYear}`, meta: e })
+    }
  }
+  logger.info(`Refreshing CDS Uber data doned`)
 }

 const refreshProtoCProgrammeToRedis = async () => {
-  try {
-    const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH'))
-    codes.forEach(async code => {
-      console.log('refreshing code', code)
-      const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
-      const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
-      const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
-      const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true', code }
-      console.log('Refreshing CDS ProtoCProgramme for code ', code)
-      await refreshProtoCProgramme(defaultQuery)
-      await refreshProtoCProgramme(onlyOld)
-      await refreshProtoCProgramme(onlyEnr)
-      await refreshProtoCProgramme(bothToggles)
-    })
-  } catch (e) {
-    console.log(e)
+  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH'))
+  for (const code of codes) {
+    logger.info(`Refreshing ProtoCProgramme code ${code}`)
+    const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
+    const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
+    const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
+    const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true', code }
+    const queries = [defaultQuery, onlyOld, onlyEnr, bothToggles]
+    for (const query of queries) {
+      try {
+        await refreshProtoCProgramme(query)
+      } catch (e) {
+        logger.error({
+          message: `Error when refreshing ProtocProgramme code ${code}, query ${Object.keys({ query })[0]}`,
+          meta: e,
+        }) // object.keys is hax to print query variable name
+      }
+    }
+    logger.info(`Refreshing ProtoCProgramme code ${code} doned`)
  }
+  logger.info(`Refreshing protocprogramme doned`)
 }

 const refreshStatistics = async () => {
-  console.log('Starting refreshing statistics')
-  await refreshStudyrightAssociations()
-  await refreshOverview()
-  await refreshNonGraduatedStudentsOfOldProgrammes()
-  await refreshTeacherLeaderboard()
-  console.log('Statistics refreshed!')
+  const statfuncs = [refreshStudyrightAssociations, refreshOverview, refreshTeacherLeaderboard]
+  logger.info('Refreshing statistics')
+  for (const func of statfuncs) {
+    await func()
+  }
+  logger.info('Statistics refreshed!')
 }

 const refreshTrends = async () => {
-  console.log('Starting refreshing trends')
-  await refreshProtoCtoRedis()
-  await refreshStatusToRedis()
-  await refreshUberToRedis()
-  await refreshProtoCProgrammeToRedis()
-  console.log('Trends refreshed!')
+  const trendfuncs = [refreshProtoCtoRedis, refreshStatusToRedis, refreshUberToRedis, refreshProtoCProgrammeToRedis]
+  logger.info('Refreshing trends')
+  for (const func of trendfuncs) {
+    await func()
+  }
+  logger.info('Trends refreshed!')
 }

 const startCron = () => {
  if (isProduction) {
-    schedule('0 6 * * *', async () => {
-      await refreshStatistics()
-      await refreshTrends()
+    // refresh 3am every day
+    schedule('0 3 * * *', async () => {
+      for (const func of [refreshStatistics, refreshTrends]) {
+        await func()
+      }
    })
  }
 }
@@ -219,4 +178,5 @@ const startCron = () => {
 module.exports = {
  startCron,
  refreshStatistics,
+  refreshTrends,
 }
@@ -1,5 +1,6 @@
 const morgan = require('morgan')
 const logger = require('../util/logger')
+const _ = require('lodash')

 // So this appears to be a hack to get neatly formatted stats like response-time etc. from morgan
 // without actually using morgan what it's used for (LOGGING REQUESTS!).
@@ -7,7 +8,8 @@ const logger = require('../util/logger')
 // so don't e.g. make it into an async function or it'll print [object Promise] on every request
 const accessLogger = morgan((tokens, req, res) => {
  const fields = ['method', 'url', 'status', 'response-time', 'remote-addr', 'remote-user', 'user-agent', 'referrer']
-  const meta = { ...req.decodedToken }
+  const { user } = req
+  const meta = { ...user }
  fields.forEach(field => (meta[field] = tokens[field](req, res)))
  meta['time'] = tokens['date'](req, res, 'iso')
  if (req.route) {
@@ -15,7 +17,7 @@ const accessLogger = morgan((tokens, req, res) => {
  }

  const message = [
-    req.decodedToken.name,
+    user.name,
    ':',
    tokens['method'](req, res),
    tokens['url'](req, res),
@@ -26,7 +28,8 @@ const accessLogger = morgan((tokens, req, res) => {
  ].join(' ')

  logger.info(message, {
-    ...meta,
+    // don't log student list which might be huge
+    ..._.omit(meta, ['studentsUserCanAccess']),
    // pass this as a custom field so we can filter by it in graylog
    isUsageStats: true,
  })
@@ -1,43 +1,8 @@
-const jwt = require('jsonwebtoken')
-const conf = require('../conf-backend')
-const blacklist = require('../services/blacklist')
-const { getUserDataFor } = require('../services/userService')
-const { ACCESS_TOKEN_HEADER_KEY } = require('../conf-backend')
-const { hasRequiredGroup, parseHyGroups } = require('../util/utils')
-
-const TOKEN_VERSION = 1 // When token structure changes, increment in userservice, backend and frontend
-
-const checkAuth = async (req, res, next) => {
-  const token = req.headers[ACCESS_TOKEN_HEADER_KEY]
-  const uid = req.headers['uid']
-
-  if (!token) {
-    return res.status(403).json({ error: 'No token in headers' })
-  }
-
-  jwt.verify(token, conf.TOKEN_SECRET, async (err, decoded) => {
-    if (err) {
-      return res.status(403).json(err)
-    }
-
-    if (decoded.version !== TOKEN_VERSION) {
-      return res.status(401).json({ error: 'Token needs to be refreshed - invalid version' })
-    }
-
-    if (decoded.mockedBy ? decoded.mockedBy !== uid : decoded.userId !== uid) {
-      return res.status(403).json({ error: 'User shibboleth id and token id did not match' })
-    }
-
-    const userData = await getUserDataFor(decoded.userId)
-    req.decodedToken = decoded
-    Object.assign(req, userData)
-    next()
-  })
-}
-
 const roles = requiredRoles => async (req, res, next) => {
-  if (req.decodedToken) {
-    const { roles } = req
+  if (req.user) {
+    const {
+      user: { roles },
+    } = req

    if (requiredRoles.every(r => roles.indexOf(r) >= 0) || roles.includes('admin')) {
      return next()
@@ -47,37 +12,6 @@ const roles = requiredRoles => async (req, res, next) => {
  res.status(403).json({ error: 'missing required roles' })
 }

-const checkRequiredGroup = async (req, res, next) => {
-  const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-  const enabled = hasRequiredGroup(hyGroups)
-  const tokenOutdated = req.decodedToken.enabled !== enabled
-
-  if (tokenOutdated) {
-    return res.status(401).json({
-      error: 'Token needs to be refreshed - enabled does not match hy-group requirement',
-    })
-  }
-
-  if (!enabled) {
-    return res.status(403).json({ error: 'User is not enabled' })
-  }
-  next()
-}
-
-const checkUserBlacklisting = async (req, res, next) => {
-  const { userId, createdAt } = req.decodedToken
-  const isBlacklisted = await blacklist.isUserBlacklisted(userId, createdAt)
-
-  if (isBlacklisted) {
-    return res.status(401).json({ error: 'Token needs to be refreshed - blacklisted' })
-  }
-
-  next()
-}
-
 module.exports = {
-  checkAuth,
  roles,
-  checkRequiredGroup,
-  checkUserBlacklisting,
 }
@@ -0,0 +1,53 @@
+const Sentry = require('@sentry/node')
+const { ApplicationError } = require('../util/customErrors')
+const { getUser, getMockedUser } = require('../services/userService')
+const { requiredGroup } = require('../conf-backend')
+const _ = require('lodash')
+
+const parseIamGroups = iamGroups => iamGroups?.split(';').filter(Boolean) ?? []
+
+const hasRequiredIamGroup = iamGroups => _.intersection(iamGroups, requiredGroup).length > 0
+
+const currentUserMiddleware = async (req, _res, next) => {
+  const {
+    'shib-session-id': sessionId,
+    'x-show-as-user': showAsUser,
+    displayname: name,
+    hygroupcn,
+    hypersonsisuid: sisId,
+    mail: email,
+    shib_logout_url: logoutUrl,
+    uid: username,
+  } = req.headers
+
+  if (!sessionId || !username) {
+    throw new ApplicationError('Not enough data in request headers', 403, { logoutUrl })
+  }
+
+  const iamGroups = parseIamGroups(hygroupcn)
+
+  if (!hasRequiredIamGroup(iamGroups)) {
+    throw new ApplicationError('User does not have required iam group', 403, { logoutUrl })
+  }
+
+  let user = await getUser({
+    username,
+    name,
+    email,
+    iamGroups,
+    sisId,
+  })
+
+  if (showAsUser && user.isAdmin) {
+    user = await getMockedUser({ userToMock: showAsUser, mockedBy: username })
+  }
+
+  Sentry.setUser({ username: user.mockedBy ?? username })
+
+  req.user = user
+  req.logoutUrl = logoutUrl
+
+  next()
+}
+
+module.exports = currentUserMiddleware
@@ -0,0 +1,19 @@
+const Sentry = require('@sentry/node')
+const { ApplicationError } = require('../util/customErrors')
+const logger = require('../util/logger')
+
+const errorHandler = (error, req, res, next) => {
+  logger.error(`${error.message} ${error.name} ${error.stack}`)
+
+  Sentry.captureException(error)
+
+  if (res.headersSent) {
+    return next(error)
+  }
+
+  const normalizedError = error instanceof ApplicationError ? error : new ApplicationError(error.message)
+
+  return res.status(normalizedError.status).json(normalizedError)
+}
+
+module.exports = errorHandler
@@ -1,13 +0,0 @@
-const Sentry = require('@sentry/node')
-
-const sentryUserIdMiddleware = (req, res, next) => {
-  Sentry.getCurrentHub().configureScope(scope => {
-    scope.setUser({
-      username: req.decodedToken && req.decodedToken.userId,
-    })
-  })
-
-  next()
-}
-
-module.exports = sentryUserIdMiddleware
@@ -0,0 +1,5 @@
+const headersMiddleware = require('unfuck-utf8-headers-middleware')
+
+const headers = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
+
+module.exports = headersMiddleware(headers)
@@ -47,6 +47,9 @@ Course.init(
    substitutions: {
      type: JSONB,
    },
+    course_unit_type: {
+      type: STRING,
+    },
  },
  {
    underscored: false,
@@ -75,6 +75,12 @@ Credit.init(
    updatedAt: {
      type: DATE,
    },
+    language: {
+      type: STRING,
+    },
+    is_open: {
+      type: BOOLEAN,
+    },
  },
  {
    underscored: false,
@@ -135,6 +135,28 @@ const CustomPopulationSearch = sequelizeKone.define('custom_population_searches'
  },
 })

+const StudyGuidanceGroupTag = sequelizeKone.define(
+  'study_guidance_group_tags',
+  {
+    id: {
+      primaryKey: true,
+      type: Sequelize.BIGINT,
+      autoIncrement: true,
+    },
+    studyGuidanceGroupId: {
+      type: Sequelize.STRING,
+      unique: true,
+    },
+    studyProgramme: {
+      type: Sequelize.STRING,
+    },
+    year: {
+      type: Sequelize.STRING,
+    },
+  },
+  { underscored: true, timestamps: true }
+)
+
 MandatoryCourse.belongsTo(MandatoryCourseLabels, {
  foreignKey: 'label',
  sourceKey: 'id',
@@ -157,4 +179,5 @@ module.exports = {
  CustomPopulationSearch,
  sequelizeKone,
  ExcludedCourse,
+  StudyGuidanceGroupTag,
 }
@@ -1,7 +1,7 @@
 const Sequelize = require('sequelize')
-const { sequelize } = require('../database/connection')
+const { sequelizeUser } = require('../database/connection')

-const UserElementDetails = sequelize.define(
+const UserElementDetails = sequelizeUser.define(
  'user_elementdetails',
  {
    userId: {
@@ -18,7 +18,7 @@ const UserElementDetails = sequelize.define(
  }
 )

-const AccessGroup = sequelize.define('access_group', {
+const AccessGroup = sequelizeUser.define('access_group', {
  id: {
    primaryKey: true,
    type: Sequelize.BIGINT,
@@ -39,7 +39,7 @@ const AccessGroup = sequelize.define('access_group', {
  },
 })

-const UserFaculties = sequelize.define('user_faculties', {
+const UserFaculties = sequelizeUser.define('user_faculties', {
  userId: {
    primaryKey: true,
    type: Sequelize.BIGINT,
@@ -59,24 +59,8 @@ const UserFaculties = sequelize.define('user_faculties', {
    type: Sequelize.DATE,
  },
 })
-const FacultyProgrammes = sequelize.define('faculty_programmes', {
-  faculty_code: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  programme_code: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})

-const User = sequelize.define(
+const User = sequelizeUser.define(
  'users',
  {
    id: {
@@ -92,47 +76,13 @@ const User = sequelize.define(
    email: { type: Sequelize.STRING },
    language: { type: Sequelize.STRING },
    sisu_person_id: { type: Sequelize.STRING },
+    last_login: { type: Sequelize.DATE },
  },
  {
    tableName: 'users',
    timestamps: false,
  }
 )
-const HyGroup = sequelize.define('hy_group', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true,
-  },
-
-  code: { type: Sequelize.STRING },
-
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})
-
-const Affiliation = sequelize.define('affiliations', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true,
-  },
-
-  code: { type: Sequelize.STRING },
-
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})

 User.hasMany(UserElementDetails, { as: 'programme' })
 UserElementDetails.belongsTo(User)
@@ -143,29 +93,12 @@ User.belongsToMany(AccessGroup, {
 })
 AccessGroup.belongsToMany(User, { through: 'user_accessgroup' })

-User.belongsToMany(HyGroup, { through: 'user_hy_group', as: 'hy_group' })
-HyGroup.belongsToMany(User, { through: 'user_hy_group' })
-
-User.belongsToMany(Affiliation, {
-  through: 'user_affiliation',
-  as: 'affiliation',
-})
-Affiliation.belongsToMany(User, { through: 'user_affiliation' })
-
 User.hasMany(UserFaculties, { as: 'faculty', foreignKey: 'userId' })
-UserFaculties.hasMany(FacultyProgrammes, {
-  as: 'programme',
-  foreignKey: 'faculty_code',
-  sourceKey: 'faculty_code',
-})

 module.exports = {
  User,
  UserElementDetails,
  AccessGroup,
-  HyGroup,
-  Affiliation,
  UserFaculties,
-  FacultyProgrammes,
-  sequelize,
+  sequelizeUser,
 }
@@ -17,7 +17,6 @@ Student.init(
    dateofuniversityenrollment: { type: DATE },
    email: { type: STRING },
    national_student_number: { type: STRING },
-    home_county_id: { type: INTEGER },
    country_fi: { type: STRING },
    country_sv: { type: STRING },
    country_en: { type: STRING },
@@ -25,6 +24,7 @@ Student.init(
    home_country_sv: { type: STRING },
    home_country_en: { type: STRING },
    gender_code: { type: INTEGER },
+    sis_person_id: { type: STRING },
    createdAt: {
      type: DATE,
    },
@@ -1,50 +1,52 @@
-const shibbolethHeadersFix = require('unfuck-utf8-headers-middleware')
-const accessLogger = require('./middleware/accesslogger')
-const sentryUserId = require('./middleware/sentryUserId')
+const Sentry = require('@sentry/node')
+const express = require('express')
+const cors = require('cors')
+const { frontUrl } = require('./conf-backend')
+const shibbolethCharsetMiddleware = require('./middleware/shibbolethCharsetMiddleware')
+const matomoInit = require('./routes/matomo-init')
+const currentUserMiddleware = require('./middleware/currentUserMiddleware')
+const accessLogger = require('./middleware/accessLogger')
+const auth = require('./middleware/auth')
+
 const courses = require('./routes/courses')
 const students = require('./routes/students')
 const population = require('./routes/population')
 const login = require('./routes/login')
-const superlogin = require('./routes/superlogin')
 const language = require('./routes/language')
 const users = require('./routes/users')
 const elementdetails = require('./routes/elementdetails')
-const auth = require('./middleware/auth')
 const teachers = require('./routes/teachers')
 const providers = require('./routes/providers')
 const faculties = require('./routes/faculties')
 const semesters = require('./routes/semesters')
 const mandatoryCourses = require('./routes/mandatorycourses')
 const mandatoryCourseLabels = require('./routes/mandatorycourselabels')
-const ping = require('./routes/ping')
 const feedback = require('./routes/feedback')
 const tags = require('./routes/tags')
 const updater = require('./routes/updater')
 const tsaAnalytics = require('./routes/tsaAnalytics')
-const matomoInit = require('./routes/matomo-init')
 const customPopulationSearch = require('./routes/customPopulationSearch')
 const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
 const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
+const studyProgramme = require('./routes/studyProgramme')
+const initializeSentry = require('./util/sentry')
+
+const errorMiddleware = require('./middleware/errorMiddleware')

 module.exports = (app, url) => {
-  app.use(url, ping)
-  app.use(
-    shibbolethHeadersFix([
-      'hyGroupCn',
-      'SHIB_LOGOUT_URL',
-      'eduPersonAffiliation',
-      'uid',
-      'displayName',
-      'mail',
-      'hyPersonSisuId',
-      'preferredLanguage',
-    ])
-  )
+  initializeSentry(app)
+  app.use(Sentry.Handlers.requestHandler())
+  app.use(Sentry.Handlers.tracingHandler())
+
+  app.use(cors({ credentials: true, origin: frontUrl }))
+  app.use(express.json())
+
+  app.use(shibbolethCharsetMiddleware)
  app.use(url, matomoInit)
+  app.use(currentUserMiddleware)
+  app.use(accessLogger)
  app.use(url, login)
-  app.use(`${url}/superlogin`, superlogin)
-  app.use(auth.checkAuth, auth.checkRequiredGroup, auth.checkUserBlacklisting, accessLogger, sentryUserId)
  app.use(url, elementdetails)
  app.use(url, courses)
  app.use(url, students)
@@ -54,10 +56,11 @@ module.exports = (app, url) => {
  app.use(url, semesters)
  app.use(url, tags)
  app.use(url, programmeModules)
+  app.use(url, studyProgramme)
  app.use(`${url}/faculties`, faculties)
  app.use(`${url}/updater`, auth.roles(['admin']), updater)
  app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
-  app.use(`${url}/users`, auth.roles(['users']), users)
+  app.use(`${url}/users`, auth.roles(['admin']), users)
  app.use(`${url}/feedback`, feedback)
  app.use(`${url}/mandatory_courses`, mandatoryCourses)
  app.use(`${url}/mandatory-course-labels`, mandatoryCourseLabels)
@@ -65,4 +68,10 @@ module.exports = (app, url) => {
  app.use(`${url}/custom-population-search`, customPopulationSearch)
  app.use(`${url}/cool-data-science`, trends)
  app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
+  app.get('*', async (_, res) => {
+    const results = { error: 'unknown endpoint' }
+    res.status(404).json(results)
+  })
+  app.use(Sentry.Handlers.errorHandler())
+  app.use(errorMiddleware)
 }
@@ -2,7 +2,6 @@ const router = require('express').Router()
 const crypto = require('crypto')
 const Course = require('../services/courses')
 const { validateParamLength } = require('../util')
-const logger = require('../util/logger')

 router.get('/v2/coursesmulti', async (req, res) => {
  let results = { courses: [] }
@@ -17,38 +16,32 @@ router.get('/v2/coursesmulti', async (req, res) => {
 })

 router.get('/v3/courseyearlystats', async (req, res) => {
-  try {
-    const { rights, roles } = req
+  const {
+    user: { rights, roles },
+  } = req

-    const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
+  const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))

-    // If user has rights to see at least one programme, then they are allowed
-    // to see all of them
-    if (!allowedRoles && rights.length < 1) {
-      return res.status(403).json({ error: 'No programmes so no access to course stats' })
-    }
+  // If user has rights to see at least one programme, then they are allowed
+  // to see all of them
+  if (!allowedRoles && rights.length < 1) {
+    return res.status(403).json({ error: 'No programmes so no access to course stats' })
+  }

-    const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
+  const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query

-    const separate = !sep ? false : JSON.parse(sep)
-    const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
-    if (!codes) {
-      res.status(422).send('Missing required query parameters')
-    } else {
-      // Studentnumbers should be obfuscated to all other users except admins
-      // and users with rights to any specific study programmes
-      const anonymize = allowedRoles !== 'admin' && rights.length < 1
-      const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
-      const results = await Course.courseYearlyStats(codes, separate, unifyOpenUniCourses, anonymizationSalt)
-      res.json(results)
-    }
-  } catch (e) {
-    logger.error(e.message)
-    console.log(e)
-    res.status(500).send('Something went wrong with handling the request.')
+  const separate = !sep ? false : JSON.parse(sep)
+  const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
+  if (!codes) {
+    res.status(422).send('Missing required query parameters')
+  } else {
+    // Studentnumbers should be obfuscated to all other users except admins
+    // and users with rights to any specific study programmes
+    const anonymize = allowedRoles !== 'admin' && rights.length < 1
+    const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
+    const results = await Course.courseYearlyStats(codes, separate, unifyOpenUniCourses, anonymizationSalt)
+    res.json(results)
  }
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -7,76 +7,56 @@ const {
 } = require('../services/customPopulationSearch')

 router.get('/', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id },
-    } = req
-    const customPopulationSearches = await getCustomPopulationSearchesByUser(id)
-    res.json(customPopulationSearches)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  const {
+    user: { id },
+  } = req
+  const customPopulationSearches = await getCustomPopulationSearchesByUser(id)
+  res.json(customPopulationSearches)
 })

 router.post('/', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id },
-      body: { name, students },
-    } = req
+  const {
+    user: { id },
+    body: { name, students },
+  } = req

-    if (!name) return res.status(400).json({ error: 'name missing' })
-    if (students && !Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
+  if (!name) return res.status(400).json({ error: 'name missing' })
+  if (students && !Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })

-    const customPopulationSearch = await createCustomPopulationSearch(name, id, students || [])
-    res.json(customPopulationSearch)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  const customPopulationSearch = await createCustomPopulationSearch(name, id, students || [])
+  res.json(customPopulationSearch)
 })

 router.put('/:id', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id: userId },
-      body: { students },
-    } = req
-    const { id } = req.params
+  const {
+    user: { id: userId },
+    body: { students },
+  } = req
+  const { id } = req.params

-    if (!id) return res.status(400).json({ error: 'id missing' })
-    if (!students) return res.status(400).json({ error: 'students missing' })
-    if (!Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
+  if (!id) return res.status(400).json({ error: 'id missing' })
+  if (!students) return res.status(400).json({ error: 'students missing' })
+  if (!Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })

-    const updatedPopulationSearch = await updateCustomPopulationSearch(userId, id, students)
-    if (!updatedPopulationSearch) return res.status(404).json({ error: 'custom population search not found' })
+  const updatedPopulationSearch = await updateCustomPopulationSearch(userId, id, students)
+  if (!updatedPopulationSearch) return res.status(404).json({ error: 'custom population search not found' })

-    res.json(updatedPopulationSearch)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  res.json(updatedPopulationSearch)
 })

 router.delete('/:id', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id: userId },
-    } = req
-    const { id } = req.params
+  const {
+    user: { id: userId },
+  } = req
+  const { id } = req.params

-    if (!id) return res.status(400).json({ error: 'id missing' })
+  if (!id) return res.status(400).json({ error: 'id missing' })

-    const deletedSuccessfully = (await deleteCustomPopulationSearch(userId, id)) > 0
+  const deletedSuccessfully = (await deleteCustomPopulationSearch(userId, id)) > 0

-    if (!deletedSuccessfully) return res.status(404).json({ error: 'custom population search not found' })
+  if (!deletedSuccessfully) return res.status(404).json({ error: 'custom population search not found' })

-    res.json(id)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  res.json(id)
 })

 module.exports = router
@@ -1,5 +1,4 @@
 const router = require('express').Router()
-const { getStudentsUserCanAccess } = require('../services/userService')
 const MandatoryCourses = require('../services/mandatoryCourses')
 const {
  productivityStatsForStudytrack,
@@ -7,7 +6,6 @@ const {
  optionData,
 } = require('../services/studyprogramme')
 const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
-
 const {
  getProductivity,
  setProductivity,
@@ -15,28 +13,20 @@ const {
  setThroughput,
  patchProductivity,
  patchThroughput,
-  getNonGraduatedStudents,
 } = require('../services/analyticsService')
 const { getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
+const logger = require('../util/logger')

 const programmeStatsSince = new Date('2017-07-31')

 router.get('/elementdetails/all', async (req, res) => {
-  try {
-    const elementdetails = await getAllElementDetails()
-    res.json(elementdetails)
-  } catch (e) {
-    res.status(500).json(e)
-  }
+  const elementdetails = await getAllElementDetails()
+  res.json(elementdetails)
 })

 router.get('/studyprogrammes', async (req, res) => {
-  try {
-    const studyrights = await getAllProgrammes()
-    res.json(studyrights)
-  } catch (err) {
-    res.status(500).json(err)
-  }
+  const studyrights = await getAllProgrammes()
+  res.json(studyrights)
 })

 router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
@@ -55,34 +45,6 @@ router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
  res.json(data)
 })

-router.get('/v2/studyprogrammes/:id/present_students', async (req, res) => {
-  try {
-    const {
-      roles,
-      decodedToken,
-      params: { id },
-    } = req
-    if (!id) return res.status(400).json({ error: 'programme id missing' })
-
-    const nonGraduatedStudents = await getNonGraduatedStudents(id)
-    if (!nonGraduatedStudents) return res.status(200).json({})
-    const {
-      data: { formattedData, studentnumbers },
-    } = nonGraduatedStudents
-
-    const filteredData = { ...formattedData }
-    const studentsUserCanAccess = await getStudentsUserCanAccess(studentnumbers, roles, decodedToken.userId)
-    Object.keys(filteredData).forEach(year => {
-      filteredData[year] = filteredData[year].filter(({ studentNumber }) => studentsUserCanAccess.has(studentNumber))
-    })
-
-    res.json(filteredData)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
 router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
  if (req.params.id) {
    const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
@@ -99,14 +61,14 @@ router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
    try {
      data = await getProductivity(code)
    } catch (e) {
-      console.error(e)
+      logger.error(`Failed to get code ${code} productivity`)
    }
    if (!data) {
      try {
        const stats = await productivityStatsForStudytrack(code, programmeStatsSince)
        data = await setProductivity(stats)
      } catch (e) {
-        console.error(e)
+        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
      }
    }
    return res.json(data)
@@ -118,14 +80,8 @@ router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
 router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
  const code = req.query.code

-  console.log('Productivity stats recalculation starting')
-  try {
-    await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-    res.status(200).end()
-  } catch (e) {
-    console.error(e)
-    return res.status(500).end()
-  }
+  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+  res.status(200).end()

  try {
    if (code.includes('MH') || code.includes('KH')) {
@@ -141,13 +97,13 @@ router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
        [code]: { status: 'RECALCULATION ERRORED' },
      })
    } catch (e) {
-      console.error(e)
+      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
      return
    }
-    console.error(e)
-    console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
+
+    logger.error(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
  }
-  console.log(`Productivity stats recalculation for studyprogramme ${code} done`)
+  logger.info(`Produdtiviy stats recalculation for studyprogramme ${code} done`)
 })

 router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
@@ -159,14 +115,14 @@ router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
    try {
      data = await getThroughput(code)
    } catch (e) {
-      console.error(e)
+      logger.error(`Failed to get code ${code} throughput`)
    }
    if (!data) {
      try {
        const result = await throughputStatsForStudytrack(req.params.id, programmeStatsSince.getFullYear())
        data = await setThroughput(result)
      } catch (e) {
-        console.error(e)
+        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
      }
    }
    return res.json({ ...data, thesis: await thesisPromise })
@@ -178,14 +134,8 @@ router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
 router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
  const code = req.query.code

-  console.log('Throughput stats recalculation starting')
-  try {
-    await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-    res.status(200).end()
-  } catch (e) {
-    console.error(e)
-    return res.status(500).end()
-  }
+  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+  res.status(200).end()

  try {
    if (code.includes('MH') || code.includes('KH')) {
@@ -199,13 +149,12 @@ router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
    try {
      await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
    } catch (e) {
-      console.error(e)
+      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
      return
    }
-    console.error(e)
-    console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
+    logger.error(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
  }
-  console.log(`Throughput stats recalculation for studyprogramme ${code} done`)
+  logger.info(`Throughput stats recalculation for studyprogramme ${code} done`)
 })

 router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {
@@ -239,6 +188,4 @@ router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
  }
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -1,23 +1,19 @@
 const router = require('express').Router()
-const userService = require('../services/userService')
 const { sendFeedbackToToska } = require('../services/mailservice')
+const logger = require('../util/logger')

-router.post('/email', async (req, res) => {
-  const { content } = req.body
-  const { uid } = req.headers
-  const { email, full_name } = await userService.byUsername(uid)
+router.post('/email', async req => {
+  const {
+    body: { content },
+    user,
+  } = req

-  const result = await sendFeedbackToToska({
+  await sendFeedbackToToska({
    feedbackContent: content,
-    userId: uid,
-    userEmail: email,
-    userFullName: full_name,
+    user,
  })
-  if (result.error) {
-    return res.status(400).json(result).end()
-  }
-  console.log('Message sent succesfully!')
-  res.status(200).json('success').end()
+
+  logger.info(`${user.userId} succesfully sent feedback to toska`)
 })

 module.exports = router
@@ -2,8 +2,10 @@ const router = require('express').Router()
 const userService = require('../services/userService')

 router.post('/language', async (req, res) => {
-  const { language } = req.body
-  const { userId } = req.decodedToken
+  const {
+    body: { language },
+    user: { userId },
+  } = req
  if (!['fi', 'sv', 'en'].includes(language)) {
    return res.status(400).json('invalid language')
  }
@@ -1,55 +1,19 @@
 const router = require('express').Router()
-const userService = require('../services/userService')
-const { parseHyGroups } = require('../util/utils')
-const { sendNotificationAboutNewUser } = require('../services/mailservice')
+const { ApplicationError } = require('../util/customErrors')
+const _ = require('lodash')

-router.post('/login', async (req, res) => {
-  try {
-    const uid = req.headers['uid']
-    if (req.headers['shib-session-id'] && uid) {
-      const full_name = req.headers.displayname || ''
-      const mail = req.headers.mail || ''
-      const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-      const affiliations = parseHyGroups(req.headers['edupersonaffiliation'])
-      const hyPersonSisuId = req.headers.hypersonsisuid || ''
-      console.log(uid, 'trying to login, referring to userservice.')
-      let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail, hyPersonSisuId)
-      if (isNew) {
-        const result = await sendNotificationAboutNewUser({ userId: uid, userFullName: full_name })
-        if (result.error) {
-          return res.status(400).json(result).end()
-        }
-      }
-      res.status(200).json({ token })
-    } else {
-      res
-        .status(401)
-        .json({
-          message: `Not enough headers login, uid: ${req.headers.uid}
-        session-id ${req.headers['shib-session-id']}`,
-        })
-        .end()
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(401).json({ message: 'problem with login', err })
-  }
-})
+router.get('/login', async (req, res) => {
+  const { user, logoutUrl } = req

-router.delete('/logout', async (req, res) => {
-  try {
-    const logoutUrl = req.headers.shib_logout_url
-    const { returnUrl } = req.body
-    if (logoutUrl) {
-      return res
-        .status(200)
-        .send({ logoutUrl: `${logoutUrl}?return=${returnUrl}` })
-        .end()
-    }
-    res.status(200).send({ logoutUrl: returnUrl }).end()
-  } catch (err) {
-    res.status(500).json({ message: 'Error with logout', err })
+  if (!user) {
+    throw new ApplicationError('User not found', 404)
  }
+
+  res.send({
+    // don't send possibly huge list of students to frontend
+    user: _.omit(user, ['studentsUserCanAccess']),
+    logoutUrl,
+  })
 })

 module.exports = router
@@ -1,5 +1,6 @@
 const router = require('express').Router()
 const MandatoryCourseLabels = require('../services/mandatoryCourseLabels')
+const logger = require('../util/logger')

 router.get('/:programme/labels', async (req, res) => {
  const { programme } = req.params
@@ -8,7 +9,7 @@ router.get('/:programme/labels', async (req, res) => {
      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
      res.status(200).json(labels)
    } catch (err) {
-      console.error(err)
+      logger.error(err)
      res.status(400).json(err.message)
    }
  } else {
@@ -25,7 +26,7 @@ router.delete('/:programme/labels', async (req, res) => {
      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
      res.status(200).json(labels)
    } catch (err) {
-      console.error(err)
+      logger.error(err)
      res.status(400).json(err.message)
    }
  } else {
@@ -42,7 +43,7 @@ router.post('/:programme/move', async (req, res) => {
      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
      res.status(200).json(labels)
    } catch (err) {
-      console.error(err)
+      logger.error(err)
      res.status(400).json(err.message)
    }
  } else {
@@ -59,7 +60,7 @@ router.post('/:programme/labels', async (req, res) => {
      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
      res.status(200).json(labels)
    } catch (err) {
-      console.error(err)
+      logger.error(err)
      res.status(400).json(err.message)
    }
  } else {
@@ -5,47 +5,32 @@ router.delete('/:programme', async (req, res) => {
  const { programme } = req.params
  const { course } = req.body
  if (!programme || !course) return res.status(400).end()
-  try {
-    await MandatoryCourses.remove(programme, course)
-    res.status(200).json(course)
-  } catch (err) {
-    console.error(err)
-    res.status(500).json(err.message)
-  }
+  await MandatoryCourses.remove(programme, course)
+  res.status(200).json(course)
 })

 router.post('/:programme', async (req, res) => {
  const { programme } = req.params
  const { course } = req.body
  if (!programme || !course) return res.status(400).end()
-  try {
-    await MandatoryCourses.create(programme, course)
-    const newCourse = await MandatoryCourses.find(programme, course)
-    if (!newCourse) return res.status(400).end()
-    res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
-  } catch (err) {
-    console.error(err)
-    res.status(500).json(err.message)
-  }
+  await MandatoryCourses.create(programme, course)
+  const newCourse = await MandatoryCourses.find(programme, course)
+  if (!newCourse) return res.status(400).end()
+  res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
 })

 router.post('/:programme/label/:course', async (req, res) => {
  const { programme, course } = req.params
  const { label } = req.body
  if (!programme || !course) return res.status(400).end()
-  try {
-    await MandatoryCourses.updateLabel(programme, course, label)
-    const updatedCourse = await MandatoryCourses.find(programme, course)
-    if (!updatedCourse) return res.status(400).end()
-    res.status(200).json({
-      name: updatedCourse.course.name,
-      code: updatedCourse.course_code,
-      label: updatedCourse.mandatory_course_label,
-    })
-  } catch (err) {
-    console.error(err)
-    res.status(500).json(err.message)
-  }
+  await MandatoryCourses.updateLabel(programme, course, label)
+  const updatedCourse = await MandatoryCourses.find(programme, course)
+  if (!updatedCourse) return res.status(400).end()
+  res.status(200).json({
+    name: updatedCourse.course.name,
+    code: updatedCourse.course_code,
+    label: updatedCourse.mandatory_course_label,
+  })
 })

 module.exports = router
@@ -28,7 +28,7 @@ _paq.push(["enableLinkTracking"]);

 const script = config.MATOMO_URL ? acual : ''

-router.get('/matomo-init', (req, res) => {
+router.get('/matomo-init', (_, res) => {
  res.contentType('text/javascript; charset=UTF-8')
  res.send(script)
 })
@@ -1,85 +0,0 @@
-/* eslint-disable */
-
-const { router, wrapper } = require('./router').routerWithWrapper()
-const axios = require('axios')
-
-wrapper.get('/ping', async (req, res) => {
-  res.json({
-    message: 'pong'
-  })
-})
-
-wrapper.get('/v3/mandatory_courses/:code', async (req, res) => {
-  const courses = []
-  const included = new Set()
-
-  const dumb_flatten = module => {
-    if (module.children) {
-      module.children.forEach(child => {
-        dumb_flatten(child)
-      })
-    } else if (module.code && !module.code.startsWith('KK') && !courses.map(c => c.code).includes(module.code)) {
-      courses.push({
-        name: {
-          fi: module.name,
-          en: module.name
-        },
-        code: module.code,
-        label: { id: `${courses.length}`, label: module.code.replace('-', '').slice(0, 4), orderNumber: courses.length }
-      })
-    }
-  }
-
-  const better_flatten = (module, label) => {
-    if (module.children) {
-      module.children.forEach(child => {
-        better_flatten(child, module.module ? module.module.name : label)
-      })
-    } else if (module.code && !module.code.startsWith('KK')) {
-      courses.push({
-        name: {
-          fi: module.name,
-          en: module.name
-        },
-        code: module.code,
-        label: { id: `${courses.length}`, label, orderNumber: courses.length }
-      })
-    }
-  }
-
-  const superFlattenFlatten = (module, label) => {
-    if (module.children) {
-      module.children.forEach(child => {
-        superFlattenFlatten(child, module.module ? module.module.name : label)
-      })
-    } else if (Array.isArray(module)) {
-      module.forEach(elem => superFlattenFlatten(elem, label))
-    } else if (module.code && !module.code.startsWith('KK')) {
-      if (!included.has(`${label}${module.code}`)) {
-        included.add(`${label}${module.code}`)
-        courses.push({
-          name: {
-            fi: module.name,
-            en: module.name
-          },
-          code: module.code,
-          label: { id: `${courses.length}`, label, orderNumber: courses.length }
-        })
-      }
-    }
-  }
-
-  const code = req.params.code
-  const response = await axios.get(`http://sis-updater-scheduler:8082/v1/courses/${code}?token=dev&superFlatten=true`)
-
-  // dumb_flatten(response.data)
-  // better_flatten(response.data)
-  superFlattenFlatten(response.data)
-
-  const byCode = (c1, c2) => (c1.code < c2.code ? -1 : 1)
-
-  courses.sort(byCode)
-  res.json(courses)
-})
-
-module.exports = router
@@ -1,13 +1,14 @@
 const crypto = require('crypto')
 const Sentry = require('@sentry/node')
 const router = require('express').Router()
+const _ = require('lodash')
 const Population = require('../services/populations')
 const Student = require('../services/students')
 const StudyrightService = require('../services/studyrights')
-const UserService = require('../services/userService')
 const TagService = require('../services/tags')
 const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
+const { ApplicationError } = require('../util/customErrors')

 const filterPersonalTags = (population, userId) => {
  return {
@@ -21,324 +22,262 @@ const filterPersonalTags = (population, userId) => {

 // POST instead of GET because of too long params and "sensitive" data
 router.post('/v2/populationstatistics/courses', async (req, res) => {
-  try {
-    if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
-      Sentry.captureException(new Error('The body should have a year, semester and study rights defined'))
-      res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
+  if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
+    Sentry.captureException(new Error('The body should have a year, semester and study rights defined'))
+    res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
+    return
+  }
+  if (!Array.isArray(req.body.studyRights)) {
+    // studyRights should always be an array
+    req.body.studyRights = [req.body.studyRights]
+  }
+
+  if (req.body.months == null) {
+    req.body.months = 12
+  }
+
+  if (req.body.years) {
+    const upperYearBound = new Date().getFullYear() + 1
+    const multicoursestatPromises = Promise.all(
+      req.body.years.map(year => {
+        if (req.body.selectedStudentsByYear) {
+          req.body.selectedStudents = req.body.selectedStudentsByYear[year]
+        }
+        const newMonths = (upperYearBound - Number(year)) * 12
+        const query = { ...req.body, year, months: newMonths }
+        const coursestatistics = Population.bottlenecksOf(query)
+        return coursestatistics
+      })
+    )
+    const multicoursestats = await multicoursestatPromises
+    const result = StatMergeService.populationCourseStatsMerger(multicoursestats)
+    if (result.error) {
+      res.status(400).json(result)
      return
    }
-    if (!Array.isArray(req.body.studyRights)) {
-      // studyRights should always be an array
-      req.body.studyRights = [req.body.studyRights]
+
+    res.json(result)
+  } else {
+    const result = await Population.bottlenecksOf(req.body)
+
+    if (result.error) {
+      Sentry.captureException(new Error(result.error))
+      res.status(400).json(result)
+      return
    }

-    if (req.body.months == null) {
-      req.body.months = 12
-    }
-
-    if (req.body.years) {
-      console.log('bottleneck on years')
-      const upperYearBound = new Date().getFullYear() + 1
-      const multicoursestatPromises = Promise.all(
-        req.body.years.map(year => {
-          if (req.body.selectedStudentsByYear) {
-            req.body.selectedStudents = req.body.selectedStudentsByYear[year]
-          }
-          const newMonths = (upperYearBound - Number(year)) * 12
-          const query = { ...req.body, year, months: newMonths }
-          const coursestatistics = Population.bottlenecksOf(query)
-          return coursestatistics
-        })
-      )
-      const multicoursestats = await multicoursestatPromises
-      const result = StatMergeService.populationCourseStatsMerger(multicoursestats)
-      console.log('result: ', result)
-      if (result.error) {
-        res.status(400).json(result)
-        return
-      }
-
-      res.json(result)
-    } else {
-      console.log('bottleneck ei ole years')
-      const result = await Population.bottlenecksOf(req.body)
-
-      if (result.error) {
-        Sentry.captureException(new Error(result.error))
-        res.status(400).json(result)
-        return
-      }
-
-      res.json(result)
-    }
-  } catch (e) {
-    console.log(e)
-    Sentry.captureException(e)
-    res.status(500).json({ error: e })
+    res.json(result)
  }
 })

 router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => {
-  try {
-    const { from, to, coursecodes } = req.body
-    if (!from || !to || !coursecodes) {
-      return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
-    }
-    const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(coursecodes)
-    if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
-      return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
-    }
-    let studentnumberlist
-    const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
-    const {
-      decodedToken: { userId },
-      roles,
-    } = req
-
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = studentnumbers
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-    }
-
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-      },
-      studentnumberlist
-    )
-
-    if (result.error) {
-      res.status(400).json(result)
-      return
-    }
-
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  const { from, to, coursecodes } = req.body
+  if (!from || !to || !coursecodes) {
+    return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
  }
+  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(coursecodes)
+  if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
+    return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
+  }
+
+  const {
+    user: { isAdmin, studentsUserCanAccess },
+  } = req
+
+  const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
+  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
+
+  const result = await Population.bottlenecksOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+    },
+    studentnumberlist
+  )
+
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).json(result)
+    return
+  }
+
+  res.json(result)
 })

 router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
-  try {
-    const { tag } = req.body
-    if (!tag) {
-      res.status(400).json({ error: 'The body should have a tag defined' })
-      return
-    }
-    let studentnumberlist
-    const studentnumbers = await Student.findByTag(tag)
-    const {
-      decodedToken: { userId },
-      roles,
-    } = req
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = studentnumbers
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-    }
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-        tag,
-      },
-      studentnumberlist
-    )
-
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  const { tag } = req.body
+  if (!tag) {
+    res.status(400).json({ error: 'The body should have a tag defined' })
+    return
  }
+  const {
+    user: { isAdmin, studentsUserCanAccess },
+  } = req
+  const studentnumbers = await Student.findByTag(tag)
+  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
+
+  const result = await Population.bottlenecksOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+      tag,
+    },
+    studentnumberlist
+  )
+
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).end()
+    return
+  }
+
+  res.json(result)
 })

 router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
-  try {
-    if (!req.body.studentnumberlist) {
-      res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
-      return
-    }
-    let studentnumberlist
-    const {
-      decodedToken: { userId },
-      roles,
-    } = req
+  const {
+    user: { isAdmin, studentsUserCanAccess },
+    body: { studentnumberlist: studentnumbersInReq },
+  } = req

-    if (roles && roles.includes('admin')) {
-      studentnumberlist = req.body.studentnumberlist
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
-    }
-
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-      },
-      studentnumberlist
-    )
-    if (result.error) {
-      res.status(400).json(result)
-      return
-    }
-
-    res.json(result)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: err })
+  if (!studentnumbersInReq) {
+    throw new ApplicationError('The body should have a studentnumberlist defined', 400)
  }
+
+  const studentnumberlist = isAdmin ? studentnumbersInReq : _.intersection(studentnumbersInReq, studentsUserCanAccess)
+
+  const result = await Population.bottlenecksOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+    },
+    studentnumberlist
+  )
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).json(result)
+    return
+  }
+
+  res.json(result)
 })

 router.get('/v3/populationstatistics', async (req, res) => {
  const { year, semesters, studyRights: studyRightsJSON } = req.query
-  const { decodedToken } = req
+  const { user } = req

+  if (!year || !semesters || !studyRightsJSON) {
+    res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
+    return
+  }
+  let studyRights = null
  try {
-    if (!year || !semesters || !studyRightsJSON) {
-      res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
+    studyRights = JSON.parse(studyRightsJSON)
+    const {
+      user: { rights, isAdmin },
+    } = req
+
+    if (!isAdmin && !rights.includes(studyRights.programme)) {
+      res.status(403).json([])
      return
    }
-    let studyRights = null
-    try {
-      studyRights = JSON.parse(studyRightsJSON)
-      const { rights, roles } = req
-
-      if (!roles || !roles.includes('admin')) {
-        if (!rights.includes(studyRights.programme)) {
-          res.status(403).json([])
-          return
-        }
-      }
-    } catch (e) {
-      console.error(e)
-      res.status(400).json({ error: 'The query had invalid studyRights' })
-      return
-    }
-
-    if (req.query.months == null) {
-      req.query.months = 12
-    }
-
-    if (req.query.years) {
-      const upperYearBound = new Date().getFullYear() + 1
-      const multipopulationstudentPromises = Promise.all(
-        req.query.years.map(year => {
-          const newMonths = (upperYearBound - Number(year)) * 12
-          const populationStudents = Population.optimizedStatisticsOf({
-            ...req.query,
-            studyRights,
-            year,
-            months: newMonths,
-          })
-          return populationStudents
-        })
-      )
-      const multipopulationstudents = await multipopulationstudentPromises
-
-      const result = StatMergeService.populationStudentsMerger(multipopulationstudents)
-
-      if (result.error) {
-        console.log(result.error)
-        res.status(400).end()
-        return
-      }
-
-      console.log(`request completed ${new Date()}`)
-      res.json(filterPersonalTags(result, decodedToken.id))
-    } else {
-      const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
-      if (result.error) {
-        console.log(result.error)
-        res.status(400).end()
-        return
-      }
-
-      console.log(`request completed ${new Date()}`)
-      res.json(filterPersonalTags(result, decodedToken.id))
-    }
  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+    res.status(400).json({ error: 'The query had invalid studyRights' })
+    return
+  }
+
+  if (req.query.months == null) {
+    req.query.months = 12
+  }
+
+  if (req.query.years) {
+    const upperYearBound = new Date().getFullYear() + 1
+    const multipopulationstudentPromises = Promise.all(
+      req.query.years.map(year => {
+        const newMonths = (upperYearBound - Number(year)) * 12
+        const populationStudents = Population.optimizedStatisticsOf({
+          ...req.query,
+          studyRights,
+          year,
+          months: newMonths,
+        })
+        return populationStudents
+      })
+    )
+    const multipopulationstudents = await multipopulationstudentPromises
+
+    const result = StatMergeService.populationStudentsMerger(multipopulationstudents)
+
+    if (result.error) {
+      Sentry.captureException(new Error(result.error))
+      res.status(400).end()
+      return
+    }
+
+    res.json(filterPersonalTags(result, user.id))
+  } else {
+    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
+    if (result.error) {
+      Sentry.captureException(new Error(result.error))
+      res.status(400).end()
+      return
+    }
+
+    res.json(filterPersonalTags(result, user.id))
  }
 })

 router.get('/v3/populationstatisticsbytag', async (req, res) => {
-  const { tag, studyRights: studyRightsJSON, months, year } = req.query
-  const { decodedToken } = req
+  const {
+    user: { id, isAdmin, studentsUserCanAccess },
+    query: { tag, studyRights: studyRightsJSON, months, year },
+  } = req

  if (!tag) return res.status(400).json({ error: 'The query should have a tag defined' })
  const foundTag = await TagService.findTagById(tag)
  if (!foundTag) return res.status(404).json({ error: 'Tag not found' })

  const semesters = ['FALL', 'SPRING']
-  let studentnumberlist
+
  const studentnumbers = await Student.findByTag(tag)
-  const {
-    decodedToken: { userId },
-    roles,
-  } = req
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = studentnumbers
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-  }
-  try {
-    const studyRights = JSON.parse(studyRightsJSON)
-    const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
-    const yearDifference = Number(year) - Number(newStartYear)
-    const newMonths = Number(months) + 12 * yearDifference
-    const result = await Population.optimizedStatisticsOf(
-      {
-        year: newStartYear,
-        studyRights,
-        semesters,
-        months: newMonths,
-        tag: foundTag,
-      },
-      studentnumberlist
-    )

-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
+  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)

-    console.log(`request completed ${new Date()}`)
-    res.json(filterPersonalTags(result, decodedToken.id))
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  const studyRights = JSON.parse(studyRightsJSON)
+  const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
+  const yearDifference = Number(year) - Number(newStartYear)
+  const newMonths = Number(months) + 12 * yearDifference
+  const result = await Population.optimizedStatisticsOf(
+    {
+      year: newStartYear,
+      studyRights,
+      semesters,
+      months: newMonths,
+      tag: foundTag,
+    },
+    studentnumberlist
+  )
+
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).end()
+    return
  }
+
+  res.json(filterPersonalTags(result, id))
 })

 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
  const { coursecodes, from, to, separate: sep } = req.query
-  const { decodedToken, roles } = req
+  const {
+    user: { id, isAdmin, studentsUserCanAccess: allStudentsUserCanAccess },
+  } = req
  const separate = sep ? JSON.parse(sep) : false

  if (!coursecodes || !from || !to) {
@@ -355,128 +294,95 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {

  const semesters = ['FALL', 'SPRING']
  const studentnumbers = await Student.findByCourseAndSemesters(JSON.parse(coursecodes), from, to, separate)
-  try {
-    const result = await Population.optimizedStatisticsOf(
-      {
-        // Useless, because studentnumbers are already filtered above by from & to.
-        // We should probably refactor this to avoid more confusement.
-        year: 1900,
-        studyRights: [],
-        semesters,
-        months: 10000,
-      },
-      studentnumbers
-    )
+  const result = await Population.optimizedStatisticsOf(
+    {
+      // Useless, because studentnumbers are already filtered above by from & to.
+      // We should probably refactor this to avoid more confusement.
+      year: 1900,
+      studyRights: [],
+      semesters,
+      months: 10000,
+    },
+    studentnumbers
+  )

-    let studentsUserCanAccess
-    if (roles && roles.includes('admin')) {
-      studentsUserCanAccess = new Set(studentnumbers)
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(decodedToken.userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentsUserCanAccess = new Set(await Student.filterStudentnumbersByAccessrights(studentnumbers, codes))
-    }
+  const studentsUserCanAccess = isAdmin
+    ? new Set(studentnumbers)
+    : new Set(_.intersection(studentnumbers, allStudentsUserCanAccess))

-    const randomHash = crypto.randomBytes(12).toString('hex')
-    const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
-      courses,
-      studyrights,
-      gender_code,
-      studentNumber: crypto.createHash('md5').update(`${studentNumber}${randomHash}`).digest('hex'),
-      firstnames: '',
-      lastname: '',
-      name: '',
-      email: '',
-      started: null,
-      credits: '',
-      tags: [],
-      obfuscated: true,
-    })
+  const randomHash = crypto.randomBytes(12).toString('hex')
+  const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
+    courses,
+    studyrights,
+    gender_code,
+    studentNumber: crypto.createHash('md5').update(`${studentNumber}${randomHash}`).digest('hex'),
+    firstnames: '',
+    lastname: '',
+    name: '',
+    email: '',
+    started: null,
+    credits: '',
+    tags: [],
+    obfuscated: true,
+  })

-    result.students = result.students.map(s => (studentsUserCanAccess.has(s.studentNumber) ? s : obfuscateStudent(s)))
+  result.students = result.students.map(s => (studentsUserCanAccess.has(s.studentNumber) ? s : obfuscateStudent(s)))

-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(filterPersonalTags(result, decodedToken.id))
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).end()
+    return
  }
+
+  res.json(filterPersonalTags(result, id))
 })

 router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
  const { studentnumberlist } = req.body
-  const { roles, decodedToken } = req
+  const {
+    user: { isAdmin, id, studentsUserCanAccess },
+  } = req

-  const studentsUserCanAccess = await UserService.getStudentsUserCanAccess(
-    studentnumberlist,
-    roles,
-    decodedToken.userId
+  const filteredStudentNumbers = isAdmin ? studentnumberlist : _.intersection(studentnumberlist, studentsUserCanAccess)
+
+  const result = await Population.optimizedStatisticsOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+    },
+    filteredStudentNumbers
  )
-  const filteredStudentNumbers = studentnumberlist.filter(s => studentsUserCanAccess.has(s))
-
-  try {
-    const result = await Population.optimizedStatisticsOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-      },
-      filteredStudentNumbers
-    )
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-    res.status(200).json(filterPersonalTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(400).end()
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    return
  }
+  res.status(200).json(filterPersonalTags(result, id))
 })

 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
-  try {
-    const { rights, roles } = req
-    if (roles && roles.includes('admin')) {
-      const studyrights = await StudyrightService.getAssociations()
-      res.json(studyrights)
-    } else {
-      const studyrights = await StudyrightService.getFilteredAssociations(rights)
-      res.json(studyrights)
-    }
-  } catch (err) {
-    res.status(500).json(err)
+  const {
+    user: { rights, roles },
+  } = req
+  if (roles?.includes('admin')) {
+    const studyrights = await StudyrightService.getAssociations()
+    res.json(studyrights)
+  } else {
+    const studyrights = await StudyrightService.getFilteredAssociations(rights)
+    res.json(studyrights)
  }
 })

 router.get('/v3/populationstatistics/studyprogrammes/unfiltered', async (req, res) => {
-  try {
-    const studyrights = await StudyrightService.getAssociations()
-    res.json(studyrights)
-  } catch (err) {
-    res.status(500).json(err)
-  }
+  const studyrights = await StudyrightService.getAssociations()
+  res.json(studyrights)
 })

 router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
-  try {
-    const { courseCodes } = req.query
-    const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(courseCodes))
-    return res.json(maxYearsToCreatePopulationFrom)
-  } catch (err) {
-    console.log('err', err)
-    res.status(500).json(err)
-  }
+  const { courseCodes } = req.query
+  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(courseCodes))
+  return res.json(maxYearsToCreatePopulationFrom)
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -8,17 +8,20 @@ const {

 router.get('/v3/programme_modules/:code', async (req, res) => {
  const { code } = req.params
-  const module = await byProgrammeCode(code)
-  if (!module) {
-    res.status(400).end()
-    return
-  }
-  res.json(module)
+  const result = await byProgrammeCode(code)
+  res.json(result)
 })

 router.get('/v3/programme_modules/:code/modules', async (req, res) => {
  const { code } = req.params
  const result = await modulesByProgrammeCode(code)
+  res.json(result)
+})
+
+router.delete('/v3/programme_modules', async (req, res) => {
+  const { programmecode, ids } = req.body
+  await removeExcludedCourses(ids)
+  const result = await byProgrammeCode(programmecode)
  if (!result) {
    res.status(400).end()
    return
@@ -26,36 +29,15 @@ router.get('/v3/programme_modules/:code/modules', async (req, res) => {
  res.json(result)
 })

-router.delete('/v3/programme_modules', async (req, res) => {
-  const { programmecode, ids } = req.body
-  try {
-    await removeExcludedCourses(ids)
-    const result = await byProgrammeCode(programmecode)
-    if (!result) {
-      res.status(400).end()
-      return
-    }
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.json(500).json(e)
-  }
-})
-
 router.post('/v3/programme_modules/:programmecode/', async (req, res) => {
  const { programmecode, coursecodes } = req.body
-  try {
-    await addExcludedCourses(programmecode, coursecodes)
-    const result = await byProgrammeCode(programmecode)
-    if (!result) {
-      res.status(400).end()
-      return
-    }
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.json(500).json(e)
+  await addExcludedCourses(programmecode, coursecodes)
+  const result = await byProgrammeCode(programmecode)
+  if (!result) {
+    res.status(400).end()
+    return
  }
+  res.json(result)
 })

 module.exports = router
@@ -6,6 +6,4 @@ router.get('/semesters/codes', async (req, res) => {
  res.json(providers)
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -1,8 +1,6 @@
 const router = require('express').Router()
 const Student = require('../services/students')
-const userService = require('../services/userService')
-const Unit = require('../services/units')
-const { mismatchedStudents } = require('../services/student_credit_total_mismatches')
+const { ApplicationError } = require('../util/customErrors')

 const filterStudentTags = (student, userId) => {
  return {
@@ -11,17 +9,9 @@ const filterStudentTags = (student, userId) => {
  }
 }

-const creditTotalMismatch = student => {
-  return {
-    ...student,
-    mismatch: mismatchedStudents.has(student.studentNumber),
-  }
-}
-
 router.get('/students', async (req, res) => {
  const {
-    roles,
-    decodedToken: { userId },
+    user: { isAdmin, studentsUserCanAccess },
    query: { searchTerm },
  } = req

@@ -33,58 +23,30 @@ router.get('/students', async (req, res) => {
      .slice(0, 2)
      .find(t => t.length > 3)
  ) {
-    return res.status(400).json({ error: 'at least one search term must be longer than 3 characters' })
+    throw new ApplicationError('at least one search term must be longer than 3 characters', 400)
  }

-  if (roles && roles.includes('admin')) {
-    let results = []
-    if (trimmedSearchTerm) {
-      results = await Student.bySearchTerm(trimmedSearchTerm)
-    }
-    return res.status(200).json(results).end()
-  } else {
-    const unitsUserCanAccess = await userService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    const matchingStudents = await Student.bySearchTermAndElements(trimmedSearchTerm, codes)
-    res.json(matchingStudents)
+  let results = []
+  if (trimmedSearchTerm) {
+    results = isAdmin
+      ? await Student.bySearchTerm(trimmedSearchTerm)
+      : await Student.bySearchTermAndStudentNumbers(trimmedSearchTerm, studentsUserCanAccess)
  }
+  res.json(results)
 })

 router.get('/students/:id', async (req, res) => {
  const { id: studentId } = req.params
-  const { roles, decodedToken } = req
+  const {
+    user: { id, isAdmin, studentsUserCanAccess },
+  } = req

-  if (roles && roles.includes('admin')) {
-    const results = await Student.withId(studentId)
-    return results.error
-      ? res.status(400).json({ error: 'error finding student' }).end()
-      : res
-          .status(200)
-          .json(creditTotalMismatch(filterStudentTags(results, decodedToken.id)))
-          .end()
+  if (!isAdmin && !studentsUserCanAccess.includes(studentId)) {
+    throw new ApplicationError('Error finding student', 400)
  }

-  const uid = req.decodedToken.userId
  const student = await Student.withId(studentId)
-  if (student.error) {
-    return res.status(400).json({ error: 'error finding student' }).end()
-  }
-  const units = await userService.getUnitsFromElementDetails(uid)
-
-  const rights = await Promise.all(
-    units.map(async unit => {
-      const jtn = await Unit.hasStudent(unit.id, student.studentNumber)
-      return jtn
-    })
-  )
-
-  if (rights.some(right => right !== null)) {
-    res.status(200).json(filterStudentTags(student, decodedToken.id)).end()
-  } else {
-    res.status(400).json({ error: 'error finding student' }).end()
-  }
+  res.json(filterStudentTags(student, id))
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -1,21 +1,39 @@
 const router = require('express').Router()
-const { getImporterClient } = require('../util/importerClient')
+const { logger } = require('../util/logger')
+const { getAllGroupsAndStudents, changeGroupTags } = require('../services/studyGuidanceGroups')

 router.get('/', async (req, res) => {
-  const personId = req.headers.hypersonsisuid
-  if (!personId) {
-    const errorMessage = 'Not possible to get groups without personId header'
-    console.log(errorMessage)
-    return res.status(400).json({ error: errorMessage })
+  const {
+    user: { uid, sisPersonId },
+  } = req
+  if (!sisPersonId) {
+    logger.error(`User ${uid} tried to get person groups but personId was ${sisPersonId} in header`)
+    return res.status(400).json({ error: 'Not possible to get groups without personId' })
  }
-  const importerClient = getImporterClient()
-  try {
-    const { data } = await importerClient.get(`/person-groups/person/${personId}`)
-    return res.json(Object.values(data))
-  } catch (e) {
-    console.log(e)
-    return res.status(500).json(e)
+  return res.json(await getAllGroupsAndStudents(sisPersonId))
+})
+
+router.put('/:id/tags', async (req, res) => {
+  const {
+    user: { uid, sisPersonId },
+    body: tags,
+    params: { id: groupId },
+  } = req
+
+  if (!sisPersonId) {
+    logger.error(`User ${uid} tried to change study guidance group tags but personId was ${sisPersonId} in header`)
+    return res.status(400).json({ error: 'Not possible to change groups without personId' })
  }
+
+  const groupsUserHasAccessTo = (await getAllGroupsAndStudents(sisPersonId)).map(group => group.id)
+
+  if (!groupsUserHasAccessTo.includes(groupId)) {
+    logger.error(
+      `User ${uid} tried to change study guidance group tags, but didn't have permission for the group ${groupId}`
+    )
+    return res.status(400).json({ error: 'Access denied' })
+  }
+  return res.status(200).json(await changeGroupTags({ groupId, tags }))
 })

 module.exports = router
@@ -0,0 +1,129 @@
+const router = require('express').Router()
+const {
+  getBasicStatsForStudytrack,
+  getCreditStatsForStudytrack,
+  getGraduationStatsForStudytrack,
+} = require('../services/studyprogrammeStats')
+const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
+
+const {
+  getBasicStats,
+  setBasicStats,
+  getCreditStats,
+  setCreditStats,
+  getGraduationStats,
+  setGraduationStats,
+  getStudytrackStats,
+  setStudytrackStats,
+} = require('../services/analyticsService')
+const logger = require('../util/logger')
+
+router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+
+  if (code) {
+    let data = null
+    try {
+      data = await getBasicStats(code, yearType)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} basic stats: ${e}`)
+    }
+    if (!data) {
+      try {
+        const result = await getBasicStatsForStudytrack({
+          studyprogramme: req.params.id,
+          yearType,
+        })
+        data = await setBasicStats(result, yearType)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} basic stats: ${e}`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+
+  if (code) {
+    let data = null
+    try {
+      data = await getCreditStats(code, yearType)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} credit stats`)
+    }
+    if (!data) {
+      try {
+        const result = await getCreditStatsForStudytrack({
+          studyprogramme: req.params.id,
+          yearType,
+        })
+        data = await setCreditStats(result, yearType)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} credit stats`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+
+  if (code) {
+    let data = null
+    try {
+      data = await getGraduationStats(code, yearType)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} graduation stats`)
+    }
+    if (!data) {
+      try {
+        const result = await getGraduationStatsForStudytrack({
+          studyprogramme: req.params.id,
+          yearType,
+        })
+        data = await setGraduationStats(result, yearType)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} graduation stats`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
+  const code = req.params.id
+
+  if (code) {
+    let data = null
+    try {
+      data = await getStudytrackStats(code)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} studytrack stats: ${e}`)
+    }
+    if (!data) {
+      try {
+        const result = await getStudytrackStatsForStudyprogramme({ studyprogramme: req.params.id })
+        data = await setStudytrackStats(result)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+module.exports = router
@@ -1,26 +0,0 @@
-const router = require('express').Router()
-const userService = require('../services/userService')
-
-router.post('/:uid', async (req, res) => {
-  try {
-    const uid = req.headers['uid']
-    if (req.headers['shib-session-id'] && uid) {
-      const asUser = req.params.uid
-      const token = await userService.superlogin(uid, asUser)
-      res.status(200).json(token)
-    } else {
-      res
-        .status(401)
-        .json({
-          message: `Not enough headers login, uid:
-        ${req.headers.uid} session-id ${req.headers['shib-session-id']}`,
-        })
-        .end()
-    }
-  } catch (err) {
-    console.log(err.message)
-    res.status(500).json({ message: 'problem with login' })
-  }
-})
-
-module.exports = router
@@ -12,190 +12,145 @@ const filterRelevantStudentTags = (studentTags, userId) => {
  return studentTags.filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
 }

-router.get('/tags', async (req, res) => {
-  try {
-    const tags = await Tags.findTags()
-    res.status(200).json(tags)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+router.get('/tags', async (_req, res) => {
+  const tags = await Tags.findTags()
+  res.status(200).json(tags)
 })

 router.get('/tags/:studytrack', async (req, res) => {
-  try {
-    const { studytrack } = req.params
-    const { rights, roles, decodedToken } = req
+  const { studytrack } = req.params
+  const {
+    user: { rights, roles, id },
+  } = req

-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()

-    const tags = await Tags.findTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantTags(tags, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const tags = await Tags.findTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantTags(tags, id))
 })

 router.post('/tags', async (req, res) => {
-  try {
-    const {
-      tag: { studytrack, tagname, year, personal_user_id },
-    } = req.body
-    const { rights, roles, decodedToken } = req
+  const {
+    tag: { studytrack, tagname, year, personal_user_id },
+  } = req.body
+  const {
+    user: { rights, roles, id },
+  } = req

-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()

-    await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
-    const tags = await Tags.findTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantTags(tags, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err.message)
-  }
+  await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
+  const tags = await Tags.findTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantTags(tags, id))
 })

 router.delete('/tags', async (req, res) => {
-  try {
-    const { tag } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tag } = req.body
+  const {
+    user: { rights, roles, id },
+  } = req

-    if (
-      !rights.includes(tag.studytrack) &&
-      !(roles && roles.includes('admin')) &&
-      !(tag.personal_user_id === decodedToken.id)
-    )
-      return res.status(403).end()
+  if (!rights.includes(tag.studytrack) && !roles?.includes('admin') && !(tag.personal_user_id === id))
+    return res.status(403).end()

-    await Tags.deleteTag(tag)
-    const t = await Tags.findTagsByStudytrack(tag.studytrack)
-    res.status(200).json(filterRelevantTags(t, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  await Tags.deleteTag(tag)
+  const t = await Tags.findTagsByStudytrack(tag.studytrack)
+  res.status(200).json(filterRelevantTags(t, id))
 })

 router.get('/studenttags', async (req, res) => {
-  try {
-    const result = await TagStudent.getStudentTags()
-    res.status(200).json(result)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const result = await TagStudent.getStudentTags()
+  res.status(200).json(result)
 })

 router.get('/studenttags/:studytrack', async (req, res) => {
-  try {
-    const { studytrack } = req.params
-    const { rights, roles, decodedToken } = req
+  const { studytrack } = req.params
+  const {
+    user: { rights, roles, id },
+  } = req

-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()

-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })

 router.get('/studenttags/:studentnumber', async (req, res) => {
-  try {
-    const { studentnumber } = req.params
-    const { decodedToken } = req
-    const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const { studentnumber } = req.params
+  const { user } = req
+  const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
+  res.status(200).json(filterRelevantStudentTags(result, user.id))
 })

 router.post('/studenttags/:studentnumber', async (req, res) => {
-  try {
-    const { tag, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tag, studytrack } = req.body
+  const {
+    user: { roles, rights, id },
+  } = req

-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()

-    await TagStudent.createStudentTag(tag)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  await TagStudent.createStudentTag(tag)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })

 router.post('/studenttags', async (req, res) => {
-  try {
-    const { tags, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tags, studytrack } = req.body
+  const {
+    user: { rights, roles, id },
+  } = req

-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()

-    const existingtags = await Tags.findTagsByStudytrack(studytrack)
-    const existingTagids = existingtags.map(t => t.tag_id)
-    const tagids = [...new Set(tags.map(t => t.tag_id))]
-    if (!tagids.find(t => existingTagids.includes(t))) return res.status(400).json({ error: 'The tag does not exist' })
+  const existingtags = await Tags.findTagsByStudytrack(studytrack)
+  const existingTagids = existingtags.map(t => t.tag_id)
+  const tagids = [...new Set(tags.map(t => t.tag_id))]
+  if (!tagids.find(t => existingTagids.includes(t))) return res.status(400).json({ error: 'The tag does not exist' })

-    const studentnumbers = tags.map(t => t.studentnumber)
-    const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
-    const missingStudents = difference(studentnumbers, students)
-    if (missingStudents.length !== 0)
-      return res
-        .status(400)
-        .json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
+  const studentnumbers = tags.map(t => t.studentnumber)
+  const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
+  const missingStudents = difference(studentnumbers, students)
+  if (missingStudents.length !== 0)
+    return res
+      .status(400)
+      .json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })

-    await TagStudent.createMultipleStudentTags(tags)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  await TagStudent.createMultipleStudentTags(tags)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })

 router.delete('/studenttags/delete_one', async (req, res) => {
-  try {
-    const { tag_id, studentnumber, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tag_id, studentnumber, studytrack } = req.body
+  const {
+    user: { rights, roles, id },
+  } = req

-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()

-    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
-    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
+  const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
+  if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })

-    await TagStudent.deleteStudentTag(studentnumber, tag_id)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  await TagStudent.deleteStudentTag(studentnumber, tag_id)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })

 router.delete('/studenttags/delete_many', async (req, res) => {
-  try {
-    const { tagId, studentnumbers, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tagId, studentnumbers, studytrack } = req.body
+  const {
+    user: { rights, roles, id },
+  } = req

-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()

-    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
-    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
+  const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
+  if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })

-    await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(400).json(err)
-  }
+  await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -25,6 +25,7 @@ router.get('/top', async (req, res) => {
  if (!yearcode) {
    return res.status(422).send('Missing required yearcode query param')
  }
+
  const result = await topteachers.getTeacherStats(category, yearcode)
  res.json(result)
 })
@@ -41,7 +42,9 @@ router.get('/top/categories', async (req, res) => {
 })

 router.get('/stats', async (req, res) => {
-  const { rights, roles } = req
+  const {
+    user: { rights, roles },
+  } = req

  const { providers, semesterStart, semesterEnd } = req.query
  if (!providers || !semesterStart) {
@@ -77,6 +80,4 @@ router.get('/:id', async (req, res) => {
  res.json(result)
 })

-router.use('*', (req, res, next) => next())
-
 module.exports = router
@@ -9,8 +9,12 @@ router.post('/event', (req, res) => {
    return res.status(400).json({ error: 'group, name and value are required' })
  }

+  const {
+    user: { userId },
+  } = req
+
  // don't await here because frontend doesn't care if it succeeds
-  tsaService.sendTsaEvent(req.decodedToken.userId, { group, name, label, value }).catch(e => {
+  tsaService.sendTsaEvent(userId, { group, name, label, value }).catch(e => {
    Sentry.withScope(scope => {
      // if error was caused by a 400, dig out the JSON response for Sentry
      if (e.isAxiosError && e.response && e.response.data) {
@@ -8,116 +8,63 @@ const {
  abort,
  updateCoursesByCourseCode,
 } = require('../services/sisUpdaterService')
-const { refreshStatistics } = require('../events')
+const { refreshStatistics, refreshTrends } = require('../events')

 router.get('/update/v2/meta', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISMetadata(req)
-    if (response) {
-      res.status(200).json('Update SIS meta scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateSISMetadata(req)
+  if (response) {
+    res.status(200).json('Update SIS meta scheduled')
  }
 })

 router.get('/update/v2/students', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISStudents()
-    if (response) {
-      res.status(200).json('Update SIS students scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateSISStudents()
+  if (response) {
+    res.status(200).json('Update SIS students scheduled')
  }
 })

 router.post('/update/v2/students', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateStudentsByStudentNumber(req.body)
-    if (response) {
-      res.status(200).json('Update SIS students scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateStudentsByStudentNumber(req.body)
+  if (response) {
+    res.status(200).json('Update SIS students scheduled')
  }
 })

 router.post('/update/v2/courses', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateCoursesByCourseCode(req.body)
-    if (response) {
-      res.status(200).json('Update SIS courses scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateCoursesByCourseCode(req.body)
+  if (response) {
+    res.status(200).json('Update SIS courses scheduled')
  }
 })

 router.get('/update/v2/programmes', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISProgrammes()
-    if (response) {
-      res.status(200).json('Update SIS programmes scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateSISProgrammes()
+  if (response) {
+    res.status(200).json('Update SIS programmes scheduled')
  }
 })

 router.get('/refresh_redis_cache', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISRedisCache()
-    if (response) {
-      res.status(200).json('Refreshing SIS redis cache scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateSISRedisCache()
+  if (response) {
+    res.status(200).json('Refreshing SIS redis cache scheduled')
  }
 })

 router.post('/refresh_statistic_v2', async (req, res) => {
-  try {
-    refreshStatistics()
-    res.status(200).json('Refreshing sis statistics')
-  } catch (err) {
-    console.error(err)
-    res.status(500).json({ error: 'error' })
-  }
+  await refreshStatistics()
+  res.status(200).json('Refreshing sis statistics')
+})
+
+router.post('/refresh_trends', async (req, res) => {
+  await refreshTrends()
+  res.status(200).json('Refreshing trends')
 })

 router.get('/abort', async (req, res) => {
-  try {
-    await abort()
-    res.status(200).json()
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
+  await abort()
+  res.status(200).json()
 })

 module.exports = router
@@ -1,16 +1,7 @@
 const router = require('express').Router()
 const userService = require('../services/userService')
-const blacklist = require('../services/blacklist')
-const { userDataCache } = require('../services/cache')
 const { sendNotificationAboutAccessToUser, previewNotificationAboutAccessToUser } = require('../services/mailservice')
-
-const addUserToBlacklist = async (user, decodedToken) => {
-  if (user) {
-    const { username } = user
-    userDataCache.del(username)
-    if (username != decodedToken.userId) await blacklist.addUserToBlacklist(username)
-  }
-}
+const logger = require('../util/logger')

 router.get('/', async (req, res) => {
  const results = await userService.findAll()
@@ -24,10 +15,7 @@ router.get('/access_groups', async (req, res) => {

 router.post('/modifyaccess', async (req, res) => {
  try {
-    const { uid } = req.body
    const result = await userService.modifyAccess(req.body)
-    const user = await userService.byId(uid)
-    await addUserToBlacklist(user, req.decodedToken)
    res.status(200).json(result)
  } catch (e) {
    res.status(400).json(e)
@@ -49,7 +37,7 @@ router.post('/email', async (req, res) => {
  if (result.error) {
    return res.status(400).json(result).end()
  }
-  console.log('Message sent successfully')
+  logger.info('Succesfully sent message about oodikone access to user')
  res.status(200).end()
 })

@@ -57,7 +45,6 @@ router.post('/:uid/elements', async (req, res) => {
  const { uid } = req.params
  const { codes } = req.body
  const user = await userService.enableElementDetails(uid, codes)
-  await addUserToBlacklist(user, req.decodedToken)
  res.json(user)
 })

@@ -65,7 +52,6 @@ router.delete('/:uid/elements', async (req, res) => {
  const { uid } = req.params
  const { codes } = req.body
  const user = await userService.removeElementDetails(uid, codes)
-  await addUserToBlacklist(user, req.decodedToken)
  res.json(user)
 })

@@ -73,7 +59,6 @@ router.post('/:uid/faculties', async (req, res) => {
  const { uid } = req.params
  const { faculties } = req.body
  const user = await userService.setFaculties(uid, faculties)
-  await addUserToBlacklist(user, req.decodedToken)
  res.json(user)
 })

@@ -3,7 +3,10 @@ const moment = require('moment')

 const createRedisKeyForProductivity = id => `PRODUCTIVITY_${id}`
 const createRedisKeyForThroughput = id => `THROUGHPUT_${id}`
-const createRedisKeyForNonGraduatedStudents = id => `NONGRADUATEDSTUDENTS_${id}`
+const createRedisKeyForBasicStats = (id, yearType) => `BASIC_STATS_${id}_${yearType}`
+const createRedisKeyForCreditStats = (id, yearType) => `CREDIT_STATS_${id}_${yearType}`
+const createRedisKeyForGraduationStats = (id, yearType) => `GRADUATION_STATS_${id}_${yearType}`
+const createRedisKeyForStudytrackStats = id => `STUDYTRACK_STATS_${id}`

 const getProductivity = async id => {
  const redisKey = createRedisKeyForProductivity(id)
@@ -88,32 +91,86 @@ const patchThroughput = async data => {
  }
 }

-const patchNonGraduatedStudents = async data => {
-  const [id, dataToPatch] = Object.entries(data)[0]
-  const redisKey = createRedisKeyForNonGraduatedStudents(id)
-  const dataFromRedisRaw = await redisClient.getAsync(redisKey)
-  const dataFromRedis = dataFromRedisRaw ? JSON.parse(dataFromRedisRaw) : { data: null }
-  const patchedData = {
-    data: {
-      ...dataFromRedis.data,
-      ...dataToPatch,
-    },
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(patchedData))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: patchedData,
-  }
-}
-
-const getNonGraduatedStudents = async id => {
-  const redisKey = createRedisKeyForNonGraduatedStudents(id)
+const getBasicStats = async (id, yearType) => {
+  const redisKey = createRedisKeyForBasicStats(id, yearType)
  const dataFromRedis = await redisClient.getAsync(redisKey)
  if (!dataFromRedis) return null
  return JSON.parse(dataFromRedis)
 }

+const setBasicStats = async (data, yearType) => {
+  const { id } = data
+  const redisKey = createRedisKeyForBasicStats(id, yearType)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getCreditStats = async (id, yearType) => {
+  const redisKey = createRedisKeyForCreditStats(id, yearType)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setCreditStats = async (data, yearType) => {
+  const { id } = data
+  const redisKey = createRedisKeyForCreditStats(id, yearType)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getGraduationStats = async (id, yearType) => {
+  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setGraduationStats = async (data, yearType) => {
+  const { id } = data
+  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getStudytrackStats = async id => {
+  const redisKey = createRedisKeyForStudytrackStats(id)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setStudytrackStats = async data => {
+  const { id } = data
+  const redisKey = createRedisKeyForStudytrackStats(id)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
 module.exports = {
  getProductivity,
  setProductivity,
@@ -121,6 +178,12 @@ module.exports = {
  getThroughput,
  setThroughput,
  patchThroughput,
-  patchNonGraduatedStudents,
-  getNonGraduatedStudents,
+  getBasicStats,
+  setBasicStats,
+  getCreditStats,
+  setCreditStats,
+  getGraduationStats,
+  setGraduationStats,
+  getStudytrackStats,
+  setStudytrackStats,
 }
@@ -1,21 +0,0 @@
-const { redisClient } = require('./redis')
-const moment = require('moment')
-
-const MAKE_REDIS_HASH_KEY = id => `BLACKLIST ${id}`
-
-const addUserToBlacklist = async userId => {
-  console.log('ADDED BLACKLIST', userId)
-  await redisClient.setAsync(MAKE_REDIS_HASH_KEY(userId), moment().toISOString(false))
-}
-
-const isUserBlacklisted = async (userId, tokenCreatedAt) => {
-  const isBlacklisted = async () => {
-    const blacklistTimestamp = await redisClient.getAsync(MAKE_REDIS_HASH_KEY(userId))
-    return tokenCreatedAt && blacklistTimestamp && moment(tokenCreatedAt) < moment(blacklistTimestamp)
-  }
-
-  const timer = () => new Promise(resolve => setTimeout(() => resolve(true), 5000))
-  return Promise.race([isBlacklisted(), timer()])
-}
-
-module.exports = { addUserToBlacklist, isUserBlacklisted }
@@ -1,12 +0,0 @@
-const LRU = require('lru-cache')
-
-// Max 25 users can be stored in the cache
-const userDataCache = new LRU({
-  max: 25,
-  length: () => 1,
-  maxAge: 1000 * 60 * 60,
-})
-
-module.exports = {
-  userDataCache,
-}
@@ -13,7 +13,7 @@ const {
 const { parseCredit } = require('./parseCredits')
 const Op = Sequelize.Op
 const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
-const _ = require('lodash')
+const { sortMainCode, getSortRank } = require('../util/utils')

 const byNameOrCode = (searchTerm, language) =>
  Course.findAll({
@@ -59,7 +59,15 @@ const findOneByCode = code => {
  })
 }

-const creditsForCourses = async (codes, anonymizationSalt) => {
+const creditsForCourses = async (codes, anonymizationSalt, unifyOpenUniCourses) => {
+  let is_open = false
+
+  if (unifyOpenUniCourses) {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
  const credits = await Credit.findAll({
    include: [
      {
@@ -105,9 +113,11 @@ const creditsForCourses = async (codes, anonymizationSalt) => {
      student_studentnumber: {
        [Op.ne]: null,
      },
+      [Op.or]: [{ is_open }, { is_open: null }],
    },
    order: [['attainment_date', 'ASC']],
  })
+
  const parsedCredits = credits.map(credit => parseCredit(credit, anonymizationSalt))
  return parsedCredits
 }
@@ -144,47 +154,20 @@ const unifyOpenUniversity = code => {
  return regexresult[1]
 }

-const allCodeAltenatives = async code => {
-  let course = await Course.findAll({
-    raw: true,
-    attributes: ['id', 'code', 'substitutions'],
-    where: {
-      code: code,
-    },
+const allCodeAlternatives = async code => {
+  const course = await Course.findOne({
+    where: { code: code },
  })

-  const allSubstitutions = _.flatten(course.map(c => c.substitutions))
-
-  let subcodes = []
-  if (allSubstitutions) {
-    subcodes = await Course.findAll({
-      raw: true,
-      attributes: ['code'],
-      where: {
-        id: {
-          [Op.in]: allSubstitutions,
-        },
-      },
-    })
-  }
-
-  const courses = [...course, ...subcodes]
-  const temp = courses
-    .map(c => c.code)
-    .map(c => {
-      if (c.match(/^A/)) return [c, 4] // open university codes come last
-      if (c.match(/^\d/)) return [c, 2] // old numeric codes come second
-      if (c.match(/^[A-Za-z]/)) return [c, 1] // new letter based codes come first
-      return [c, 3] // unknown, comes before open uni?
-    })
-    .sort((a, b) => a[1] - b[1])
-    .map(c => c[0])
-
-  return allSubstitutions ? _.uniq(temp) : [code]
+  return sortMainCode([...course.substitutions, code])
 }

 const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anonymizationSalt) => {
-  let codes = await allCodeAltenatives(coursecode)
+  const courseForSubs = await Course.findOne({
+    where: { code: coursecode },
+  })
+
+  let codes = courseForSubs.substitutions ? sortMainCode([...courseForSubs.substitutions, coursecode]) : [coursecode]

  if (isOpenUniCourseCode(coursecode) && !unifyOpenUniCourses) {
    codes = [coursecode]
@@ -195,7 +178,7 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
  }

  const [credits, course] = await Promise.all([
-    creditsForCourses(codes, anonymizationSalt),
+    creditsForCourses(codes, anonymizationSalt, unifyOpenUniCourses),
    Course.findOne({
      where: {
        code: coursecode,
@@ -204,6 +187,7 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
  ])

  const counter = new CourseYearlyStatsCounter()
+
  for (let credit of credits) {
    const {
      studentnumber,
@@ -357,10 +341,6 @@ const codeLikeTerm = code =>

 const byNameAndOrCodeLike = async (name, code) => {
  let rawCourses = await Course.findAll({
-    include: {
-      model: Organization,
-      required: true,
-    },
    where: {
      ...nameLikeTerm(name),
      ...codeLikeTerm(code),
@@ -368,40 +348,10 @@ const byNameAndOrCodeLike = async (name, code) => {
  })

  const courses = rawCourses
-    .map(
-      ({
-        id,
-        code,
-        name,
-        latest_instance_date,
-        is_study_module,
-        coursetypecode,
-        startdate,
-        max_attainment_date,
-        min_attainment_date,
-        createdAt,
-        updatedAt,
-        substitutions,
-        organizations,
-      }) => {
-        return {
-          id,
-          code,
-          name,
-          latest_instance_date,
-          is_study_module,
-          coursetypecode,
-          startdate,
-          max_attainment_date,
-          min_attainment_date,
-          createdAt,
-          updatedAt,
-          substitutions,
-          organizations: organizations.map(o => o.id),
-        }
-      }
-    )
-    .sort(a => (a.code.match(/^[A-Za-z]{3}[0-9]{1}/) ? -1 : 1))
+    .map(course => {
+      return { ...course.dataValues }
+    })
+    .sort((x, y) => getSortRank(y.code) - getSortRank(x.code))

  let substitutionGroupIndex = 0
  const visited = []
@@ -409,7 +359,11 @@ const byNameAndOrCodeLike = async (name, code) => {
  const organizeSubgroups = course => {
    if (visited.includes(course.code)) return

-    let temp = courses.filter(c => (course.substitutions ? course.substitutions.includes(c.id) : false))
+    let temp = []
+    if (course.substitutions !== null) {
+      temp = courses.filter(c => course.substitutions.includes(c.code))
+    }
+
    temp.unshift(course)
    temp.forEach(cu => {
      if (visited.includes(course.code)) return
@@ -437,6 +391,6 @@ module.exports = {
  byCodes,
  maxYearsToCreatePopulationFrom,
  unifyOpenUniversity,
-  allCodeAltenatives,
+  allCodeAlternatives,
  findOneByCode,
 }
@@ -1,5 +1,6 @@
 const axios = require('axios')
 const { isProduction, pateToken } = require('../conf-backend')
+const { ApplicationError } = require('../util/customErrors')

 const pateClient = axios.create({
  baseURL: 'https://pate.toska.cs.helsinki.fi',
@@ -19,33 +20,22 @@ const baseSettings = {
 }

 const sendEmail = async (options = {}) => {
-  if (!isProduction) {
-    const errorMessage = 'Skipped sending email in non-production environment'
-    console.log(errorMessage, options)
-    return { error: errorMessage }
-  }
-
-  if (!pateToken) {
-    const errorMessage = 'Skipped sending email since pate token was not available'
-    console.log(errorMessage, options)
-    return { error: errorMessage }
-  }
-
-  const { data } = await pateClient.post('/', options)
-
-  return data
+  if (!isProduction) throw new ApplicationError('Email sending is disabled in development mode.')
+  if (!pateToken) throw new ApplicationError('Email sending failed because pate token is missing.')
+  await pateClient.post('/', options)
 }

-const sendFeedbackToToska = ({ feedbackContent, userId, userEmail, userFullName }) => {
-  const userDetails = `Sent by ${userFullName}, userid: ${userId}, email: ${userEmail}`
+const sendFeedbackToToska = async ({ feedbackContent, user }) => {
+  const { name, userId, email } = user
+  const userDetails = `Sent by ${name}, userid: ${userId}, email: ${email}`
  const text = [feedbackContent, userDetails].join('<br />')
-  return sendEmail({
+  await sendEmail({
    template,
    emails: [
      {
        to: 'Toska <grp-toska@helsinki.fi>',
-        replyTo: userEmail,
-        subject: `Oodikone feedback from ${userFullName}`,
+        replyTo: email,
+        subject: `Oodikone feedback from ${name}`,
        text,
      },
    ],
@@ -0,0 +1,338 @@
+const sequelize = require('sequelize')
+const { Op } = sequelize
+const {
+  Credit,
+  Course,
+  Organization,
+  Studyright,
+  StudyrightElement,
+  ElementDetail,
+  Transfer,
+  Student,
+  SemesterEnrollment,
+  Semester,
+} = require('../models')
+const { formatStudyright, formatStudent } = require('./studyprogrammeHelpers')
+
+const whereStudents = studentnumbers => {
+  return studentnumbers ? studentnumbers : { [Op.not]: null }
+}
+
+const studytrackStudents = async studentnumbers =>
+  (
+    await Student.findAll({
+      where: {
+        studentnumber: {
+          [Op.in]: studentnumbers,
+        },
+      },
+    })
+  ).map(formatStudent)
+
+const enrolledStudyrights = async studytrack => {
+  const studyrights = await Studyright.findAll({
+    attributes: ['studyrightid', 'studystartdate', 'enddate', 'prioritycode'],
+    include: [
+      {
+        model: StudyrightElement,
+        required: true,
+        where: { code: studytrack },
+      },
+      {
+        model: Student,
+        attributes: ['studentnumber'],
+        required: true,
+      },
+    ],
+  })
+
+  const studentnumbers = studyrights.map(s => s.student.studentnumber)
+
+  const students = await Student.findAll({
+    attributes: ['studentnumber'],
+    include: [
+      {
+        model: Studyright,
+        include: [
+          {
+            model: StudyrightElement,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        ],
+        attributes: ['studyrightid'],
+      },
+      {
+        model: SemesterEnrollment,
+        attributes: ['semestercode'],
+        include: [
+          {
+            model: Semester,
+            required: true,
+          },
+        ],
+        where: {
+          enrollmenttype: 1,
+        },
+      },
+    ],
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumbers,
+      },
+    },
+  })
+
+  let enrollments = new Set()
+
+  students.forEach(student => {
+    student.semester_enrollments.forEach(e => {
+      const enrollmentStartDate = e.semester.startdate
+      const enrollmentEndDate = e.semester.enddate
+      const correctStudyRight = studyrights.find(
+        s =>
+          student.studentnumber === s.student.studentnumber &&
+          s.studystartdate <= enrollmentStartDate && // Studying in the programme should have started on the same day or earlier than the enrollment
+          s.enddate >= enrollmentEndDate && // Studying in the programme should have ended on the same day or later than the enrollment
+          (!s.canceldate || s.canceldate > enrollmentStartDate) && // Studyright should not be canceled OR the possible canceldate should be after the enrollmentStartDate
+          (s.prioritycode === 1 || s.prioritycode === 30) // Studyright should be primary or the student should have graduated
+      )
+      if (correctStudyRight) {
+        enrollments.add({
+          studentnumber: student.studentnumber,
+          startdate: e.semester.startdate,
+        })
+      }
+    })
+  })
+
+  return enrollments
+}
+
+const startedStudyrights = async (studytrack, since, studentnumbers) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: {
+            model: ElementDetail,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        studystartdate: {
+          [Op.gte]: since,
+        },
+        student_studentnumber: whereStudents(studentnumbers),
+      },
+    })
+  ).map(formatStudyright)
+
+const graduatedStudyRights = async (studytrack, since, studentnumbers) =>
+  await Studyright.findAll({
+    include: {
+      model: StudyrightElement,
+      required: true,
+      include: {
+        model: ElementDetail,
+        required: true,
+        where: {
+          code: studytrack,
+        },
+      },
+    },
+    where: {
+      graduated: 1,
+      enddate: {
+        [Op.gte]: since,
+      },
+      student_studentnumber: whereStudents(studentnumbers),
+    },
+  })
+
+const cancelledStudyRights = async (studytrack, since) => {
+  return await Studyright.findAll({
+    include: {
+      model: StudyrightElement,
+      required: true,
+      where: {
+        code: {
+          [Op.eq]: studytrack,
+        },
+      },
+    },
+    where: {
+      canceldate: {
+        [Op.gte]: since,
+      },
+    },
+  })
+}
+
+const transfersAway = async (studytrack, since) => {
+  return await Transfer.findAll({
+    where: {
+      transferdate: {
+        [Op.gte]: since,
+      },
+      sourcecode: studytrack,
+    },
+    distinct: true,
+    col: 'studentnumber',
+  })
+}
+
+const transfersTo = async (studytrack, since) => {
+  return await Transfer.findAll({
+    where: {
+      transferdate: {
+        [Op.gte]: since,
+      },
+      targetcode: studytrack,
+    },
+    distinct: true,
+    col: 'studentnumber',
+  })
+}
+
+const getProgrammesStudents = async studyprogramme =>
+  (
+    await Studyright.findAll({
+      attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode'],
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          where: {
+            code: {
+              [Op.in]: [studyprogramme],
+            },
+          },
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+    })
+  ).map(formatStudyright)
+
+const getCreditsForStudyProgramme = async (provider, since) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      where: {
+        is_study_module: false,
+      },
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.notIn]: [10, 9, 7],
+      },
+      isStudyModule: {
+        [Op.not]: true,
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+    },
+  })
+
+const getTransferredCredits = async (provider, since) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'credittypecode', 'student_studentnumber'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      where: {
+        is_study_module: false,
+      },
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.eq]: [9],
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+    },
+  })
+
+const getThesisCredits = async (provider, since, thesisType) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      where: {
+        course_unit_type: {
+          [Op.in]: thesisType,
+        },
+      },
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.notIn]: [10, 9, 7],
+      },
+      isStudyModule: {
+        [Op.not]: true,
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+    },
+  })
+
+module.exports = {
+  studytrackStudents,
+  enrolledStudyrights,
+  startedStudyrights,
+  graduatedStudyRights,
+  cancelledStudyRights,
+  transfersAway,
+  transfersTo,
+  getProgrammesStudents,
+  getCreditsForStudyProgramme,
+  getTransferredCredits,
+  getThesisCredits,
+}
@@ -3,6 +3,154 @@ const { Organization, ProgrammeModule } = require('../models')
 const Op = Sequelize.Op
 const { dbConnections } = require('../database/connection')

+// temp refactoring faculty <-> userservice mappings away from userservice
+const facultiesAndProgrammesInUserService = [
+  ['H10', 'MH10_001'],
+  ['H10', 'MH40_011'],
+  ['H10', 'KH10_001'],
+  ['H20', 'KH20_001'],
+  ['H20', 'MH20_001'],
+  ['H20', 'MH20_002'],
+  ['H30', 'KH30_001'],
+  ['H30', 'KH30_002'],
+  ['H30', 'MH30_002'],
+  ['H30', 'MH30_003'],
+  ['H30', 'MH30_004'],
+  ['H30', 'MH30_005'],
+  ['H30', 'MH30_001'],
+  ['H40', 'KH40_001'],
+  ['H40', 'KH40_002'],
+  ['H40', 'KH40_003'],
+  ['H40', 'KH40_004'],
+  ['H40', 'KH40_005'],
+  ['H40', 'KH40_006'],
+  ['H40', 'MH40_001'],
+  ['H40', 'MH40_002'],
+  ['H40', 'MH40_003'],
+  ['H40', 'MH40_004'],
+  ['H40', 'MH40_005'],
+  ['H40', 'MH40_006'],
+  ['H40', 'MH40_007'],
+  ['H40', 'MH40_008'],
+  ['H40', 'MH40_009'],
+  ['H40', 'MH40_010'],
+  ['H40', 'MH40_011'],
+  ['H40', 'MH40_012'],
+  ['H40', 'MH40_013'],
+  ['H40', 'MH40_014'],
+  ['H40', 'MH40_015'],
+  ['H40', 'MH70_001'],
+  ['H40', 'MH70_006'],
+  ['H50', 'KH50_001'],
+  ['H50', 'KH50_002'],
+  ['H50', 'KH50_003'],
+  ['H50', 'KH50_004'],
+  ['H50', 'KH50_005'],
+  ['H50', 'KH50_006'],
+  ['H50', 'KH50_007'],
+  ['H50', 'MH50_001'],
+  ['H50', 'MH50_002'],
+  ['H50', 'MH50_003'],
+  ['H50', 'MH50_004'],
+  ['H50', 'MH50_005'],
+  ['H50', 'MH50_006'],
+  ['H50', 'MH50_007'],
+  ['H50', 'MH50_008'],
+  ['H50', 'MH50_009'],
+  ['H50', 'MH50_010'],
+  ['H50', 'MH50_011'],
+  ['H50', 'MH50_012'],
+  ['H50', 'MH50_013'],
+  ['H55', 'KH55_001'],
+  ['H55', 'MH55_001'],
+  ['H57', 'KH57_001'],
+  ['H57', 'KH57_002'],
+  ['H57', 'KH57_003'],
+  ['H57', 'MH50_002'],
+  ['H57', 'MH50_013'],
+  ['H57', 'MH57_001'],
+  ['H57', 'MH57_002'],
+  ['H57', 'MH57_003'],
+  ['H57', 'MH57_004'],
+  ['H57', 'MH57_005'],
+  ['H57', 'MH80_007'],
+  ['H60', 'KH60_001'],
+  ['H60', 'MH60_001'],
+  ['H70', 'KH40_001'],
+  ['H70', 'KH50_001'],
+  ['H70', 'KH70_001'],
+  ['H70', 'KH70_002'],
+  ['H70', 'KH70_003'],
+  ['H70', 'KH70_004'],
+  ['H70', 'MH50_001'],
+  ['H70', 'MH57_005'],
+  ['H70', 'MH70_001'],
+  ['H70', 'MH70_002'],
+  ['H70', 'MH70_003'],
+  ['H70', 'MH70_004'],
+  ['H70', 'MH70_005'],
+  ['H70', 'MH70_006'],
+  ['H70', 'MH70_007'],
+  ['H70', 'MH70_008'],
+  ['H70', 'MH70_009'],
+  ['H70', 'MH70_010'],
+  ['H74', 'KH74_001'],
+  ['H80', 'KH57_002'],
+  ['H80', 'KH80_001'],
+  ['H80', 'KH80_002'],
+  ['H80', 'KH80_003'],
+  ['H80', 'KH80_004'],
+  ['H80', 'MH57_002'],
+  ['H80', 'MH57_003'],
+  ['H80', 'MH57_005'],
+  ['H80', 'MH80_001'],
+  ['H80', 'MH80_002'],
+  ['H80', 'MH80_003'],
+  ['H80', 'MH80_004'],
+  ['H80', 'MH80_005'],
+  ['H80', 'MH80_006'],
+  ['H80', 'MH80_007'],
+  ['H90', 'KH90_001'],
+  ['H30', '447000'],
+  ['H30', '447001'],
+  ['H30', '447002'],
+  ['H60', 'EDUK730'],
+  ['H60', 'K-MUUT-ERIL'],
+  ['H60', 'ED400'],
+  ['H60', 'ED200'],
+  ['H60', 'ED700'],
+  ['H60', 'ED100'],
+  ['H60', 'ED300'],
+  ['H60', 'ED600'],
+  ['H60', '8009'],
+  ['H60', '8118'],
+  ['H60', '8600'],
+  ['H60', 'DUK-PED'],
+  ['H60', '65'],
+  ['H60', '1102'],
+  ['H60', '20018'],
+  ['H60', '8601'],
+  ['H60', '9290'],
+  ['H60', '8172'],
+  ['H60', '83600'],
+  ['H60', 'DUM-PED'],
+  ['H60', 'ED500'],
+  ['H60', '83601'],
+  ['H60', '0130'],
+  ['H60', '20005-e'],
+  ['H60', '1040'],
+  ['H60', '8703'],
+  ['H60', '0005'],
+  ['H60', '7900'],
+  ['H60', '0152 '],
+  ['H90', 'MH90_001'],
+]
+
+const facultiesAndProgrammesForTrends = facultiesAndProgrammesInUserService.map(f => ({
+  faculty_code: f[0],
+  programme_code: f[1],
+}))
+
 // Have facultyfetching to work like it worked during oodi-db time
 const facultiesInOodi = [
  'H10',
@@ -66,4 +214,5 @@ module.exports = {
  degreeProgrammeCodesOfFaculty,
  isFaculty,
  providersOfFaculty,
+  facultiesAndProgrammesForTrends,
 }
@@ -1,6 +1,7 @@
 const { Op } = require('sequelize')
 const moment = require('moment')
 const { sortBy } = require('lodash')
+const { sortMainCode } = require('../util/utils')

 const {
  Student,
@@ -19,7 +20,6 @@ const {
  dbConnections: { sequelize },
 } = require('../database/connection')
 const { Tag, TagStudent } = require('../models/models_kone')
-const { /*allCodeAltenatives, */ findOneByCode } = require('./courses')
 const { CourseStatsCounter } = require('./course_stats_counter')
 const { getPassingSemester, semesterEnd, semesterStart } = require('../util/semester')
 const { getAllProgrammes } = require('./studyrights')
@@ -65,7 +65,7 @@ const formatStudentForPopulationStatistics = (
  startDateMoment,
  endDateMoment
 ) => {
-  const toCourse = ({ grade, attainment_date, credits, course_code, credittypecode, isStudyModule }) => {
+  const toCourse = ({ grade, attainment_date, credits, course_code, credittypecode, isStudyModule, language }) => {
    const attainment_date_normailized =
      attainment_date < startDate ? startDateMoment.clone().add(1, 'day').toISOString() : attainment_date
    const passed = Credit.passed({ credittypecode })
@@ -78,6 +78,7 @@ const formatStudentForPopulationStatistics = (
      credits,
      isStudyModuleCredit: isStudyModule,
      credittypecode,
+      language,
    }
  }

@@ -265,6 +266,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
        'isStudyModule',
        'student_studentnumber',
        'course_code',
+        'language',
      ],
      where: creditsOfStudent,
      raw: true,
@@ -494,6 +496,7 @@ const studentnumbersWithAllStudyrightElements = async (
    if (!newestStudytrack) return false
    return studyRights.includes(newestStudytrack.code)
  })
+
  const studentnumberlist = filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers

  // fetch students that have transferred out of the programme and filter out these studentnumbers
@@ -816,10 +819,6 @@ const findCourses = async (studentnumbers, beforeDate) => {
          },
        },
      },
-      /* {
-        attributes: ['discipline_id', 'name'],
-        model: Discipline
-      }, */
      {
        attributes: ['coursetypecode', 'name'],
        model: CourseType,
@@ -842,12 +841,24 @@ const parseCreditInfo = credit => ({
  date: credit.attainment_date,
 })

-const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+const findMainCourse = async code => {
+  const course = await Course.findOne({
+    where: { code: code },
+  })

-const unifyOpenUniversity = code => {
-  const regexresult = isOpenUniCourseCode(code)
-  if (!regexresult) return code
-  return regexresult[1]
+  let result = []
+
+  if (course.substitutions) {
+    const sortedCourses = sortMainCode([...course.substitutions, code])
+    result = await Course.findOne({
+      attributes: ['code', 'name'],
+      where: { code: sortedCourses[0] },
+    })
+  } else {
+    return course
+  }
+
+  return result
 }

 const bottlenecksOf = async (query, studentnumberlist) => {
@@ -918,6 +929,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
          transferredStudents,
          tag
        ))
+
      const allstudents = studentnumbers.reduce((numbers, num) => {
        numbers[num] = true
        return numbers
@@ -954,14 +966,14 @@ const bottlenecksOf = async (query, studentnumberlist) => {
  for (const course of courses) {
    let { course_type } = course
    let maincourse = course
-    if (isOpenUniCourseCode) {
+    const sortedMaincourse = await findMainCourse(course.code)
+    if (sortedMaincourse) maincourse = sortedMaincourse
+
+    /*  if (isOpenUniCourseCode) {
      const nonOpen = unifyOpenUniversity(course.code)
      const response = await findOneByCode(nonOpen)
+ */

-      if (response) {
-        maincourse = response
-      }
-    }
    if (!stats[maincourse.code]) {
      stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudentslength)
    }
@@ -982,7 +994,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {

  bottlenecks.coursestatistics = Object.values(stats).map(coursestatistics => coursestatistics.getFinalStats())
  bottlenecks.allStudents = allstudentslength
-  console.log('bottleneck: ', bottlenecks)
+
  return bottlenecks
 }

--- a/Show More
+++ b/Show More