Add some more tests

If something goes wrong, this commit could be reverted to restore the
old studyprogramme views

.github/workflows/release-oodikone.yaml

@@ -1,4 +1,4 @@
-name: "Release oodikone to production"
+name: 'Release oodikone to production'
 
 on:
   release:
@@ -6,23 +6,23 @@ on:
 
 jobs:
   build_and_release_backend:
-    name: "Build and release backend"
+    name: 'Build and release backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
+      - name: 'Extract release version'
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
         run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
         uses: docker/build-push-action@v2
         with:
           build-args: |
@@ -38,26 +38,26 @@ jobs:
           target: production
 
   build_and_release_frontend:
-    name: "Build and release frontend"
+    name: 'Build and release frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Get current time"
+      - name: 'Get current time'
         id: get_current_time
         run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
-      - name: "Extract release version"
+      - name: 'Extract release version'
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
         run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
         uses: docker/build-push-action@v2
         with:
           build-args: |
@@ -72,11 +72,11 @@ jobs:
           push: true
           tags: toska/oodikone2-frontend:latest
           target: production
-      - name: "Create a container from built image "
+      - name: 'Create a container from built image '
         run: docker create --name frontend toska/oodikone2-frontend:latest
-      - name: "Copy assets from the container"
+      - name: 'Copy assets from the container'
         run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
+      - name: 'Create Sentry release'
         uses: getsentry/action-release@v1
         env:
           SENTRY_URL: ${{ secrets.SENTRY_URL }}
@@ -90,49 +90,16 @@ jobs:
           url_prefix: '~/static/js'
           version: ${{ steps.extract_version.outputs.VERSION }}
 
-  build_and_release_userservice:
-    name: "Build and release userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
-        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
-        # This steps extracts release version from GITHUB_REF env var
-        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
-        id: extract_version
-        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: true
-          tags: toska/oodikone2-userservice:latest
-          target: production
-
   report_success:
     runs-on: ubuntu-latest
-    name: "Report deployment success to oodikone notifications"
+    name: 'Report deployment success to oodikone notifications'
     needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
     if: success()
     steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -141,15 +108,14 @@ jobs:
 
   report_failure:
     runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
+    name: 'Report deployment failure to oodikone notifications'
     needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
     if: failure()
     steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure

.github/workflows/release-sis-updater-scheduler.yaml

@@ -1,54 +0,0 @@
-name: "Release sis-updater-scheduler"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-sis-updater-scheduler.yaml'
-      - 'updater/sis-updater-scheduler/**'
-
-jobs:
-  build_and_release:
-    name: "Build and release sis-updater-scheduler"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: docker/setup-buildx-action@v1
-    - name: "Login to DockerHub"
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: "Build and push"
-      uses: docker/build-push-action@v2
-      with:
-        cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
-        cache-to: type=inline
-        context: updater/sis-updater-scheduler/
-        push: true
-        tags: toska/sis-updater-scheduler:latest
-        target: production
-    - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-      with:
-        webhook-url: ${{ secrets.WEBHOOK_URL }}
-        message-type: deployment
-        softa-url: https://oodikone.cs.helsinki.fi
-        deployment-target: production
-        softa-to-deploy: sis-updater-scheduler
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: production
-          softa-to-deploy: sis-updater-scheduler

.github/workflows/release-sis-updater-worker.yaml

@@ -1,54 +0,0 @@
-name: "Release sis-updater-worker"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-sis-updater-worker.yaml'
-      - 'updater/sis-updater-worker/**'
-
-jobs:
-  build_and_release:
-    name: "Build and release sis-updater-worker"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: docker/setup-buildx-action@v1
-    - name: "Login to DockerHub"
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: "Build and push"
-      uses: docker/build-push-action@v2
-      with:
-        cache-from: type=registry,ref=toska/sis-updater-worker:latest
-        cache-to: type=inline
-        context: updater/sis-updater-worker/
-        push: true
-        tags: toska/sis-updater-worker:latest
-        target: production
-    - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-      with:
-        webhook-url: ${{ secrets.WEBHOOK_URL }}
-        message-type: deployment
-        softa-url: https://oodikone.cs.helsinki.fi
-        deployment-target: production
-        softa-to-deploy: sis-updater-worker
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: production
-          softa-to-deploy: sis-updater-worker

.github/workflows/test-and-release-updater.yaml

@@ -0,0 +1,138 @@
+name: 'Test and release updater'
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - '.github/workflows/test-and-release-updater.yaml'
+      - 'updater/sis-updater-worker/**'
+      - 'updater/sis-updater-scheduler/**'
+
+jobs:
+  build_worker:
+    name: 'Build sis-updater-worker'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Build the image'
+        uses: docker/build-push-action@v2
+        with:
+          context: updater/sis-updater-worker/
+          target: production
+          push: false
+          outputs: type=docker,dest=/tmp/worker.tar
+          tags: toska/sis-updater-worker:latest
+          cache-from: type=registry,ref=toska/sis-updater-worker:latest
+          cache-to: type=inline
+      - name: 'Store built image'
+        uses: actions/upload-artifact@v2
+        with:
+          path: /tmp/worker.tar
+
+  build_scheduler:
+    name: 'Build sis-updater-scheduler'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Build the image'
+        uses: docker/build-push-action@v2
+        with:
+          context: updater/sis-updater-scheduler/
+          target: production
+          push: false
+          outputs: type=docker,dest=/tmp/scheduler.tar
+          tags: toska/sis-updater-scheduler:latest
+          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
+          cache-to: type=inline
+      - name: 'Store built image'
+        uses: actions/upload-artifact@v2
+        with:
+          path: /tmp/scheduler.tar
+
+  test_updater:
+    name: 'Test updater'
+    runs-on: ubuntu-latest
+    needs:
+      - build_worker
+      - build_scheduler
+    steps:
+      - uses: actions/checkout@v2
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Download built images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load worker image'
+        run: docker load --input /tmp/artifact/worker.tar
+      - name: 'Load scheduler image'
+        run: docker load --input /tmp/artifact/scheduler.tar
+      - name: 'Start worker'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
+      - name: 'Run hourly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: hourly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run weekly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: weekly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run prepurge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: prepurge
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run purge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: purge
+          EXIT_AFTER_IMMEDIATES: yes
+
+  release_updater:
+    name: 'Release updater'
+    runs-on: ubuntu-latest
+    needs:
+      - test_updater
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Login to DockerHub'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: 'Build and push scheduler'
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
+          cache-to: type=inline
+          context: updater/sis-updater-scheduler/
+          push: true
+          tags: toska/sis-updater-scheduler:latest
+          target: production
+      - name: 'Build and push worker'
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=registry,ref=toska/sis-updater-worker:latest
+          cache-to: type=inline
+          context: updater/sis-updater-worker/
+          push: true
+          tags: toska/sis-updater-worker:latest
+          target: production
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone.cs.helsinki.fi
+          deployment-target: production
+          softa-to-deploy: sis-updater

.github/workflows/test-oodikone.yaml

@@ -9,6 +9,7 @@ on:
       - 'cypress/**'
       - 'cypress.json'
       - 'services/**'
+      - '!services/backend/src/rapodiff.js'
       - 'docker-compose.ci.yml'
       - 'docker-compose.test.yml'
       - 'nginx.conf'
@@ -45,21 +46,7 @@ jobs:
           'eslint --ext .jsx,.js services/frontend' \
           'stylelint services/frontend/src/**/*.css'
 
-  lint_userservice:
-    name: 'Lint userservice'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: 14
-      - name: 'Install dependencies'
-        run: npm ci && cd services/userservice && npm ci
-      - name: 'Run frontend linter'
-        run: npm run eslint services/userservice
-
   # === Build images ===
-
   build_backend:
     name: 'Build backend'
     runs-on: ubuntu-latest
@@ -116,50 +103,106 @@ jobs:
         with:
           path: /tmp/frontend.tar
 
-  build_userservice:
-    name: 'Build userservice'
+  build_and_push_anon_sis_db:
+    name: 'Build anon sis-db and push to toska hub if new migrations have been added'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: 'Build image and export it'
-        uses: docker/build-push-action@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
         with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: false
-          tags: toska/oodikone2-userservice:staging
-          target: production
-          outputs: type=docker,dest=/tmp/userservice.tar
-      - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v2
+          filters: |
+            migrations:
+              - 'updater/sis-updater-worker/src/db/migrations/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
         with:
-          path: /tmp/userservice.tar
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach sis-updater-worker
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
 
-  # === Integration tests ===
-  #
-  run_userservice_integration_tests:
-    name: 'Run userservice integration tests'
+  build_and_push_anon_kone_db:
+    name: 'Build anon kone-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: 'Run tests included in docker build'
-        run: docker-compose --file docker-compose.test.yml up --build userservice-test
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_kone/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
+
+  build_and_push_anon_user_db:
+    name: 'Build anon user-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_user/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
 
   # === End-to-end tests ===
-
   test_course_statistics:
     name: 'Test course statistics'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -171,7 +214,6 @@ jobs:
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
@@ -202,7 +244,9 @@ jobs:
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -214,7 +258,6 @@ jobs:
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
@@ -245,7 +288,9 @@ jobs:
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -257,7 +302,6 @@ jobs:
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
@@ -288,7 +332,9 @@ jobs:
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -300,7 +346,6 @@ jobs:
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
@@ -331,7 +376,9 @@ jobs:
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -343,7 +390,6 @@ jobs:
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
@@ -374,7 +420,9 @@ jobs:
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -386,7 +434,6 @@ jobs:
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
@@ -417,7 +464,9 @@ jobs:
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -429,7 +478,6 @@ jobs:
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
@@ -464,7 +512,9 @@ jobs:
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -476,7 +526,6 @@ jobs:
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
@@ -504,8 +553,6 @@ jobs:
     needs:
       - lint_backend
       - lint_frontend
-      - lint_userservice
-      - run_userservice_integration_tests
       - test_course_statistics
       - test_custom_population
       - test_filters
@@ -524,7 +571,6 @@ jobs:
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
       - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
@@ -551,7 +597,6 @@ jobs:
         run: |
           docker push toska/oodikone2-backend:staging
           docker push toska/oodikone2-frontend:staging
-          docker push toska/oodikone2-userservice:staging
       - name: 'Send deployment success notification to slack'
         if: success()
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
@@ -574,8 +619,6 @@ jobs:
     needs:
       - lint_backend
       - lint_frontend
-      - lint_userservice
-      - run_userservice_integration_tests
       - test_course_statistics
       - test_custom_population
       - test_filters

.lintstagedrc.js

@@ -7,8 +7,8 @@ module.exports = {
   '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
   '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
   '*.css': files => `stylelint --fix ${files.join(' ')}`,
-  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0-alpine hadolint ${relativeFilePaths(files)}`,
-  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck-alpine:v0.7.2 ${relativeFilePaths(files)} -x`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0 hadolint ${relativeFilePaths(files)}`,
+  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck:v0.7.2 ${relativeFilePaths(files)} -x`,
   '.github/workflows/*': files => `${dockerCmdBase} rhysd/actionlint:1.6.2 ${relativeFilePaths(files)}`,
   'docker-compose*': files => {
     const composeFiles = file => {

README.md

@@ -43,8 +43,8 @@ What different CLI options do:
    - Cleans up any previous real data databases
    - Downloads needed database dumps from production servers.
    - Creates real data databases and populates them with downloaded dumps.
-3. Reset sis-importer data
-   - Like option 2, but only for sis-importer database. Useful when developing updater microservice.
+3. Reset single database
+   - Like option 2, but only for single database. Leaves other databases untouched.
 
 ## ⌨️ Development
 
@@ -79,7 +79,7 @@ It is recommended to spend some time to become familiar with `docker` and `docke
 - `docker-compose ps`: view the containers in the running environment
 - `docker-compose logs frontend`: print logs for just frontend
 - `docker-compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
-- `docker exec -it userservice sh`: open bash terminal inside userservice container
+- `docker exec -it backend sh`: open bash terminal inside backend container
 - `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.
 
 ### User types for development
@@ -124,12 +124,16 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 
 ### Modules are missing after updating package.json
 
-You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container.
 
 ### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data
 
 Make sure you have your VPN on
 
+### I made a change to sis-db models aand created a migration, but tests are failing in CI!
+
+Migrations are handled by sis-importer-worker, since oodikone is using only replica. Since CI is testing only services included in oodikone, your new changes aren't migrated and thus are failing. Open anonyymioodi-folder in this repo to find more about how to to update the anon sis-db.
+
 ### Everything is broken, can't get oodikone running, data is not there etc.
 
 Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.

anonyymioodi/.gitignore

@@ -0,0 +1,2 @@
+*.sqz
+

anonyymioodi/README.md

@@ -0,0 +1,65 @@
+# Anonyymioodi - anonymous database setup for oodikone
+
+(name anonyymioodi for :culture: reasons)
+
+This folder contains a guide and scripts to create anonymous database images for oodikone, mainly used for testing.
+
+## How to create / modify database images
+
+Creating / modifying images varies a bit depending on the db. Generally, to modify images, you can either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose. 
+
+By default, previous db images from Toska Hub are used as a base. If needed, comment out Toska hub image and uncomment corresponding clean postgres image in docker-compose before running other steps.
+
+See more details on how to create / modify databases below.
+
+### Sis-importer-db
+
+The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
+- Move `sis-importer-db.sqz` to this folder
+- Run `./create_container_from_dump.sh sis-importer-db`
+- Enrich db with command in file `enrich_data.sql` either by piping command to container or by running command inside container.
+
+### Sis-db:
+
+The following steps are for building a clean sis-db image from scratch using oodikone's updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
+- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
+- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
+- Start updater, see oodikone readme for more info.
+- Populate sis-db from sis-importer-db:
+
+```
+curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
+```
+(or run these from frontend. You might also want to follow sis-updater-workers logs to ensure things are running smoothly.)
+- Clean up any extra stuff you don't want to keep in sis-db.
+- Bonus: check that cypress tests run okay
+
+### User-db / kone-db
+
+- Temporarily replace the db service in oodikone's docker-compose.yml with the same db service from this folder's docker-compose.yml
+- Ensure the newest user-db/kone-db image is pulled from the registry with `docker-compose pull`
+- Modify db to wanted state. Tips:
+  - Run oodikone if you want to upgrade something from frontend (e.g. tags in kone-db, courses in sis-db etc.).
+  - Easy way to create new users to user-db is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
+
+## Publishing the image
+
+After completing the steps above and making sure your local version has the correct data, run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the db image to Toska Hub.
+
+## How to update Postgres versions
+
+When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
+
+- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
+- Start database with docker-compose
+- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
+- Close container with docker-compose down.
+- Update the Postgres version in docker-compose.yml in this folder.
+- Uncomment row with Postgres image and comment row with Toskahub image.
+- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`

anonyymioodi/commit_and_push_to_toska_hub.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+infomsg "Committing and pushing to registry"
+database=${args[0]}
+docker commit "$database" "registry.toska.cs.helsinki.fi/$database"
+docker push "registry.toska.cs.helsinki.fi/$database"
+successmsg "Succesfully pushed"

anonyymioodi/create_container_from_dump.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+# === Create images ===
+
+retry () {
+  sleep 5
+  for i in {1..60}; do
+    "$@" && break || warningmsg "Retry attempt $i failed, waiting..." && sleep 5;
+  done
+}
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+
+infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
+database=${args[0]}
+docker-compose down --remove-orphans
+docker-compose up -d "$database"
+
+database_dump="$database.sqz"
+infomsg "Attempting to restore database $database from dump $database_dump"
+retry docker-compose exec "$database" pg_isready --dbname="$database"
+docker exec -i "$database" /bin/bash -c "pg_restore --username=postgres \
+--format=custom --dbname=$database --no-owner" < "$database_dump"
+successmsg "Database $database succesfully created"

anonyymioodi/docker-compose.yml

@@ -0,0 +1,50 @@
+x-defaults: &defaults
+  restart: unless-stopped
+
+services:
+  kone-db:
+    <<: *defaults
+    container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/kone-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone
+
+  sis-db:
+    <<: *defaults
+    container_name: sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-db
+    image: postgres:13.2
+    shm_size: 2gb
+
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-importer-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-importer-db
+    image: postgres:13.1
+    profiles:
+      - updater
+
+  user-db:
+    <<: *defaults
+    container_name: user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone

anonyymioodi/enrich_sampledata.sql

@@ -0,0 +1,133 @@
+-- add these to sis-importer db by running:
+-- docker exec -i sis-importer-db psql -U postgres sis-importer-db < enrich_sampledata.sql
+
+-- if you need to run updater for these students only, see studentnumberlist in the end
+-- this file
+
+-- Add admission types (valintaperusteet) to studyrights for some of the computer
+-- science 2018-2019 population students. See person_ids / student_number below 
+-- (in case you need to update dumps). 
+
+-- TODO: this should be moved into the sampledata creator script in
+-- importer repo, but since this file contains studentnumbers and person ids, it's 
+-- better to keep this here
+
+-- KP (koepisteet)
+-- hy-hlo-1412620   | 013683126
+-- hy-hlo-1485246   | 014340523
+-- hy-hlo-32230770  | 010589388
+-- hy-hlo-35626233  | 010775888
+-- hy-hlo-40150144  | 011065995
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:kp' 
+WHERE person_id IN (
+'hy-hlo-1412620',
+'hy-hlo-1485246',
+'hy-hlo-32230770',
+'hy-hlo-35626233',
+'hy-hlo-40150144'
+);
+
+-- TV (todistusvalinta)
+-- hy-hlo-45783433  | 011321150
+-- hy-hlo-48537946  | 010764653
+-- hy-hlo-49801905  | 011194400
+-- hy-hlo-53321302  | 011338871
+-- hy-hlo-51223300  | 013951984
+-- hy-hlo-53310137  | 012520688
+-- hy-hlo-53621583  | 011196796
+-- hy-hlo-53863064  | 011736884
+-- hy-hlo-57711592  | 010262566
+-- hy-hlo-57732872  | 013670647
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:tv' 
+WHERE person_id IN (
+'hy-hlo-45783433',
+'hy-hlo-48537946',
+'hy-hlo-49801905',
+'hy-hlo-53321302',
+'hy-hlo-51223300',
+'hy-hlo-53310137',
+'hy-hlo-53621583',
+'hy-hlo-53863064',
+'hy-hlo-57711592',
+'hy-hlo-57732872'
+);
+
+-- YP:
+-- hy-hlo-67768022  | 011952325
+-- hy-hlo-73740025  | 011736606
+-- hy-hlo-74780917  | 012775433
+-- hy-hlo-79615371  | 014113941
+-- hy-hlo-78118455  | 014300615
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:yp' 
+WHERE person_id IN (
+'hy-hlo-67768022',
+'hy-hlo-73740025',
+'hy-hlo-74780917',
+'hy-hlo-79615371',
+'hy-hlo-78118455'
+);
+-- AV:
+-- hy-hlo-78777265  | 010779392
+-- hy-hlo-81909550  | 011894153
+-- hy-hlo-84571890  | 013897158
+-- hy-hlo-80330834  | 015085926
+-- hy-hlo-84775722  | 011791272
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:av' 
+WHERE person_id IN (
+'hy-hlo-78777265',
+'hy-hlo-81909550',
+'hy-hlo-84571890',
+'hy-hlo-80330834',
+'hy-hlo-84775722'
+);
+-- M:
+-- hy-hlo-87439827  | 011928456
+-- hy-hlo-96612371  | 014418361
+-- hy-hlo-84840261  | 010975895
+-- hy-hlo-90330681  | 010522688
+-- hy-hlo-96594603  | 013951573
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:m' 
+WHERE person_id IN (
+'hy-hlo-87439827',
+'hy-hlo-96612371',
+'hy-hlo-84840261',
+'hy-hlo-90330681',
+'hy-hlo-96594603'
+);
+
+-- all studentnumbers for running updater:
+013683126
+014340523
+010589388
+010775888
+011065995
+011321150
+010764653
+011194400
+011338871
+013951984
+012520688
+011196796
+011736884
+010262566
+013670647
+011952325
+011736606
+012775433
+014113941
+014300615
+010779392
+011894153
+013897158
+015085926
+011791272
+011928456
+014418361
+010975895
+010522688
+013951573

cypress/integration/Course_statistics.js

@@ -20,15 +20,13 @@ describe('Course Statistics tests', () => {
 
       cy.contains('.tabular.menu a', 'Table').click()
       cy.contains('All')
-      cy.contains('.modeSelectorRow a', 'Attempts').click()
-      cy.contains('.modeSelectorRow a', 'Students').click()
+      cy.cs('viewMode-Attempts').click()
+      cy.cs('viewMode-Students').click()
 
       cy.contains('.tabular.menu a', 'Pass rate chart').click()
-      cy.get('div.modeSelectorContainer').click()
       cy.contains('svg', 'Pass rate chart')
 
       cy.contains('.tabular.menu a', 'Grade distribution chart').click()
-      cy.get('div.modeSelectorContainer').click()
       cy.contains('svg', 'Grades')
 
       cy.contains('a', 'New query').click()
@@ -186,6 +184,18 @@ describe('Course Statistics tests', () => {
       cy.contains('td', '010674989').siblings().contains('1')
     })
 
+    it('Language distribution is correct', () => {
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
+      cy.contains(/^TKT20003$/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
+      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.contains('Language distribution').click()
+      cy.contains('td', 'finnish').siblings().contains('107')
+      cy.contains('td', 'english').siblings().contains('5')
+    })
+
     describe('When searching unified course stats', () => {
       beforeEach(() => {
         cy.url().should('include', '/coursestatistics')
@@ -292,7 +302,7 @@ describe('Course Statistics tests', () => {
     })
   })
 
-  it('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
+  it.only('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
     cy.init('/coursestatistics', 'onlycoursestatistics')
     cy.get('[data-cy=navbar-courseStatistics]').click()
     cy.get('[data-cy=course-code-input]').type('TKT20003')
@@ -302,10 +312,14 @@ describe('Course Statistics tests', () => {
     cy.contains('Faculty statistics').should('not.exist')
     cy.contains('Show population').should('not.exist')
 
+    cy.cs('viewMode-Attempts').click()
+
     const attemptsTableContents = [
       // [time, passed, failed, passrate]
       ['Total*', 288, 213, 75, '73.96 %'],
+      // ['Total', 295, 216, 79, '73.22 %'],
       ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
+      //['2020-2021', 5, 2, 3, '40.00 %'],
       ['2019-2020', 164, 121, 43, '73.78 %'],
       ['2018-2019', 85, 60, 25, '70.59 %'],
       ['2017-2018', 39, 32, 7, '82.05 %'],

cypress/integration/Filters.js

@@ -9,23 +9,38 @@ const checkFilteringResult = studentCount => {
 // Semantic UI doesn't allow injection of data-cy:s for single multiple dropdown selection.
 // This function tries to click "x" on all selections inside dropdown matching given attribute
 const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.delete').click()
+  cy.cs(dataCyAttribute).find('i.delete').click({ force: true })
+  cy.cs(dataCyAttribute).click()
 }
 
 const clearSingleDropdownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.clear').click()
+  cy.cs(dataCyAttribute).find('i.clear').click({ force: true })
 }
 
 // Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
 // to every it-function. Reason behind using test function wrapper is that Cypresses internal beforeEach / afterEach
 // functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
 const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
-  return (dataCyAttributeOfHeaderToClick, testStepFunctionToRun) => {
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
-    checkFilteringResult(amountWithoutFiltering)
-    testStepFunctionToRun()
-    checkFilteringResult(amountWithoutFiltering)
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
+  return (filterName, testStepFunctionToRun) => {
+    const card = cy.cs(`${filterName}-filter-card`)
+
+    card.invoke('attr', 'data-open').then(open => {
+      console.log('open', open)
+
+      const getHeader = () => cy.cs(`${filterName}-header`)
+
+      if (open === 'false') {
+        getHeader().click({ force: true })
+      }
+
+      checkFilteringResult(amountWithoutFiltering)
+      testStepFunctionToRun()
+      checkFilteringResult(amountWithoutFiltering)
+
+      if (open === 'false') {
+        getHeader().click({ force: true })
+      }
+    })
   }
 }
 
@@ -41,91 +56,121 @@ describe('Population Statistics', () => {
 
   it('Transfer filter is on not transferred by default', () => {
     cy.contains('By default only students who have not transferred to this study programme are shown.')
-    cy.cs('transferredToProgrammeFilter-havenot').find('input').should('be.checked')
+
+    runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
+      const card = cy.cs('TransferredToProgramme-filter-card')
+      card.get('[data-cy="option-havenot"] input').should('be.checked')
+    })
+  })
+
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(5)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
   })
 
   it('Graduation filter works', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
+
       const graduated = 11
-      cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+      getCard().cs('option-graduated-true').click()
       checkFilteringResult(graduated)
-      cy.cs('graduatedFromProgrammeFilter-graduated-false').click()
+      getCard().cs('option-graduated-false').click()
       checkFilteringResult(defaultAmountOfStudents - graduated)
-      cy.cs('graduatedFromProgrammeFilter-all').click()
+      getCard().cs('option-all').click()
     })
   })
 
   it('Transfer filter works', () => {
-    runTestStepWithPreAndPostParts('transferredToProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
       const transferred = 1
-      cy.cs('transferredToProgrammeFilter-have').click()
+      cy.cs('TransferredToProgramme-filter-card').cs('option-have').click()
       checkFilteringResult(transferred)
-      cy.cs('transferredToProgrammeFilter-all').click()
+      cy.cs('TransferredToProgramme-filter-card').cs('option-all').click()
       checkFilteringResult(defaultAmountOfStudents + transferred)
-      cy.cs('transferredToProgrammeFilter-havenot').click()
+      cy.cs('TransferredToProgramme-filter-card').cs('option-havenot').click()
     })
   })
 
   it('Enrollment filter works', () => {
-    runTestStepWithPreAndPostParts('enrollmentStatusFilter-header', () => {
-      cy.selectFromDropdown('enrollmentStatusFilter-status', 0)
-      cy.selectFromDropdown('enrollmentStatusFilter-semesters', [0])
+    runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
+      cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
+      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(0)
       checkFilteringResult(137)
       clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
     })
   })
 
   it('Credit filter works', () => {
-    runTestStepWithPreAndPostParts('credit-filter-header', () => {
+    runTestStepWithPreAndPostParts('CreditsEarned', () => {
       cy.cs('credit-filter-min').type('50')
       cy.cs('credit-filter-max').type('150')
       checkFilteringResult(116)
-      cy.cs('credit-filter-min').find('input').clear()
-      cy.cs('credit-filter-max').find('input').clear()
+      cy.cs('credit-filter-min').clear()
+      cy.cs('credit-filter-max').clear()
     })
   })
 
   it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(42)
+      checkFilteringResult(40)
       cy.cs('ageFilter-min').find('input').clear()
       cy.cs('ageFilter-max').find('input').clear()
     })
   })
 
   it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(36)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
       checkFilteringResult(112)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
       checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
       checkFilteringResult(0)
       clearSingleDropdownSelection('genderFilter-dropdown')
     })
   })
 
   it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', 0)
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 14)
-      checkFilteringResult(121)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(7)
+      checkFilteringResult(141)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
 
   // Return when we have +2020 population with admission types in data
   it.skip('Admission type filter works', () => {
-    runTestStepWithPreAndPostParts('admissionTypeFilter-header', () => {
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('AdmissionTypeFilter', () => {
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(5)
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 2)
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(2)
       checkFilteringResult(10)
       clearSingleDropdownSelection('admissionTypeFilter-dropdown')
     })
@@ -135,31 +180,32 @@ describe('Population Statistics', () => {
   it.skip('Courses filter works', () => {
     // courses takes some time to load, wait for it to complete
     cy.wait(10000)
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
       cy.cs('courseFilter-course-dropdown').click()
       const courses = ['TKT20001', 'MAT11002']
       cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
       checkFilteringResult(140)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
       checkFilteringResult(131)
       cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
       checkFilteringResult(56)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 2)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
       checkFilteringResult(2)
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })
   })
 
   it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
-        cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
+        const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
+        getCard().cs('option-graduated-true').click()
         cy.cs('ageFilter-min').type('20')
         cy.cs('ageFilter-max').type('30')
         checkFilteringResult(11)
         cy.cs('ageFilter-min').find('input').clear()
         cy.cs('ageFilter-max').find('input').clear()
-        cy.cs('graduatedFromProgrammeFilter-all').click()
+        getCard().cs('option-all').click()
       })
     })
   })
@@ -176,7 +222,7 @@ describe('Course Statistics', () => {
   })
 
   it('Grade filter works', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
       cy.cs('gradeFilter-5').click()
       checkFilteringResult(20)
       cy.cs('gradeFilter-3').click()
@@ -186,44 +232,68 @@ describe('Course Statistics', () => {
     })
   })
 
+  it('Programme defaults to "Attainment" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Attainment')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(116)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
   it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(32)
+      checkFilteringResult(31)
       cy.cs('ageFilter-min').find('input').clear()
       cy.cs('ageFilter-max').find('input').clear()
     })
   })
 
   it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(27)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
       checkFilteringResult(91)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
       checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
       checkFilteringResult(0)
       clearSingleDropdownSelection('genderFilter-dropdown')
     })
   })
 
   it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 12)
-      checkFilteringResult(29)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(3)
+      checkFilteringResult(84)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
 
   it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
         cy.cs('gradeFilter-3').click()
         cy.cs('ageFilter-min').type('20')
         cy.cs('ageFilter-max').type('30')
@@ -245,13 +315,14 @@ describe('Custom Population Statistics', () => {
     cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
       const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
       runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
-      cy.cs('student-no-input').click().type(students.join('/n'))
+      cy.cs('student-no-input').click().type(students.join('\n'))
+      cy.wait(10000)
     })
     cy.cs('search-button').click()
   })
 
   it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('30')
       checkFilteringResult(1)
@@ -260,36 +331,59 @@ describe('Custom Population Statistics', () => {
     })
   })
 
-  it('Gender filter works', () => {
-    cy.cs('genderFilter-header').click()
-    cy.selectFromDropdown('genderFilter-dropdown', 0)
-    checkFilteringResult(3)
-    cy.selectFromDropdown('genderFilter-dropdown', 1)
-    checkFilteringResult(2)
-    cy.selectFromDropdown('genderFilter-dropdown', 2)
-    checkFilteringResult(0)
-    clearSingleDropdownSelection('genderFilter-dropdown')
-    checkFilteringResult(5, true)
-  })
-
-  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
-      checkFilteringResult(1)
-      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
     })
   })
 
-  it('Courses filter works', () => {
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
-      const courses = ['MAT11001', 'TKT20004']
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
       checkFilteringResult(5)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
-      checkFilteringResult(5)
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(3)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+      checkFilteringResult(5, true)
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.cs('startYearAtUni-dropdown').click()
+    })
+  })
+
+  it.skip('Courses filter works', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
+      const courses = ['MAT11001', 'TKT20004']
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[0] + ' ')
+        .click()
+      checkFilteringResult(5)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
+      checkFilteringResult(5)
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[1] + ' ')
+        .click()
+      checkFilteringResult(3)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(3)
       checkFilteringResult(1)
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })

cypress/integration/Population_statistics.js

@@ -80,7 +80,7 @@ describe('Population Statistics tests', () => {
 
       cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
       // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000) // a bit hacky way, wait until ui is ready
+      cy.wait(15000) // a bit hacky way, wait until ui is ready
       cy.get('[data-cy=expand-TKT1]').click()
       cy.get('[data-cy=coursestats-link-TKT10002]').click()
       cy.wait('@coursePage')

cypress/integration/Student_statistics.js

@@ -13,7 +13,7 @@ describe('Student Statistics tests', () => {
     studentnumber: '010654019',
   }
 
-  it.only('Student statistics search form is usable', () => {
+  it('Student statistics search form is usable', () => {
     cy.contains('Show student names')
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.lastname)
@@ -38,7 +38,7 @@ describe('Student Statistics tests', () => {
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.studentnumber)
     cy.contains(student.studentnumber).click()
-    cy.contains('Started: 01.08.2011')
+    cy.contains('Started: 01.08.2017')
     cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 01.08.2025')
     cy.contains(student.lastname).should('not.exist')
     cy.contains(student.firstnames).should('not.exist')
@@ -80,7 +80,7 @@ describe('Student Statistics tests', () => {
     cy.contains('Student number').should('not.exist')
   })
 
-  it.only('Can jump to population page', () => {
+  it('Can jump to population page', () => {
     cy.get('.prompt').type(student.studentnumber)
     cy.contains(student.studentnumber).click()
     cy.get('i.level.up.alternate.icon').eq(0).click()
@@ -88,3 +88,13 @@ describe('Student Statistics tests', () => {
     cy.contains('Tietojenkäsittelytieteen kandiohjelma')
   })
 })
+
+// Use admin to see all students
+describe('Testing with admin rights', () => {
+  it('Does not crash if student has no studyright or courses', () => {
+    const studentNumber = '011825096'
+    cy.init(`/students/${studentNumber}`, 'admin') // use admin to see all students
+    cy.contains('Started: Unavailable')
+    cy.contains('Credits: 0')
+  })
+})

cypress/integration/Studyprogramme_overview.js

@@ -7,161 +7,83 @@ const deleteTag = name => {
   cy.contains(name).should('not.exist')
 }
 
-describe('Studyprogramme overview', () => {
-  // check is used by couple of tests
-  const checkProgressAndProductivity = (user = 'basic') => {
-    // Column amount depends on user. Note that with dev user, you'll see even more
-    // stuff than with admin or basic.
-    const CSBachPopulationProgress2018ForAdmin = [
-      '162',
-      '44 (27%)',
-      '118 (72%)',
-      '161 (99%)',
-      '161',
-      '1',
-      '11',
-      '11',
-      '152',
-      '125',
-      '78',
-      '42',
-      '16',
-    ]
-    const CSBachPopulationProgress2018ForBasic = [
-      '162',
-      '44 (27%)',
-      '118 (72%)',
-      '161 (99%)',
-      '161',
-      '1',
-      '11',
-      '11',
-      '152',
-      '125',
-      '78',
-      '42',
-      '16',
-    ] // cancelled and transferred from are missing
-
-    const population = user === 'basic' ? CSBachPopulationProgress2018ForBasic : CSBachPopulationProgress2018ForAdmin
-    cy.contains('Population progress')
-    cy.contains('Yearly productivity')
-
-    cy.contains('2018-2019')
-      .siblings()
-      .each((elem, index) => {
-        cy.wrap(elem).contains(population[index])
-      })
-    const populationproductivity2019 = ['7388.0', '17', '7388.00', '0.00', '349.00']
-    cy.get('table')
-      .eq(1)
-      .contains('2019')
-      .siblings()
-      .each((elem, index) => {
-        cy.wrap(elem).contains(populationproductivity2019[index])
-      })
+const getEmptyYears = () => {
+  const thisYear = new Date().getFullYear()
+  const years = []
+  for (let year = thisYear; year > 2021; year--) {
+    years.push(year)
   }
+  return years
+}
 
-  describe('when opening programme page with admin user', () => {
-    beforeEach(() => {
-      cy.init('/study-programme/KH50_005', 'admin')
-    })
-
-    it('progress should not be recalculating when opened for the first time', () => {
-      cy.contains('Recalculating').should('not.exist')
-      cy.contains('Last updated').should('exist')
-    })
-
-    // Following vars are used to compare values before and after stats calculation
-    // Taken from https://docs.cypress.io/api/commands/should#Compare-text-values-of-two-elements
-    const normalizeText = s => s.replace(/\s/g, '').toLowerCase()
-    let originalProgressCalculatedText
-    let originalProductivityCalculatedText
-    const doRecalculation = () => {
-      // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
-      cy.wait(5000)
-      cy.get('.attached').contains('Admin').click()
-      cy.contains('recalculate productivity').click()
-      cy.contains('recalculate throughput').click()
-      cy.get('.attached').contains('Overview').click() // TODO: add data-cy tags for tabs
-      // Wait to "recalculating" to disappear, TODO: change this to cy.intercept
-      cy.wait(5000)
-      cy.reload()
-    }
-
-    it('renders progress and productivity tables with calculated status', () => {
-      doRecalculation()
-
-      // check contains also study programme options for admin
-      cy.contains('Study programme options')
-      cy.get('table').should('have.length', 3)
-
-      checkProgressAndProductivity('admin')
-
-      // Grab update dates to be compared later
-      cy.cs('throughputUpdateStatus')
-        .invoke('text')
-        .then(text => {
-          originalProgressCalculatedText = normalizeText(text)
-          expect(originalProgressCalculatedText).not.to.contain('recalculating')
-          expect(originalProgressCalculatedText).not.to.contain('refresh')
-        })
-
-      cy.cs('productivityUpdateStatus')
-        .invoke('text')
-        .then(text => {
-          originalProductivityCalculatedText = normalizeText(text)
-          expect(originalProductivityCalculatedText).not.to.contain('recalculating')
-          expect(originalProgressCalculatedText).not.to.contain('refresh')
-        })
-    })
-
-    it('renders progress and productivity tables with calculated status after recalculating stats again', () => {
-      doRecalculation()
-      checkProgressAndProductivity('admin')
-
-      // Check new calculation statuses are reported
-      const newProgressCalculatedTextElement = cy.cs('throughputUpdateStatus').invoke('text')
-      const newProductivityCalculatedTextElement = cy.cs('productivityUpdateStatus').invoke('text')
-
-      newProgressCalculatedTextElement.should(text => {
-        const newProgressCalculatedText = normalizeText(text)
-        expect(newProgressCalculatedText).not.to.contain('recalculating')
-        expect(newProgressCalculatedText).not.to.contain('refresh')
-        expect(newProgressCalculatedText).not.to.contain(originalProgressCalculatedText)
-      })
-
-      newProductivityCalculatedTextElement.should(text => {
-        const newProductivityCalculatedText = normalizeText(text)
-        expect(newProductivityCalculatedText).not.to.contain('recalculating')
-        expect(newProductivityCalculatedText).not.to.contain('refresh')
-        expect(newProductivityCalculatedText).not.to.contain(originalProgressCalculatedText)
-      })
-    })
-  })
-
+describe('Studyprogramme overview', () => {
   describe('when opening programme page with basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme/KH50_005')
     })
 
-    it('renders progress and productivity tables with previosly calculated values', () => {
-      // check doesn't contain study programme options for admin
-      cy.contains('Study programme options').should('not.exist')
-      cy.get('table').should('have.length', 2)
-      checkProgressAndProductivity()
-    })
-
-    it('can open Thesis page', () => {
+    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
+    it('Basic information -tab loads', () => {
       cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.contains('Thesis Courses').click()
-      cy.contains('Add thesis course').click()
-      cy.contains('No results')
+      cy.get('[data-cy=Section-StudentsOfTheStudyprogramme]')
+      cy.get('[data-cy=Section-CreditsProducedByTheStudyprogramme]')
+      cy.get('[data-cy=Section-GraduatedAndThesisWritersOfTheProgramme]')
+      cy.get('[data-cy=Section-ProgrammesAfterGraduation]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
     })
 
-    it('can move to Population statistics page by clickin', () => {
-      cy.get('i.level.up.alternate.icon').eq(0).click()
-      cy.contains('Students (10)')
+    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
+    it('Studytracks and student populations -tab loads', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      cy.get('[data-cy=Section-StudytrackOverview]')
+      cy.get('[data-cy=Section-StudytrackProgress]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
+    })
+
+    it('Basic information contains correct students', () => {
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, All, Started, Graduated, Cancelled, Transferred Away, Transferred to]
+        ...years.map(year => [year, 0, 0, 0, 0, 0, 0]),
+        [2021, 0, 0, 0, 0, 1, 0],
+        [2020, 12, 10, 36, 16, 3, 2],
+        [2019, 87, 86, 17, 14, 0, 1],
+        [2018, 161, 162, 1, 3, 0, 0],
+        [2017, 171, 173, 0, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudentsOfTheStudyprogramme')
+    })
+
+    it('Basic information contains correct credits', () => {
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Total, Major credits, Non-major credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        [2021, 0, 0, 0, 0],
+        [2020, 4928, 548, 4213, 167],
+        [2019, 7737, 1644, 5744, 349],
+        [2018, 6730, 1820, 4638, 272],
+        [2017, 2900, 788, 1724, 388],
+      ]
+
+      cy.checkTableStats(tableContents, 'CreditsProducedByTheStudyprogramme')
+    })
+
+    it('Basic information contains correct thesis writers and graduates', () => {
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Graduated, Wrote thesis]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 36, 35],
+        [2019, 17, 25],
+        [2018, 1, 5],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(tableContents, 'GraduatedAndThesisWritersOfTheProgramme')
     })
 
     it('can create and delete tags for population', () => {

cypress/integration/Users.js

@@ -39,22 +39,17 @@ describe('Users tests', () => {
 
     it("mocking normal user shows only the mocked user's programmes", () => {
       cy.contains('mocking').should('not.exist')
-      cy.contains('tr', 'basic').within($row => {
+      cy.contains('tr', 'basic').within(() => {
         cy.contains('.button', 'Edit').click()
       })
-      cy.intercept({
-        method: 'POST',
-        path: '/api/superlogin/basic',
-      }).as('superlogin')
       cy.get('i.spy').click()
-      cy.wait('@superlogin')
       cy.contains('mocking as basic')
       cy.wait(1000)
       cy.contains('Study programme').click().siblings().contains('Search by class').click()
       cy.contains('label', 'Study programme')
       cy.contains('label', 'Study programme')
         .siblings()
-        .within($row => {
+        .within(() => {
           cy.get("div[role='option']").should('have.length', 2).contains('Tietojenkäsittelytieteen kandiohjelma')
         })
     })

cypress/support/commands.js

@@ -7,7 +7,7 @@ const adminUserHeaders = {
   uid: 'admin',
   displayname: 'Admin User',
   'shib-session-id': 'mock-cypress-session',
-  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users;grp-toska',
   edupersonaffiliation: 'member;employee;faculty',
   mail: 'grp-toska+mockadminuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-6666666',
@@ -54,8 +54,6 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
     if (!headersToUse) throw Error(`${userId} is not valid user id!`)
     req.headers = headersToUse
   })
-  // Hide SIS warning flag to avoid screen size problems when running headlessly
-  window.localStorage.setItem('SIS_WARNING_FLAG', 'true')
   const baseUrl = Cypress.config().baseUrl
   cy.visit(baseUrl.concat(path))
 })
@@ -63,24 +61,37 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
 /**
  * Shorthand for using "Cypress Selectors" (CS), i.e., `data-cy` attributes.
  */
-Cypress.Commands.add('cs', name => {
-  return cy.get(`[data-cy='${name}']`)
+Cypress.Commands.add('cs', { prevSubject: 'optional' }, (subject, name) => {
+  const selector = `[data-cy="${name}"]`
+
+  if (subject === undefined) {
+    return cy.get(selector)
+  } else {
+    return cy.wrap(subject).find(selector)
+  }
 })
 
 /**
  * Select item specified by `index` number from a semantic-ui dropdown with
  * `data-cy` attribute value `name`.
  */
-Cypress.Commands.add('selectFromDropdown', (name, index) => {
+Cypress.Commands.add('selectFromDropdown', { prevSubject: true }, (s, index) => {
+  const subject = cy.wrap(s)
   const indexes = Array.isArray(index) ? index : [index]
 
   indexes.forEach(i => {
-    cy.cs(name).click().children('.menu').children().eq(i).click({ force: true })
+    subject.click()
+
+    if (typeof i === 'number') {
+      subject.children('.menu').children().eq(i).click({ force: true })
+    } else {
+      subject.children('.menu').containing(i).eq(0).click({ force: true })
+    }
   })
 
   // Close multiple selection so it does not block elements underneath it.
   if (Array.isArray(index)) {
-    cy.cs(name).children('.icon').click({ force: true })
+    cy.wrap(s).children('.icon').click({ force: true })
   }
 })
 
@@ -93,3 +104,18 @@ Cypress.Commands.add('selectStudyProgramme', name => {
   cy.cs('select-study-programme').click().children().contains(name).click()
   cy.contains('See population').click()
 })
+
+Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {
+  cy.get(`[data-cy=${tableName}]>tbody`).within(() => {
+    correctStats.forEach((values, trIndex) => {
+      cy.get('tr')
+        .eq(trIndex)
+        .within(() => {
+          values.forEach((value, tdIndex) => {
+            cy.get('td').eq(tdIndex).contains(value)
+          })
+        })
+    })
+    cy.get('tr').should('have.length', correctStats.length)
+  })
+})

docker-compose.ci.yml

@@ -2,7 +2,6 @@ x-defaults: &defaults
   restart: unless-stopped
 
 services:
-
   backend:
     <<: *defaults
     container_name: backend
@@ -10,17 +9,16 @@ services:
       - kone-db
       - redis
       - sis-db
+      - user-db
     environment:
-      CI: "true"
+      CI: 'true'
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:1337
       REDIS: redis
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
     image: toska/oodikone2-backend:staging
 
   frontend:
@@ -31,11 +29,15 @@ services:
   kone-db:
     <<: *defaults
     container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
     image: registry.toska.cs.helsinki.fi/kone-db
 
   redis:
     <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
     container_name: redis
     image: redis
     ports:
@@ -45,23 +47,20 @@ services:
     <<: *defaults
     container_name: sis-db
     image: registry.toska.cs.helsinki.fi/sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    shm_size: 2gb
 
   user-db:
     <<: *defaults
     container_name: user-db
-    image: registry.toska.cs.helsinki.fi/user-db
-
-  userservice:
-    <<: *defaults
-    container_name: userservice
-    depends_on:
-      - user-db
     environment:
-      CI: "true"
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    image: toska/oodikone2-userservice:staging
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db
 
   nginx:
     image: nginx:latest
@@ -69,8 +68,71 @@ services:
     depends_on:
       - frontend
       - backend
-      - userservice
     ports:
-      -  1337:80
+      - 1337:80
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf
+
+  # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    image: registry.toska.cs.helsinki.fi/sis-importer-db
+    profiles:
+      - updater
+
+  sis-updater-worker:
+    <<: *defaults
+    build:
+      target: development
+      context: updater/sis-updater-worker
+    depends_on:
+      - redis
+      - sis-db
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_URI: //redis:6379
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+    profiles:
+      - updater
+
+  sis-updater-scheduler:
+    <<: *defaults
+    build:
+      target: development
+      context: updater/sis-updater-scheduler
+    container_name: sis-updater-scheduler
+    depends_on:
+      - redis
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      REDIS_URI: //redis:6379
+      SECRET_TOKEN: dev
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
+    ports:
+      - 8082:8082
+    profiles:
+      - updater
+
+  sis-updater-nats:
+    <<: *defaults
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_channels 1000 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    container_name: sis-updater-nats
+    image: nats-streaming:0.16.2
+    profiles:
+      - updater

docker-compose.real.yml

@@ -1,7 +1,7 @@
 services:
-
   backend:
     environment:
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db-real
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db-real
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db-real
 
@@ -18,9 +18,5 @@ services:
       DB_URL: postgres://postgres@sis-db:5432/sis-db-real
       SIS_IMPORTER_DATABASE: sis-importer-db-real
 
-  userservice:
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db-real
-
 volumes:
   redis-data-real:

docker-compose.test.yml

@@ -1,10 +0,0 @@
-services:
-
-  userservice-test:
-    build:
-      context: services/userservice
-      target: test
-    container_name: userservice-test
-    environment:
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest

docker-compose.yml

@@ -1,3 +1,5 @@
+version: '2.1'
+
 x-defaults: &defaults
   restart: unless-stopped
 
@@ -26,18 +28,17 @@ services:
       - kone-db
       - redis
       - sis-db
+      - user-db
     env_file:
       - .env
     environment:
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
       FRONT_URL: http://localhost:3000
       REDIS: redis
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
     ports:
       - 8080:8080
     profiles:
@@ -98,7 +99,7 @@ services:
 
   sis-updater-nats:
     <<: *defaults
-    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_channels 1000 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:
@@ -123,6 +124,8 @@ services:
       SIS_IMPORTER_USER: postgres
       SIS_NATS_TOKEN: dev
       SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
     ports:
       - 8082:8082
     profiles:
@@ -165,26 +168,6 @@ services:
     volumes:
       - user-db-data:/data
 
-  userservice:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/userservice
-    container_name: userservice
-    depends_on:
-      - user-db
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    ports:
-      - 4567:4567
-    profiles:
-      - oodikone
-    volumes:
-      - userservice-nodemod:/usr/src/app/node_modules
-      - ./services/userservice/:/usr/src/app
-
 volumes:
   backend-nodemod:
   frontend-nodemod:
@@ -195,4 +178,3 @@ volumes:
   sis-updater-scheduler-nodemod:
   sis-updater-worker-nodemod:
   user-db-data:
-  userservice-nodemod:

package.json

@@ -15,7 +15,8 @@
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
     "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "rapodiff": "docker exec backend node /usr/src/app/src/rapodiff.js"
   },
   "repository": {
     "type": "git",

scripts/functions_for_setup.sh

@@ -165,7 +165,6 @@ set_up_oodikone_from_scratch() {
   folders_to_set_up=(
     "$PROJECT_ROOT"
     "$PROJECT_ROOT/services/frontend"
-    "$PROJECT_ROOT/services/userservice"
     "$PROJECT_ROOT/services/backend"
     "$PROJECT_ROOT/updater/sis-updater-scheduler"
     "$PROJECT_ROOT/updater/sis-updater-worker"

services/backend/.dockerignore

@@ -1,6 +1,3 @@
 .dockerignore
 Dockerfile
 node_modules/
-requests/
-scripts/
-*.log

services/backend/.gitignore

@@ -1,3 +0,0 @@
-# Student diff output
-/student_diff_*
-/diff_partial_att_data.json

services/backend/Dockerfile

@@ -17,7 +17,7 @@ ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
 ARG SENTRY_RELEASE
 ENV SENTRY_RELEASE=$SENTRY_RELEASE
 COPY --chown=node:node ./package* /usr/src/app/
-RUN npm ci --production
+RUN npm ci --production && apk add --no-cache vim curl nano
 COPY --chown=node:node . /usr/src/app/
 USER node
 CMD ["npm", "start"]

services/backend/package-lock.json

@@ -332,11 +332,6 @@
         "concat-map": "0.0.1"
       }
     },
-    "buffer-equal-constant-time": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
-      "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
-    },
     "buffer-writer": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
@@ -704,14 +699,6 @@
       "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=",
       "dev": true
     },
-    "ecdsa-sig-formatter": {
-      "version": "1.0.11",
-      "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
-      "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
-      "requires": {
-        "safe-buffer": "^5.0.1"
-      }
-    },
     "ee-first": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -1330,49 +1317,6 @@
       "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=",
       "dev": true
     },
-    "jsonwebtoken": {
-      "version": "8.5.1",
-      "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz",
-      "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==",
-      "requires": {
-        "jws": "^3.2.2",
-        "lodash.includes": "^4.3.0",
-        "lodash.isboolean": "^3.0.3",
-        "lodash.isinteger": "^4.0.4",
-        "lodash.isnumber": "^3.0.3",
-        "lodash.isplainobject": "^4.0.6",
-        "lodash.isstring": "^4.0.1",
-        "lodash.once": "^4.0.0",
-        "ms": "^2.1.1",
-        "semver": "^5.6.0"
-      },
-      "dependencies": {
-        "semver": {
-          "version": "5.7.1",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
-        }
-      }
-    },
-    "jwa": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz",
-      "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==",
-      "requires": {
-        "buffer-equal-constant-time": "1.0.1",
-        "ecdsa-sig-formatter": "1.0.11",
-        "safe-buffer": "^5.0.1"
-      }
-    },
-    "jws": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz",
-      "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==",
-      "requires": {
-        "jwa": "^1.4.1",
-        "safe-buffer": "^5.0.1"
-      }
-    },
     "kind-of": {
       "version": "6.0.3",
       "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
@@ -1389,41 +1333,6 @@
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
       "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
     },
-    "lodash.includes": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
-      "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8="
-    },
-    "lodash.isboolean": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
-      "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY="
-    },
-    "lodash.isinteger": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
-      "integrity": "sha1-YZwK89A/iwTDH1iChAt3sRzWg0M="
-    },
-    "lodash.isnumber": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
-      "integrity": "sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w="
-    },
-    "lodash.isplainobject": {
-      "version": "4.0.6",
-      "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
-      "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs="
-    },
-    "lodash.isstring": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
-      "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE="
-    },
-    "lodash.once": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
-      "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w="
-    },
     "logform": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz",

services/backend/package.json

@@ -4,11 +4,7 @@
   "scripts": {
     "start": "node --max_old_space_size=4096 index.js",
     "dev": "nodemon --max_old_space_size=4096 index.js",
-    "diff:students": "node /usr/src/app/src/sisoodi_diff/students/index.js",
-    "diff:populations": "node /usr/src/app/src/sisoodi_diff/populations.js",
-    "diff:courses": "node /usr/src/app/src/sisoodi_diff/courses/index.js",
-    "diff:transfers": "node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "node /usr/src/app/src/sisoodi_diff/trends.js"
+    "rapodiff": "node /usr/src/app/src/rapodiff.js"
   },
   "dependencies": {
     "@sentry/node": "^5.30.0",
@@ -21,7 +17,6 @@
     "cron": "^1.8.2",
     "express": "^4.17.1",
     "express-async-errors": "^3.1.1",
-    "jsonwebtoken": "^8.5.1",
     "lodash": "^4.17.21",
     "lru-cache": "^5.1.1",
     "moment": "^2.29.1",

services/backend/src/app.js

@@ -1,13 +1,9 @@
 require('express-async-errors')
 const express = require('express')
-const cors = require('cors')
-const Sentry = require('@sentry/node')
-const { baseUrl, backendPort, frontUrl } = require('./conf-backend')
+const { baseUrl, backendPort } = require('./conf-backend')
 const routes = require('./routes')
 const { startCron } = require('./events')
 const { initializeDatabaseConnection, dbConnections } = require('./database/connection')
-const initializeSentry = require('./util/sentry')
-const errorMiddleware = require('./middleware/errormiddleware')
 const logger = require('./util/logger')
 
 initializeDatabaseConnection()
@@ -21,28 +17,11 @@ initializeDatabaseConnection()
     dbConnections.on('error', () => {
       logger.error('Failed to connect to sis db!')
     })
-
-    const app = express()
-    initializeSentry(app)
-    app.use(Sentry.Handlers.requestHandler())
-    app.use(Sentry.Handlers.tracingHandler())
-
     startCron()
 
-    app.use(cors({ credentials: true, origin: frontUrl }))
-    app.use(express.json())
-
+    const app = express()
     routes(app, baseUrl)
 
-    app.get('*', async (_, res) => {
-      const results = { error: 'unknown endpoint' }
-      res.status(404).json(results)
-    })
-
-    app.use(Sentry.Handlers.errorHandler())
-
-    app.use(errorMiddleware)
-
     const server = app.listen(backendPort, () => logger.info(`Backend listening on port ${backendPort}!`))
     process.on('SIGTERM', () => {
       server.close(() => {

services/backend/src/conf-backend.js

@@ -36,6 +36,7 @@ const MATOMO_URL = process.env.MATOMO_URL
 
 // Other stuff
 const DB_URL_KONE = process.env.DB_URL_KONE
+const DB_URL_USER = process.env.DB_URL_USER
 
 let DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS, 10)
 if (isNaN(DB_MAX_CONNECTIONS)) {
@@ -47,10 +48,8 @@ if (DB_MAX_CRON_CONNECTIONS < 1) {
 }
 
 const redis = process.env.REDIS
-const TOKEN_SECRET = process.env.TOKEN_SECRET
 const SECRET_TOKEN = process.env.SECRET_TOKEN
 const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
-const USERSERVICE_URL = process.env.USERSERVICE_URL
 const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL
 
 const formatURL = url => {
@@ -59,18 +58,14 @@ const formatURL = url => {
 
 const addSlashToEnd = url => (url.endsWith('/') ? url : url + '/')
 
-const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
-
 module.exports = {
+  DB_URL_USER,
   DB_URL_KONE,
   DB_MAX_CONNECTIONS,
   DB_MAX_CRON_CONNECTIONS,
   redis,
-  TOKEN_SECRET,
   SECRET_TOKEN,
   DB_SCHEMA_KONE,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL),
-  ACCESS_TOKEN_HEADER_KEY,
   requiredGroup,
   SIS_UPDATER_URL,
   ANALYTICS_INFLUXDB_URL: formatURL(ANALYTICS_INFLUXDB_URL),

services/backend/src/database/connection.js

@@ -16,7 +16,7 @@ class DbConnection extends EventEmitter {
       pool: {
         max: 25,
         min: 0,
-        acquire: 10000,
+        acquire: 30000,
         idle: 300000000,
       },
       logging: false,
@@ -39,54 +39,73 @@ class DbConnection extends EventEmitter {
   }
 }
 
+// Old-style kone + user db connections
 const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
   schema: conf.DB_SCHEMA_KONE,
   searchPath: conf.DB_SCHEMA_KONE,
   logging: false,
 })
+
 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)
 
+const sequelizeUser = new Sequelize(conf.DB_URL_USER, {
+  logging: false,
+})
+
 const initializeDatabaseConnection = async () => {
   const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-  const waitSeconds = 60
-  for (let i = 1; i <= waitSeconds; i++) {
-    try {
-      await sequelizeKone.authenticate()
-      break
-    } catch (e) {
-      if (i === waitSeconds) {
-        logger.error(`Could not connect to database in ${waitSeconds} seconds`)
-        throw e
+  const rounds = 60
+
+  for (const [seq, dbName] of [
+    [sequelizeKone, 'kone-db'],
+    [sequelizeUser, 'user-db'],
+  ]) {
+    logger.info(`Connecting to ${dbName}...`)
+    for (let round = 1; round <= rounds; round++) {
+      try {
+        await seq.authenticate()
+        break
+      } catch (error) {
+        if (round === rounds) {
+          logger.error(`${dbName} database connection failed!`)
+          throw error
+        }
+        await sleep(1000)
       }
-      await sleep(1000)
     }
-  }
-  try {
+    logger.info(`${dbName} database connection established`)
+
+    const schema = dbName === 'kone-db' ? conf.DB_SCHEMA_KONE : undefined
+    const migrationsFolder = dbName === 'kone-db' ? 'migrations_kone' : 'migrations_user'
     const migrator = new Umzug({
       storage: 'sequelize',
       storageOptions: {
-        sequelize: sequelizeKone,
+        sequelize: seq,
         tableName: 'migrations',
-        schema: conf.DB_SCHEMA_KONE,
+        schema,
       },
       migrations: {
-        params: [sequelizeKone.getQueryInterface(), Sequelize],
-        path: `${process.cwd()}/src/database/migrations_kone`,
+        params: [seq.getQueryInterface(), Sequelize],
+        path: `${process.cwd()}/src/database/${migrationsFolder}`,
         pattern: /\.js$/,
-        schema: conf.DB_SCHEMA_KONE,
+        schema,
       },
     })
-    const migrations = await migrator.up()
-    logger.info({ message: 'Kone Migrations up to date', meta: migrations })
-  } catch (e) {
-    logger.error({ message: 'Kone Migration error: ', meta: e })
-    throw e
+    try {
+      const migrations = await migrator.up()
+      logger.info({ message: `${dbName} migrations up to date: `, meta: migrations.map(m => m.file) })
+    } catch (error) {
+      logger.error({ message: `${dbName} migrations failed`, meta: error })
+      throw error
+    }
   }
 }
 
 const dbConnections = new DbConnection()
+
 module.exports = {
   dbConnections,
   sequelizeKone,
+  sequelizeUser,
   initializeDatabaseConnection,
 }

services/backend/src/database/migrations_user/20211227_00_drop_affiliations.js

@@ -0,0 +1,9 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('user_affiliation')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS user_affilation_id_seq')
+    await queryInterface.dropTable('affiliations')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS affiliations_id_seq')
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_user/20220112_00_drop_hy_groups.js

@@ -0,0 +1,9 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('user_hy_group')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS user_hy_group_id_seq')
+    await queryInterface.dropTable('hy_groups')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS hy_groups_id_seq')
+  },
+  down: async () => {},
+}

services/backend/src/events.js

@@ -10,9 +10,8 @@ const {
 } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
-const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studyprogramme')
+const { updateBasicView, updateStudytrackView } = require('./services/studyprogrammeUpdates')
 const { findAndSaveTeachers } = require('./services/topteachers')
-const { setProductivity, setThroughput, patchProductivity, patchThroughput } = require('./services/analyticsService')
 const { isProduction } = require('./conf-backend')
 const { getCurrentSemester } = require('./services/semesters')
 const logger = require('./util/logger')
@@ -24,47 +23,21 @@ const refreshStudyrightAssociations = async () => {
   logger.info('Studyright associations refreshed!')
 }
 
-const refreshOverview = async () => {
-  logger.info('Refreshing throughput and productivity for programmes...')
+const refreshNewOverviews = async () => {
+  logger.info('Refreshing studyprogramme and studytrack overview statistics for all programmes')
   const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH') || code.includes('MH'))
   let ready = 0
   for (const code of codes) {
-    let programmeStatsSince = new Date('2017-07-31')
-    if (code.includes('MH') || code.includes('KH')) {
-      programmeStatsSince = new Date('2017-07-31')
-    } else {
-      programmeStatsSince = new Date('2000-07-31')
-    }
     try {
-      await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-      const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
-      await setThroughput(data)
+      await updateBasicView(code)
+      await updateStudytrackView(code)
+      ready += 1
     } catch (e) {
-      logger.error({ message: `failed to recalculate throughput for ${code}`, meta: e })
-      try {
-        await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-      } catch (e) {
-        logger.error({ message: 'failed to update throughtput status to error', meta: e })
-      }
+      logger.error({ message: `Failed to update overview stats for programme ${code}`, meta: e })
     }
-    try {
-      await patchProductivity({ [code]: { status: 'RECALCULATING' } })
-      const data = await productivityStatsForStudytrack(code, programmeStatsSince)
-      await setProductivity(data)
-    } catch (e) {
-      logger.error({ message: `failed to recalculate productivity for ${code}`, meta: e })
-      try {
-        await patchProductivity({
-          [code]: { status: 'RECALCULATION ERRORED' },
-        })
-      } catch (e) {
-        logger.error({ message: 'failed to update productivity status to error', meta: e })
-      }
-    }
-    ready += 1
     logger.info(`${ready}/${codes.length} programmes done`)
   }
-  logger.info('Throughput and productivity for programmes refreshed')
+  logger.info('Studyprogramme and studytrack overview stats refreshed!')
 }
 
 const refreshTeacherLeaderboard = async () => {
@@ -147,7 +120,7 @@ const refreshProtoCProgrammeToRedis = async () => {
 }
 
 const refreshStatistics = async () => {
-  const statfuncs = [refreshStudyrightAssociations, refreshOverview, refreshTeacherLeaderboard]
+  const statfuncs = [refreshStudyrightAssociations, refreshTeacherLeaderboard]
   logger.info('Refreshing statistics')
   for (const func of statfuncs) {
     await func()
@@ -166,9 +139,10 @@ const refreshTrends = async () => {
 
 const startCron = () => {
   if (isProduction) {
+    logger.info('Cronjob for refreshing stats started: runs at 3am every day.')
     // refresh 3am every day
     schedule('0 3 * * *', async () => {
-      for (const func of [refreshStatistics, refreshTrends]) {
+      for (const func of [refreshStatistics, refreshTrends, refreshNewOverviews]) {
         await func()
       }
     })

services/backend/src/middleware/accessLogger.js

@@ -1,5 +1,6 @@
 const morgan = require('morgan')
 const logger = require('../util/logger')
+const _ = require('lodash')
 
 // So this appears to be a hack to get neatly formatted stats like response-time etc. from morgan
 // without actually using morgan what it's used for (LOGGING REQUESTS!).
@@ -7,7 +8,8 @@ const logger = require('../util/logger')
 // so don't e.g. make it into an async function or it'll print [object Promise] on every request
 const accessLogger = morgan((tokens, req, res) => {
   const fields = ['method', 'url', 'status', 'response-time', 'remote-addr', 'remote-user', 'user-agent', 'referrer']
-  const meta = { ...req.decodedToken }
+  const { user } = req
+  const meta = { ...user }
   fields.forEach(field => (meta[field] = tokens[field](req, res)))
   meta['time'] = tokens['date'](req, res, 'iso')
   if (req.route) {
@@ -15,7 +17,7 @@ const accessLogger = morgan((tokens, req, res) => {
   }
 
   const message = [
-    req.decodedToken.name,
+    user.name,
     ':',
     tokens['method'](req, res),
     tokens['url'](req, res),
@@ -26,7 +28,8 @@ const accessLogger = morgan((tokens, req, res) => {
   ].join(' ')
 
   logger.info(message, {
-    ...meta,
+    // don't log student list which might be huge
+    ..._.omit(meta, ['studentsUserCanAccess']),
     // pass this as a custom field so we can filter by it in graylog
     isUsageStats: true,
   })

services/backend/src/middleware/auth.js

@@ -1,43 +1,8 @@
-const jwt = require('jsonwebtoken')
-const conf = require('../conf-backend')
-const blacklist = require('../services/blacklist')
-const { getUserDataFor } = require('../services/userService')
-const { ACCESS_TOKEN_HEADER_KEY } = require('../conf-backend')
-const { hasRequiredGroup, parseHyGroups } = require('../util/utils')
-
-const TOKEN_VERSION = 1.1 // When token structure changes, increment in userservice, backend and frontend
-
-const checkAuth = async (req, res, next) => {
-  const token = req.headers[ACCESS_TOKEN_HEADER_KEY]
-  const uid = req.headers['uid']
-
-  if (!token) {
-    return res.status(403).json({ error: 'No token in headers' })
-  }
-
-  jwt.verify(token, conf.TOKEN_SECRET, async (err, decoded) => {
-    if (err) {
-      return res.status(403).json(err)
-    }
-
-    if (decoded.version !== TOKEN_VERSION) {
-      return res.status(401).json({ error: 'Token needs to be refreshed - invalid version' })
-    }
-
-    if (decoded.mockedBy ? decoded.mockedBy !== uid : decoded.userId !== uid) {
-      return res.status(403).json({ error: 'User shibboleth id and token id did not match' })
-    }
-
-    const userData = await getUserDataFor(decoded.userId)
-    req.decodedToken = decoded
-    Object.assign(req, userData)
-    next()
-  })
-}
-
 const roles = requiredRoles => async (req, res, next) => {
-  if (req.decodedToken) {
-    const { roles } = req
+  if (req.user) {
+    const {
+      user: { roles },
+    } = req
 
     if (requiredRoles.every(r => roles.indexOf(r) >= 0) || roles.includes('admin')) {
       return next()
@@ -47,37 +12,6 @@ const roles = requiredRoles => async (req, res, next) => {
   res.status(403).json({ error: 'missing required roles' })
 }
 
-const checkRequiredGroup = async (req, res, next) => {
-  const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-  const enabled = hasRequiredGroup(hyGroups)
-  const tokenOutdated = req.decodedToken.enabled !== enabled
-
-  if (tokenOutdated) {
-    return res.status(401).json({
-      error: 'Token needs to be refreshed - enabled does not match hy-group requirement',
-    })
-  }
-
-  if (!enabled) {
-    return res.status(403).json({ error: 'User is not enabled' })
-  }
-  next()
-}
-
-const checkUserBlacklisting = async (req, res, next) => {
-  const { userId, createdAt } = req.decodedToken
-  const isBlacklisted = await blacklist.isUserBlacklisted(userId, createdAt)
-
-  if (isBlacklisted) {
-    return res.status(401).json({ error: 'Token needs to be refreshed - blacklisted' })
-  }
-
-  next()
-}
-
 module.exports = {
-  checkAuth,
   roles,
-  checkRequiredGroup,
-  checkUserBlacklisting,
 }

services/backend/src/middleware/currentUserMiddleware.js

@@ -0,0 +1,53 @@
+const Sentry = require('@sentry/node')
+const { ApplicationError } = require('../util/customErrors')
+const { getUser, getMockedUser } = require('../services/userService')
+const { requiredGroup } = require('../conf-backend')
+const _ = require('lodash')
+
+const parseIamGroups = iamGroups => iamGroups?.split(';').filter(Boolean) ?? []
+
+const hasRequiredIamGroup = iamGroups => _.intersection(iamGroups, requiredGroup).length > 0
+
+const currentUserMiddleware = async (req, _res, next) => {
+  const {
+    'shib-session-id': sessionId,
+    'x-show-as-user': showAsUser,
+    displayname: name,
+    hygroupcn,
+    hypersonsisuid: sisId,
+    mail: email,
+    shib_logout_url: logoutUrl,
+    uid: username,
+  } = req.headers
+
+  if (!sessionId || !username) {
+    throw new ApplicationError('Not enough data in request headers', 403, { logoutUrl })
+  }
+
+  const iamGroups = parseIamGroups(hygroupcn)
+
+  if (!hasRequiredIamGroup(iamGroups)) {
+    throw new ApplicationError('User does not have required iam group', 403, { logoutUrl })
+  }
+
+  let user = await getUser({
+    username,
+    name,
+    email,
+    iamGroups,
+    sisId,
+  })
+
+  if (showAsUser && user.isAdmin) {
+    user = await getMockedUser({ userToMock: showAsUser, mockedBy: username })
+  }
+
+  Sentry.setUser({ username: user.mockedBy ?? username })
+
+  req.user = user
+  req.logoutUrl = logoutUrl
+
+  next()
+}
+
+module.exports = currentUserMiddleware

services/backend/src/middleware/sentryUserId.js

@@ -1,13 +0,0 @@
-const Sentry = require('@sentry/node')
-
-const sentryUserIdMiddleware = (req, res, next) => {
-  Sentry.getCurrentHub().configureScope(scope => {
-    scope.setUser({
-      username: req.decodedToken && req.decodedToken.userId,
-    })
-  })
-
-  next()
-}
-
-module.exports = sentryUserIdMiddleware

services/backend/src/middleware/shibbolethCharsetMiddleware.js

@@ -0,0 +1,5 @@
+const headersMiddleware = require('unfuck-utf8-headers-middleware')
+
+const headers = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
+
+module.exports = headersMiddleware(headers)

services/backend/src/models/course.js

@@ -47,6 +47,9 @@ Course.init(
     substitutions: {
       type: JSONB,
     },
+    course_unit_type: {
+      type: STRING,
+    },
   },
   {
     underscored: false,

services/backend/src/models/credit.js

@@ -75,6 +75,12 @@ Credit.init(
     updatedAt: {
       type: DATE,
     },
+    language: {
+      type: STRING,
+    },
+    is_open: {
+      type: BOOLEAN,
+    },
   },
   {
     underscored: false,

services/backend/src/models/models_user.js

@@ -1,7 +1,7 @@
 const Sequelize = require('sequelize')
-const { sequelize } = require('../database/connection')
+const { sequelizeUser } = require('../database/connection')
 
-const UserElementDetails = sequelize.define(
+const UserElementDetails = sequelizeUser.define(
   'user_elementdetails',
   {
     userId: {
@@ -18,7 +18,7 @@ const UserElementDetails = sequelize.define(
   }
 )
 
-const AccessGroup = sequelize.define('access_group', {
+const AccessGroup = sequelizeUser.define('access_group', {
   id: {
     primaryKey: true,
     type: Sequelize.BIGINT,
@@ -39,7 +39,7 @@ const AccessGroup = sequelize.define('access_group', {
   },
 })
 
-const UserFaculties = sequelize.define('user_faculties', {
+const UserFaculties = sequelizeUser.define('user_faculties', {
   userId: {
     primaryKey: true,
     type: Sequelize.BIGINT,
@@ -60,7 +60,7 @@ const UserFaculties = sequelize.define('user_faculties', {
   },
 })
 
-const User = sequelize.define(
+const User = sequelizeUser.define(
   'users',
   {
     id: {
@@ -83,41 +83,6 @@ const User = sequelize.define(
     timestamps: false,
   }
 )
-const HyGroup = sequelize.define('hy_group', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true,
-  },
-
-  code: { type: Sequelize.STRING },
-
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})
-
-const Affiliation = sequelize.define('affiliations', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true,
-  },
-
-  code: { type: Sequelize.STRING },
-
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})
 
 User.hasMany(UserElementDetails, { as: 'programme' })
 UserElementDetails.belongsTo(User)
@@ -128,23 +93,12 @@ User.belongsToMany(AccessGroup, {
 })
 AccessGroup.belongsToMany(User, { through: 'user_accessgroup' })
 
-User.belongsToMany(HyGroup, { through: 'user_hy_group', as: 'hy_group' })
-HyGroup.belongsToMany(User, { through: 'user_hy_group' })
-
-User.belongsToMany(Affiliation, {
-  through: 'user_affiliation',
-  as: 'affiliation',
-})
-Affiliation.belongsToMany(User, { through: 'user_affiliation' })
-
 User.hasMany(UserFaculties, { as: 'faculty', foreignKey: 'userId' })
 
 module.exports = {
   User,
   UserElementDetails,
   AccessGroup,
-  HyGroup,
-  Affiliation,
   UserFaculties,
-  sequelize,
+  sequelizeUser,
 }

services/backend/src/models/student.js

@@ -17,7 +17,6 @@ Student.init(
     dateofuniversityenrollment: { type: DATE },
     email: { type: STRING },
     national_student_number: { type: STRING },
-    home_county_id: { type: INTEGER },
     country_fi: { type: STRING },
     country_sv: { type: STRING },
     country_en: { type: STRING },
@@ -25,6 +24,7 @@ Student.init(
     home_country_sv: { type: STRING },
     home_country_en: { type: STRING },
     gender_code: { type: INTEGER },
+    sis_person_id: { type: STRING },
     createdAt: {
       type: DATE,
     },

services/backend/src/rapodiff.js

@@ -0,0 +1,64 @@
+/* eslint no-console: 0 */
+
+// script needs rapo certs, so you need to run this in production pannu
+//
+// possible workflows:
+// - go to oodikone pannu, then "docker exec -it backend sh", then run this script with "npm
+// run rapodiff". Modify with vim / nano until you're happy, then copy changes to local version and commit.
+// - or bind mount a file to backend container in pannu, overriding this file. Modify that file
+// inside or outside container, however you want. Keep file in pannu, or put it to pannu's git repo (oodikone-server-setup).
+
+const axios = require('axios').default
+const fs = require('fs')
+const https = require('https')
+const _ = require('lodash')
+
+const { optimizedStatisticsOf } = require('./services/populations')
+
+const { RAPO_CERT_PATH, RAPO_KEY_PATH, RAPO_API_KEY } = process.env
+
+const agent = new https.Agent({
+  cert: fs.readFileSync(RAPO_CERT_PATH, 'utf8'),
+  key: fs.readFileSync(RAPO_KEY_PATH, 'utf8'),
+})
+
+const api = axios.create({
+  baseURL: 'https://gw.api.helsinki.fi/secure/rapo/report/',
+  headers: { 'X-Api-Key': RAPO_API_KEY },
+  httpsAgent: agent,
+})
+
+const main = async () => {
+  const facultyCode = 'H50'
+  const programme = 'KH50_005'
+  const studyrightStartDate = '2020-08-01'
+  const presence = 'LÄSNÄOLEVA'
+
+  const months = 12 // some amount of months, dunno
+  const year = 2020
+
+  const oodikoneQuery = {
+    semesters: ['FALL', 'SPRING'],
+    months,
+    studyRights: { programme },
+    year,
+  }
+
+  const oodikoneStuff = await optimizedStatisticsOf(oodikoneQuery)
+  const oodikoneStudents = oodikoneStuff.students.map(s => s.studentNumber)
+
+  const rapoUrl = encodeURI(
+    `studyrights?faculty=${facultyCode}&presence=${presence}&education=${programme}&studyrightStartDate=${studyrightStartDate}`
+  )
+
+  const { data: rapoStuff } = await api.get(rapoUrl)
+  const rapoStudents = rapoStuff.map(s => s.opiskelijanumero)
+
+  const onlyInOodikone = _.difference(oodikoneStudents, rapoStudents)
+  const onlyInRapo = _.difference(rapoStudents, oodikoneStudents)
+
+  console.log('Only in Oodikone:', onlyInOodikone)
+  console.log('Only in Rapo:', onlyInRapo)
+}
+
+main().then(() => process.exit(0))

services/backend/src/routes.js

@@ -1,50 +1,52 @@
-const shibbolethHeadersFix = require('unfuck-utf8-headers-middleware')
-const accessLogger = require('./middleware/accesslogger')
-const sentryUserId = require('./middleware/sentryUserId')
+const Sentry = require('@sentry/node')
+const express = require('express')
+const cors = require('cors')
+const { frontUrl } = require('./conf-backend')
+const shibbolethCharsetMiddleware = require('./middleware/shibbolethCharsetMiddleware')
+const matomoInit = require('./routes/matomo-init')
+const currentUserMiddleware = require('./middleware/currentUserMiddleware')
+const accessLogger = require('./middleware/accessLogger')
+const auth = require('./middleware/auth')
+
 const courses = require('./routes/courses')
 const students = require('./routes/students')
 const population = require('./routes/population')
 const login = require('./routes/login')
-const superlogin = require('./routes/superlogin')
 const language = require('./routes/language')
 const users = require('./routes/users')
 const elementdetails = require('./routes/elementdetails')
-const auth = require('./middleware/auth')
 const teachers = require('./routes/teachers')
 const providers = require('./routes/providers')
 const faculties = require('./routes/faculties')
 const semesters = require('./routes/semesters')
 const mandatoryCourses = require('./routes/mandatorycourses')
 const mandatoryCourseLabels = require('./routes/mandatorycourselabels')
-const ping = require('./routes/ping')
 const feedback = require('./routes/feedback')
 const tags = require('./routes/tags')
 const updater = require('./routes/updater')
 const tsaAnalytics = require('./routes/tsaAnalytics')
-const matomoInit = require('./routes/matomo-init')
 const customPopulationSearch = require('./routes/customPopulationSearch')
 const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
 const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
+const studyProgramme = require('./routes/studyProgramme')
+const initializeSentry = require('./util/sentry')
+
+const errorMiddleware = require('./middleware/errorMiddleware')
 
 module.exports = (app, url) => {
-  app.use(url, ping)
-  app.use(
-    shibbolethHeadersFix([
-      'hyGroupCn',
-      'SHIB_LOGOUT_URL',
-      'eduPersonAffiliation',
-      'uid',
-      'displayName',
-      'mail',
-      'hyPersonSisuId',
-      'preferredLanguage',
-    ])
-  )
+  initializeSentry(app)
+  app.use(Sentry.Handlers.requestHandler())
+  app.use(Sentry.Handlers.tracingHandler())
+
+  app.use(cors({ credentials: true, origin: frontUrl }))
+  app.use(express.json())
+
+  app.use(shibbolethCharsetMiddleware)
   app.use(url, matomoInit)
+  app.use(currentUserMiddleware)
+  app.use(accessLogger)
   app.use(url, login)
-  app.use(`${url}/superlogin`, superlogin)
-  app.use(auth.checkAuth, auth.checkRequiredGroup, auth.checkUserBlacklisting, accessLogger, sentryUserId)
   app.use(url, elementdetails)
   app.use(url, courses)
   app.use(url, students)
@@ -54,6 +56,7 @@ module.exports = (app, url) => {
   app.use(url, semesters)
   app.use(url, tags)
   app.use(url, programmeModules)
+  app.use(url, studyProgramme)
   app.use(`${url}/faculties`, faculties)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
@@ -65,4 +68,10 @@ module.exports = (app, url) => {
   app.use(`${url}/custom-population-search`, customPopulationSearch)
   app.use(`${url}/cool-data-science`, trends)
   app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
+  app.get('*', async (_, res) => {
+    const results = { error: 'unknown endpoint' }
+    res.status(404).json(results)
+  })
+  app.use(Sentry.Handlers.errorHandler())
+  app.use(errorMiddleware)
 }

services/backend/src/routes/courses.js

@@ -16,7 +16,9 @@ router.get('/v2/coursesmulti', async (req, res) => {
 })
 
 router.get('/v3/courseyearlystats', async (req, res) => {
-  const { rights, roles } = req
+  const {
+    user: { rights, roles },
+  } = req
 
   const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
 
@@ -42,6 +44,4 @@ router.get('/v3/courseyearlystats', async (req, res) => {
   }
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/customPopulationSearch.js

@@ -8,7 +8,7 @@ const {
 
 router.get('/', async (req, res) => {
   const {
-    decodedToken: { id },
+    user: { id },
   } = req
   const customPopulationSearches = await getCustomPopulationSearchesByUser(id)
   res.json(customPopulationSearches)
@@ -16,7 +16,7 @@ router.get('/', async (req, res) => {
 
 router.post('/', async (req, res) => {
   const {
-    decodedToken: { id },
+    user: { id },
     body: { name, students },
   } = req
 
@@ -29,7 +29,7 @@ router.post('/', async (req, res) => {
 
 router.put('/:id', async (req, res) => {
   const {
-    decodedToken: { id: userId },
+    user: { id: userId },
     body: { students },
   } = req
   const { id } = req.params
@@ -46,7 +46,7 @@ router.put('/:id', async (req, res) => {
 
 router.delete('/:id', async (req, res) => {
   const {
-    decodedToken: { id: userId },
+    user: { id: userId },
   } = req
   const { id } = req.params
 

services/backend/src/routes/elementdetails.js

@@ -188,6 +188,4 @@ router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
   }
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/feedback.js

@@ -1,28 +1,19 @@
 const router = require('express').Router()
-const { getUserDataFor } = require('../services/userService')
 const { sendFeedbackToToska } = require('../services/mailservice')
 const logger = require('../util/logger')
 
-router.post('/email', async (req, res) => {
-  const { content } = req.body
+router.post('/email', async req => {
   const {
-    decodedToken: { userId },
+    body: { content },
+    user,
   } = req
 
-  const { email, full_name } = await getUserDataFor(userId)
-
-  const result = await sendFeedbackToToska({
+  await sendFeedbackToToska({
     feedbackContent: content,
-    userId,
-    userEmail: email,
-    userFullName: full_name,
+    user,
   })
-  if (result.error) {
-    return res.status(400).json(result).end()
-  }
-  logger.info(`${userId} succesfully sent some feedback mail to toska`)
 
-  res.status(200).json('success').end()
+  logger.info(`${user.userId} succesfully sent feedback to toska`)
 })
 
 module.exports = router

services/backend/src/routes/language.js

@@ -2,8 +2,10 @@ const router = require('express').Router()
 const userService = require('../services/userService')
 
 router.post('/language', async (req, res) => {
-  const { language } = req.body
-  const { userId } = req.decodedToken
+  const {
+    body: { language },
+    user: { userId },
+  } = req
   if (!['fi', 'sv', 'en'].includes(language)) {
     return res.status(400).json('invalid language')
   }

services/backend/src/routes/login.js

@@ -1,47 +1,19 @@
 const router = require('express').Router()
-const userService = require('../services/userService')
-const { parseHyGroups } = require('../util/utils')
-const { sendNotificationAboutNewUser } = require('../services/mailservice')
-const logger = require('../util/logger')
+const { ApplicationError } = require('../util/customErrors')
+const _ = require('lodash')
 
-router.post('/login', async (req, res) => {
-  const uid = req.headers['uid']
-  if (req.headers['shib-session-id'] && uid) {
-    const full_name = req.headers.displayname || ''
-    const mail = req.headers.mail || ''
-    const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-    const affiliations = parseHyGroups(req.headers['edupersonaffiliation'])
-    const hyPersonSisuId = req.headers.hypersonsisuid || ''
-    logger.info(`${uid} trying to login, referring to userservice`)
-    let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail, hyPersonSisuId)
-    if (isNew) {
-      const result = await sendNotificationAboutNewUser({ userId: uid, userFullName: full_name })
-      if (result.error) {
-        return res.status(400).json(result).end()
-      }
-    }
-    res.status(200).json({ token })
-  } else {
-    res
-      .status(401)
-      .json({
-        message: `Not enough headers login, uid: ${req.headers.uid}
-        session-id ${req.headers['shib-session-id']}`,
-      })
-      .end()
-  }
-})
+router.get('/login', async (req, res) => {
+  const { user, logoutUrl } = req
 
-router.delete('/logout', async (req, res) => {
-  const logoutUrl = req.headers.shib_logout_url
-  const { returnUrl } = req.body
-  if (logoutUrl) {
-    return res
-      .status(200)
-      .send({ logoutUrl: `${logoutUrl}?return=${returnUrl}` })
-      .end()
+  if (!user) {
+    throw new ApplicationError('User not found', 404)
   }
-  res.status(200).send({ logoutUrl: returnUrl }).end()
+
+  res.send({
+    // don't send possibly huge list of students to frontend
+    user: _.omit(user, ['studentsUserCanAccess']),
+    logoutUrl,
+  })
 })
 
 module.exports = router

services/backend/src/routes/matomo-init.js

@@ -28,7 +28,7 @@ _paq.push(["enableLinkTracking"]);
 
 const script = config.MATOMO_URL ? acual : ''
 
-router.get('/matomo-init', (req, res) => {
+router.get('/matomo-init', (_, res) => {
   res.contentType('text/javascript; charset=UTF-8')
   res.send(script)
 })

services/backend/src/routes/ping.js

@@ -1,85 +0,0 @@
-/* eslint-disable */
-
-const { router, wrapper } = require('./router').routerWithWrapper()
-const axios = require('axios')
-
-wrapper.get('/ping', async (req, res) => {
-  res.json({
-    message: 'pong'
-  })
-})
-
-wrapper.get('/v3/mandatory_courses/:code', async (req, res) => {
-  const courses = []
-  const included = new Set()
-
-  const dumb_flatten = module => {
-    if (module.children) {
-      module.children.forEach(child => {
-        dumb_flatten(child)
-      })
-    } else if (module.code && !module.code.startsWith('KK') && !courses.map(c => c.code).includes(module.code)) {
-      courses.push({
-        name: {
-          fi: module.name,
-          en: module.name
-        },
-        code: module.code,
-        label: { id: `${courses.length}`, label: module.code.replace('-', '').slice(0, 4), orderNumber: courses.length }
-      })
-    }
-  }
-
-  const better_flatten = (module, label) => {
-    if (module.children) {
-      module.children.forEach(child => {
-        better_flatten(child, module.module ? module.module.name : label)
-      })
-    } else if (module.code && !module.code.startsWith('KK')) {
-      courses.push({
-        name: {
-          fi: module.name,
-          en: module.name
-        },
-        code: module.code,
-        label: { id: `${courses.length}`, label, orderNumber: courses.length }
-      })
-    }
-  }
-
-  const superFlattenFlatten = (module, label) => {
-    if (module.children) {
-      module.children.forEach(child => {
-        superFlattenFlatten(child, module.module ? module.module.name : label)
-      })
-    } else if (Array.isArray(module)) {
-      module.forEach(elem => superFlattenFlatten(elem, label))
-    } else if (module.code && !module.code.startsWith('KK')) {
-      if (!included.has(`${label}${module.code}`)) {
-        included.add(`${label}${module.code}`)
-        courses.push({
-          name: {
-            fi: module.name,
-            en: module.name
-          },
-          code: module.code,
-          label: { id: `${courses.length}`, label, orderNumber: courses.length }
-        })
-      }
-    }
-  }
-
-  const code = req.params.code
-  const response = await axios.get(`http://sis-updater-scheduler:8082/v1/courses/${code}?token=dev&superFlatten=true`)
-
-  // dumb_flatten(response.data)
-  // better_flatten(response.data)
-  superFlattenFlatten(response.data)
-
-  const byCode = (c1, c2) => (c1.code < c2.code ? -1 : 1)
-
-  courses.sort(byCode)
-  res.json(courses)
-})
-
-module.exports = router

services/backend/src/routes/population.js

@@ -1,15 +1,14 @@
 const crypto = require('crypto')
 const Sentry = require('@sentry/node')
 const router = require('express').Router()
+const _ = require('lodash')
 const Population = require('../services/populations')
 const Student = require('../services/students')
 const StudyrightService = require('../services/studyrights')
-const UserService = require('../services/userService')
 const TagService = require('../services/tags')
 const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
-const { getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')
-const { logger } = require('../util/logger')
+const { ApplicationError } = require('../util/customErrors')
 
 const filterPersonalTags = (population, userId) => {
   return {
@@ -80,20 +79,13 @@ router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) =>
   if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
     return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
   }
-  let studentnumberlist
-  const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
+
   const {
-    decodedToken: { userId },
-    roles,
+    user: { isAdmin, studentsUserCanAccess },
   } = req
 
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = studentnumbers
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-  }
+  const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
+  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
 
   const result = await Population.bottlenecksOf(
     {
@@ -120,19 +112,12 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
     res.status(400).json({ error: 'The body should have a tag defined' })
     return
   }
-  let studentnumberlist
-  const studentnumbers = await Student.findByTag(tag)
   const {
-    decodedToken: { userId },
-    roles,
+    user: { isAdmin, studentsUserCanAccess },
   } = req
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = studentnumbers
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-  }
+  const studentnumbers = await Student.findByTag(tag)
+  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
+
   const result = await Population.bottlenecksOf(
     {
       year: 1900,
@@ -154,27 +139,17 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
 })
 
 router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
-  if (!req.body.studentnumberlist) {
-    res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
-    return
-  }
-  let studentnumberlist
   const {
-    decodedToken: { userId, sisPersonId },
-    roles,
+    user: { isAdmin, studentsUserCanAccess },
+    body: { studentnumberlist: studentnumbersInReq },
   } = req
 
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = req.body.studentnumberlist
-  } else if (req.body.usingStudyGuidanceGroups) {
-    const studentsUserCanAccess = await getAllStudentsUserHasInGroups(sisPersonId)
-    studentnumberlist = req.body.studentnumberlist.filter(s => studentsUserCanAccess.has(s))
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
+  if (!studentnumbersInReq) {
+    throw new ApplicationError('The body should have a studentnumberlist defined', 400)
   }
 
+  const studentnumberlist = isAdmin ? studentnumbersInReq : _.intersection(studentnumbersInReq, studentsUserCanAccess)
+
   const result = await Population.bottlenecksOf(
     {
       year: 1900,
@@ -195,7 +170,7 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
 
 router.get('/v3/populationstatistics', async (req, res) => {
   const { year, semesters, studyRights: studyRightsJSON } = req.query
-  const { decodedToken } = req
+  const { user } = req
 
   if (!year || !semesters || !studyRightsJSON) {
     res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
@@ -204,13 +179,13 @@ router.get('/v3/populationstatistics', async (req, res) => {
   let studyRights = null
   try {
     studyRights = JSON.parse(studyRightsJSON)
-    const { rights, roles } = req
+    const {
+      user: { rights, isAdmin },
+    } = req
 
-    if (!roles || !roles.includes('admin')) {
-      if (!rights.includes(studyRights.programme)) {
-        res.status(403).json([])
-        return
-      }
+    if (!isAdmin && !rights.includes(studyRights.programme)) {
+      res.status(403).json([])
+      return
     }
   } catch (e) {
     res.status(400).json({ error: 'The query had invalid studyRights' })
@@ -245,7 +220,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
       return
     }
 
-    res.json(filterPersonalTags(result, decodedToken.id))
+    res.json(filterPersonalTags(result, user.id))
   } else {
     const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
     if (result.error) {
@@ -254,32 +229,26 @@ router.get('/v3/populationstatistics', async (req, res) => {
       return
     }
 
-    res.json(filterPersonalTags(result, decodedToken.id))
+    res.json(filterPersonalTags(result, user.id))
   }
 })
 
 router.get('/v3/populationstatisticsbytag', async (req, res) => {
-  const { tag, studyRights: studyRightsJSON, months, year } = req.query
-  const { decodedToken } = req
+  const {
+    user: { id, isAdmin, studentsUserCanAccess },
+    query: { tag, studyRights: studyRightsJSON, months, year },
+  } = req
 
   if (!tag) return res.status(400).json({ error: 'The query should have a tag defined' })
   const foundTag = await TagService.findTagById(tag)
   if (!foundTag) return res.status(404).json({ error: 'Tag not found' })
 
   const semesters = ['FALL', 'SPRING']
-  let studentnumberlist
+
   const studentnumbers = await Student.findByTag(tag)
-  const {
-    decodedToken: { userId },
-    roles,
-  } = req
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = studentnumbers
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-  }
+
+  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
+
   const studyRights = JSON.parse(studyRightsJSON)
   const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
   const yearDifference = Number(year) - Number(newStartYear)
@@ -301,12 +270,14 @@ router.get('/v3/populationstatisticsbytag', async (req, res) => {
     return
   }
 
-  res.json(filterPersonalTags(result, decodedToken.id))
+  res.json(filterPersonalTags(result, id))
 })
 
 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
   const { coursecodes, from, to, separate: sep } = req.query
-  const { decodedToken, roles } = req
+  const {
+    user: { id, isAdmin, studentsUserCanAccess: allStudentsUserCanAccess },
+  } = req
   const separate = sep ? JSON.parse(sep) : false
 
   if (!coursecodes || !from || !to) {
@@ -335,14 +306,9 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
     studentnumbers
   )
 
-  let studentsUserCanAccess
-  if (roles && roles.includes('admin')) {
-    studentsUserCanAccess = new Set(studentnumbers)
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(decodedToken.userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentsUserCanAccess = new Set(await Student.filterStudentnumbersByAccessrights(studentnumbers, codes))
-  }
+  const studentsUserCanAccess = isAdmin
+    ? new Set(studentnumbers)
+    : new Set(_.intersection(studentnumbers, allStudentsUserCanAccess))
 
   const randomHash = crypto.randomBytes(12).toString('hex')
   const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
@@ -368,26 +334,16 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
     return
   }
 
-  res.json(filterPersonalTags(result, decodedToken.id))
+  res.json(filterPersonalTags(result, id))
 })
 
 router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
-  const { studentnumberlist, usingStudyGuidanceGroups } = req.body
-  const { roles, decodedToken } = req
+  const { studentnumberlist } = req.body
+  const {
+    user: { isAdmin, id, studentsUserCanAccess },
+  } = req
 
-  if (usingStudyGuidanceGroups) {
-    if (!decodedToken.sisPersonId) {
-      logger.error(
-        `User ${decodedToken.userId} tried to get person groups but personId was ${decodedToken.sisPersonId} in header`
-      )
-      return res.status(400).json({ error: 'Not possible to get groups without personId' })
-    }
-  }
-
-  const studentsUserCanAccess = usingStudyGuidanceGroups
-    ? await getAllStudentsUserHasInGroups(decodedToken.sisPersonId)
-    : await UserService.getStudentsUserCanAccess(studentnumberlist, roles, decodedToken.userId)
-  const filteredStudentNumbers = studentnumberlist.filter(s => studentsUserCanAccess.has(s))
+  const filteredStudentNumbers = isAdmin ? studentnumberlist : _.intersection(studentnumberlist, studentsUserCanAccess)
 
   const result = await Population.optimizedStatisticsOf(
     {
@@ -402,12 +358,14 @@ router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
     Sentry.captureException(new Error(result.error))
     return
   }
-  res.status(200).json(filterPersonalTags(result, decodedToken.id))
+  res.status(200).json(filterPersonalTags(result, id))
 })
 
 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
-  const { rights, roles } = req
-  if (roles && roles.includes('admin')) {
+  const {
+    user: { rights, roles },
+  } = req
+  if (roles?.includes('admin')) {
     const studyrights = await StudyrightService.getAssociations()
     res.json(studyrights)
   } else {
@@ -427,6 +385,4 @@ router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req
   return res.json(maxYearsToCreatePopulationFrom)
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/semesters.js

@@ -6,6 +6,4 @@ router.get('/semesters/codes', async (req, res) => {
   res.json(providers)
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/students.js

@@ -1,7 +1,6 @@
 const router = require('express').Router()
 const Student = require('../services/students')
-const userService = require('../services/userService')
-const Unit = require('../services/units')
+const { ApplicationError } = require('../util/customErrors')
 
 const filterStudentTags = (student, userId) => {
   return {
@@ -12,8 +11,7 @@ const filterStudentTags = (student, userId) => {
 
 router.get('/students', async (req, res) => {
   const {
-    roles,
-    decodedToken: { userId },
+    user: { isAdmin, studentsUserCanAccess },
     query: { searchTerm },
   } = req
 
@@ -25,55 +23,30 @@ router.get('/students', async (req, res) => {
       .slice(0, 2)
       .find(t => t.length > 3)
   ) {
-    return res.status(400).json({ error: 'at least one search term must be longer than 3 characters' })
+    throw new ApplicationError('at least one search term must be longer than 3 characters', 400)
   }
 
-  if (roles && roles.includes('admin')) {
-    let results = []
-    if (trimmedSearchTerm) {
-      results = await Student.bySearchTerm(trimmedSearchTerm)
-    }
-    return res.status(200).json(results).end()
-  } else {
-    const unitsUserCanAccess = await userService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    const matchingStudents = await Student.bySearchTermAndElements(trimmedSearchTerm, codes)
-    res.json(matchingStudents)
+  let results = []
+  if (trimmedSearchTerm) {
+    results = isAdmin
+      ? await Student.bySearchTerm(trimmedSearchTerm)
+      : await Student.bySearchTermAndStudentNumbers(trimmedSearchTerm, studentsUserCanAccess)
   }
+  res.json(results)
 })
 
 router.get('/students/:id', async (req, res) => {
   const { id: studentId } = req.params
-  const { roles, decodedToken } = req
+  const {
+    user: { id, isAdmin, studentsUserCanAccess },
+  } = req
 
-  if (roles && roles.includes('admin')) {
-    const results = await Student.withId(studentId)
-    return results.error
-      ? res.status(400).json({ error: 'error finding student' }).end()
-      : res.status(200).json(filterStudentTags(results, decodedToken.id)).end()
+  if (!isAdmin && !studentsUserCanAccess.includes(studentId)) {
+    throw new ApplicationError('Error finding student', 400)
   }
 
-  const uid = req.decodedToken.userId
   const student = await Student.withId(studentId)
-  if (student.error) {
-    return res.status(400).json({ error: 'error finding student' }).end()
-  }
-  const units = await userService.getUnitsFromElementDetails(uid)
-
-  const rights = await Promise.all(
-    units.map(async unit => {
-      const jtn = await Unit.hasStudent(unit.id, student.studentNumber)
-      return jtn
-    })
-  )
-
-  if (rights.some(right => right !== null)) {
-    res.status(200).json(filterStudentTags(student, decodedToken.id)).end()
-  } else {
-    res.status(400).json({ error: 'error finding student' }).end()
-  }
+  res.json(filterStudentTags(student, id))
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/studyGuidanceGroups.js

@@ -4,7 +4,7 @@ const { getAllGroupsAndStudents, changeGroupTags } = require('../services/studyG
 
 router.get('/', async (req, res) => {
   const {
-    decodedToken: { uid, sisPersonId },
+    user: { uid, sisPersonId },
   } = req
   if (!sisPersonId) {
     logger.error(`User ${uid} tried to get person groups but personId was ${sisPersonId} in header`)
@@ -15,7 +15,7 @@ router.get('/', async (req, res) => {
 
 router.put('/:id/tags', async (req, res) => {
   const {
-    decodedToken: { uid, sisPersonId },
+    user: { uid, sisPersonId },
     body: tags,
     params: { id: groupId },
   } = req

services/backend/src/routes/studyProgramme.js

@@ -0,0 +1,165 @@
+const router = require('express').Router()
+const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics')
+const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
+const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
+const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
+const {
+  getBasicStats,
+  setBasicStats,
+  getCreditStats,
+  setCreditStats,
+  getGraduationStats,
+  setGraduationStats,
+  getStudytrackStats,
+  setStudytrackStats,
+} = require('../services/analyticsService')
+const { updateBasicView, updateStudytrackView } = require('../services/studyprogrammeUpdates')
+const logger = require('../util/logger')
+
+router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
+
+  if (code) {
+    let data = null
+    try {
+      data = await getBasicStats(code, yearType, specialGroups)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} basic stats: ${e}`)
+    }
+    if (!data) {
+      try {
+        const result = await getBasicStatsForStudytrack({
+          studyprogramme: req.params.id,
+          yearType,
+          specialGroups,
+        })
+        data = await setBasicStats(result, yearType, specialGroups)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} basic stats: ${e}`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
+
+  if (code) {
+    let data = null
+    try {
+      data = await getCreditStats(code, yearType, specialGroups)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} credit stats`)
+    }
+    if (!data) {
+      try {
+        const result = await getCreditStatsForStudytrack({
+          studyprogramme: req.params.id,
+          yearType,
+          specialGroups,
+        })
+        data = await setCreditStats(result, yearType, specialGroups)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} credit stats`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
+
+  if (code) {
+    let data = null
+    try {
+      data = await getGraduationStats(code, yearType, specialGroups)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} graduation stats`)
+    }
+    if (!data) {
+      try {
+        const result = await getGraduationStatsForStudytrack({
+          studyprogramme: req.params.id,
+          yearType,
+          specialGroups,
+        })
+        data = await setGraduationStats(result, yearType, specialGroups)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} graduation stats`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
+  const code = req.params.id
+  const graduated = req.query?.graduated
+  const specialGroups = req.query?.special_groups
+
+  if (code) {
+    let data = null
+    try {
+      data = await getStudytrackStats(code, graduated, specialGroups)
+    } catch (e) {
+      logger.error(`Failed to get code ${code} studytrack stats: ${e}`)
+    }
+    if (!data) {
+      try {
+        const result = await getStudytrackStatsForStudyprogramme({ studyprogramme: code, specialGroups, graduated })
+        data = await setStudytrackStats(result, graduated, specialGroups)
+      } catch (e) {
+        logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
+      }
+    }
+    return res.json(data)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/update_basicview', async (req, res) => {
+  const code = req.params.id
+  if (code) {
+    let result = null
+    try {
+      result = await updateBasicView(code)
+    } catch (e) {
+      logger.error(`Failed to update code ${code} basic stats: ${e}`)
+    }
+    return res.json(result)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/update_studytrackview', async (req, res) => {
+  const code = req.params.id
+  if (code) {
+    let result = null
+    try {
+      result = await updateStudytrackView(code)
+    } catch (e) {
+      logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
+    }
+    return res.json(result)
+  } else {
+    res.status(422).end()
+  }
+})
+
+module.exports = router

services/backend/src/routes/superlogin.js

@@ -1,21 +0,0 @@
-const router = require('express').Router()
-const userService = require('../services/userService')
-
-router.post('/:uid', async (req, res) => {
-  const uid = req.headers['uid']
-  if (req.headers['shib-session-id'] && uid) {
-    const asUser = req.params.uid
-    const token = await userService.superlogin(uid, asUser)
-    res.status(200).json(token)
-  } else {
-    res
-      .status(401)
-      .json({
-        message: `Not enough headers login, uid:
-        ${req.headers.uid} session-id ${req.headers['shib-session-id']}`,
-      })
-      .end()
-  }
-})
-
-module.exports = router

services/backend/src/routes/tags.js

@@ -12,48 +12,50 @@ const filterRelevantStudentTags = (studentTags, userId) => {
   return studentTags.filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
 }
 
-router.get('/tags', async (req, res) => {
+router.get('/tags', async (_req, res) => {
   const tags = await Tags.findTags()
   res.status(200).json(tags)
 })
 
 router.get('/tags/:studytrack', async (req, res) => {
   const { studytrack } = req.params
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const tags = await Tags.findTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantTags(tags, decodedToken.id))
+  res.status(200).json(filterRelevantTags(tags, id))
 })
 
 router.post('/tags', async (req, res) => {
   const {
     tag: { studytrack, tagname, year, personal_user_id },
   } = req.body
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
   const tags = await Tags.findTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantTags(tags, decodedToken.id))
+  res.status(200).json(filterRelevantTags(tags, id))
 })
 
 router.delete('/tags', async (req, res) => {
   const { tag } = req.body
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (
-    !rights.includes(tag.studytrack) &&
-    !(roles && roles.includes('admin')) &&
-    !(tag.personal_user_id === decodedToken.id)
-  )
+  if (!rights.includes(tag.studytrack) && !roles?.includes('admin') && !(tag.personal_user_id === id))
     return res.status(403).end()
 
   await Tags.deleteTag(tag)
   const t = await Tags.findTagsByStudytrack(tag.studytrack)
-  res.status(200).json(filterRelevantTags(t, decodedToken.id))
+  res.status(200).json(filterRelevantTags(t, id))
 })
 
 router.get('/studenttags', async (req, res) => {
@@ -63,37 +65,43 @@ router.get('/studenttags', async (req, res) => {
 
 router.get('/studenttags/:studytrack', async (req, res) => {
   const { studytrack } = req.params
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.get('/studenttags/:studentnumber', async (req, res) => {
   const { studentnumber } = req.params
-  const { decodedToken } = req
+  const { user } = req
   const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, user.id))
 })
 
 router.post('/studenttags/:studentnumber', async (req, res) => {
   const { tag, studytrack } = req.body
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { roles, rights, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   await TagStudent.createStudentTag(tag)
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.post('/studenttags', async (req, res) => {
   const { tags, studytrack } = req.body
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const existingtags = await Tags.findTagsByStudytrack(studytrack)
   const existingTagids = existingtags.map(t => t.tag_id)
@@ -110,37 +118,39 @@ router.post('/studenttags', async (req, res) => {
 
   await TagStudent.createMultipleStudentTags(tags)
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.delete('/studenttags/delete_one', async (req, res) => {
   const { tag_id, studentnumber, studytrack } = req.body
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
   if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
 
   await TagStudent.deleteStudentTag(studentnumber, tag_id)
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.delete('/studenttags/delete_many', async (req, res) => {
   const { tagId, studentnumbers, studytrack } = req.body
-  const { rights, roles, decodedToken } = req
+  const {
+    user: { rights, roles, id },
+  } = req
 
-  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
 
   const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
   if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
 
   await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
+  res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/teachers.js

@@ -42,7 +42,9 @@ router.get('/top/categories', async (req, res) => {
 })
 
 router.get('/stats', async (req, res) => {
-  const { rights, roles } = req
+  const {
+    user: { rights, roles },
+  } = req
 
   const { providers, semesterStart, semesterEnd } = req.query
   if (!providers || !semesterStart) {
@@ -78,6 +80,4 @@ router.get('/:id', async (req, res) => {
   res.json(result)
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/tsaAnalytics.js

@@ -9,8 +9,12 @@ router.post('/event', (req, res) => {
     return res.status(400).json({ error: 'group, name and value are required' })
   }
 
+  const {
+    user: { userId },
+  } = req
+
   // don't await here because frontend doesn't care if it succeeds
-  tsaService.sendTsaEvent(req.decodedToken.userId, { group, name, label, value }).catch(e => {
+  tsaService.sendTsaEvent(userId, { group, name, label, value }).catch(e => {
     Sentry.withScope(scope => {
       // if error was caused by a 400, dig out the JSON response for Sentry
       if (e.isAxiosError && e.response && e.response.data) {

services/backend/src/routes/users.js

@@ -1,18 +1,8 @@
 const router = require('express').Router()
 const userService = require('../services/userService')
-const blacklist = require('../services/blacklist')
-const { userDataCache } = require('../services/cache')
 const { sendNotificationAboutAccessToUser, previewNotificationAboutAccessToUser } = require('../services/mailservice')
 const logger = require('../util/logger')
 
-const addUserToBlacklist = async (user, decodedToken) => {
-  if (user) {
-    const { username } = user
-    userDataCache.del(username)
-    if (username != decodedToken.userId) await blacklist.addUserToBlacklist(username)
-  }
-}
-
 router.get('/', async (req, res) => {
   const results = await userService.findAll()
   res.json(results)
@@ -25,12 +15,7 @@ router.get('/access_groups', async (req, res) => {
 
 router.post('/modifyaccess', async (req, res) => {
   try {
-    const { username } = req.body
     const result = await userService.modifyAccess(req.body)
-    const user = {
-      username,
-    }
-    await addUserToBlacklist(user, req.decodedToken)
     res.status(200).json(result)
   } catch (e) {
     res.status(400).json(e)
@@ -60,7 +45,6 @@ router.post('/:uid/elements', async (req, res) => {
   const { uid } = req.params
   const { codes } = req.body
   const user = await userService.enableElementDetails(uid, codes)
-  await addUserToBlacklist(user, req.decodedToken)
   res.json(user)
 })
 
@@ -68,7 +52,6 @@ router.delete('/:uid/elements', async (req, res) => {
   const { uid } = req.params
   const { codes } = req.body
   const user = await userService.removeElementDetails(uid, codes)
-  await addUserToBlacklist(user, req.decodedToken)
   res.json(user)
 })
 
@@ -76,7 +59,6 @@ router.post('/:uid/faculties', async (req, res) => {
   const { uid } = req.params
   const { faculties } = req.body
   const user = await userService.setFaculties(uid, faculties)
-  await addUserToBlacklist(user, req.decodedToken)
   res.json(user)
 })
 

services/backend/src/services/analyticsService.js

@@ -3,6 +3,12 @@ const moment = require('moment')
 
 const createRedisKeyForProductivity = id => `PRODUCTIVITY_${id}`
 const createRedisKeyForThroughput = id => `THROUGHPUT_${id}`
+const createRedisKeyForBasicStats = (id, yearType, specialGroups) => `BASIC_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForCreditStats = (id, yearType, specialGroups) => `CREDIT_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForGraduationStats = (id, yearType, specialGroups) =>
+  `GRADUATION_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForStudytrackStats = (id, graduated, specialGroups) =>
+  `STUDYTRACK_STATS_${id}_${graduated}_${specialGroups}`
 
 const getProductivity = async id => {
   const redisKey = createRedisKeyForProductivity(id)
@@ -87,6 +93,86 @@ const patchThroughput = async data => {
   }
 }
 
+const getBasicStats = async (id, yearType, specialGroups) => {
+  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setBasicStats = async (data, yearType, specialGroups) => {
+  const { id } = data
+  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getCreditStats = async (id, yearType, specialGroups) => {
+  const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setCreditStats = async (data, yearType, specialGroups) => {
+  const { id } = data
+  const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getGraduationStats = async (id, yearType, specialGroups) => {
+  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setGraduationStats = async (data, yearType, specialGroups) => {
+  const { id } = data
+  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getStudytrackStats = async (id, graduated, specialGroups) => {
+  const redisKey = createRedisKeyForStudytrackStats(id, graduated, specialGroups)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setStudytrackStats = async (data, graduated, specialGroups) => {
+  const { id } = data
+  const redisKey = createRedisKeyForStudytrackStats(id, graduated, specialGroups)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
 module.exports = {
   getProductivity,
   setProductivity,
@@ -94,4 +180,12 @@ module.exports = {
   getThroughput,
   setThroughput,
   patchThroughput,
+  getBasicStats,
+  setBasicStats,
+  getCreditStats,
+  setCreditStats,
+  getGraduationStats,
+  setGraduationStats,
+  getStudytrackStats,
+  setStudytrackStats,
 }

services/backend/src/services/blacklist.js

@@ -1,22 +0,0 @@
-const { redisClient } = require('./redis')
-const moment = require('moment')
-const logger = require('../util/logger')
-
-const MAKE_REDIS_HASH_KEY = id => `BLACKLIST ${id}`
-
-const addUserToBlacklist = async userId => {
-  logger.info(`user ${userId} added to token blacklist`)
-  await redisClient.setAsync(MAKE_REDIS_HASH_KEY(userId), moment().toISOString(false))
-}
-
-const isUserBlacklisted = async (userId, tokenCreatedAt) => {
-  const isBlacklisted = async () => {
-    const blacklistTimestamp = await redisClient.getAsync(MAKE_REDIS_HASH_KEY(userId))
-    return tokenCreatedAt && blacklistTimestamp && moment(tokenCreatedAt) < moment(blacklistTimestamp)
-  }
-
-  const timer = () => new Promise(resolve => setTimeout(() => resolve(true), 5000))
-  return Promise.race([isBlacklisted(), timer()])
-}
-
-module.exports = { addUserToBlacklist, isUserBlacklisted }

services/backend/src/services/cache.js

@@ -1,12 +0,0 @@
-const LRU = require('lru-cache')
-
-// Max 25 users can be stored in the cache
-const userDataCache = new LRU({
-  max: 25,
-  length: () => 1,
-  maxAge: 1000 * 60 * 60,
-})
-
-module.exports = {
-  userDataCache,
-}

services/backend/src/services/courses.js

@@ -13,7 +13,7 @@ const {
 const { parseCredit } = require('./parseCredits')
 const Op = Sequelize.Op
 const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
-const _ = require('lodash')
+const { sortMainCode, getSortRank } = require('../util/utils')
 
 const byNameOrCode = (searchTerm, language) =>
   Course.findAll({
@@ -59,7 +59,15 @@ const findOneByCode = code => {
   })
 }
 
-const creditsForCourses = async (codes, anonymizationSalt) => {
+const creditsForCourses = async (codes, anonymizationSalt, unifyOpenUniCourses) => {
+  let is_open = false
+
+  if (unifyOpenUniCourses) {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
   const credits = await Credit.findAll({
     include: [
       {
@@ -105,9 +113,11 @@ const creditsForCourses = async (codes, anonymizationSalt) => {
       student_studentnumber: {
         [Op.ne]: null,
       },
+      [Op.or]: [{ is_open }, { is_open: null }],
     },
     order: [['attainment_date', 'ASC']],
   })
+
   const parsedCredits = credits.map(credit => parseCredit(credit, anonymizationSalt))
   return parsedCredits
 }
@@ -144,47 +154,20 @@ const unifyOpenUniversity = code => {
   return regexresult[1]
 }
 
-const allCodeAltenatives = async code => {
-  let course = await Course.findAll({
-    raw: true,
-    attributes: ['id', 'code', 'substitutions'],
-    where: {
-      code: code,
-    },
+const allCodeAlternatives = async code => {
+  const course = await Course.findOne({
+    where: { code: code },
   })
 
-  const allSubstitutions = _.flatten(course.map(c => c.substitutions))
-
-  let subcodes = []
-  if (allSubstitutions) {
-    subcodes = await Course.findAll({
-      raw: true,
-      attributes: ['code'],
-      where: {
-        id: {
-          [Op.in]: allSubstitutions,
-        },
-      },
-    })
-  }
-
-  const courses = [...course, ...subcodes]
-  const temp = courses
-    .map(c => c.code)
-    .map(c => {
-      if (c.match(/^AY?/)) return [c, 4] // open university codes come last
-      if (c.match(/^\d/)) return [c, 2] // old numeric codes come second
-      if (c.match(/^[A-Za-z]/)) return [c, 1] // new letter based codes come first
-      return [c, 3] // unknown, comes before open uni?
-    })
-    .sort((a, b) => a[1] - b[1])
-    .map(c => c[0])
-
-  return allSubstitutions ? _.uniq(temp) : [code]
+  return sortMainCode([...course.substitutions, code])
 }
 
 const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anonymizationSalt) => {
-  let codes = await allCodeAltenatives(coursecode)
+  const courseForSubs = await Course.findOne({
+    where: { code: coursecode },
+  })
+
+  let codes = courseForSubs.substitutions ? sortMainCode([...courseForSubs.substitutions, coursecode]) : [coursecode]
 
   if (isOpenUniCourseCode(coursecode) && !unifyOpenUniCourses) {
     codes = [coursecode]
@@ -195,7 +178,7 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
   }
 
   const [credits, course] = await Promise.all([
-    creditsForCourses(codes, anonymizationSalt),
+    creditsForCourses(codes, anonymizationSalt, unifyOpenUniCourses),
     Course.findOne({
       where: {
         code: coursecode,
@@ -204,6 +187,7 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses, anony
   ])
 
   const counter = new CourseYearlyStatsCounter()
+
   for (let credit of credits) {
     const {
       studentnumber,
@@ -357,10 +341,6 @@ const codeLikeTerm = code =>
 
 const byNameAndOrCodeLike = async (name, code) => {
   let rawCourses = await Course.findAll({
-    include: {
-      model: Organization,
-      required: true,
-    },
     where: {
       ...nameLikeTerm(name),
       ...codeLikeTerm(code),
@@ -369,9 +349,9 @@ const byNameAndOrCodeLike = async (name, code) => {
 
   const courses = rawCourses
     .map(course => {
-      return { ...course.dataValues, organizations: course.organizations.map(o => o.id) }
+      return { ...course.dataValues }
     })
-    .sort(a => (a.code.match(/^[A-Za-z]{3}[0-9]{1}/) ? -1 : 1))
+    .sort((x, y) => getSortRank(y.code) - getSortRank(x.code))
 
   let substitutionGroupIndex = 0
   const visited = []
@@ -379,7 +359,11 @@ const byNameAndOrCodeLike = async (name, code) => {
   const organizeSubgroups = course => {
     if (visited.includes(course.code)) return
 
-    let temp = courses.filter(c => (course.substitutions ? course.substitutions.includes(c.id) : false))
+    let temp = []
+    if (course.substitutions !== null) {
+      temp = courses.filter(c => course.substitutions.includes(c.code))
+    }
+
     temp.unshift(course)
     temp.forEach(cu => {
       if (visited.includes(course.code)) return
@@ -407,6 +391,6 @@ module.exports = {
   byCodes,
   maxYearsToCreatePopulationFrom,
   unifyOpenUniversity,
-  allCodeAltenatives,
+  allCodeAlternatives,
   findOneByCode,
 }

services/backend/src/services/mailservice.js

@@ -1,6 +1,6 @@
 const axios = require('axios')
 const { isProduction, pateToken } = require('../conf-backend')
-const logger = require('../util/logger')
+const { ApplicationError } = require('../util/customErrors')
 
 const pateClient = axios.create({
   baseURL: 'https://pate.toska.cs.helsinki.fi',
@@ -20,33 +20,22 @@ const baseSettings = {
 }
 
 const sendEmail = async (options = {}) => {
-  if (!isProduction) {
-    const errorMessage = 'Skipped sending email in non-production environment'
-    logger.error({ message: errorMessage, meta: options })
-    return { error: errorMessage }
-  }
-
-  if (!pateToken) {
-    const errorMessage = 'Skipped sending email since pate token was not available'
-    logger.error({ message: errorMessage, meta: options })
-    return { error: errorMessage }
-  }
-
-  const { data } = await pateClient.post('/', options)
-
-  return data
+  if (!isProduction) throw new ApplicationError('Email sending is disabled in development mode.')
+  if (!pateToken) throw new ApplicationError('Email sending failed because pate token is missing.')
+  await pateClient.post('/', options)
 }
 
-const sendFeedbackToToska = ({ feedbackContent, userId, userEmail, userFullName }) => {
-  const userDetails = `Sent by ${userFullName}, userid: ${userId}, email: ${userEmail}`
+const sendFeedbackToToska = async ({ feedbackContent, user }) => {
+  const { name, userId, email } = user
+  const userDetails = `Sent by ${name}, userid: ${userId}, email: ${email}`
   const text = [feedbackContent, userDetails].join('<br />')
-  return sendEmail({
+  await sendEmail({
     template,
     emails: [
       {
         to: 'Toska <grp-toska@helsinki.fi>',
-        replyTo: userEmail,
-        subject: `Oodikone feedback from ${userFullName}`,
+        replyTo: email,
+        subject: `Oodikone feedback from ${name}`,
         text,
       },
     ],

services/backend/src/services/newStudyprogramme.js

@@ -0,0 +1,402 @@
+const sequelize = require('sequelize')
+const { Op } = sequelize
+const {
+  Credit,
+  Course,
+  Organization,
+  Studyright,
+  StudyrightElement,
+  ElementDetail,
+  Transfer,
+  Student,
+  SemesterEnrollment,
+  Semester,
+} = require('../models')
+const { formatStudyright, formatStudent, formatTransfer } = require('./studyprogrammeHelpers')
+const { getCurrentSemester } = require('./semesters')
+
+const whereStudents = studentnumbers => {
+  return studentnumbers ? studentnumbers : { [Op.not]: null }
+}
+
+const sinceDate = since => {
+  return since ? { [Op.gte]: since } : { [Op.not]: null }
+}
+
+const studytrackStudents = async studentnumbers =>
+  (
+    await Student.findAll({
+      where: {
+        studentnumber: {
+          [Op.in]: studentnumbers,
+        },
+      },
+    })
+  ).map(formatStudent)
+
+const enrolledStudents = async (studytrack, since, studentnumbers) => {
+  const currentSemester = await getCurrentSemester()
+
+  const students = await Student.findAll({
+    attributes: ['studentnumber'],
+    include: [
+      {
+        model: Studyright,
+        include: [
+          {
+            model: StudyrightElement,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        ],
+        attributes: ['studyrightid'],
+      },
+      {
+        model: SemesterEnrollment,
+        attributes: ['semestercode'],
+        include: [
+          {
+            model: Semester,
+            required: true,
+            where: {
+              semestercode: currentSemester.semestercode,
+            },
+          },
+        ],
+        where: {
+          enrollmenttype: 1,
+        },
+      },
+    ],
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumbers,
+      },
+    },
+  })
+
+  return students.filter(s => s.semester_enrollments?.length)
+}
+
+const absentStudents = async (studytrack, studentnumbers) => {
+  const currentSemester = await getCurrentSemester()
+  const students = await Student.findAll({
+    attributes: ['studentnumber'],
+    include: [
+      {
+        model: Studyright,
+        include: [
+          {
+            model: StudyrightElement,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        ],
+        attributes: ['studyrightid'],
+      },
+      {
+        model: SemesterEnrollment,
+        attributes: ['semestercode'],
+        where: {
+          enrollmenttype: 2,
+          semestercode: currentSemester.semestercode,
+        },
+      },
+    ],
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumbers,
+      },
+    },
+  })
+
+  return students
+}
+
+const allStudyrights = async (studytrack, since, studentnumbers) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: {
+            model: ElementDetail,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        student_studentnumber: whereStudents(studentnumbers),
+      },
+    })
+  ).map(formatStudyright)
+
+const startedStudyrights = async (studytrack, since, studentnumbers) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: {
+            model: ElementDetail,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        studystartdate: {
+          [Op.gte]: since,
+        },
+        student_studentnumber: whereStudents(studentnumbers),
+      },
+    })
+  ).map(formatStudyright)
+
+const graduatedStudyRights = async (studytrack, since, studentnumbers) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: {
+            model: ElementDetail,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        graduated: 1,
+        enddate: sinceDate(since),
+        student_studentnumber: whereStudents(studentnumbers),
+      },
+    })
+  ).map(formatStudyright)
+
+const followingStudyrights = async (since, studentnumbers) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: [
+            {
+              model: ElementDetail,
+              attributes: ['name'],
+            },
+          ],
+          attributes: ['code'],
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        studystartdate: {
+          [Op.gte]: since,
+        },
+        extentcode: 2,
+        student_studentnumber: whereStudents(studentnumbers),
+      },
+    })
+  ).map(formatStudyright)
+
+const transfersAway = async (studytrack, since) =>
+  (
+    await Transfer.findAll({
+      where: {
+        transferdate: {
+          [Op.gte]: since,
+        },
+        sourcecode: studytrack,
+      },
+      distinct: true,
+      col: 'studentnumber',
+    })
+  ).map(formatTransfer)
+
+const transfersTo = async (studytrack, since) =>
+  (
+    await Transfer.findAll({
+      where: {
+        transferdate: {
+          [Op.gte]: since,
+        },
+        targetcode: studytrack,
+      },
+      distinct: true,
+      col: 'studentnumber',
+    })
+  ).map(formatTransfer)
+
+const allTransfers = async (studytrack, since) => {
+  const transferredTo = await transfersTo(studytrack, since)
+  const transferredAway = await transfersAway(studytrack, since)
+  return [...transferredTo, ...transferredAway]
+}
+
+const getProgrammesStudyrights = async studyprogramme =>
+  (
+    await Studyright.findAll({
+      attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode'],
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          where: {
+            code: {
+              [Op.in]: [studyprogramme],
+            },
+          },
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+    })
+  ).map(formatStudyright)
+
+const getCreditsForStudyProgramme = async (provider, since) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      where: {
+        is_study_module: false,
+      },
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.notIn]: [10, 9, 7],
+      },
+      isStudyModule: {
+        [Op.not]: true,
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+    },
+  })
+
+const getTransferredCredits = async (provider, since) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.eq]: [9],
+      },
+      isStudyModule: {
+        [Op.not]: true,
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+    },
+  })
+
+const getThesisCredits = async (provider, since, thesisType, studentnumbers) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      where: {
+        course_unit_type: {
+          [Op.in]: thesisType,
+        },
+      },
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.notIn]: [10, 9, 7],
+      },
+      isStudyModule: {
+        [Op.not]: true,
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+      student_studentnumber: whereStudents(studentnumbers),
+    },
+  })
+
+module.exports = {
+  studytrackStudents,
+  enrolledStudents,
+  absentStudents,
+  allStudyrights,
+  startedStudyrights,
+  graduatedStudyRights,
+  followingStudyrights,
+  transfersAway,
+  transfersTo,
+  allTransfers,
+  getProgrammesStudyrights,
+  getCreditsForStudyProgramme,
+  getTransferredCredits,
+  getThesisCredits,
+}

services/backend/src/services/populations.js

@@ -1,6 +1,7 @@
 const { Op } = require('sequelize')
 const moment = require('moment')
 const { sortBy } = require('lodash')
+const { sortMainCode } = require('../util/utils')
 
 const {
   Student,
@@ -19,7 +20,6 @@ const {
   dbConnections: { sequelize },
 } = require('../database/connection')
 const { Tag, TagStudent } = require('../models/models_kone')
-const { /*allCodeAltenatives, */ findOneByCode } = require('./courses')
 const { CourseStatsCounter } = require('./course_stats_counter')
 const { getPassingSemester, semesterEnd, semesterStart } = require('../util/semester')
 const { getAllProgrammes } = require('./studyrights')
@@ -65,7 +65,7 @@ const formatStudentForPopulationStatistics = (
   startDateMoment,
   endDateMoment
 ) => {
-  const toCourse = ({ grade, attainment_date, credits, course_code, credittypecode, isStudyModule }) => {
+  const toCourse = ({ grade, attainment_date, credits, course_code, credittypecode, isStudyModule, language }) => {
     const attainment_date_normailized =
       attainment_date < startDate ? startDateMoment.clone().add(1, 'day').toISOString() : attainment_date
     const passed = Credit.passed({ credittypecode })
@@ -78,6 +78,7 @@ const formatStudentForPopulationStatistics = (
       credits,
       isStudyModuleCredit: isStudyModule,
       credittypecode,
+      language,
     }
   }
 
@@ -265,6 +266,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
         'isStudyModule',
         'student_studentnumber',
         'course_code',
+        'language',
       ],
       where: creditsOfStudent,
       raw: true,
@@ -361,8 +363,10 @@ const studentnumbersWithAllStudyrightElements = async (
   exchangeStudents,
   cancelledStudents,
   nondegreeStudents,
-  transferredStudents,
-  tag
+  transferredOutStudents,
+  tag,
+  transferredToStudents,
+  graduatedStudents
 ) => {
   // eslint-disable-line
 
@@ -494,11 +498,12 @@ const studentnumbersWithAllStudyrightElements = async (
     if (!newestStudytrack) return false
     return studyRights.includes(newestStudytrack.code)
   })
-  const studentnumberlist = filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers
+
+  let studentnumberlist = filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers
 
   // fetch students that have transferred out of the programme and filter out these studentnumbers
-  if (!transferredStudents) {
-    const transferredOutStudents = (
+  if (!transferredOutStudents) {
+    const transfersOut = (
       await Transfer.findAll({
         attributes: ['studentnumber'],
         where: {
@@ -516,8 +521,63 @@ const studentnumbersWithAllStudyrightElements = async (
       })
     ).map(s => s.studentnumber)
 
-    const notTransferredStudents = studentnumberlist.filter(sn => !transferredOutStudents.includes(sn))
-    return notTransferredStudents
+    studentnumberlist = studentnumberlist.filter(sn => !transfersOut.includes(sn))
+  }
+
+  // fetch students that have transferred to the programme and filter out these studentnumbers
+  if (transferredToStudents) {
+    const transfersTo = (
+      await Transfer.findAll({
+        attributes: ['studentnumber'],
+        where: {
+          targetcode: {
+            [Op.in]: studyRights,
+          },
+          transferdate: {
+            [Op.gt]: startDate,
+          },
+          studentnumber: {
+            [Op.in]: studentnumberlist,
+          },
+        },
+        raw: true,
+      })
+    ).map(s => s.studentnumber)
+    studentnumberlist = studentnumberlist.filter(sn => !transfersTo.includes(sn))
+  }
+
+  // fetch students that have graduated from the programme and filter out these studentnumbers
+  if (graduatedStudents) {
+    const graduated = (
+      await Student.findAll({
+        attributes: ['studentnumber'],
+        include: [
+          {
+            model: Studyright,
+            include: [
+              {
+                model: StudyrightElement,
+                required: true,
+                where: {
+                  code: {
+                    [Op.in]: studyRights,
+                  },
+                },
+              },
+            ],
+            where: {
+              graduated: 1,
+            },
+          },
+        ],
+        where: {
+          studentnumber: {
+            [Op.in]: studentnumbers,
+          },
+        },
+      })
+    ).map(s => s.studentnumber)
+    studentnumberlist = studentnumberlist.filter(sn => !graduated.includes(sn))
   }
 
   return studentnumberlist
@@ -816,10 +876,6 @@ const findCourses = async (studentnumbers, beforeDate) => {
           },
         },
       },
-      /* {
-        attributes: ['discipline_id', 'name'],
-        model: Discipline
-      }, */
       {
         attributes: ['coursetypecode', 'name'],
         model: CourseType,
@@ -842,12 +898,24 @@ const parseCreditInfo = credit => ({
   date: credit.attainment_date,
 })
 
-const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+const findMainCourse = async code => {
+  const course = await Course.findOne({
+    where: { code: code },
+  })
 
-const unifyOpenUniversity = code => {
-  const regexresult = isOpenUniCourseCode(code)
-  if (!regexresult) return code
-  return regexresult[1]
+  let result = []
+
+  if (course.substitutions) {
+    const sortedCourses = sortMainCode([...course.substitutions, code])
+    result = await Course.findOne({
+      attributes: ['code', 'name'],
+      where: { code: sortedCourses[0] },
+    })
+  } else {
+    return course
+  }
+
+  return result
 }
 
 const bottlenecksOf = async (query, studentnumberlist) => {
@@ -918,6 +986,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
           transferredStudents,
           tag
         ))
+
       const allstudents = studentnumbers.reduce((numbers, num) => {
         numbers[num] = true
         return numbers
@@ -954,14 +1023,14 @@ const bottlenecksOf = async (query, studentnumberlist) => {
   for (const course of courses) {
     let { course_type } = course
     let maincourse = course
-    if (isOpenUniCourseCode) {
+    const sortedMaincourse = await findMainCourse(course.code)
+    if (sortedMaincourse) maincourse = sortedMaincourse
+
+    /*  if (isOpenUniCourseCode) {
       const nonOpen = unifyOpenUniversity(course.code)
       const response = await findOneByCode(nonOpen)
+ */
 
-      if (response) {
-        maincourse = response
-      }
-    }
     if (!stats[maincourse.code]) {
       stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudentslength)
     }
@@ -982,6 +1051,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
 
   bottlenecks.coursestatistics = Object.values(stats).map(coursestatistics => coursestatistics.getFinalStats())
   bottlenecks.allStudents = allstudentslength
+
   return bottlenecks
 }
 

services/backend/src/services/students.js

@@ -288,56 +288,71 @@ const bySearchTerm = async searchterm => {
   return matches.map(formatStudent)
 }
 
-const bySearchTermAndElements = async (searchterm, codes) => {
+const bySearchTermAndStudentNumbers = async (searchterm, studentnumbers) => {
   const terms = splitByEmptySpace(searchterm)
   const matches = await Student.findAll({
-    include: {
-      model: StudyrightElement,
-      required: true,
-      where: {
-        code: {
-          [Op.in]: codes,
+    where: {
+      [Op.and]: {
+        [Op.or]: [nameLike(terms), studentnumberLike(terms)],
+        studentnumber: {
+          [Op.in]: studentnumbers,
         },
       },
     },
-    where: {
-      [Op.or]: [nameLike(terms), studentnumberLike(terms)],
-    },
   })
   return matches.map(formatStudent)
 }
 
-const filterStudentnumbersByAccessrights = async (studentnumbers, codes) => {
-  const students = await Student.findAll({
-    attributes: ['studentnumber'],
-    include: {
-      attributes: [],
-      model: StudyrightElement,
-      required: true,
-      where: {
-        code: {
-          [Op.in]: codes,
+const filterStudentnumbersByAccessrights = async (studentnumbers, codes) =>
+  (
+    await Student.findAll({
+      attributes: ['studentnumber'],
+      include: {
+        attributes: [],
+        model: StudyrightElement,
+        required: true,
+        where: {
+          code: {
+            [Op.in]: codes,
+          },
         },
       },
-    },
-    where: {
-      studentnumber: {
-        [Op.in]: studentnumbers,
+      where: {
+        studentnumber: {
+          [Op.in]: studentnumbers,
+        },
       },
-    },
-    raw: true,
-  })
-  return students.map(student => student.studentnumber)
-}
+      raw: true,
+    })
+  ).map(({ studentnumber }) => studentnumber)
+
+const getStudentnumbersByElementdetails = async codes =>
+  (
+    await Student.findAll({
+      attributes: ['studentnumber'],
+      include: {
+        attributes: [],
+        model: StudyrightElement,
+        required: true,
+        where: {
+          code: {
+            [Op.in]: codes,
+          },
+        },
+      },
+      raw: true,
+    })
+  ).map(({ studentnumber }) => studentnumber)
 
 module.exports = {
   withId,
   bySearchTerm,
   createStudent,
   updateStudent,
-  bySearchTermAndElements,
+  bySearchTermAndStudentNumbers,
   filterStudentnumbersByAccessrights,
   findByCourseAndSemesters,
   findByTag,
   splitByEmptySpace,
+  getStudentnumbersByElementdetails,
 }

services/backend/src/services/studyGuidanceGroups.js

@@ -1,31 +1,42 @@
 const { getImporterClient } = require('../util/importerClient')
 const importerClient = getImporterClient()
 const { StudyGuidanceGroupTag } = require('../models/models_kone')
+const logger = require('../util/logger')
+const _ = require('lodash')
 
-const getAllStudentsUserHasInGroups = async sisPersonId => {
+const getGroupsFromImporter = async sisPersonId => {
   if (!importerClient) {
-    return new Set()
+    return []
   }
-  const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
-  return new Set(
-    Object.values(data)
+  const answerTimeout = new Promise(resolve => setTimeout(resolve, 2000))
+  try {
+    const result = await Promise.race([importerClient.get(`/person-groups/person/${sisPersonId}`), answerTimeout])
+    if (!result) return []
+    return Object.values(result.data)
+  } catch (error) {
+    logger.error("Couldn't fetch users studyguidance groups")
+    return []
+  }
+}
+
+const getAllGroupsAndStudents = async sisPersonId => {
+  const tagsByGroupId = (await StudyGuidanceGroupTag.findAll()).reduce((acc, curr) => {
+    const { studyProgramme, year, studyGuidanceGroupId } = curr
+    return { ...acc, [studyGuidanceGroupId]: { studyProgramme, year } }
+  }, {})
+  return (await getGroupsFromImporter(sisPersonId)).map(group => ({
+    ...group,
+    tags: tagsByGroupId[group.id],
+  }))
+}
+
+const getAllStudentsUserHasInGroups = async sisPersonId =>
+  _.uniqBy(
+    (await getGroupsFromImporter(sisPersonId))
       .map(group => group.members)
       .flat()
       .map(member => member.personStudentNumber)
   )
-}
-
-const getAllGroupsAndStudents = async sisPersonId => {
-  if (!importerClient) {
-    return []
-  }
-  const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
-  const tagsByGroupId = (await StudyGuidanceGroupTag.findAll()).reduce((acc, curr) => {
-    const { studyProgramme, year } = curr
-    return { ...acc, [curr.studyGuidanceGroupId]: { studyProgramme, year } }
-  }, {})
-  return Object.values(data).map(group => ({ ...group, tags: tagsByGroupId[group.id] }))
-}
 
 const changeGroupTags = async ({ groupId, tags }) => {
   const { studyProgramme, year } = tags
@@ -45,8 +56,11 @@ const changeGroupTags = async ({ groupId, tags }) => {
   return result
 }
 
+const checkStudyGuidanceGroupsAccess = async hyPersonSisuId => (await getGroupsFromImporter(hyPersonSisuId)).length > 0
+
 module.exports = {
   getAllStudentsUserHasInGroups,
   getAllGroupsAndStudents,
   changeGroupTags,
+  checkStudyGuidanceGroupsAccess,
 }

services/backend/src/services/studyprogramme.js

@@ -211,7 +211,7 @@ const findProgrammeThesisCredits = async code => {
     },
     where: {
       credittypecode: {
-        [Op.ne]: 10,
+        [Op.eq]: 4,
       },
     },
   })

services/backend/src/services/studyprogrammeBasics.js

@@ -0,0 +1,202 @@
+const { indexOf } = require('lodash')
+const {
+  getCorrectStudentnumbers,
+  getStatsBasis,
+  defineYear,
+  getYearsArray,
+  getStartDate,
+  tableTitles,
+} = require('./studyprogrammeHelpers')
+const {
+  allStudyrights,
+  graduatedStudyRights,
+  startedStudyrights,
+  transfersAway,
+  transfersTo,
+} = require('./newStudyprogramme')
+const { getYearStartAndEndDates } = require('../util/semester')
+
+const getAllStats = async ({ studyprogramme, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+      const all = await allStudyrights(studyprogramme, null, studentnumbersOfTheYear)
+
+      graphStats[indexOf(years, year)] = all.length
+      tableStats[year] += all.length
+    })
+  )
+  return { graphStats, tableStats }
+}
+
+const getStartedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+      const studyrights = await startedStudyrights(studyprogramme, since, studentnumbersOfTheYear)
+
+      studyrights.forEach(({ studystartdate }) => {
+        const startYear = defineYear(studystartdate, isAcademicYear)
+        graphStats[indexOf(years, startYear)] += 1
+        tableStats[startYear] += 1
+      })
+    })
+  )
+  return { graphStats, tableStats }
+}
+
+const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+
+      const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
+      studyrights.forEach(({ enddate }) => {
+        const graduationYear = defineYear(enddate, isAcademicYear)
+        graphStats[indexOf(years, graduationYear)] += 1
+        tableStats[graduationYear] += 1
+      })
+    })
+  )
+  return { graphStats, tableStats }
+}
+
+const getCancelledStats = async ({ studyprogramme, years, isAcademicYear }) => {
+  const studyrights = await allStudyrights(studyprogramme)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ canceldate }) => {
+    if (canceldate !== null && canceldate !== undefined) {
+      const cancelYear = defineYear(canceldate, isAcademicYear)
+      graphStats[indexOf(years, cancelYear)] += 1
+      tableStats[cancelYear] += 1
+    }
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredAwayStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const studyrights = await transfersAway(studyprogramme, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredToStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const studyrights = await transfersTo(studyprogramme, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getBasicStatsForStudytrack = async ({ studyprogramme, yearType, specialGroups }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const all = await getAllStats(queryParameters)
+  const started = await getStartedStats(queryParameters)
+  const graduated = await getGraduatedStats(queryParameters)
+  const cancelled = await getCancelledStats(queryParameters)
+  const transferredAway = await getTransferredAwayStats(queryParameters)
+  const transferredTo = await getTransferredToStats(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const titles = tableTitles['basics'][specialGroups]
+  const tableStats = reversedYears.map(year =>
+    includeAllSpecials
+      ? [
+          year,
+          all.tableStats[year],
+          started.tableStats[year],
+          graduated.tableStats[year],
+          cancelled.tableStats[year],
+          transferredAway.tableStats[year],
+          transferredTo.tableStats[year],
+        ]
+      : [year, all.tableStats[year], started.tableStats[year], graduated.tableStats[year]]
+  )
+
+  return {
+    id: studyprogramme,
+    years,
+    graphStats: includeAllSpecials
+      ? [
+          {
+            name: 'Started',
+            data: started.graphStats,
+          },
+          {
+            name: 'Graduated',
+            data: graduated.graphStats,
+          },
+          {
+            name: 'Cancelled',
+            data: cancelled.graphStats,
+          },
+          {
+            name: 'Transferred away',
+            data: transferredAway.graphStats,
+          },
+          {
+            name: 'Transferred To',
+            data: transferredTo.graphStats,
+          },
+        ]
+      : [
+          {
+            name: 'Started',
+            data: started.graphStats,
+          },
+          {
+            name: 'Graduated',
+            data: graduated.graphStats,
+          },
+        ],
+    tableStats,
+    titles,
+  }
+}
+
+module.exports = {
+  getBasicStatsForStudytrack,
+}

services/backend/src/services/studyprogrammeCredits.js

@@ -0,0 +1,126 @@
+const { indexOf } = require('lodash')
+
+const { mapToProviders } = require('../util/utils')
+const {
+  getStatsBasis,
+  defineYear,
+  isMajorStudentCredit,
+  getYearsArray,
+  getStartDate,
+  isSpecialGroupCredit,
+  tableTitles,
+} = require('./studyprogrammeHelpers')
+const {
+  getProgrammesStudyrights,
+  getCreditsForStudyProgramme,
+  getTransferredCredits,
+  allTransfers,
+} = require('./newStudyprogramme')
+
+// Fetches all the credits for the studyprogramme and divides them into major-students and non-major students credits
+// Division is done on the basis that whether the student had a primary studyright to the programme on the attainment_date
+// If special groups are excluded, the transfer students are filtered away from the major-students as well
+const getRegularCreditStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  let studyrights = await getProgrammesStudyrights(studyprogramme, since)
+  const transfers = (await allTransfers(studyprogramme, since)).map(t => t.studyrightid)
+
+  if (!includeAllSpecials) {
+    studyrights.filter(s => !transfers.includes(s.studyrightid))
+  }
+
+  const credits = await getCreditsForStudyProgramme(providercode, since)
+
+  let majors = getStatsBasis(years)
+  let nonMajors = getStatsBasis(years)
+
+  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
+    const studyright = studyrights.find(studyright => studyright.studentnumber == student_studentnumber)
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+
+    if (!includeAllSpecials && isSpecialGroupCredit(studyright, attainment_date, transfers)) {
+      return
+    }
+
+    if (isMajorStudentCredit(studyright, attainment_date)) {
+      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      majors.tableStats[attainmentYear] += credits || 0
+    } else {
+      nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      nonMajors.tableStats[attainmentYear] += credits || 0
+    }
+  })
+
+  return { majors, nonMajors }
+}
+
+// Fetches all the credits with the type "transferred / hyväksiluettu" and divides them by year
+const getTransferredCreditStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const credits = await getTransferredCredits(providercode, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  credits.forEach(({ attainment_date, credits }) => {
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+    graphStats[indexOf(years, attainmentYear)] += credits || 0
+    tableStats[attainmentYear] += credits || 0
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getCreditStatsForStudytrack = async ({ studyprogramme, yearType, specialGroups }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const { majors, nonMajors } = await getRegularCreditStats(queryParameters)
+  const transferred = await getTransferredCreditStats(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const titles = tableTitles['credits'][specialGroups]
+
+  const tableStats = reversedYears.map(year =>
+    includeAllSpecials
+      ? [
+          year,
+          majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
+          majors.tableStats[year],
+          nonMajors.tableStats[year],
+          transferred.tableStats[year],
+        ]
+      : [
+          year,
+          majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
+          majors.tableStats[year],
+          transferred.tableStats[year],
+        ]
+  )
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    graphStats: [
+      {
+        name: 'Major students credits',
+        data: majors.graphStats,
+      },
+      {
+        name: 'Non-major students credits',
+        data: nonMajors.graphStats,
+      },
+      {
+        name: 'Transferred credits',
+        data: transferred.graphStats,
+      },
+    ],
+    titles,
+  }
+}
+
+module.exports = {
+  getCreditStatsForStudytrack,
+}

services/backend/src/services/studyprogrammeGraduations.js

@@ -0,0 +1,205 @@
+const moment = require('moment')
+const { indexOf, uniqBy } = require('lodash')
+
+const { mapToProviders } = require('../util/utils')
+const {
+  getCorrectStudentnumbers,
+  getStatsBasis,
+  defineYear,
+  getYearsObject,
+  getAcademicYearsObject,
+  getYearsArray,
+  getMedian,
+  getMean,
+  getStartDate,
+  getThesisType,
+} = require('./studyprogrammeHelpers')
+const {
+  graduatedStudyRights,
+  transfersAway,
+  transfersTo,
+  getThesisCredits,
+  followingStudyrights,
+} = require('./newStudyprogramme')
+const { getYearStartAndEndDates } = require('../util/semester')
+
+const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+
+      const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
+      studyrights.forEach(({ enddate }) => {
+        const graduationYear = defineYear(enddate, isAcademicYear)
+        graphStats[indexOf(years, graduationYear)] += 1
+        tableStats[graduationYear] += 1
+      })
+    })
+  )
+  return { graphStats, tableStats }
+}
+
+const getThesisStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const thesisType = getThesisType(studyprogramme)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+
+      const credits = await getThesisCredits(providercode, since, thesisType, studentnumbersOfTheYear)
+
+      credits?.forEach(({ attainment_date }) => {
+        const attainmentYear = defineYear(attainment_date, isAcademicYear)
+        graphStats[indexOf(years, attainmentYear)] += 1
+        tableStats[attainmentYear] += 1
+      })
+    })
+  )
+
+  return { graphStats, tableStats }
+}
+
+const getGraduationTimeStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  let graduationAmounts = getYearsObject(years)
+  let graduationTimes = getYearsObject(years, true)
+  let totalAmounts = isAcademicYear ? getAcademicYearsObject(years) : getYearsObject(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+
+      const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
+      totalAmounts[year] = studentnumbersOfTheYear.length
+
+      studyrights.forEach(({ enddate, studystartdate }) => {
+        const graduationYear = defineYear(enddate, isAcademicYear)
+        const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+        graduationAmounts[graduationYear] += 1
+        graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
+      })
+    })
+  )
+  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
+  const comparisonValue = studyprogramme.includes('KH') ? 72 : 48
+
+  const medians = getYearsObject(years, true)
+  const means = getYearsObject(years, true)
+
+  // HighCharts graph require the data to have this format (ie. actual value, "empty value")
+  years.forEach(year => {
+    const median = getMedian(graduationTimes[year])
+    const mean = getMean(graduationTimes[year])
+    medians[year] = [
+      ['', median],
+      ['', comparisonValue - median],
+    ]
+    means[year] = [
+      ['', mean],
+      ['', comparisonValue - mean],
+    ]
+  })
+  return { medians, means, graduationAmounts, totalAmounts }
+}
+
+const getProgrammesAfterGraduation = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const graduated = await graduatedStudyRights(studyprogramme, since)
+  const graduatedStudents = graduated.map(s => s.studentnumber)
+  let studyrightsAfterThisProgramme = await followingStudyrights(since, graduatedStudents)
+  const studyprogrammeCodes = uniqBy(studyrightsAfterThisProgramme, 'code').map(s => ({ code: s.code, name: s.name }))
+
+  const stats = {}
+  studyprogrammeCodes.forEach(c => (stats[c.code] = { ...c, ...getYearsObject(years) }))
+
+  if (!includeAllSpecials) {
+    const transferredAway = await transfersAway(studyprogramme, since)
+    const transferredTo = await transfersTo(studyprogramme, since)
+    const transfers = [...transferredAway, ...transferredTo].map(s => s.studyrightid)
+    studyrightsAfterThisProgramme.filter(s => !transfers.includes(s.studyrightid) && !s.canceldate)
+  }
+
+  studyrightsAfterThisProgramme.forEach(({ studystartdate, code }) => {
+    const startYear = defineYear(studystartdate, isAcademicYear)
+    if (years.includes(startYear) && code) {
+      stats[code][startYear] += 1
+    }
+  })
+
+  const tableStats = Object.values(stats)
+    .map(p => [p.code, p.name['fi'], ...years.map(year => p[year])])
+    .filter(p => p[0].includes('MH'))
+
+  const graphStats = Object.values(stats)
+    .map(p => ({ name: p.name['fi'], code: p.code, data: years.map(year => p[year]) }))
+    .filter(p => p.code.includes('MH'))
+
+  return { tableStats, graphStats }
+}
+
+const getGraduationStatsForStudytrack = async ({ studyprogramme, yearType, specialGroups }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const thesis = await getThesisStats(queryParameters)
+  const graduated = await getGraduatedStats(queryParameters)
+  const graduationTimeStats = await getGraduationTimeStats(queryParameters)
+  const programmesAfterGraduation = await getProgrammesAfterGraduation(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+
+  const titles = ['', 'Graduated', 'Wrote thesis']
+  const programmesAfterTitles = ['Code', 'Programme', ...years]
+  const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    titles,
+    graphStats: [
+      {
+        name: 'Graduated students',
+        data: graduated.graphStats,
+      },
+      {
+        name: 'Wrote thesis',
+        data: thesis.graphStats,
+      },
+    ],
+    graduationMedianTime: graduationTimeStats.medians,
+    graduationMeanTime: graduationTimeStats.means,
+    graduationAmounts: graduationTimeStats.graduationAmounts,
+    totalAmounts: graduationTimeStats.totalAmounts,
+    programmesAfterTableStats: programmesAfterGraduation.tableStats,
+    programmesAfterGraphStats: programmesAfterGraduation.graphStats,
+    programmesAfterTitles,
+  }
+}
+
+module.exports = {
+  getGraduationStatsForStudytrack,
+}

services/backend/src/services/studyprogrammeHelpers.js

@@ -0,0 +1,330 @@
+const { mean } = require('lodash')
+const { studentnumbersWithAllStudyrightElements } = require('./populations')
+
+// Helper functions
+
+const getCorrectStudentnumbers = async ({ codes, startDate, endDate, includeAllSpecials, includeGraduated = true }) => {
+  let studentnumbers = []
+  const exchangeStudents = includeAllSpecials
+  const cancelledStudents = includeAllSpecials
+  const nondegreeStudents = includeAllSpecials
+  const transferredOutStudents = includeAllSpecials
+  const transferredToStudents = !includeAllSpecials
+  const graduatedStudents = !includeGraduated
+
+  studentnumbers = await studentnumbersWithAllStudyrightElements(
+    codes,
+    startDate,
+    endDate,
+    exchangeStudents,
+    cancelledStudents,
+    nondegreeStudents,
+    transferredOutStudents,
+    null,
+    transferredToStudents,
+    graduatedStudents
+  )
+
+  return studentnumbers
+}
+
+const formatStudyright = studyright => {
+  const {
+    studyrightid,
+    studystartdate,
+    enddate,
+    canceldate,
+    graduated,
+    prioritycode,
+    extentcode,
+    student,
+    studyright_elements,
+  } = studyright
+
+  return {
+    studyrightid,
+    studystartdate,
+    enddate,
+    canceldate,
+    graduated,
+    prioritycode,
+    extentcode,
+    studentnumber: student.studentnumber,
+    code: studyright_elements?.length ? studyright_elements[0].code : null,
+    name:
+      studyright_elements?.length && studyright_elements[0].element_detail && studyright_elements[0].element_detail.name
+        ? studyright_elements[0].element_detail.name
+        : null,
+  }
+}
+
+const formatStudent = student => {
+  const { studentnumber, gender_code, home_country_en, creditcount } = student
+  return {
+    studentnumber,
+    gender_code,
+    home_country_en,
+    creditcount,
+  }
+}
+
+const formatTransfer = transfer => {
+  const { sourcecode, transfercode, transferdate, studyrightid } = transfer
+  return {
+    sourcecode,
+    transfercode,
+    transferdate,
+    studyrightid,
+  }
+}
+
+const getYearsArray = (since, isAcademicYear) => {
+  const years = []
+  for (let i = since; i <= new Date().getFullYear(); i++) {
+    const year = isAcademicYear ? `${i} - ${i + 1}` : i
+    years.push(year)
+  }
+  return years
+}
+
+const getYearsObject = (years, emptyArrays = false) => {
+  let yearsObject = {}
+  for (const year of years) {
+    yearsObject = { ...yearsObject, [year]: emptyArrays ? [] : 0 }
+  }
+  return yearsObject
+}
+
+const getAcademicYearsObject = (years, emptyArrays = false) => {
+  let yearsObject = {}
+  for (const year of years) {
+    yearsObject = { ...yearsObject, [`${year} - ${year + 1}`]: emptyArrays ? [] : 0 }
+  }
+  return yearsObject
+}
+
+const getStatsBasis = years => {
+  return {
+    graphStats: new Array(years.length).fill(0),
+    tableStats: getYearsObject(years),
+  }
+}
+
+const isMajorStudentCredit = (studyright, attainment_date) =>
+  studyright &&
+  (studyright.prioritycode === 1 || studyright.prioritycode === 30) && // Is studyright state = MAIN or state = GRADUATED
+  studyright.studystartdate && // The student has started studying in the programme
+  studyright.studystartdate <= attainment_date && // Has the credit been attained after studying in the programme started
+  studyright.enddate >= attainment_date && // Has the credit been attained before the studyright ended
+  (!studyright.canceldate || studyright.canceldate >= attainment_date) // If the studyright was cancelled, was the credit attained before it was cancelled
+
+const isSpecialGroupCredit = (studyright, attainment_date, transfers) => {
+  if (!studyright) return true // If there is no studyright matching the credit, is not a major student credit
+  if (studyright.canceldate) return true // Cancelled studyrights are in the special groups
+  if (studyright.studystartdate > attainment_date) return true // Credits before the studyright started are not major student credits
+  if (studyright.enddate && attainment_date > studyright.enddate) return true // Credits after studyright are not major student credits
+  if ([7, 9, 16, 34, 33, 99, 14, 13].includes(studyright.extentcode)) return true // Excludes non-degree studyrights and exchange students
+  if (transfers.includes(studyright.studyrightid)) return true // Excludes both transfers in and out of the programme
+  return false
+}
+
+const getMedian = values => {
+  if (values.length === 0) return 0
+  values.sort((a, b) => a - b)
+  const half = Math.floor(values.length / 2)
+  if (values.length % 2) return values[half]
+  return (values[half - 1] + values[half]) / 2.0
+}
+
+const getMean = values => {
+  if (values.length === 0) return 0
+  return Math.round(mean(values))
+}
+
+const defineYear = (date, isAcademicYear) => {
+  if (!date) return ''
+  const year = date.getFullYear()
+  if (!isAcademicYear) return year
+  if (date < new Date(`${year}-07-31`)) return `${year - 1} - ${year}`
+  return `${year} - ${year + 1}`
+}
+
+const getStartDate = (studyprogramme, isAcademicYear) => {
+  if ((studyprogramme.includes('KH') || studyprogramme.includes('MH')) && isAcademicYear) return new Date('2017-08-01')
+  if (studyprogramme.includes('KH') || studyprogramme.includes('MH')) return new Date('2017-01-01')
+  if (isAcademicYear) return new Date('2017-08-01')
+  return new Date('2017-01-01')
+}
+
+// There are 9 course_unit_types
+// 1. urn:code:course-unit-type:regular
+// 2. urn:code:course-unit-type:bachelors-thesis
+// 3. urn:code:course-unit-type:masters-thesis
+// 4. urn:code:course-unit-type:doctors-thesis
+// 5. urn:code:course-unit-type:licentiate-thesis
+// 6. urn:code:course-unit-type:bachelors-maturity-examination
+// 7. urn:code:course-unit-type:masters-maturity-examination
+// 8. urn:code:course-unit-type:communication-and-linguistic-studies
+// 9. urn:code:course-unit-type:practical-training-homeland
+// Four of these are thesis types
+
+const getThesisType = studyprogramme => {
+  if (studyprogramme.includes('MH') || studyprogramme.includes('ma'))
+    return ['urn:code:course-unit-type:masters-thesis']
+  if (studyprogramme.includes('KH') || studyprogramme.includes('ba'))
+    return ['urn:code:course-unit-type:bachelors-thesis']
+  if (/^(T)[0-9]{6}$/.test(studyprogramme))
+    return ['urn:code:course-unit-type:doctors-thesis', 'urn:code:course-unit-type:licentiate-thesis']
+  return 'thesis-type-not-defined'
+}
+
+const getPercentage = (value, total) => {
+  if (typeof value !== 'number' || typeof total !== 'number') return 'NA'
+  if (total === 0) return 'NA'
+  if (value === 0) return '0 %'
+  return `${((value / total) * 100).toFixed(1)} %`
+}
+
+const getEmptyArray = length => new Array(length).fill(0)
+
+const getBachelorCreditGraphStats = years => ({
+  lte30: {
+    name: 'Less than 30 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte60: {
+    name: '30-59 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte90: {
+    name: '60-89 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte120: {
+    name: '90-119 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte150: {
+    name: '120-149 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte150: {
+    name: 'More than 150 credits',
+    data: getEmptyArray(years.length),
+  },
+})
+
+const getMasterCreditGraphStats = years => ({
+  lte200: {
+    name: 'Less than 200 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte220: {
+    name: '200-219 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte240: {
+    name: '220-239 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte260: {
+    name: '240-259 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte280: {
+    name: '260-279 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte280: {
+    name: 'More than 280 credits',
+    data: getEmptyArray(years.length),
+  },
+})
+
+const getCreditGraphStats = (studyprogramme, years) =>
+  studyprogramme.includes('KH') ? getBachelorCreditGraphStats(years) : getMasterCreditGraphStats(years)
+
+const bachelorCreditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'mte150']
+const masterCreditThresholds = ['lte200', 'lte220', 'lte240', 'lte260', 'lte280', 'mte280']
+const bachelorCreditAmounts = [30, 60, 90, 120, 150, 150]
+const masterCreditAmounts = [200, 220, 240, 260, 280, 280]
+
+const getCreditThresholds = studyprogramme => {
+  if (studyprogramme.includes('KH')) {
+    return { creditThresholdKeys: bachelorCreditThresholds, creditThresholdAmounts: bachelorCreditAmounts }
+  }
+  return { creditThresholdKeys: masterCreditThresholds, creditThresholdAmounts: masterCreditAmounts }
+}
+
+const tableTitles = {
+  basics: {
+    SPECIAL_EXCLUDED: ['', 'All', 'Started studying', 'Graduated'],
+    SPECIAL_INCLUDED: ['', 'All', 'Started studying', 'Graduated', 'Cancelled', 'Transferred away', 'Transferred to'],
+  },
+  credits: {
+    SPECIAL_EXCLUDED: ['', 'Total', 'Major students credits', 'Transferred credits'],
+    SPECIAL_INCLUDED: ['', 'Total', 'Major students credits', 'Non major students credits', 'Transferred credits'],
+  },
+  creditProgress: {
+    bachelor: [
+      '',
+      'All',
+      '< 30 credits',
+      '30-59 credits',
+      '60-89 credits',
+      '90-119 credits',
+      '120-149 credits',
+      '> 150 credits',
+    ],
+    master: [
+      '',
+      'All',
+      '< 200 credits',
+      '200-219 credits',
+      '220-239 credits',
+      '240-259 credits',
+      '260-279 credits',
+      '> 280 credits',
+    ],
+  },
+  studytracks: [
+    '',
+    'All',
+    'Started studying',
+    'Currently enrolled',
+    'Absent',
+    'Cancelled',
+    'Graduated',
+    'Men',
+    'Women',
+    'Finnish',
+  ],
+}
+
+const getCreditProgressTableTitles = studyprogramme =>
+  studyprogramme.includes('KH') ? tableTitles.bachelorCreditProgress : tableTitles.masterCreditProgress
+
+module.exports = {
+  getCorrectStudentnumbers,
+  formatStudyright,
+  formatStudent,
+  formatTransfer,
+  getYearsArray,
+  getYearsObject,
+  getAcademicYearsObject,
+  getStatsBasis,
+  isMajorStudentCredit,
+  isSpecialGroupCredit,
+  getMedian,
+  getMean,
+  defineYear,
+  getStartDate,
+  getThesisType,
+  getPercentage,
+  getBachelorCreditGraphStats,
+  getMasterCreditGraphStats,
+  getCreditGraphStats,
+  getCreditThresholds,
+  tableTitles,
+  getCreditProgressTableTitles,
+}

services/backend/src/services/studyprogrammeUpdates.js

@@ -0,0 +1,84 @@
+const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics')
+const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
+const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
+const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
+const {
+  setBasicStats,
+  setCreditStats,
+  setGraduationStats,
+  setStudytrackStats,
+} = require('../services/analyticsService')
+
+const updateBasicView = async code => {
+  const yearTypes = ['ACADEMIC_YEAR', 'CALENDAR_YEAR']
+  const special = ['SPECIAL_INCLUDED', 'SPECIAL_EXCLUDED']
+
+  yearTypes.forEach(async yearType => {
+    special.forEach(async specialGroups => {
+      const basicStats = await getBasicStatsForStudytrack({
+        studyprogramme: code,
+        yearType,
+        specialGroups,
+      })
+      await setBasicStats(basicStats, yearType, specialGroups)
+
+      const creditStats = await getCreditStatsForStudytrack({
+        studyprogramme: code,
+        yearType,
+        specialGroups,
+      })
+      await setCreditStats(creditStats, yearType, specialGroups)
+
+      const graduationStats = await getGraduationStatsForStudytrack({
+        studyprogramme: code,
+        yearType,
+        specialGroups,
+      })
+      await setGraduationStats(graduationStats, yearType, specialGroups)
+    })
+  })
+
+  return 'OK'
+}
+
+const updateStudytrackView = async code => {
+  const specialIncluded = 'SPECIAL_INCLUDED'
+  const specialExcluded = 'SPECIAL_EXCLUDED'
+  const graduatedIncluded = 'GRADUATED_INCLUDED'
+  const graduatedExcluded = 'GRADUATED_EXCLUDED'
+
+  const statsSpecialAndGraduated = await getStudytrackStatsForStudyprogramme({
+    studyprogramme: code,
+    specialGroups: specialIncluded,
+    graduated: graduatedIncluded,
+  })
+  await setStudytrackStats(statsSpecialAndGraduated, graduatedIncluded, specialIncluded)
+
+  const statsSpecialNotGraduated = await getStudytrackStatsForStudyprogramme({
+    studyprogramme: code,
+    specialGroups: specialIncluded,
+    graduated: graduatedExcluded,
+  })
+  await setStudytrackStats(statsSpecialNotGraduated, graduatedExcluded, specialIncluded)
+
+  const statsNotSpecialAndGraduated = await getStudytrackStatsForStudyprogramme({
+    studyprogramme: code,
+    specialGroups: specialExcluded,
+    graduated: graduatedIncluded,
+  })
+  await setStudytrackStats(statsNotSpecialAndGraduated, graduatedIncluded, specialExcluded)
+
+  const statsNotSpecialNotGraduated = await getStudytrackStatsForStudyprogramme({
+    studyprogramme: code,
+    specialGroups: specialExcluded,
+    graduated: graduatedExcluded,
+  })
+  await setStudytrackStats(statsNotSpecialNotGraduated, graduatedExcluded, specialExcluded)
+
+  return 'OK'
+}
+
+module.exports = {
+  updateBasicView,
+  updateStudytrackView,
+}

services/backend/src/services/studytrackStats.js

@@ -0,0 +1,317 @@
+const { indexOf } = require('lodash')
+const moment = require('moment')
+
+const { getAssociations } = require('./studyrights')
+const {
+  getMedian,
+  getMean,
+  getStartDate,
+  getYearsArray,
+  getPercentage,
+  getYearsObject,
+  getAcademicYearsObject,
+  getCorrectStudentnumbers,
+  getCreditThresholds,
+  getCreditGraphStats,
+  tableTitles,
+} = require('./studyprogrammeHelpers')
+const {
+  studytrackStudents,
+  allStudyrights,
+  startedStudyrights,
+  enrolledStudents,
+  absentStudents,
+  graduatedStudyRights,
+} = require('./newStudyprogramme')
+const { getAcademicYearDates } = require('../util/semester')
+
+const getStudentData = (students, thresholdKeys, thresholdAmounts) => {
+  let data = { female: 0, male: 0, finnish: 0 }
+  thresholdKeys.forEach(t => (data[t] = 0))
+
+  students.forEach(({ gender_code, home_country_en, creditcount }) => {
+    data.male += gender_code === '1' ? 1 : 0
+    data.female += gender_code === '2' ? 1 : 0
+    data.finnish += home_country_en === 'Finland' ? 1 : 0
+    data[thresholdKeys[0]] += creditcount < thresholdAmounts[0] ? 1 : 0
+    data[thresholdKeys[1]] += creditcount >= thresholdAmounts[0] && creditcount < thresholdAmounts[1] ? 1 : 0
+    data[thresholdKeys[2]] += creditcount >= thresholdAmounts[1] && creditcount < thresholdAmounts[2] ? 1 : 0
+    data[thresholdKeys[3]] += creditcount >= thresholdAmounts[2] && creditcount < thresholdAmounts[3] ? 1 : 0
+    data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
+    data[thresholdKeys[5]] += creditcount >= thresholdAmounts[5] ? 1 : 0
+  })
+  return data
+}
+
+const getGraduationTimeStats = async ({
+  year,
+  graduated,
+  studyprogramme,
+  track,
+  graduationMeanTime,
+  graduationMedianTime,
+  graduationAmounts,
+}) => {
+  // Count how long each student took to graduate
+  let graduationTimes = []
+  graduated.forEach(({ enddate, studystartdate }) => {
+    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+    graduationAmounts[track][year] += 1
+    graduationTimes = [...graduationTimes, timeToGraduation]
+  })
+
+  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
+  const comparisonValue = studyprogramme.includes('KH') ? 72 : 48
+
+  // HighCharts graph requires the data to have this format (ie. actual value, "empty value")
+  const median = getMedian(graduationTimes)
+  const mean = getMean(graduationTimes)
+  graduationMedianTime[track][year] = [
+    ['', median],
+    ['', comparisonValue - median],
+  ]
+  graduationMeanTime[track][year] = [
+    ['', mean],
+    ['', comparisonValue - mean],
+  ]
+}
+
+// Goes through the programme and all its studytracks for the said year and adds the wanted stats to the data objects
+const getStudytrackDataForTheYear = async ({
+  studyprogramme,
+  special,
+  includeGraduated,
+  studytracks,
+  studytrackNames,
+  year,
+  years,
+  data,
+  creditThresholdKeys,
+  creditThresholdAmounts,
+}) => {
+  const {
+    mainStatsByYear,
+    mainStatsByTrack,
+    creditGraphStats,
+    creditTableStats,
+    graduationMeanTime,
+    graduationMedianTime,
+    graduationAmounts,
+    totalAmounts,
+    emptyTracks,
+  } = data
+  const { startDate, endDate } = getAcademicYearDates(year)
+
+  await Promise.all(
+    studytracks.map(async track => {
+      const codes = studyprogramme === track ? [studyprogramme] : [studyprogramme, track]
+
+      const studentnumbers = await getCorrectStudentnumbers({
+        codes,
+        startDate,
+        endDate,
+        includeAllSpecials: special,
+        includeGraduated,
+      })
+
+      // Get all the studyrights and students for the calculations
+      const all = await allStudyrights(track, null, studentnumbers)
+      const students = await studytrackStudents(studentnumbers)
+      const studentData = getStudentData(students, creditThresholdKeys, creditThresholdAmounts)
+      const started = await startedStudyrights(track, startDate, studentnumbers)
+      const enrolled = await enrolledStudents(track, startDate, studentnumbers)
+      const absent = await absentStudents(track, studentnumbers)
+      const cancelled = all.filter(s => s.canceldate !== null && s.canceldate !== undefined)
+      const graduated = await graduatedStudyRights(track, null, studentnumbers)
+
+      // If the track has no stats for that year, it should be removed from the table and dropdown options
+      if (all.length === 0) {
+        emptyTracks.has(track) ? emptyTracks.set(track, emptyTracks.get(track) + 1) : emptyTracks.set(track, 1)
+        return
+      }
+
+      // Count stats for creditgraph for the year per track
+      creditThresholdKeys.forEach(
+        threshold => (creditGraphStats[track][threshold].data[indexOf(years, year)] = studentData[threshold])
+      )
+      // Count stats for the credit progress table for the year per track
+      creditTableStats[track] = [
+        ...creditTableStats[track],
+        [`${year} - ${year + 1}`, all.length, ...creditThresholdKeys.map(threshold => studentData[threshold])],
+      ]
+
+      // Count stats for the main studytrack table grouped by tracks
+      mainStatsByTrack[track] = [
+        ...mainStatsByTrack[track],
+        [
+          `${year} - ${year + 1}`,
+          all.length,
+          getPercentage(all.length, all.length),
+          started.length,
+          getPercentage(started.length, all.length),
+          enrolled.length,
+          getPercentage(enrolled.length, all.length),
+          absent.length,
+          getPercentage(absent.length, all.length),
+          cancelled.length,
+          getPercentage(cancelled.length, all.length),
+          graduated.length,
+          getPercentage(graduated.length, all.length),
+          studentData.male,
+          getPercentage(studentData.male, all.length),
+          studentData.female,
+          getPercentage(studentData.female, all.length),
+          studentData.finnish,
+          getPercentage(studentData.finnish, all.length),
+        ],
+      ]
+
+      // Count stats for the main studytrack table grouped by year
+      mainStatsByYear[year] = [
+        ...mainStatsByYear[year],
+        [
+          studytrackNames[track]?.name['fi']
+            ? `${studytrackNames[track]?.name['fi']}, ${track}`
+            : `${year} - ${year + 1}`,
+          all.length,
+          getPercentage(all.length, all.length),
+          started.length,
+          getPercentage(started.length, all.length),
+          enrolled.length,
+          getPercentage(enrolled.length, all.length),
+          absent.length,
+          getPercentage(absent.length, all.length),
+          cancelled.length,
+          getPercentage(cancelled.length, all.length),
+          graduated.length,
+          getPercentage(graduated.length, all.length),
+          studentData.male,
+          getPercentage(studentData.male, all.length),
+          studentData.female,
+          getPercentage(studentData.female, all.length),
+          studentData.finnish,
+          getPercentage(studentData.finnish, all.length),
+        ],
+      ]
+
+      // Count stats for the graduation time charts grouped by year
+      totalAmounts[track][`${year} - ${year + 1}`] = await all.length
+      await getGraduationTimeStats({
+        year: `${year} - ${year + 1}`,
+        graduated,
+        studyprogramme,
+        track,
+        graduationMedianTime,
+        graduationMeanTime,
+        graduationAmounts,
+      })
+    })
+  )
+}
+
+// Defines the studytrack names for the studytrack selector
+// If the track has no stats for any year, it should be removed the dropdown options
+const getStudytrackOptions = (studyprogramme, studytrackNames, studytracks, emptyTracks, years) => {
+  const names = { [studyprogramme]: 'All students of the programme' }
+  studytracks.forEach(track => {
+    const trackName = studytrackNames[track]?.name['fi']
+    if (trackName && emptyTracks.get(track) !== years.length) {
+      names[track] = trackName
+    }
+  })
+  return names
+}
+
+// Creates empty objects for each statistic type, which are then updated with the studytrack data
+const getEmptyStatsObjects = (years, studytracks, studyprogramme) => {
+  const mainStatsByYear = getYearsObject(years, true)
+  const mainStatsByTrack = {}
+  const creditGraphStats = {}
+  const creditTableStats = {}
+  const graduationMedianTime = {}
+  const graduationMeanTime = {}
+  const graduationAmounts = {}
+  const totalAmounts = {}
+  const emptyTracks = new Map()
+
+  studytracks.forEach(async track => {
+    mainStatsByTrack[track] = []
+    creditGraphStats[track] = getCreditGraphStats(studyprogramme, years)
+    creditTableStats[track] = []
+    graduationMedianTime[track] = getAcademicYearsObject(years, true)
+    graduationMeanTime[track] = getAcademicYearsObject(years, true)
+    graduationAmounts[track] = getAcademicYearsObject(years)
+    totalAmounts[track] = getAcademicYearsObject(years)
+  })
+
+  return {
+    mainStatsByYear,
+    mainStatsByTrack,
+    creditGraphStats,
+    creditTableStats,
+    graduationMedianTime,
+    graduationMeanTime,
+    graduationAmounts,
+    totalAmounts,
+    emptyTracks,
+  }
+}
+
+// Combines all the data for the Populations and Studytracks -view
+const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, graduated, specialGroups }) => {
+  const isAcademicYear = true
+  const special = specialGroups === 'SPECIAL_INCLUDED'
+  const includeGraduated = graduated === 'GRADUATED_INCLUDED'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear())
+
+  const associations = await getAssociations()
+  const studytracks = associations.programmes[studyprogramme]
+    ? [studyprogramme, ...associations.programmes[studyprogramme].studytracks]
+    : [studyprogramme]
+
+  const studytrackNames = associations.studyTracks
+
+  const data = getEmptyStatsObjects(years, studytracks, studyprogramme)
+  const { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(studyprogramme)
+
+  await Promise.all(
+    years.map(async year => {
+      return await getStudytrackDataForTheYear({
+        studyprogramme,
+        special,
+        includeGraduated,
+        studytracks,
+        studytrackNames,
+        year,
+        years,
+        data,
+        creditThresholdKeys,
+        creditThresholdAmounts,
+      })
+    })
+  )
+
+  const studytrackOptions = getStudytrackOptions(studyprogramme, studytrackNames, studytracks, data.emptyTracks, years)
+
+  return {
+    id: studyprogramme,
+    years: getYearsArray(since.getFullYear(), isAcademicYear),
+    mainStatsByTrack: data.mainStatsByTrack,
+    mainStatsByYear: data.mainStatsByYear,
+    creditTableStats: data.creditTableStats,
+    creditGraphStats: data.creditGraphStats,
+    graduationMedianTime: data.graduationMedianTime,
+    graduationMeanTime: data.graduationMeanTime,
+    graduationAmounts: data.graduationAmounts,
+    totalAmounts: data.totalAmounts,
+    studytrackOptions,
+    includeGraduated,
+    populationTitles: tableTitles['studytracks'],
+    creditTableTitles: tableTitles['creditProgress'][studyprogramme.includes('KH') ? 'bachelor' : 'master'],
+  }
+}
+
+module.exports = {
+  getStudytrackStatsForStudyprogramme,
+}