FacultyStats: Show difference to previous year

Need to decrypt in backend to make courses of population tab work

.eslintrc.json

@@ -14,6 +14,7 @@
     "no-console": "error",
     "react/no-unescaped-entities": "off",
     "no-async-promise-executor": "off",
-    "prettier/prettier": "error"
+    "prettier/prettier": "error",
+    "cypress/no-assigning-return-values": "off"
   }
 }

.github/actions/run-tests/action.yml

@@ -0,0 +1,52 @@
+name: 'Run tests'
+description: 'Runs Cypress tests for individual spec'
+
+inputs:
+  username:
+    required: true
+    description: 'Toska registry username'
+  password:
+    required: true
+    description: 'Toska registry password'
+  spec:
+    required: true
+    description: 'Path to Cypress file'
+
+runs:
+  using: composite
+  steps:
+    - name: 'Download images'
+      uses: actions/download-artifact@v3
+      with:
+        path: /tmp/
+    - name: 'Load built images'
+      shell: bash
+      run: |
+        docker load --input /tmp/artifact/backend.tar
+        docker load --input /tmp/artifact/frontend.tar
+    - name: 'Login to toska docker registry'
+      uses: docker/login-action@v2
+      with:
+        registry: registry.toska.cs.helsinki.fi
+        username: ${{ inputs.username }}
+        password: ${{ inputs.password }}
+    - name: 'Run services, pull rest of the needed images'
+      shell: bash
+      run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+    - name: 'Run cypress'
+      uses: cypress-io/github-action@v4
+      with:
+        browser: chrome
+        spec: ${{ inputs.spec }}
+      env:
+        CYPRESS_baseUrl: http://localhost:1337
+    - name: 'Upload video if tests fail'
+      uses: actions/upload-artifact@v3
+      if: failure()
+      with:
+        name: Cypress videos
+        path: ./cypress/videos/*
+    - name: 'Print logs on failure'
+      shell: bash
+      if: failure()
+      run: docker-compose --file docker-compose.ci.yml logs

.github/dependabot.yml

@@ -1,15 +1,68 @@
 version: 2
 updates:
-- package-ecosystem: npm
-  directory: "/"
-  schedule:
-    interval: weekly
-    time: "03:00"
-  open-pull-requests-limit: 10
-  target-branch: "trunk"
-- package-ecosystem: "github-actions"
-  directory: "/"
-  schedule:
-    interval: weekly
-    time: "03:00"
-  target-branch: "trunk"
+  - package-ecosystem: npm
+    directory: '/'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'infra'
+    ignore:
+      - dependency-name: '*'
+        update-types: ['version-update:semver-major']
+
+  - package-ecosystem: npm
+    directory: '/updater/sis-updater-scheduler'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'updater'
+    ignore:
+      - dependency-name: '*'
+        update-types: ['version-update:semver-major']
+
+  - package-ecosystem: npm
+    directory: '/updater/sis-updater-worker'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'updater'
+    ignore:
+      - dependency-name: '*'
+        update-types: ['version-update:semver-major']
+
+  - package-ecosystem: npm
+    directory: '/services/backend'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'back'
+    ignore:
+      - dependency-name: '*'
+        update-types: ['version-update:semver-major']
+
+  - package-ecosystem: npm
+    directory: '/services/frontend'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    labels:
+      - 'dependencies'
+      - 'front'
+    ignore:
+      - dependency-name: '*'
+        update-types: ['version-update:semver-major']
+
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: weekly
+      time: '03:00'
+    open-pull-requests-limit: 10

.github/workflows/release-oodikone.yaml

@@ -1,4 +1,4 @@
-name: "Release oodikone to production"
+name: 'Release oodikone to production'
 
 on:
   release:
@@ -6,24 +6,24 @@ on:
 
 jobs:
   build_and_release_backend:
-    name: "Build and release backend"
+    name: 'Build and release backend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
+      - uses: actions/checkout@v3
+      - uses: docker/setup-buildx-action@v2
+      - name: 'Login to DockerHub'
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
+      - name: 'Extract release version'
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
         run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
+      - name: 'Build and push'
+        uses: docker/build-push-action@v3
         with:
           build-args: |
             SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
@@ -38,27 +38,27 @@ jobs:
           target: production
 
   build_and_release_frontend:
-    name: "Build and release frontend"
+    name: 'Build and release frontend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
+      - uses: actions/checkout@v3
+      - uses: docker/setup-buildx-action@v2
+      - name: 'Login to DockerHub'
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Get current time"
+      - name: 'Get current time'
         id: get_current_time
         run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
-      - name: "Extract release version"
+      - name: 'Extract release version'
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
         run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
+      - name: 'Build and push'
+        uses: docker/build-push-action@v3
         with:
           build-args: |
             SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
@@ -72,11 +72,11 @@ jobs:
           push: true
           tags: toska/oodikone2-frontend:latest
           target: production
-      - name: "Create a container from built image "
+      - name: 'Create a container from built image '
         run: docker create --name frontend toska/oodikone2-frontend:latest
-      - name: "Copy assets from the container"
+      - name: 'Copy assets from the container'
         run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
+      - name: 'Create Sentry release'
         uses: getsentry/action-release@v1
         env:
           SENTRY_URL: ${{ secrets.SENTRY_URL }}
@@ -90,49 +90,16 @@ jobs:
           url_prefix: '~/static/js'
           version: ${{ steps.extract_version.outputs.VERSION }}
 
-  build_and_release_userservice:
-    name: "Build and release userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
-        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
-        # This steps extracts release version from GITHUB_REF env var
-        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
-        id: extract_version
-        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: true
-          tags: toska/oodikone2-userservice:latest
-          target: production
-
   report_success:
     runs-on: ubuntu-latest
-    name: "Report deployment success to oodikone notifications"
+    name: 'Report deployment success to oodikone notifications'
     needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
     if: success()
     steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -141,15 +108,14 @@ jobs:
 
   report_failure:
     runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
+    name: 'Report deployment failure to oodikone notifications'
     needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
     if: failure()
     steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure

.github/workflows/release-sis-updater-scheduler.yaml

@@ -1,54 +0,0 @@
-name: "Release sis-updater-scheduler"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-sis-updater-scheduler.yaml'
-      - 'updater/sis-updater-scheduler/**'
-
-jobs:
-  build_and_release:
-    name: "Build and release sis-updater-scheduler"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: docker/setup-buildx-action@v1
-    - name: "Login to DockerHub"
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: "Build and push"
-      uses: docker/build-push-action@v2
-      with:
-        cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
-        cache-to: type=inline
-        context: updater/sis-updater-scheduler/
-        push: true
-        tags: toska/sis-updater-scheduler:latest
-        target: production
-    - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-      with:
-        webhook-url: ${{ secrets.WEBHOOK_URL }}
-        message-type: deployment
-        softa-url: https://oodikone.cs.helsinki.fi
-        deployment-target: production
-        softa-to-deploy: sis-updater-scheduler
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: production
-          softa-to-deploy: sis-updater-scheduler

.github/workflows/release-sis-updater-worker.yaml

@@ -1,54 +0,0 @@
-name: "Release sis-updater-worker"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-sis-updater-worker.yaml'
-      - 'updater/sis-updater-worker/**'
-
-jobs:
-  build_and_release:
-    name: "Build and release sis-updater-worker"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: docker/setup-buildx-action@v1
-    - name: "Login to DockerHub"
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: "Build and push"
-      uses: docker/build-push-action@v2
-      with:
-        cache-from: type=registry,ref=toska/sis-updater-worker:latest
-        cache-to: type=inline
-        context: updater/sis-updater-worker/
-        push: true
-        tags: toska/sis-updater-worker:latest
-        target: production
-    - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-      with:
-        webhook-url: ${{ secrets.WEBHOOK_URL }}
-        message-type: deployment
-        softa-url: https://oodikone.cs.helsinki.fi
-        deployment-target: production
-        softa-to-deploy: sis-updater-worker
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: production
-          softa-to-deploy: sis-updater-worker

.github/workflows/test-and-release-updater.yaml

@@ -0,0 +1,162 @@
+name: 'Test and release updater'
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - '.github/workflows/test-and-release-updater.yaml'
+      - 'updater/sis-updater-worker/**'
+      - 'updater/sis-updater-scheduler/**'
+
+jobs:
+  build_worker:
+    name: 'Build sis-updater-worker'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: docker/setup-buildx-action@v2
+      - name: 'Build the image'
+        uses: docker/build-push-action@v3
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=production
+          context: updater/sis-updater-worker/
+          target: production
+          push: false
+          outputs: type=docker,dest=/tmp/worker.tar
+          tags: toska/sis-updater-worker:latest
+          cache-from: type=registry,ref=toska/sis-updater-worker:latest
+          cache-to: type=inline
+      - name: 'Store built image'
+        uses: actions/upload-artifact@v3
+        with:
+          path: /tmp/worker.tar
+
+  build_scheduler:
+    name: 'Build sis-updater-scheduler'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: docker/setup-buildx-action@v2
+      - name: 'Build the image'
+        uses: docker/build-push-action@v3
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=production
+          context: updater/sis-updater-scheduler/
+          target: production
+          push: false
+          outputs: type=docker,dest=/tmp/scheduler.tar
+          tags: toska/sis-updater-scheduler:latest
+          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
+          cache-to: type=inline
+      - name: 'Store built image'
+        uses: actions/upload-artifact@v3
+        with:
+          path: /tmp/scheduler.tar
+
+  test_updater:
+    name: 'Test updater'
+    runs-on: ubuntu-latest
+    needs:
+      - build_worker
+      - build_scheduler
+    steps:
+      - uses: actions/checkout@v3
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v2
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Download built images'
+        uses: actions/download-artifact@v3
+        with:
+          path: /tmp/
+      - name: 'Load worker image'
+        run: docker load --input /tmp/artifact/worker.tar
+      - name: 'Load scheduler image'
+        run: docker load --input /tmp/artifact/scheduler.tar
+      - name: 'Start worker'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
+      - name: 'Run hourly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: hourly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run daily jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: daily
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run weekly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: weekly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run prepurge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: prepurge
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run purge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: purge
+          EXIT_AFTER_IMMEDIATES: yes
+
+  release_updater:
+    name: 'Release updater'
+    runs-on: ubuntu-latest
+    needs:
+      - test_updater
+    steps:
+      - uses: actions/checkout@v3
+      - uses: docker/setup-buildx-action@v2
+      - name: 'Login to DockerHub'
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: 'Build and push scheduler'
+        uses: docker/build-push-action@v3
+        with:
+          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
+          cache-to: type=inline
+          context: updater/sis-updater-scheduler/
+          push: true
+          tags: toska/sis-updater-scheduler:latest
+          target: production
+      - name: 'Build and push worker'
+        uses: docker/build-push-action@v3
+        with:
+          cache-from: type=registry,ref=toska/sis-updater-worker:latest
+          cache-to: type=inline
+          context: updater/sis-updater-worker/
+          push: true
+          tags: toska/sis-updater-worker:latest
+          target: production
+      - name: 'Create Sentry release'
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: updater
+        with:
+          environment: production
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/static/js'
+          version: ${{ github.sha }}
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone.cs.helsinki.fi
+          deployment-target: production
+          softa-to-deploy: sis-updater

.github/workflows/test-oodikone.yaml

@@ -6,13 +6,16 @@ on:
       - master
     paths:
       - '.github/workflows/test-oodikone.yaml'
+      - '.github/actions/**'
       - 'cypress/**'
-      - 'cypress.json'
+      - 'cypress.config.json'
       - 'services/**'
+      - '!services/backend/src/rapodiff.js'
       - 'docker-compose.ci.yml'
       - 'docker-compose.test.yml'
       - 'nginx.conf'
       - 'package*.json'
+      - 'updater/sis-updater-worker/src/db/migrations/**'
 
 jobs:
   # === Static tests / linting ===
@@ -21,8 +24,8 @@ jobs:
     name: 'Lint backend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3.4.1
         with:
           node-version: 14
       - name: 'Install dependencies'
@@ -34,8 +37,8 @@ jobs:
     name: 'Lint frontend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3.4.1
         with:
           node-version: 14
       - name: 'Install dependencies'
@@ -45,29 +48,15 @@ jobs:
           'eslint --ext .jsx,.js services/frontend' \
           'stylelint services/frontend/src/**/*.css'
 
-  lint_userservice:
-    name: 'Lint userservice'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: 14
-      - name: 'Install dependencies'
-        run: npm ci && cd services/userservice && npm ci
-      - name: 'Run frontend linter'
-        run: npm run eslint services/userservice
-
   # === Build images ===
-
   build_backend:
     name: 'Build backend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
+      - uses: actions/checkout@v3
+      - uses: docker/setup-buildx-action@v2
       - name: 'Build image and export it'
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           build-args: |
             SENTRY_RELEASE=${{ github.sha }}
@@ -82,7 +71,7 @@ jobs:
           target: production
           outputs: type=docker,dest=/tmp/backend.tar
       - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           path: /tmp/backend.tar
 
@@ -90,13 +79,13 @@ jobs:
     name: 'Build frontend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
+      - uses: actions/checkout@v3
+      - uses: docker/setup-buildx-action@v2
       - name: 'Get current time'
         id: get_current_time
         run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
       - name: 'Build image and export it'
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           build-args: |
             SENTRY_RELEASE=${{ github.sha }}
@@ -112,350 +101,247 @@ jobs:
           target: production
           outputs: type=docker,dest=/tmp/frontend.tar
       - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           path: /tmp/frontend.tar
 
-  build_userservice:
-    name: 'Build userservice'
+  build_and_push_anon_sis_db:
+    name: 'Build anon sis-db and push to toska hub if new migrations have been added'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: 'Build image and export it'
-        uses: docker/build-push-action@v2
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
         with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: false
-          tags: toska/oodikone2-userservice:staging
-          target: production
-          outputs: type=docker,dest=/tmp/userservice.tar
-      - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v2
+          filters: |
+            migrations:
+              - 'updater/sis-updater-worker/src/db/migrations/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v2
         with:
-          path: /tmp/userservice.tar
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach sis-updater-worker
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
 
-  # === Integration tests ===
-  #
-  run_userservice_integration_tests:
-    name: 'Run userservice integration tests'
+  build_and_push_anon_kone_db:
+    name: 'Build anon kone-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: 'Run tests included in docker build'
-        run: docker-compose --file docker-compose.test.yml up --build userservice-test
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_kone/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v3
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v2
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
+
+  build_and_push_anon_user_db:
+    name: 'Build anon user-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_user/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v3
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v2
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
 
   # === End-to-end tests ===
-
   test_course_statistics:
     name: 'Test course statistics'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: 'Download images'
-        uses: actions/download-artifact@v2
+      - uses: actions/checkout@v3
+      - name: 'Run tests'
+        uses: ./.github/actions/run-tests
         with:
-          path: /tmp/
-      - name: 'Load built images'
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v1
-        with:
-          registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: 'Run services, pull rest of the needed images'
-        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: 'Run cypress'
-        uses: cypress-io/github-action@v2
-        with:
-          browser: chrome
-          spec: cypress/integration/Course_statistics.js
-        env:
-          CYPRESS_baseUrl: http://localhost:1337
-      - name: 'Upload video if tests fail'
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: Cypress videos
-          path: ./cypress/videos/*
-      - name: 'Print logs on failure'
-        if: failure()
-        run: docker-compose --file docker-compose.ci.yml logs
+          spec: cypress/e2e/Course_statistics.js
 
   test_custom_population:
     name: 'Test custom population'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: 'Download images'
-        uses: actions/download-artifact@v2
+      - uses: actions/checkout@v3
+      - name: 'Run tests'
+        uses: ./.github/actions/run-tests
         with:
-          path: /tmp/
-      - name: 'Load built images'
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v1
-        with:
-          registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: 'Run services, pull rest of the needed images'
-        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: 'Run cypress'
-        uses: cypress-io/github-action@v2
-        with:
-          browser: chrome
-          spec: cypress/integration/Custom_population.js
-        env:
-          CYPRESS_baseUrl: http://localhost:1337
-      - name: 'Upload video if tests fail'
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: Cypress videos
-          path: ./cypress/videos/*
-      - name: 'Print logs on failure'
-        if: failure()
-        run: docker-compose --file docker-compose.ci.yml logs
+          spec: cypress/e2e/Custom_population.js
 
   test_population_statistics:
     name: 'Test population statistics'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: 'Download images'
-        uses: actions/download-artifact@v2
+      - uses: actions/checkout@v3
+      - name: 'Run tests'
+        uses: ./.github/actions/run-tests
         with:
-          path: /tmp/
-      - name: 'Load built images'
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v1
-        with:
-          registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: 'Run services, pull rest of the needed images'
-        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: 'Run cypress'
-        uses: cypress-io/github-action@v2
-        with:
-          browser: chrome
-          spec: cypress/integration/Population_statistics.js
-        env:
-          CYPRESS_baseUrl: http://localhost:1337
-      - name: 'Upload video if tests fail'
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: Cypress videos
-          path: ./cypress/videos/*
-      - name: 'Print logs on failure'
-        if: failure()
-        run: docker-compose --file docker-compose.ci.yml logs
+          spec: cypress/e2e/Population_statistics.js
 
   test_filters:
     name: 'Test filters'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: 'Download images'
-        uses: actions/download-artifact@v2
+      - uses: actions/checkout@v3
+      - name: 'Run tests'
+        uses: ./.github/actions/run-tests
         with:
-          path: /tmp/
-      - name: 'Load built images'
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v1
-        with:
-          registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: 'Run services, pull rest of the needed images'
-        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: 'Run cypress'
-        uses: cypress-io/github-action@v2
-        with:
-          browser: chrome
-          spec: cypress/integration/Filters.js
-        env:
-          CYPRESS_baseUrl: http://localhost:1337
-      - name: 'Upload video if tests fail'
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: Cypress videos
-          path: ./cypress/videos/*
-      - name: 'Print logs on failure'
-        if: failure()
-        run: docker-compose --file docker-compose.ci.yml logs
+          spec: cypress/e2e/Filters.js
 
   test_student_statistics:
     name: 'Test student statistics'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: 'Download images'
-        uses: actions/download-artifact@v2
+      - uses: actions/checkout@v3
+      - name: 'Run tests'
+        uses: ./.github/actions/run-tests
         with:
-          path: /tmp/
-      - name: 'Load built images'
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v1
-        with:
-          registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: 'Run services, pull rest of the needed images'
-        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: 'Run cypress'
-        uses: cypress-io/github-action@v2
-        with:
-          browser: chrome
-          spec: cypress/integration/Student_statistics.js
-        env:
-          CYPRESS_baseUrl: http://localhost:1337
-      - name: 'Upload video if tests fail'
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: Cypress videos
-          path: ./cypress/videos/*
-      - name: 'Print logs on failure'
-        if: failure()
-        run: docker-compose --file docker-compose.ci.yml logs
+          spec: cypress/e2e/Student_statistics.js
 
   test_studyprogramme_overview:
     name: 'Test studyprogramme overview'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: 'Download images'
-        uses: actions/download-artifact@v2
+      - uses: actions/checkout@v3
+      - name: 'Run tests'
+        uses: ./.github/actions/run-tests
         with:
-          path: /tmp/
-      - name: 'Load built images'
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v1
-        with:
-          registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: 'Run services, pull rest of the needed images'
-        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: 'Run cypress'
-        uses: cypress-io/github-action@v2
-        with:
-          browser: chrome
-          spec: cypress/integration/Studyprogramme_overview.js
-        env:
-          CYPRESS_baseUrl: http://localhost:1337
-      - name: 'Upload video if tests fail'
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: Cypress videos
-          path: ./cypress/videos/*
-      - name: 'Print logs on failure'
-        if: failure()
-        run: docker-compose --file docker-compose.ci.yml logs
+          spec: cypress/e2e/Studyprogramme_overview.js
 
   test_teachers_and_users:
     name: 'Test teachers and users'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: 'Download images'
-        uses: actions/download-artifact@v2
+      - uses: actions/checkout@v3
+      - name: 'Run tests'
+        uses: ./.github/actions/run-tests
         with:
-          path: /tmp/
-      - name: 'Load built images'
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: 'Login to toska docker registry'
-        uses: docker/login-action@v1
-        with:
-          registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: 'Run services, pull rest of the needed images'
-        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: 'Run cypress'
-        uses: cypress-io/github-action@v2
-        with:
-          browser: chrome
           spec: |
-            cypress/integration/Teachers.js
-            cypress/integration/Users.js
-        env:
-          CYPRESS_baseUrl: http://localhost:1337
-      - name: 'Upload video if tests fail'
-        uses: actions/upload-artifact@v2
-        if: failure()
+            cypress/e2e/Teachers.js
+            cypress/e2e/Users.js
+
+  test_faculty_statistics:
+    name: 'Test faculty statistics'
+    needs:
+      - build_backend
+      - build_frontend
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: 'Run tests'
+        uses: ./.github/actions/run-tests
         with:
-          name: Cypress videos
-          path: ./cypress/videos/*
-      - name: 'Print logs on failure'
-        if: failure()
-        run: docker-compose --file docker-compose.ci.yml logs
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          spec: cypress/e2e/Faculty_statistics.js
 
   # === Other tests ===
 
@@ -464,21 +350,22 @@ jobs:
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: 'Download images'
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: /tmp/
       - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
@@ -504,8 +391,6 @@ jobs:
     needs:
       - lint_backend
       - lint_frontend
-      - lint_userservice
-      - run_userservice_integration_tests
       - test_course_statistics
       - test_custom_population
       - test_filters
@@ -513,20 +398,20 @@ jobs:
       - test_student_statistics
       - test_studyprogramme_overview
       - test_teachers_and_users
+      - test_faculty_statistics
       - test_service_exit_statuses
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: 'Download images'
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: /tmp/
       - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
       - name: 'Login to DockerHub'
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -551,7 +436,6 @@ jobs:
         run: |
           docker push toska/oodikone2-backend:staging
           docker push toska/oodikone2-frontend:staging
-          docker push toska/oodikone2-userservice:staging
       - name: 'Send deployment success notification to slack'
         if: success()
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
@@ -574,8 +458,6 @@ jobs:
     needs:
       - lint_backend
       - lint_frontend
-      - lint_userservice
-      - run_userservice_integration_tests
       - test_course_statistics
       - test_custom_population
       - test_filters
@@ -583,10 +465,11 @@ jobs:
       - test_student_statistics
       - test_studyprogramme_overview
       - test_teachers_and_users
+      - test_faculty_statistics
       - test_service_exit_statuses
     if: failure()
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: 'Send notification to slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:

.lintstagedrc.js

@@ -7,9 +7,9 @@ module.exports = {
   '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
   '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
   '*.css': files => `stylelint --fix ${files.join(' ')}`,
-  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0-alpine hadolint ${relativeFilePaths(files)}`,
-  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck-alpine:v0.7.2 ${relativeFilePaths(files)} -x`,
-  '.github/workflows/*': files => `${dockerCmdBase} rhysd/actionlint:1.6.2 ${relativeFilePaths(files)}`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0 hadolint ${relativeFilePaths(files)}`,
+  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck:v0.7.2 ${relativeFilePaths(files)} -x`,
+  '.github/{workflows,actions}/*': files => `npm run actionlint ${relativeFilePaths(files)}`,
   'docker-compose*': files => {
     const composeFiles = file => {
       if (['docker-compose.ci.yml', 'docker-compose.test.yml'].some(f => file.includes(f))) {

.prettierrc

@@ -5,5 +5,5 @@
   "printWidth": 120,
   "tabWidth": 2,
   "useTabs": false,
-  "jsxSingleQuote": false,
+  "jsxSingleQuote": false
 }

.stylelintrc

@@ -13,6 +13,10 @@
         "ignorePseudoClasses": ["global"]
       }
     ],
-    "prettier/prettier": true
+    "prettier/prettier": true,
+    "selector-class-pattern": null,
+    "custom-property-pattern": null,
+    "value-no-vendor-prefix": null,
+    "property-no-vendor-prefix": null
   }
 }

README.md

@@ -8,7 +8,7 @@ To run Oodikone locally, you will need the following:
 
 1. Applications:
    - [Docker](https://docs.docker.com/install/) (version 20.10+),
-   - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
+   - [Docker Compose](https://docs.docker.com/compose/install/)(version 1.29+) and
    - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
 2. Access to Toska Docker Hub,
 3. For real data setup, access to Toskas production servers.
@@ -43,11 +43,15 @@ What different CLI options do:
    - Cleans up any previous real data databases
    - Downloads needed database dumps from production servers.
    - Creates real data databases and populates them with downloaded dumps.
-3. Reset sis-importer data
-   - Like option 2, but only for sis-importer database. Useful when developing updater microservice.
+3. Reset single database
+   - Like option 2, but only for single database. Leaves other databases untouched.
 
 ## ⌨️ Development
 
+### Architecture
+
+![Oodikone architecture](Oodikone.png)
+
 ### Basics
 
 The development environment is entirely configured in the `docker-compose.yml` file located in this repository. The file defines the services for oodikone's two main components: **oodikone** and **updater**.
@@ -79,7 +83,7 @@ It is recommended to spend some time to become familiar with `docker` and `docke
 - `docker-compose ps`: view the containers in the running environment
 - `docker-compose logs frontend`: print logs for just frontend
 - `docker-compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
-- `docker exec -it userservice sh`: open bash terminal inside userservice container
+- `docker exec -it backend sh`: open bash terminal inside backend container
 - `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.
 
 ### User types for development
@@ -104,7 +108,7 @@ There are three types of tests in this project: static tests with eslint/prettie
 
 - Cypress
   - can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
-  - are defined in cypress -folder and cypress.json
+  - are defined in cypress -folder and cypress.config.json
   - whole cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
   - There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.
 - Jest
@@ -124,12 +128,16 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 
 ### Modules are missing after updating package.json
 
-You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container.
 
 ### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data
 
 Make sure you have your VPN on
 
+### I made a change to sis-db models aand created a migration, but tests are failing in CI!
+
+Migrations are handled by sis-importer-worker, since oodikone is using only replica. Since CI is testing only services included in oodikone, your new changes aren't migrated and thus are failing. Open anonyymioodi-folder in this repo to find more about how to to update the anon sis-db.
+
 ### Everything is broken, can't get oodikone running, data is not there etc.
 
 Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.

anonyymioodi/.gitignore

@@ -0,0 +1,2 @@
+*.sqz
+

anonyymioodi/README.md

@@ -0,0 +1,65 @@
+# Anonyymioodi - anonymous database setup for oodikone
+
+(name anonyymioodi for :culture: reasons)
+
+This folder contains a guide and scripts to create anonymous database images for oodikone, mainly used for testing.
+
+## How to create / modify database images
+
+Creating / modifying images varies a bit depending on the db. Generally, to modify images, you can either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose. 
+
+By default, previous db images from Toska Hub are used as a base. If needed, comment out Toska hub image and uncomment corresponding clean postgres image in docker-compose before running other steps.
+
+See more details on how to create / modify databases below.
+
+### Sis-importer-db
+
+The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
+- Move `sis-importer-db.sqz` to this folder
+- Run `./create_container_from_dump.sh sis-importer-db`
+- Enrich db with command in file `enrich_data.sql` either by piping command to container or by running command inside container.
+
+### Sis-db:
+
+The following steps are for building a clean sis-db image from scratch using oodikone's updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
+- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
+- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
+- Start updater, see oodikone readme for more info.
+- Populate sis-db from sis-importer-db:
+
+```
+curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
+```
+(or run these from frontend. You might also want to follow sis-updater-workers logs to ensure things are running smoothly.)
+- Clean up any extra stuff you don't want to keep in sis-db.
+- Bonus: check that cypress tests run okay
+
+### User-db / kone-db
+
+- Temporarily replace the db service in oodikone's docker-compose.yml with the same db service from this folder's docker-compose.yml
+- Ensure the newest user-db/kone-db image is pulled from the registry with `docker-compose pull`
+- Modify db to wanted state. Tips:
+  - Run oodikone if you want to upgrade something from frontend (e.g. tags in kone-db, courses in sis-db etc.).
+  - Easy way to create new users to user-db is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
+
+## Publishing the image
+
+After completing the steps above and making sure your local version has the correct data, run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the db image to Toska Hub.
+
+## How to update Postgres versions
+
+When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
+
+- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
+- Start database with docker-compose
+- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
+- Close container with docker-compose down.
+- Update the Postgres version in docker-compose.yml in this folder.
+- Uncomment row with Postgres image and comment row with Toskahub image.
+- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`

anonyymioodi/commit_and_push_to_toska_hub.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+infomsg "Committing and pushing to registry"
+database=${args[0]}
+docker commit "$database" "registry.toska.cs.helsinki.fi/$database"
+docker push "registry.toska.cs.helsinki.fi/$database"
+successmsg "Succesfully pushed"

anonyymioodi/create_container_from_dump.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+# === Create images ===
+
+retry () {
+  sleep 5
+  for i in {1..60}; do
+    "$@" && break || warningmsg "Retry attempt $i failed, waiting..." && sleep 5;
+  done
+}
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+
+infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
+database=${args[0]}
+docker-compose down --remove-orphans
+docker-compose up -d "$database"
+
+database_dump="$database.sqz"
+infomsg "Attempting to restore database $database from dump $database_dump"
+retry docker-compose exec "$database" pg_isready --dbname="$database"
+docker exec -i "$database" /bin/bash -c "pg_restore --username=postgres \
+--format=custom --dbname=$database --no-owner" < "$database_dump"
+successmsg "Database $database succesfully created"

anonyymioodi/docker-compose.yml

@@ -0,0 +1,50 @@
+x-defaults: &defaults
+  restart: unless-stopped
+
+services:
+  kone-db:
+    <<: *defaults
+    container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/kone-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone
+
+  sis-db:
+    <<: *defaults
+    container_name: sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-db
+    image: postgres:13.2
+    shm_size: 2gb
+
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-importer-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-importer-db
+    image: postgres:13.1
+    profiles:
+      - updater
+
+  user-db:
+    <<: *defaults
+    container_name: user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone

anonyymioodi/enrich_sampledata.sql

@@ -0,0 +1,133 @@
+-- add these to sis-importer db by running:
+-- docker exec -i sis-importer-db psql -U postgres sis-importer-db < enrich_sampledata.sql
+
+-- if you need to run updater for these students only, see studentnumberlist in the end
+-- this file
+
+-- Add admission types (valintaperusteet) to studyrights for some of the computer
+-- science 2018-2019 population students. See person_ids / student_number below 
+-- (in case you need to update dumps). 
+
+-- TODO: this should be moved into the sampledata creator script in
+-- importer repo, but since this file contains studentnumbers and person ids, it's 
+-- better to keep this here
+
+-- KP (koepisteet)
+-- hy-hlo-1412620   | 013683126
+-- hy-hlo-1485246   | 014340523
+-- hy-hlo-32230770  | 010589388
+-- hy-hlo-35626233  | 010775888
+-- hy-hlo-40150144  | 011065995
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:kp' 
+WHERE person_id IN (
+'hy-hlo-1412620',
+'hy-hlo-1485246',
+'hy-hlo-32230770',
+'hy-hlo-35626233',
+'hy-hlo-40150144'
+);
+
+-- TV (todistusvalinta)
+-- hy-hlo-45783433  | 011321150
+-- hy-hlo-48537946  | 010764653
+-- hy-hlo-49801905  | 011194400
+-- hy-hlo-53321302  | 011338871
+-- hy-hlo-51223300  | 013951984
+-- hy-hlo-53310137  | 012520688
+-- hy-hlo-53621583  | 011196796
+-- hy-hlo-53863064  | 011736884
+-- hy-hlo-57711592  | 010262566
+-- hy-hlo-57732872  | 013670647
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:tv' 
+WHERE person_id IN (
+'hy-hlo-45783433',
+'hy-hlo-48537946',
+'hy-hlo-49801905',
+'hy-hlo-53321302',
+'hy-hlo-51223300',
+'hy-hlo-53310137',
+'hy-hlo-53621583',
+'hy-hlo-53863064',
+'hy-hlo-57711592',
+'hy-hlo-57732872'
+);
+
+-- YP:
+-- hy-hlo-67768022  | 011952325
+-- hy-hlo-73740025  | 011736606
+-- hy-hlo-74780917  | 012775433
+-- hy-hlo-79615371  | 014113941
+-- hy-hlo-78118455  | 014300615
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:yp' 
+WHERE person_id IN (
+'hy-hlo-67768022',
+'hy-hlo-73740025',
+'hy-hlo-74780917',
+'hy-hlo-79615371',
+'hy-hlo-78118455'
+);
+-- AV:
+-- hy-hlo-78777265  | 010779392
+-- hy-hlo-81909550  | 011894153
+-- hy-hlo-84571890  | 013897158
+-- hy-hlo-80330834  | 015085926
+-- hy-hlo-84775722  | 011791272
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:av' 
+WHERE person_id IN (
+'hy-hlo-78777265',
+'hy-hlo-81909550',
+'hy-hlo-84571890',
+'hy-hlo-80330834',
+'hy-hlo-84775722'
+);
+-- M:
+-- hy-hlo-87439827  | 011928456
+-- hy-hlo-96612371  | 014418361
+-- hy-hlo-84840261  | 010975895
+-- hy-hlo-90330681  | 010522688
+-- hy-hlo-96594603  | 013951573
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:m' 
+WHERE person_id IN (
+'hy-hlo-87439827',
+'hy-hlo-96612371',
+'hy-hlo-84840261',
+'hy-hlo-90330681',
+'hy-hlo-96594603'
+);
+
+-- all studentnumbers for running updater:
+013683126
+014340523
+010589388
+010775888
+011065995
+011321150
+010764653
+011194400
+011338871
+013951984
+012520688
+011196796
+011736884
+010262566
+013670647
+011952325
+011736606
+012775433
+014113941
+014300615
+010779392
+011894153
+013897158
+015085926
+011791272
+011928456
+014418361
+010975895
+010522688
+013951573

cypress.config.js

@@ -0,0 +1,19 @@
+const { defineConfig } = require('cypress')
+
+module.exports = defineConfig({
+  projectId: 'c3jsph',
+  defaultCommandTimeout: 30000,
+  requestTimeout: 30000,
+  videoCompression: false,
+  viewportWidth: 1800,
+  viewportHeight: 1200,
+  e2e: {
+    // We've imported your old cypress plugins here.
+    // You may want to clean this up later by importing these.
+    setupNodeEvents(on, config) {
+      return require('./cypress/plugins/index.js')(on, config)
+    },
+    baseUrl: 'http://localhost:3000',
+    specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}',
+  },
+})

cypress.json

@@ -1,7 +0,0 @@
-{
-  "projectId": "c3jsph",
-  "baseUrl": "http://localhost:3000",
-  "defaultCommandTimeout": 30000,
-  "requestTimeout": 30000,
-  "videoCompression": false
-}

cypress/e2e/Course_statistics.js

@@ -20,15 +20,13 @@ describe('Course Statistics tests', () => {
 
       cy.contains('.tabular.menu a', 'Table').click()
       cy.contains('All')
-      cy.contains('.modeSelectorRow a', 'Attempts').click()
-      cy.contains('.modeSelectorRow a', 'Students').click()
+      cy.cs('viewMode-Attempts').click()
+      cy.cs('viewMode-Students').click()
 
       cy.contains('.tabular.menu a', 'Pass rate chart').click()
-      cy.get('div.modeSelectorContainer').click()
       cy.contains('svg', 'Pass rate chart')
 
       cy.contains('.tabular.menu a', 'Grade distribution chart').click()
-      cy.get('div.modeSelectorContainer').click()
       cy.contains('svg', 'Grades')
 
       cy.contains('a', 'New query').click()
@@ -84,8 +82,8 @@ describe('Course Statistics tests', () => {
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by a course code']").type(courseCode)
       cy.contains('td', courseCode).click()
-      cy.contains('Separate statistics for Spring and Fall semesters').click()
       cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.get('[data-cy=separateToggle]', { force: true }).click({ force: true })
       cy.contains('Syksy 2019')
       cy.contains('Syksy 2017')
       cy.get('tbody').contains('19')
@@ -98,7 +96,7 @@ describe('Course Statistics tests', () => {
       cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
 
       cy.contains('Tietokantojen perusteet')
-      cy.contains('Avoin yo: Tietokantojen perusteet')
+      cy.contains('Tietokantojen perusteet')
       cy.contains('A581328')
       cy.contains('AYTKT10004')
       cy.contains('TKT10004, 581328')
@@ -107,46 +105,39 @@ describe('Course Statistics tests', () => {
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
 
-      cy.contains('TKT10004, 581328 Tietokantojen perusteet')
+      cy.contains('TKT10004, 581328, AYTKT10004, A581328 Tietokantojen perusteet')
       cy.get('.right').click()
       cy.contains('No results')
+    })
 
+    it('Provider organization toggle works', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
       cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
-      cy.contains('td', /^AYTKT10004/).click()
 
+      cy.contains('td', /^TKT10004/).click()
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
-      cy.contains('AYTKT10004 Avoin yo: Tietokantojen perusteet')
+
+      cy.contains('TKT10004, 581328, AYTKT10004, A581328 Tietokantojen perusteet')
+      cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.contains('TKT10004, 581328 Tietokantojen perusteet')
     })
 
     it('Searching course by name displays right courses, 10 credit courses', () => {
       cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
-
       cy.contains('Tietorakenteet ja algoritmit')
-      cy.contains('Avoin yo: Tietorakenteet ja algoritmit')
       cy.contains('AYTKT20001')
       cy.contains('TKT20001, 58131')
       cy.contains('td', /^TKT20001/).click()
-
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
 
-      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
       cy.get('.right').click()
       cy.contains('No results')
-
-      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
-      cy.contains('td', /^AYTKT20001/).click()
-
-      cy.contains('Fetch statistics').should('be.enabled').click()
-      cy.contains('Search for courses').should('not.exist')
-      cy.contains('AYTKT20001 Avoin yo: Tietorakenteet ja algoritmit')
-
-      cy.get('.right').click()
-      cy.contains('No results')
-      cy.get("input[name='unifyOpenUniCourses']").parent().click()
       cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
       cy.contains('td', 'TKT20001, 58131, AYTKT20001').click()
 
@@ -161,12 +152,11 @@ describe('Course Statistics tests', () => {
       cy.get("input[placeholder='Search by a course code']").type('TKT20003')
       cy.contains('tr', 'TKT20003').click()
       cy.contains('Fetch statistics').should('be.enabled').click()
-
       cy.contains('Käyttöjärjestelmät')
       cy.contains('TKT20003')
 
-      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
-      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020 (open and normal')
       cy.contains('TKT20003')
 
       cy.contains('Students (127)').click()
@@ -176,21 +166,33 @@ describe('Course Statistics tests', () => {
 
     it('Population of course shows grades for each student', () => {
       cy.get("input[placeholder='Search by a course code']").type('TKT20001')
-      cy.contains(/^TKT20001$/).click()
+      cy.contains(/^TKT20001, AYTKT20001$/).click()
       cy.contains('Fetch statistics').should('be.enabled').click()
-      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
-      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
-      cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019')
-      cy.contains('Students (197)').click()
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
+      cy.get('tbody >:nth-child(3) > :nth-child(2) .level').click()
+      cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019 (open and normal)')
+      cy.contains('Students (204)').click()
       cy.contains('td', '010262566').siblings().contains('4')
       cy.contains('td', '010674989').siblings().contains('1')
     })
 
+    it('Language distribution is correct', () => {
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
+      cy.contains(/^TKT20003$/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
+      cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.contains('Language distribution').click()
+      cy.contains('td', 'finnish').siblings().contains('122')
+      cy.contains('td', 'english').siblings().contains('5')
+    })
+
     describe('When searching unified course stats', () => {
       beforeEach(() => {
         cy.url().should('include', '/coursestatistics')
         cy.contains('Search for courses')
-        cy.get("input[name='unifyOpenUniCourses']").parent().click()
         cy.get("input[placeholder='Search by a course code']").type('TKT10002')
         cy.contains('td', 'TKT10002, AYTKT10002').click()
         cy.contains('Fetch statistics').should('be.enabled').click()
@@ -202,18 +204,28 @@ describe('Course Statistics tests', () => {
       const yearRange = { from: '2000-2001', to: '2020-2021' }
       const attemptsTableContents = [
         // [time, passed, failed, passrate]
-        ['Total', 511, 486, 25],
+        ['Total', null, 511, 486, 25],
+      ]
+
+      const attemptsTableContentsOpen = [
+        // [time, passed, failed, passrate]
+        ['Total', null, 175, 175, 0],
+      ]
+
+      const attemptsTableContentsReqular = [
+        // [time, passed, failed, passrate]
+        ['Total', null, 312, 298, 14],
       ]
 
       const gradesTableContents = [
         // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-        ['Total', 511, 25, 11, 17, 21, 59, 365, 13],
-        ['2020-2021', 1, 0, 0, 0, 0, 0, 1, 0],
-        ['2019-2020', 54, 4, 3, 1, 3, 4, 39, 0],
-        ['2018-2019', 154, 7, 2, 6, 5, 15, 119, 0],
-        ['2017-2018', 186, 9, 3, 5, 6, 16, 139, 8],
-        ['2016-2017', 71, 2, 1, 2, 5, 13, 48, 0],
-        ['2015-2016', 21, 1, 1, 1, 0, 7, 8, 3],
+        ['Total', null, 511, 25, 11, 17, 21, 59, 365, 13],
+        ['2020-2021', null, 1, 0, 0, 0, 0, 0, 1, 0],
+        ['2019-2020', null, 54, 4, 3, 1, 3, 4, 39, 0],
+        ['2018-2019', null, 154, 7, 2, 6, 5, 15, 119, 0],
+        ['2017-2018', null, 186, 9, 3, 5, 6, 16, 139, 8],
+        ['2016-2017', null, 71, 2, 1, 2, 5, 13, 48, 0],
+        ['2015-2016', null, 21, 1, 1, 1, 0, 7, 8, 3],
       ]
 
       it('shows stats', () => {
@@ -240,6 +252,7 @@ describe('Course Statistics tests', () => {
               .eq(trIndex)
               .within(() => {
                 values.forEach((value, tdIndex) => {
+                  if (value === null) return
                   cy.get('td').eq(tdIndex).contains(value)
                 })
               })
@@ -253,6 +266,7 @@ describe('Course Statistics tests', () => {
               .eq(trIndex)
               .within(() => {
                 values.forEach((value, tdIndex) => {
+                  if (value === null) return
                   cy.get('td').eq(tdIndex).contains(value)
                 })
               })
@@ -261,7 +275,45 @@ describe('Course Statistics tests', () => {
         })
       })
 
-      it('after changing time range shows same stats', () => {
+      it('Toggling course provider changes stats correctly', () => {
+        cy.get('[data-cy=unify_radio_reqular]').click()
+        cy.get("div[name='toYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', '2020-2021')
+          cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', yearRange.from)
+          cy.get("div[role='option']").should('have.length', 21)
+        })
+        cy.contains('#CourseStatPanes a.item', 'Table').click()
+        cy.contains('#CourseStatPanes a.item', 'Attempts').click()
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          attemptsTableContentsReqular.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  if (value === null) return
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
+              })
+          })
+        })
+        cy.get('[data-cy=unify_radio_open]').click()
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          attemptsTableContentsOpen.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  if (value === null) return
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
+              })
+          })
+        })
+        cy.get('[data-cy=unify_radio_unify]').click()
+      })
+
+      it('After changing time range shows same stats', () => {
         const newYearRange = { from: '2016-2017', to: '2019-2020' }
         cy.get("div[name='fromYear']")
           .click()
@@ -290,9 +342,25 @@ describe('Course Statistics tests', () => {
         cy.contains('Show population').should('be.enabled')
       })
     })
+
+    it('If no data available, unify toggle is disabled', () => {
+      cy.get("input[placeholder='Search by a course code']").type('AYCSM14111')
+      cy.contains('AYCSM14111').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('AYCSM14111 Avoin yo: Full Stack -websovelluskehitys: React Native')
+      cy.get('[data-cy=unify_radio_reqular]').find('input').should('be.disabled')
+      cy.get('[data-cy=unify_radio_unify]').find('input').should('not.be.disabled')
+    })
+
+    it('Has right to see all the sudents, because course provider is TKT', () => {
+      cy.visit('coursestatistics?courseCodes=%5B%22TKT10004%22%5D&cs_tab=0&separate=false')
+      cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
+      cy.contains('Students (154)').click()
+      cy.contains('015224224')
+    })
   })
 
-  it('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
+  it.skip('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
     cy.init('/coursestatistics', 'onlycoursestatistics')
     cy.get('[data-cy=navbar-courseStatistics]').click()
     cy.get('[data-cy=course-code-input]').type('TKT20003')
@@ -302,13 +370,17 @@ describe('Course Statistics tests', () => {
     cy.contains('Faculty statistics').should('not.exist')
     cy.contains('Show population').should('not.exist')
 
+    cy.cs('viewMode-Attempts').click()
+
     const attemptsTableContents = [
       // [time, passed, failed, passrate]
-      ['Total*', 288, 213, 75, '73.96 %'],
+      ['Total', 288, 213, 75, '73.96 %'], //Somehow cypress in CI can't find the number even if it is present in the artifact video
       ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
-      ['2019-2020', 164, 121, 43, '73.78 %'],
-      ['2018-2019', 85, 60, 25, '70.59 %'],
-      ['2017-2018', 39, 32, 7, '82.05 %'],
+      // ['Total', null, 295, 216, 79, '73.22 %'],
+      // ['2020-2021', null, 5, 2, 3, '40.00 %'],
+      ['2019-2020', null, 164, 121, 43, '73.78 %'],
+      ['2018-2019', null, 85, 60, 25, '70.59 %'],
+      ['2017-2018', null, 39, 32, 7, '82.05 %'],
     ]
 
     cy.contains('#CourseStatPanes a.item', 'Attempts').click()
@@ -318,6 +390,7 @@ describe('Course Statistics tests', () => {
           .eq(trIndex)
           .within(() => {
             values.forEach((value, tdIndex) => {
+              if (value === null) return
               cy.get('td').eq(tdIndex).contains(value)
             })
           })

cypress/e2e/Faculty_statistics.js

@@ -0,0 +1,70 @@
+/// <reference types="Cypress" />
+
+// Change admin to regular user when feature is ready for general use
+
+describe('Faculty overview', () => {
+  describe('Faculty can be selected', () => {
+    it('Faculties are listed and one can be chosen', () => {
+      cy.init('/faculties', 'admin')
+      cy.get('[data-cy=select-faculty]').contains('td', 'H10')
+      cy.contains('td', 'H99').should('not.exist')
+      cy.contains('td', 'H90').click()
+      cy.contains('.header', 'Eläinlääketieteellinen tiedekunta')
+    })
+  })
+
+  describe('Faculty basic information', () => {
+    beforeEach(() => {
+      cy.init('/faculties', 'admin')
+      cy.contains('td', 'H90').click()
+    })
+    it('Credits produced by faculty are shown', () => {
+      cy.get(['data-cy=Section-CreditsProducedByTheFaculty'])
+      cy.get(['data-cy=Graph-CreditsProducedByTheFaculty'])
+    })
+  })
+
+  describe('Study programme information', () => {
+    beforeEach(() => {
+      cy.init('/faculties', 'admin')
+      cy.contains('td', 'H80').click()
+    })
+
+    it('Study programme credit information is not visible in the beginning', () => {
+      cy.get('table[data-cy="Table-CreditsProducedByTheFaculty"]').should('be.visible')
+
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-2"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-3"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-4"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-5"]').should('not.be.visible')
+    })
+
+    it('Study programme credit information can be toggled', () => {
+      cy.get('table[data-cy="Table-CreditsProducedByTheFaculty"]').should('be.visible')
+
+      cy.get('div[data-cy="Button-Show-CreditsProducedByTheFaculty-0"]').click()
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-2"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-3"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-4"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-5"]').should('not.be.visible')
+      cy.get('div[data-cy="Button-Show-CreditsProducedByTheFaculty-3"]').click()
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-2"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-3"]').should('be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-4"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-5"]').should('not.be.visible')
+      cy.get('div[data-cy="Button-Hide-CreditsProducedByTheFaculty-0"]').click()
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-2"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-3"]').should('be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-4"]').should('not.be.visible')
+      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-5"]').should('not.be.visible')
+    })
+  })
+})

cypress/e2e/Filters.js

@@ -0,0 +1,402 @@
+/// <reference types="Cypress" />
+
+// const baseUrl = Cypress.config().baseUrl
+
+const MOCKED_DATE = new Date(2022, 6, 12).getTime()
+
+const checkFilteringResult = studentCount => {
+  cy.contains(`Students (${studentCount})`)
+}
+
+// Semantic UI doesn't allow injection of data-cy:s for single multiple dropdown selection.
+// This function tries to click "x" on all selections inside dropdown matching given attribute
+const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
+  cy.cs(dataCyAttribute).find('i.delete').click({ force: true })
+  cy.cs(dataCyAttribute).click()
+}
+
+const clearSingleDropdownSelection = dataCyAttribute => {
+  cy.cs(dataCyAttribute).find('i.clear').click({ force: true })
+}
+
+const testRangeFilter = (parentEl, min, max, expected) => {
+  cy.cs(parentEl)
+    .cs('range-selector-min')
+    .find('input')
+    .invoke('val')
+    .then(initialMin => {
+      cy.cs(parentEl)
+        .cs('range-selector-max')
+        .find('input')
+        .invoke('val')
+        .then(initialMax => {
+          cy.cs(parentEl).cs('range-selector-min').find('input').type(`{selectall}{backspace}${min}`).blur()
+          cy.cs(parentEl).cs('range-selector-max').find('input').type(`{selectall}{backspace}${max}`).blur()
+
+          checkFilteringResult(expected)
+
+          cy.cs(parentEl).cs('range-selector-min').find('input').type(`{selectall}{backspace}${initialMin}`).blur()
+          cy.cs(parentEl).cs('range-selector-max').find('input').type(`{selectall}{backspace}${initialMax}`).blur()
+        })
+    })
+}
+
+// Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
+// to every it-function. Reason behind using test function wrapper is that Cypresses internal beforeEach / afterEach
+// functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
+const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
+  return (filterName, testStepFunctionToRun) => {
+    const card = cy.cs(`${filterName}-filter-card`)
+
+    card.invoke('attr', 'data-open').then(open => {
+      console.log('open', open)
+
+      const getHeader = () => cy.cs(`${filterName}-header`)
+
+      if (open === 'false') {
+        getHeader().click({ force: true })
+      }
+
+      checkFilteringResult(amountWithoutFiltering)
+      testStepFunctionToRun()
+      checkFilteringResult(amountWithoutFiltering)
+
+      if (open === 'false') {
+        getHeader().click({ force: true })
+      }
+    })
+  }
+}
+
+describe('Population Statistics', () => {
+  const pathToCSBach2018 =
+    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2018'
+  const defaultAmountOfStudents = 159
+  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
+
+  before(() => {
+    cy.init(pathToCSBach2018)
+  })
+
+  it('Transfer filter is on not transferred by default', () => {
+    cy.contains('By default only students who have not transferred to this study programme are shown.')
+
+    runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
+      const card = cy.cs('TransferredToProgramme-filter-card')
+      card.get('[data-cy="option-havenot"] input').should('be.checked')
+    })
+  })
+
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(5)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
+  // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
+  // So fix this
+  it.skip('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Graduation filter works', () => {
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
+
+      const graduated = 11
+      getCard().cs('option-graduated-true').click()
+      checkFilteringResult(graduated)
+      getCard().cs('option-graduated-false').click()
+      checkFilteringResult(defaultAmountOfStudents - graduated)
+      getCard().cs('option-all').click()
+    })
+  })
+
+  it('Transfer filter works', () => {
+    runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
+      const transferred = 1
+      cy.cs('TransferredToProgramme-filter-card').cs('option-have').click()
+      checkFilteringResult(transferred)
+      cy.cs('TransferredToProgramme-filter-card').cs('option-all').click()
+      checkFilteringResult(defaultAmountOfStudents + transferred)
+      cy.cs('TransferredToProgramme-filter-card').cs('option-havenot').click()
+    })
+  })
+
+  it('Enrollment filter works', () => {
+    runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
+      cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
+      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(0)
+      checkFilteringResult(148)
+      clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
+    })
+  })
+
+  it('Credit filter works', () => {
+    runTestStepWithPreAndPostParts('CreditsEarned', () => {
+      testRangeFilter('CreditsEarned-filter-card', 50, 150, 118)
+    })
+  })
+
+  it('Age filter works', () => {
+    cy.clock(MOCKED_DATE, ['Date'])
+    runTestStepWithPreAndPostParts('Age', () => {
+      testRangeFilter('Age-filter-card', 20, 40, 47)
+    })
+  })
+
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(42)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(117)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
+      checkFilteringResult(0)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(7)
+      checkFilteringResult(152)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+    })
+  })
+
+  // Return when we have +2020 population with admission types in data
+  it.skip('Admission type filter works', () => {
+    runTestStepWithPreAndPostParts('AdmissionTypeFilter', () => {
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(5)
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(2)
+      checkFilteringResult(10)
+      clearSingleDropdownSelection('admissionTypeFilter-dropdown')
+    })
+  })
+
+  // plz fix
+  it.skip('Courses filter works', () => {
+    // courses takes some time to load, wait for it to complete
+    cy.wait(10000)
+    runTestStepWithPreAndPostParts('Courses', () => {
+      cy.cs('courseFilter-course-dropdown').click()
+      const courses = ['TKT20001', 'MAT11002']
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
+      checkFilteringResult(140)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
+      checkFilteringResult(131)
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+      checkFilteringResult(56)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
+      checkFilteringResult(2)
+      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
+    })
+  })
+
+  it('Filter combinations work', () => {
+    cy.clock(MOCKED_DATE, ['Date'])
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
+        const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
+        getCard().cs('option-graduated-true').click()
+        testRangeFilter('Age-filter-card', 20, 30, 1)
+        getCard().cs('option-all').click()
+      })
+    })
+  })
+})
+
+describe('Course Statistics', () => {
+  const pathToDSAndAlgoSpring2019 =
+    '/coursepopulation?coursecodes=%5B%22TKT20001%22%2C%2258131%22%5D&from=138&separate=true&to=138&years=Kev%C3%A4t%202019&unifyCourses=unifyStats'
+  const defaultAmountOfStudents = 118
+  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
+
+  before(() => {
+    cy.init(pathToDSAndAlgoSpring2019)
+  })
+
+  it('Grade filter works', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
+      cy.cs('gradeFilter-5').click()
+      checkFilteringResult(20)
+      cy.cs('gradeFilter-3').click()
+      checkFilteringResult(38)
+      cy.cs('gradeFilter-3').click()
+      cy.cs('gradeFilter-5').click()
+    })
+  })
+
+  it('Programme defaults to "Attainment" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Attainment')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(116)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
+  // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
+  // So fix this
+  it.skip('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Age filter works', () => {
+    cy.clock(MOCKED_DATE, ['Date'])
+    runTestStepWithPreAndPostParts('Age', () => {
+      testRangeFilter('Age-filter-card', 20, 40, 33)
+    })
+  })
+
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(27)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(91)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
+      checkFilteringResult(0)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(3)
+      checkFilteringResult(84)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+    })
+  })
+
+  it('Filter combinations work', () => {
+    cy.clock(MOCKED_DATE, ['Date'])
+    runTestStepWithPreAndPostParts('Grade', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
+        cy.cs('gradeFilter-3').click()
+        testRangeFilter('Age-filter-card', 20, 30, 1)
+        cy.cs('gradeFilter-3').click()
+      })
+    })
+  })
+})
+
+describe('Custom Population Statistics', () => {
+  let runTestStepWithPreAndPostParts
+
+  before(() => {
+    cy.init('/custompopulation')
+    cy.cs('custom-pop-search-button').click()
+    cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
+      const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
+      runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
+      cy.cs('student-no-input').click().type(students.join('\n'))
+      cy.wait(10000)
+    })
+    cy.cs('search-button').click()
+  })
+
+  it('Age filter works', () => {
+    cy.clock(MOCKED_DATE, ['Date'])
+    runTestStepWithPreAndPostParts('Age', () => {
+      testRangeFilter('Age-filter-card', 20, 30, 1)
+    })
+  })
+
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(5)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+      checkFilteringResult(5, true)
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.cs('startYearAtUni-dropdown').click()
+    })
+  })
+
+  it.skip('Courses filter works', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
+      const courses = ['MAT11001', 'TKT20004']
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[0] + ' ')
+        .click()
+      checkFilteringResult(5)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
+      checkFilteringResult(5)
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[1] + ' ')
+        .click()
+      checkFilteringResult(3)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(3)
+      checkFilteringResult(1)
+      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
+    })
+  })
+})

cypress/e2e/Population_statistics.js

@@ -3,15 +3,15 @@
 const moment = require('moment')
 const _ = require('lodash')
 
-const setPopStatsUntil = (until, includeSettings = []) => {
-  cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
-  includeSettings.forEach(setting => {
-    cy.contains('Advanced settings').parent().siblings().contains(setting).click()
-  })
-  cy.get('.adv-stats-until > .form-control').click().clear().type(until)
-  cy.contains('Fetch population with new settings').click()
-  cy.contains('Advanced settings')
-}
+// const setPopStatsUntil = (until, includeSettings = []) => {
+//   cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
+//   includeSettings.forEach(setting => {
+//     cy.contains('Advanced settings').parent().siblings().contains(setting).click()
+//   })
+//   cy.get('.adv-stats-until > .form-control').click().clear().type(until)
+//   cy.contains('Fetch population with new settings').click()
+//   cy.contains('Advanced settings')
+// }
 
 describe('Population Statistics tests', () => {
   const pathToCSBach2017 =
@@ -65,13 +65,14 @@ describe('Population Statistics tests', () => {
     it('Population statistics is usable on general level', () => {
       cy.visit(pathToCSBach2017)
       cy.cs('filtered-students')
-      setPopStatsUntil('toukokuu 2020')
+      // This advanced settings does not work correctly
+      // see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3552
+      // setPopStatsUntil('toukokuu 2020')
 
       cy.get('.card').within(() => {
         cy.contains('Tietojenkäsittelytieteen kandiohjelma')
-        cy.contains('Sample size: 149 students')
+        cy.contains('Sample size: 170 students')
         cy.contains('Excludes exchange students')
-        cy.contains("Excludes students who haven't enrolled present nor absent")
         cy.contains('Excludes students with non-degree study right')
         cy.contains('Excludes students who have transferred out of this programme')
       })
@@ -80,17 +81,17 @@ describe('Population Statistics tests', () => {
 
       cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
       // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000) // a bit hacky way, wait until ui is ready
-      cy.get('[data-cy=expand-TKT1]').click()
-      cy.get('[data-cy=coursestats-link-TKT10002]').click()
+      cy.wait(25000) // a bit hacky way, wait until ui is ready
+      cy.get('[data-cy=toggle-group-module-TKT1]').click()
+      cy.contains('td', 'TKT10002').siblings().find('i.level.up').click()
       cy.wait('@coursePage')
       cy.url().should('include', '/coursestatistics')
-      cy.contains('TKT10002, 581325 Ohjelmoinnin perusteet')
+      cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
     })
 
     it("Empty 'tags' tab has a link to the page where tags can be created", () => {
       cy.visit(pathToCSBach2017)
-      cy.contains('Students (149)').click()
+      cy.contains('Students (170)').click()
       cy.get('[data-cy=student-table-tabs]').contains('Tags').click()
       cy.contains('No tags defined. You can define them here.').find('a').click()
       cy.contains('Tags').click()
@@ -119,12 +120,11 @@ describe('Population Statistics tests', () => {
 
       cy.contains('Fetch population').click()
 
-      cy.contains('Credit accumulation (for 149 students)')
+      cy.contains('Credit accumulation (for 170 students)')
 
-      // spring + fall and include cancelled
+      // spring + fall
       cy.get('[data-cy=advanced-toggle]').click()
       cy.cs('toggle-spring').click()
-      cy.cs('toggle-cancelled').click()
       cy.contains('Fetch population').click()
 
       cy.contains('Credit accumulation (for 170 students)')
@@ -133,6 +133,7 @@ describe('Population Statistics tests', () => {
     it('Credit Statistics, Credits Gained tab works', () => {
       cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
       cy.contains('Credit statistics').click()
+      cy.contains('Credits Gained').click()
       cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the population')
 
       const months = Math.ceil(moment.duration(moment().diff(moment('2017-08-1'))).asMonths())
@@ -140,7 +141,7 @@ describe('Population Statistics tests', () => {
       const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
       const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))
 
-      cy.get('.credits-gained-table').should('contain', '(n=149)')
+      cy.get('.credits-gained-table').should('contain', '(n=170)')
 
       for (const [start, end] of ranges) {
         let value = 'credits'
@@ -159,7 +160,7 @@ describe('Population Statistics tests', () => {
       cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
         const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)
 
-        expect(sum).to.equal(149)
+        expect(sum).to.equal(170)
       })
 
       cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('not.exist')
@@ -170,6 +171,7 @@ describe('Population Statistics tests', () => {
       cy.cs('filtered-students')
       cy.contains('Credit statistics')
       cy.get("[data-cy='credit-statistics']").click()
+      cy.contains('Credits Gained').click()
       cy.get('.credits-gained-divider').click()
       cy.get("[data-cy='credits-gained-table-Avoin väylä']").should('exist')
       cy.get("[data-cy='credits-gained-table-Yhteispisteet']").should('exist')
@@ -182,10 +184,10 @@ describe('Population Statistics tests', () => {
       cy.contains('Credit statistics').click({ force: true })
       cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
 
-      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
-      cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
-      cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
-      cy.get("[data-cy='credit-stats-min']").should('contain', '11')
+      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 170 Students')
+      cy.get("[data-cy='credit-stats-mean']").should('contain', '114.75')
+      cy.get("[data-cy='credit-stats-stdev']").should('contain', '65.53')
+      cy.get("[data-cy='credit-stats-min']").should('contain', '5')
       cy.get("[data-cy='credit-stats-max']").should('contain', '314')
     })
   })
@@ -198,7 +200,7 @@ describe('Population Statistics tests', () => {
       const existing = '010113437'
       const nonExisting = '66666666'
       cy.visit(pathToCSBach2017)
-      cy.contains('Students (149)').click()
+      cy.contains('Students (170)').click()
       cy.contains(existing)
       cy.contains(nonExisting).should('not.exist')
       cy.contains('button', 'Check studentnumbers').click()
@@ -215,4 +217,30 @@ describe('Population Statistics tests', () => {
       })
     })
   })
+
+  describe('when using IAM user', () => {
+    beforeEach(() => {
+      cy.init(pathToCSBach2017, 'onlyiamrights')
+    })
+
+    it.skip('Population statistics is visible', () => {
+      cy.reload()
+
+      cy.get('.card').within(() => {
+        cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+        cy.contains('Sample size: 170 students')
+        cy.contains('Excludes exchange students')
+        cy.contains('Excludes students with non-degree study right')
+        cy.contains('Excludes students who have transferred out of this programme')
+      })
+
+      cy.contains('Credit accumulation (for 170 students)')
+      cy.contains('Credit statistics')
+    })
+
+    it.skip('Students tab is not available', () => {
+      cy.contains('Students (170)').should('not.exist')
+      // TODO: somehow check that obfuscation works
+    })
+  })
 })

cypress/e2e/Student_statistics.js

@@ -11,9 +11,10 @@ describe('Student Statistics tests', () => {
     firstnames: 'Matti Jaakko',
     lastname: 'Nieminen',
     studentnumber: '010654019',
+    sis_person_id: 'hy-hlo-84830887',
   }
 
-  it.only('Student statistics search form is usable', () => {
+  it('Student statistics search form is usable', () => {
     cy.contains('Show student names')
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.lastname)
@@ -37,9 +38,9 @@ describe('Student Statistics tests', () => {
   it('Can get student specific page by clicking student', () => {
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
-    cy.contains('Started: 01.08.2011')
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 01.08.2025')
+    cy.contains('td a', student.studentnumber).click()
+    cy.contains('Started: 01.08.2017')
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 31.07.2025')
     cy.contains(student.lastname).should('not.exist')
     cy.contains(student.firstnames).should('not.exist')
 
@@ -54,7 +55,7 @@ describe('Student Statistics tests', () => {
 
   it('Can get back to search menu', () => {
     cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
     cy.get('.remove').click()
     cy.contains('Student number').should('not.exist')
     cy.contains('Credits').should('not.exist')
@@ -62,7 +63,7 @@ describe('Student Statistics tests', () => {
 
   it('Can jump to course', () => {
     cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
     cy.contains('Introduction to Machine Learning (DATA11002)')
       .siblings()
       .within(() => {
@@ -72,6 +73,14 @@ describe('Student Statistics tests', () => {
     cy.contains('Introduction to Machine Learning')
   })
 
+  it('Has correct Sisu link', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains('td a', student.studentnumber).click()
+    cy.get('[data-cy=sisulink] > a')
+      .should('have.attr', 'href')
+      .and('include', 'https://sis-helsinki.funidata.fi/tutor/role/staff/student/hy-hlo-84830887/basic/basic-info')
+  })
+
   it('Searching with bad inputs doesnt yield results', () => {
     cy.get('.prompt').type('SWAG LITTINEN')
     cy.contains('Student number').should('not.exist')
@@ -80,11 +89,21 @@ describe('Student Statistics tests', () => {
     cy.contains('Student number').should('not.exist')
   })
 
-  it.only('Can jump to population page', () => {
+  it('Can jump to population page', () => {
     cy.get('.prompt').type(student.studentnumber)
-    cy.contains(student.studentnumber).click()
+    cy.contains('td a', student.studentnumber).click()
     cy.get('i.level.up.alternate.icon').eq(0).click()
     cy.contains('Population statistics')
     cy.contains('Tietojenkäsittelytieteen kandiohjelma')
   })
 })
+
+// Use admin to see all students
+describe('Testing with admin rights', () => {
+  it('Does not crash if student has no studyright or courses', () => {
+    const studentNumber = '011825096'
+    cy.init(`/students/${studentNumber}`, 'admin') // use admin to see all students
+    cy.contains('Started: Unavailable')
+    cy.contains('Credits: 0')
+  })
+})

cypress/e2e/Studyprogramme_overview.js

@@ -0,0 +1,540 @@
+/// <reference types="Cypress" />
+
+const deleteTag = name => {
+  cy.contains('td', name).siblings().contains('Delete').click()
+  cy.contains('Are you sure you want to delete tag')
+  cy.contains('Confirm').click()
+  cy.contains('td', name).should('not.exist')
+}
+
+const getEmptyYears = isAcademicYear => {
+  const thisYear = new Date().getFullYear()
+  const years = []
+  for (let year = thisYear; year > 2021; year--) {
+    if (isAcademicYear) {
+      years.push(`${year} - ${year + 1}`)
+    } else {
+      years.push(year)
+    }
+  }
+  return years
+}
+
+describe('Studyprogramme overview', () => {
+  /* Basic information overview -tests*/
+  describe('Basic information -view works for basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme')
+      cy.viewport(1536, 960)
+    })
+
+    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
+    it('Basic information -tab loads', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('[data-cy=Section-StudentsOfTheStudyprogramme]')
+      cy.get('[data-cy=Section-CreditsProducedByTheStudyprogramme]')
+      cy.get('[data-cy=Section-GraduatedAndThesisWritersOfTheProgramme]')
+      cy.get('[data-cy=Section-ProgrammesBeforeOrAfter]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
+    })
+
+    it('Basic information contains correct students', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Started, Graduated, Transferred Away, Transferred to]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        [2021, 0, 0, 1, 0],
+        [2020, 12, 35, 3, 2],
+        [2019, 87, 17, 0, 1],
+        [2018, 161, 1, 0, 0],
+        [2017, 171, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudentsOfTheStudyprogramme')
+    })
+
+    it('Basic information contains correct credits', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Total, Major credits, Non-major credits, Non-degree credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0, 0, 0]),
+        [2021, 0, 0, 0, 0, 0],
+        [2020, 5860, 5591, 102, 0, 167],
+        [2019, 8696, 7737, 587, 23, 349],
+        [2018, 7818, 6584, 857, 105, 272],
+        [2017, 3085, 2492, 158, 47, 388],
+      ]
+
+      cy.checkTableStats(tableContents, 'CreditsProducedByTheStudyprogramme')
+    })
+
+    it('Basic information contains correct thesis writers and graduates', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Graduated, Wrote thesis]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 35, 35],
+        [2019, 17, 25],
+        [2018, 1, 5],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(tableContents, 'GraduatedAndThesisWritersOfTheProgramme')
+    })
+
+    it.skip('Special studyrights can be excluded and basic data changes accordingly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen maisteriohjelma').click()
+      cy.get('[data-cy=StudentToggle]').click()
+      const years = getEmptyYears()
+      const studentTableContents = [
+        // [Year, Started, Graduated]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 22, 0],
+        [2019, 12, 0],
+        [2018, 0, 0],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
+
+      const creditTableContents = [
+        // [Year, Total, Major students credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0]),
+        [2021, 0, 0, 0],
+        [2020, 117.5, 0, 7.5],
+        [2019, 67, 0, 0],
+        [2018, 0, 0, 0],
+        [2017, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+
+      const graduatedTableContents = [
+        // [Year, Graduated, Wrote thesis]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 0, 0],
+        [2019, 0, 0],
+        [2018, 0, 0],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(graduatedTableContents, 'GraduatedAndThesisWritersOfTheProgramme')
+      cy.get('[data-cy=StudentToggle]').click()
+    })
+
+    it('Year can be changed to academic year, and data changes accordingly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen maisteriohjelma').click()
+      cy.get('[data-cy=YearToggle]').click()
+      const isAcademicYear = true
+      const years = getEmptyYears(isAcademicYear)
+      const studentTableContents = [
+        // [Year, Started, Graduated, Transferred away, Transferred to]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0],
+        ['2020 - 2021', 5, 0, 0, 0],
+        ['2019 - 2020', 27, 0, 0, 0],
+        ['2018 - 2019', 4, 0, 0, 0],
+        ['2017 - 2018', 0, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
+
+      const creditTableContents = [
+        // [Year, Total, Major students credits, Non-major students credits, Non-degree credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0],
+        ['2020 - 2021', 0, 0, 0, 0],
+        ['2019 - 2020', 338.5, 177, 154, 0, 7.5],
+        ['2018 - 2019', 173, 0, 173, 0, 0],
+        ['2017 - 2018', 42, 0, 42, 0, 0],
+      ]
+
+      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+      cy.get('[data-cy=YearToggle]').click()
+    })
+
+    it('Basic information graphs render', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+
+      cy.get('[data-cy=Graph-StudentsOfTheStudyprogramme')
+        .should('contain', 'Started')
+        .should('contain', 'Graduated')
+        .should('contain', 'Transferred away')
+
+      cy.get('[data-cy=Graph-CreditsProducedByTheStudyprogramme')
+        .should('contain', 'Major students credits')
+        .should('contain', 'Non-major students credits')
+        .should('contain', 'Transferred credits')
+        .should('contain', 857)
+        .should('contain', 587)
+
+      cy.get('[data-cy=Graph-GraduatedAndThesisWritersOfTheProgramme')
+        .should('contain', 'Graduated students')
+        .should('contain', 'Wrote thesis')
+        .should('contain', 1)
+        .should('contain', 5)
+        .should('contain', 17)
+        .should('contain', 25)
+        .should('contain', 35)
+
+      cy.get('[data-cy=Graph-2019-AverageGraduationTimes]')
+        .should('contain', '2019')
+        .should('contain', 'n = 17')
+        .should('contain', '23 kk')
+
+      cy.get('[data-cy=Graph-ProgrammesBeforeOrAfter')
+        .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
+        .should('contain', 'Datatieteen maisteriohjelma')
+        .should('contain', 'Matematiikan ja tilastotieteen maisteriohjelma')
+        .should('contain', 1)
+        .should('contain', 4)
+        .should('contain', 13)
+    })
+  })
+
+  /* Studytrack overview -tests*/
+  describe('Studytrack overview works for basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme')
+    })
+
+    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
+    it('Studytracks and student populations -tab loads', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      cy.get('[data-cy=Section-StudytrackOverview]')
+      cy.get('[data-cy=Section-StudytrackProgress]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
+    })
+
+    it('Students of the studyprogramme are shown correctly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      const tableContents = [
+        // [Year, All, Started studying, Currently enrolled, Absent, Cancelled, Graduated, Men, Women, Finnish]
+        [
+          '2020 - 2021',
+          12,
+          '100.0 %',
+          10,
+          '83.3 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          10,
+          '83.3 %',
+          2,
+          '16.7 %',
+          12,
+          '100.0 %',
+        ],
+        [
+          '2019 - 2020',
+          86,
+          '100.0 %',
+          86,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          1,
+          '1.2 %',
+          1,
+          '1.2 %',
+          65,
+          '75.6 %',
+          21,
+          '24.4 %',
+          86,
+          '100.0 %',
+        ],
+        [
+          '2018 - 2019',
+          162,
+          '100.0 %',
+          161,
+          '99.4 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          11,
+          '6.8 %',
+          118,
+          '72.8 %',
+          44,
+          '27.2 %',
+          161,
+          '99.4 %',
+        ],
+        [
+          '2017 - 2018',
+          171,
+          '100.0 %',
+          171,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          4,
+          '2.3 %',
+          42,
+          '24.6 %',
+          135,
+          '78.9 %',
+          36,
+          '21.1 %',
+          166,
+          '97.1 %',
+        ],
+        [
+          'Total',
+          431,
+          '100.0 %',
+          431,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          5,
+          '1.2 %',
+          54,
+          '12.5 %',
+          328,
+          '76.1 %',
+          103,
+          '23.9 %',
+          425,
+          '98.6 %',
+        ],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudytrackOverview')
+    })
+
+    it('Student progress data is shown correctly', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      const tableContents = [
+        ['2020 - 2021', 12, 5, 2, 2, 0, 2, 0, 1],
+        ['2019 - 2020', 86, 13, 15, 32, 11, 2, 0, 13],
+        ['2018 - 2019', 162, 7, 16, 35, 33, 27, 18, 26],
+        ['2017 - 2018', 171, 10, 22, 18, 22, 24, 27, 48],
+        ['Total', 431, 35, 55, 87, 66, 55, 45, 88],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudytrackProgress')
+    })
+
+    it('Studytrack overview graphs render', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+
+      cy.get('[data-cy=Graph-StudytrackProgress]')
+        .should('contain', 'Less than 30 credits')
+        .should('contain', '30-59 credits')
+        .should('contain', 'More than 180 credits')
+        .should('contain', '5.8%') // The percentage for less than 30 credits in 2017-2018, to check that the graph renders
+        .should('contain', '4.3%') // The percentage for less than 30 credits in 2018-2019
+        .should('contain', '15.1%') // The percentage for less than 30 credits in 2019-2020
+
+      cy.get('[data-cy=Graph-2018-AverageGraduationTimes]')
+        .should('contain', '2018 - 2019')
+        .should('contain', 'n = 11 / 162')
+        .should('contain', '22 kk')
+    })
+  })
+
+  /* Programme courses -tests*/
+  describe('Programme courses works for basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme')
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Programme courses').click()
+    })
+
+    it('content loads', () => {
+      cy.get('[data-cy=CoursesYearFilter]')
+      cy.get('[data-cy=CourseTabs]')
+    })
+
+    it('time range selection works', () => {
+      cy.get('[data-cy=fromYear]').click().contains('2018').click()
+
+      cy.get('[data-cy=toYear').click().contains('2019').click()
+
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('684')
+      })
+    })
+
+    it('calendar year -> academic year toggle works', () => {
+      cy.get('[data-cy=fromYear]').click().contains('2018').click()
+
+      cy.get('[data-cy=toYear').click().contains('2019').click()
+
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('684')
+      })
+
+      cy.get('[data-cy=calendarAcademicYearToggle]').first().click()
+
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('772')
+      })
+    })
+
+    it('contains correct courses in alphabetical order', () => {
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('Aineopintojen harjoitustyö: Tietokantasovellus')
+        cy.get('tr').eq(-1).contains('Äidinkielinen viestintä')
+      })
+    })
+
+    it('different sorting options work', () => {
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        // Test sorting by different columns
+        cy.get('th').eq(0).click()
+        cy.get('tr').eq(1).contains('Tietojenkäsittelytieteen kisälliopetus: Java-ohjelmointi')
+        cy.get('th').eq(1).click()
+        cy.get('tr').eq(1).contains('Äidinkielinen viestintä')
+        cy.get('th').eq(2).click()
+        cy.get('tr').eq(1).contains('Tietorakenteet ja algoritmit')
+      })
+    })
+
+    it('show credits -> show students toggle works', () => {
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('1068')
+      })
+
+      cy.get('[data-cy=creditsStudentsToggle]').first().click()
+
+      cy.get('[data-cy=CoursesSortableTable]').within(() => {
+        cy.get('tr').eq(1).contains('267')
+      })
+    })
+  })
+
+  /* Tag-tests*/
+  describe('Basic information -view works for basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme')
+    })
+
+    it('can create and delete tags for population', () => {
+      const name = `tag-${new Date().getTime()}`
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+      cy.contains('2018')
+      deleteTag(name)
+    })
+
+    it('can create personal tags', () => {
+      const name = `tag-${new Date().getTime()}`
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+
+      cy.get('.ui > label').click()
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+      deleteTag(name)
+    })
+
+    it('can add tags to students', () => {
+      const name = `tag-${new Date().getTime()}`
+
+      const student = '010113437'
+
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+
+      cy.contains('Add tags to students').click()
+      cy.get('.form > .field > .dropdown').click().get('.ui > input.search').type(name).click()
+
+      cy.get('.form > .field > .dropdown > .visible').contains(name).click()
+
+      cy.get('textarea').type('010113437')
+      cy.get('.positive').click()
+
+      cy.contains('Student statistics').click()
+      cy.get('.prompt').type(student)
+      cy.contains('a', student).click()
+      cy.contains(name)
+
+      cy.go('back')
+      cy.go('back')
+
+      deleteTag(name)
+
+      cy.contains('Student statistics').click()
+      cy.get('.prompt').type(student)
+      cy.contains('a', student).click()
+      cy.contains(name).should('not.exist')
+    })
+  })
+
+  describe('IAM user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme', 'onlyiamrights')
+    })
+
+    it.skip('can access programme and correct tabs are visible', () => {
+      cy.reload()
+
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+
+      cy.contains('Basic information')
+      cy.contains('Studytracks and student populations')
+
+      cy.contains('Update statistics').should('not.exist')
+      cy.contains('Degree Courses').should('not.exist')
+    })
+
+    it.skip('can access basic information', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('Basic information').click()
+
+      cy.get('[data-cy=Section-StudentsOfTheStudyprogramme]')
+      cy.get('[data-cy=Section-CreditsProducedByTheStudyprogramme]')
+      cy.get('[data-cy=Section-GraduatedAndThesisWritersOfTheProgramme]')
+      cy.get('[data-cy=Section-ProgrammesBeforeOrAfter]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
+    })
+
+    it.skip('can access studytracks', () => {
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+
+      cy.get('[data-cy=Section-StudytrackOverview]')
+      cy.get('[data-cy=Section-StudytrackProgress]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
+    })
+  })
+})

cypress/e2e/Teachers.js

@@ -16,7 +16,7 @@ describe('Teachers page tests', () => {
     cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
     cy.cs('course-providers').children('.icon').click()
     cy.get('.form > .fluid').click()
-    cy.contains('Name')
+    cy.contains('Teacher')
     cy.contains(teacher1)
     cy.contains(teacher2)
   })
@@ -32,21 +32,13 @@ describe('Teachers page tests', () => {
     cy.url().should('include', '/teachers')
     cy.get('.borderless > :nth-child(3)').click()
     cy.get('.prompt').type(teacher1.split()[0])
-    cy.contains(teacher1).click()
+    cy.contains('td', teacher1).click()
     cy.contains('Aineopintojen harjoitustyö: Tietokantasovellus')
   })
 
   it('Check leaderboad works', () => {
     cy.get('.borderless > :nth-child(2)').click({ force: true })
     cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.contains('2019-2020').click({ force: true })
-    cy.contains('Recalculate this year').click({ force: true })
-    cy.wait(5000)
-    cy.reload()
-    cy.contains('2019-2020').click({ force: true })
-    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.contains('Passed')
-    cy.contains('Ahokas Jarmo Ilkka')
+    cy.contains('2020-2021').click({ force: true })
   })
 })

cypress/e2e/Users.js

@@ -39,22 +39,15 @@ describe('Users tests', () => {
 
     it("mocking normal user shows only the mocked user's programmes", () => {
       cy.contains('mocking').should('not.exist')
-      cy.contains('tr', 'basic').within($row => {
-        cy.contains('.button', 'Edit').click()
-      })
-      cy.intercept({
-        method: 'POST',
-        path: '/api/superlogin/basic',
-      }).as('superlogin')
+      cy.cs('user-edit-button-basic').click()
       cy.get('i.spy').click()
-      cy.wait('@superlogin')
       cy.contains('mocking as basic')
       cy.wait(1000)
       cy.contains('Study programme').click().siblings().contains('Search by class').click()
       cy.contains('label', 'Study programme')
       cy.contains('label', 'Study programme')
         .siblings()
-        .within($row => {
+        .within(() => {
           cy.get("div[role='option']").should('have.length', 2).contains('Tietojenkäsittelytieteen kandiohjelma')
         })
     })

cypress/integration/Filters.js

@@ -1,297 +0,0 @@
-/// <reference types="Cypress" />
-
-// const baseUrl = Cypress.config().baseUrl
-
-const checkFilteringResult = studentCount => {
-  cy.contains(`Students (${studentCount})`)
-}
-
-// Semantic UI doesn't allow injection of data-cy:s for single multiple dropdown selection.
-// This function tries to click "x" on all selections inside dropdown matching given attribute
-const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.delete').click()
-}
-
-const clearSingleDropdownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.clear').click()
-}
-
-// Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
-// to every it-function. Reason behind using test function wrapper is that Cypresses internal beforeEach / afterEach
-// functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
-const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
-  return (dataCyAttributeOfHeaderToClick, testStepFunctionToRun) => {
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
-    checkFilteringResult(amountWithoutFiltering)
-    testStepFunctionToRun()
-    checkFilteringResult(amountWithoutFiltering)
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
-  }
-}
-
-describe('Population Statistics', () => {
-  const pathToCSBach2018 =
-    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2018'
-  const defaultAmountOfStudents = 148
-  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
-
-  before(() => {
-    cy.init(pathToCSBach2018)
-  })
-
-  it('Transfer filter is on not transferred by default', () => {
-    cy.contains('By default only students who have not transferred to this study programme are shown.')
-    cy.cs('transferredToProgrammeFilter-havenot').find('input').should('be.checked')
-  })
-
-  it('Graduation filter works', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
-      const graduated = 11
-      cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
-      checkFilteringResult(graduated)
-      cy.cs('graduatedFromProgrammeFilter-graduated-false').click()
-      checkFilteringResult(defaultAmountOfStudents - graduated)
-      cy.cs('graduatedFromProgrammeFilter-all').click()
-    })
-  })
-
-  it('Transfer filter works', () => {
-    runTestStepWithPreAndPostParts('transferredToProgrammeFilter-header', () => {
-      const transferred = 1
-      cy.cs('transferredToProgrammeFilter-have').click()
-      checkFilteringResult(transferred)
-      cy.cs('transferredToProgrammeFilter-all').click()
-      checkFilteringResult(defaultAmountOfStudents + transferred)
-      cy.cs('transferredToProgrammeFilter-havenot').click()
-    })
-  })
-
-  it('Enrollment filter works', () => {
-    runTestStepWithPreAndPostParts('enrollmentStatusFilter-header', () => {
-      cy.selectFromDropdown('enrollmentStatusFilter-status', 0)
-      cy.selectFromDropdown('enrollmentStatusFilter-semesters', [0])
-      checkFilteringResult(137)
-      clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
-    })
-  })
-
-  it('Credit filter works', () => {
-    runTestStepWithPreAndPostParts('credit-filter-header', () => {
-      cy.cs('credit-filter-min').type('50')
-      cy.cs('credit-filter-max').type('150')
-      checkFilteringResult(116)
-      cy.cs('credit-filter-min').find('input').clear()
-      cy.cs('credit-filter-max').find('input').clear()
-    })
-  })
-
-  it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
-      cy.cs('ageFilter-min').type('20')
-      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(42)
-      cy.cs('ageFilter-min').find('input').clear()
-      cy.cs('ageFilter-max').find('input').clear()
-    })
-  })
-
-  it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
-      checkFilteringResult(36)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
-      checkFilteringResult(112)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
-      checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
-      checkFilteringResult(0)
-      clearSingleDropdownSelection('genderFilter-dropdown')
-    })
-  })
-
-  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', 0)
-      checkFilteringResult(1)
-      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 14)
-      checkFilteringResult(121)
-      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-    })
-  })
-
-  // Return when we have +2020 population with admission types in data
-  it.skip('Admission type filter works', () => {
-    runTestStepWithPreAndPostParts('admissionTypeFilter-header', () => {
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 0)
-      checkFilteringResult(5)
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 2)
-      checkFilteringResult(10)
-      clearSingleDropdownSelection('admissionTypeFilter-dropdown')
-    })
-  })
-
-  // plz fix
-  it.skip('Courses filter works', () => {
-    // courses takes some time to load, wait for it to complete
-    cy.wait(10000)
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
-      cy.cs('courseFilter-course-dropdown').click()
-      const courses = ['TKT20001', 'MAT11002']
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
-      checkFilteringResult(140)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
-      checkFilteringResult(131)
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
-      checkFilteringResult(56)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 2)
-      checkFilteringResult(2)
-      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
-    })
-  })
-
-  it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
-        cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
-        cy.cs('ageFilter-min').type('20')
-        cy.cs('ageFilter-max').type('30')
-        checkFilteringResult(11)
-        cy.cs('ageFilter-min').find('input').clear()
-        cy.cs('ageFilter-max').find('input').clear()
-        cy.cs('graduatedFromProgrammeFilter-all').click()
-      })
-    })
-  })
-})
-
-describe('Course Statistics', () => {
-  const pathToDSAndAlgoSpring2019 =
-    '/coursepopulation?coursecodes=%5B%22TKT20001%22%2C%2258131%22%5D&from=138&separate=true&to=138&years=Kev%C3%A4t%202019'
-  const defaultAmountOfStudents = 118
-  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
-
-  before(() => {
-    cy.init(pathToDSAndAlgoSpring2019)
-  })
-
-  it('Grade filter works', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
-      cy.cs('gradeFilter-5').click()
-      checkFilteringResult(20)
-      cy.cs('gradeFilter-3').click()
-      checkFilteringResult(38)
-      cy.cs('gradeFilter-3').click()
-      cy.cs('gradeFilter-5').click()
-    })
-  })
-
-  it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
-      cy.cs('ageFilter-min').type('20')
-      cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(32)
-      cy.cs('ageFilter-min').find('input').clear()
-      cy.cs('ageFilter-max').find('input').clear()
-    })
-  })
-
-  it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
-      checkFilteringResult(27)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
-      checkFilteringResult(91)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
-      checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
-      checkFilteringResult(0)
-      clearSingleDropdownSelection('genderFilter-dropdown')
-    })
-  })
-
-  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
-      checkFilteringResult(1)
-      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 12)
-      checkFilteringResult(29)
-      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-    })
-  })
-
-  it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
-        cy.cs('gradeFilter-3').click()
-        cy.cs('ageFilter-min').type('20')
-        cy.cs('ageFilter-max').type('30')
-        checkFilteringResult(1)
-        cy.cs('ageFilter-min').find('input').clear()
-        cy.cs('ageFilter-max').find('input').clear()
-        cy.cs('gradeFilter-3').click()
-      })
-    })
-  })
-})
-
-describe('Custom Population Statistics', () => {
-  let runTestStepWithPreAndPostParts
-
-  before(() => {
-    cy.init('/custompopulation')
-    cy.cs('custom-pop-search-button').click()
-    cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
-      const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
-      runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
-      cy.cs('student-no-input').click().type(students.join('/n'))
-    })
-    cy.cs('search-button').click()
-  })
-
-  it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
-      cy.cs('ageFilter-min').type('20')
-      cy.cs('ageFilter-max').type('30')
-      checkFilteringResult(1)
-      cy.cs('ageFilter-min').find('input').clear()
-      cy.cs('ageFilter-max').find('input').clear()
-    })
-  })
-
-  it('Gender filter works', () => {
-    cy.cs('genderFilter-header').click()
-    cy.selectFromDropdown('genderFilter-dropdown', 0)
-    checkFilteringResult(3)
-    cy.selectFromDropdown('genderFilter-dropdown', 1)
-    checkFilteringResult(2)
-    cy.selectFromDropdown('genderFilter-dropdown', 2)
-    checkFilteringResult(0)
-    clearSingleDropdownSelection('genderFilter-dropdown')
-    checkFilteringResult(5, true)
-  })
-
-  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
-      checkFilteringResult(1)
-      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-    })
-  })
-
-  it('Courses filter works', () => {
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
-      const courses = ['MAT11001', 'TKT20004']
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
-      checkFilteringResult(5)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
-      checkFilteringResult(5)
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
-      checkFilteringResult(3)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)
-      checkFilteringResult(1)
-      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
-    })
-  })
-})

cypress/integration/Studyprogramme_overview.js

@@ -1,228 +0,0 @@
-/// <reference types="Cypress" />
-
-const deleteTag = name => {
-  cy.contains(name).siblings().contains('Delete').click()
-  cy.contains('Are you sure you want to delete tag')
-  cy.contains('Confirm').click()
-  cy.contains(name).should('not.exist')
-}
-
-describe('Studyprogramme overview', () => {
-  // check is used by couple of tests
-  const checkProgressAndProductivity = (user = 'basic') => {
-    // Column amount depends on user. Note that with dev user, you'll see even more
-    // stuff than with admin or basic.
-    const CSBachPopulationProgress2018ForAdmin = [
-      '162',
-      '44 (27%)',
-      '118 (72%)',
-      '161 (99%)',
-      '161',
-      '1',
-      '11',
-      '11',
-      '152',
-      '125',
-      '78',
-      '42',
-      '16',
-    ]
-    const CSBachPopulationProgress2018ForBasic = [
-      '162',
-      '44 (27%)',
-      '118 (72%)',
-      '161 (99%)',
-      '161',
-      '1',
-      '11',
-      '11',
-      '152',
-      '125',
-      '78',
-      '42',
-      '16',
-    ] // cancelled and transferred from are missing
-
-    const population = user === 'basic' ? CSBachPopulationProgress2018ForBasic : CSBachPopulationProgress2018ForAdmin
-    cy.contains('Population progress')
-    cy.contains('Yearly productivity')
-
-    cy.contains('2018-2019')
-      .siblings()
-      .each((elem, index) => {
-        cy.wrap(elem).contains(population[index])
-      })
-    const populationproductivity2019 = ['7388.0', '17', '7388.00', '0.00', '349.00']
-    cy.get('table')
-      .eq(1)
-      .contains('2019')
-      .siblings()
-      .each((elem, index) => {
-        cy.wrap(elem).contains(populationproductivity2019[index])
-      })
-  }
-
-  describe('when opening programme page with admin user', () => {
-    beforeEach(() => {
-      cy.init('/study-programme/KH50_005', 'admin')
-    })
-
-    it('progress should not be recalculating when opened for the first time', () => {
-      cy.contains('Recalculating').should('not.exist')
-      cy.contains('Last updated').should('exist')
-    })
-
-    // Following vars are used to compare values before and after stats calculation
-    // Taken from https://docs.cypress.io/api/commands/should#Compare-text-values-of-two-elements
-    const normalizeText = s => s.replace(/\s/g, '').toLowerCase()
-    let originalProgressCalculatedText
-    let originalProductivityCalculatedText
-    const doRecalculation = () => {
-      // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
-      cy.wait(5000)
-      cy.get('.attached').contains('Admin').click()
-      cy.contains('recalculate productivity').click()
-      cy.contains('recalculate throughput').click()
-      cy.get('.attached').contains('Overview').click() // TODO: add data-cy tags for tabs
-      // Wait to "recalculating" to disappear, TODO: change this to cy.intercept
-      cy.wait(5000)
-      cy.reload()
-    }
-
-    it('renders progress and productivity tables with calculated status', () => {
-      doRecalculation()
-
-      // check contains also study programme options for admin
-      cy.contains('Study programme options')
-      cy.get('table').should('have.length', 3)
-
-      checkProgressAndProductivity('admin')
-
-      // Grab update dates to be compared later
-      cy.cs('throughputUpdateStatus')
-        .invoke('text')
-        .then(text => {
-          originalProgressCalculatedText = normalizeText(text)
-          expect(originalProgressCalculatedText).not.to.contain('recalculating')
-          expect(originalProgressCalculatedText).not.to.contain('refresh')
-        })
-
-      cy.cs('productivityUpdateStatus')
-        .invoke('text')
-        .then(text => {
-          originalProductivityCalculatedText = normalizeText(text)
-          expect(originalProductivityCalculatedText).not.to.contain('recalculating')
-          expect(originalProgressCalculatedText).not.to.contain('refresh')
-        })
-    })
-
-    it('renders progress and productivity tables with calculated status after recalculating stats again', () => {
-      doRecalculation()
-      checkProgressAndProductivity('admin')
-
-      // Check new calculation statuses are reported
-      const newProgressCalculatedTextElement = cy.cs('throughputUpdateStatus').invoke('text')
-      const newProductivityCalculatedTextElement = cy.cs('productivityUpdateStatus').invoke('text')
-
-      newProgressCalculatedTextElement.should(text => {
-        const newProgressCalculatedText = normalizeText(text)
-        expect(newProgressCalculatedText).not.to.contain('recalculating')
-        expect(newProgressCalculatedText).not.to.contain('refresh')
-        expect(newProgressCalculatedText).not.to.contain(originalProgressCalculatedText)
-      })
-
-      newProductivityCalculatedTextElement.should(text => {
-        const newProductivityCalculatedText = normalizeText(text)
-        expect(newProductivityCalculatedText).not.to.contain('recalculating')
-        expect(newProductivityCalculatedText).not.to.contain('refresh')
-        expect(newProductivityCalculatedText).not.to.contain(originalProgressCalculatedText)
-      })
-    })
-  })
-
-  describe('when opening programme page with basic user', () => {
-    beforeEach(() => {
-      cy.init('/study-programme/KH50_005')
-    })
-
-    it('renders progress and productivity tables with previosly calculated values', () => {
-      // check doesn't contain study programme options for admin
-      cy.contains('Study programme options').should('not.exist')
-      cy.get('table').should('have.length', 2)
-      checkProgressAndProductivity()
-    })
-
-    it('can open Thesis page', () => {
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.contains('Thesis Courses').click()
-      cy.contains('Add thesis course').click()
-      cy.contains('No results')
-    })
-
-    it('can move to Population statistics page by clickin', () => {
-      cy.get('i.level.up.alternate.icon').eq(0).click()
-      cy.contains('Students (10)')
-    })
-
-    it('can create and delete tags for population', () => {
-      const name = `tag-${new Date().getTime()}`
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Tags').click()
-      cy.get('.tagNameSelectInput > .ui > input').type(name)
-      cy.get('.yearSelectInput').type('2018')
-      cy.contains('Create new tag').click()
-      cy.contains(name)
-      cy.contains('2018')
-      deleteTag(name)
-    })
-
-    it('can create personal tags', () => {
-      const name = `tag-${new Date().getTime()}`
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Tags').click()
-      cy.get('.tagNameSelectInput > .ui > input').type(name)
-      cy.get('.yearSelectInput').type('2018')
-
-      cy.get('.ui > label').click()
-      cy.contains('Create new tag').click()
-      cy.contains(name)
-      deleteTag(name)
-    })
-
-    it('can add tags to students', () => {
-      const name = `tag-${new Date().getTime()}`
-
-      const student = '010113437'
-
-      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Tags').click()
-      cy.get('.tagNameSelectInput > .ui > input').type(name)
-      cy.get('.yearSelectInput').type('2018')
-      cy.contains('Create new tag').click()
-      cy.contains(name)
-
-      cy.contains('Add tags to students').click()
-      cy.get('.form > .field > .dropdown').click().get('.ui > .search').type(name).click()
-
-      cy.get('.form > .field > .dropdown > .visible').contains(name).click()
-
-      cy.get('textarea').type('010113437')
-      cy.get('.positive').click()
-
-      cy.contains('Student statistics').click()
-      cy.get('.prompt').type(student)
-      cy.contains(student).click()
-      cy.contains(name)
-
-      cy.go('back')
-      cy.go('back')
-
-      deleteTag(name)
-
-      cy.contains('Student statistics').click()
-      cy.get('.prompt').type(student)
-      cy.contains(student).click()
-      cy.contains(name).should('not.exist')
-    })
-  })
-})

cypress/support/commands.js

@@ -7,7 +7,7 @@ const adminUserHeaders = {
   uid: 'admin',
   displayname: 'Admin User',
   'shib-session-id': 'mock-cypress-session',
-  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users;grp-toska',
   edupersonaffiliation: 'member;employee;faculty',
   mail: 'grp-toska+mockadminuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-6666666',
@@ -43,7 +43,23 @@ const norightsUserHeaders = {
   hypersonsisuid: 'hy-hlo-0000000',
 }
 
-const userHeaders = [adminUserHeaders, basicUserHeaders, onlycoursestatisticsUserHeaders, norightsUserHeaders]
+const onlyIamRightsUserHeaders = {
+  uid: 'onlyiamrights',
+  displayname: 'Only IAM rights user',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'hy-employees;hy-mltdk-tkt-jory',
+  edupersonaffiliation: 'member',
+  mail: 'grp-toska+mockonlyiamrightsuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-1111111',
+}
+
+const userHeaders = [
+  adminUserHeaders,
+  basicUserHeaders,
+  onlycoursestatisticsUserHeaders,
+  norightsUserHeaders,
+  onlyIamRightsUserHeaders,
+]
 
 /**
  Set up headers to login, set up correct user (admin / basic / etc.) and open given path.
@@ -54,8 +70,6 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
     if (!headersToUse) throw Error(`${userId} is not valid user id!`)
     req.headers = headersToUse
   })
-  // Hide SIS warning flag to avoid screen size problems when running headlessly
-  window.localStorage.setItem('SIS_WARNING_FLAG', 'true')
   const baseUrl = Cypress.config().baseUrl
   cy.visit(baseUrl.concat(path))
 })
@@ -63,24 +77,37 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
 /**
  * Shorthand for using "Cypress Selectors" (CS), i.e., `data-cy` attributes.
  */
-Cypress.Commands.add('cs', name => {
-  return cy.get(`[data-cy='${name}']`)
+Cypress.Commands.add('cs', { prevSubject: 'optional' }, (subject, name) => {
+  const selector = `[data-cy="${name}"]`
+
+  if (subject === undefined) {
+    return cy.get(selector)
+  } else {
+    return cy.wrap(subject).find(selector)
+  }
 })
 
 /**
  * Select item specified by `index` number from a semantic-ui dropdown with
  * `data-cy` attribute value `name`.
  */
-Cypress.Commands.add('selectFromDropdown', (name, index) => {
+Cypress.Commands.add('selectFromDropdown', { prevSubject: true }, (s, index) => {
+  const subject = cy.wrap(s)
   const indexes = Array.isArray(index) ? index : [index]
 
   indexes.forEach(i => {
-    cy.cs(name).click().children('.menu').children().eq(i).click({ force: true })
+    subject.click()
+
+    if (typeof i === 'number') {
+      subject.children('.menu').children().eq(i).click({ force: true })
+    } else {
+      subject.children('.menu').containing(i).eq(0).click({ force: true })
+    }
   })
 
   // Close multiple selection so it does not block elements underneath it.
   if (Array.isArray(index)) {
-    cy.cs(name).children('.icon').click({ force: true })
+    cy.wrap(s).children('.icon').click({ force: true })
   }
 })
 
@@ -93,3 +120,18 @@ Cypress.Commands.add('selectStudyProgramme', name => {
   cy.cs('select-study-programme').click().children().contains(name).click()
   cy.contains('See population').click()
 })
+
+Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {
+  cy.get(`[data-cy=Table-${tableName}]>tbody`).within(() => {
+    correctStats.forEach((values, trIndex) => {
+      cy.get('tr')
+        .eq(trIndex)
+        .within(() => {
+          values.forEach((value, tdIndex) => {
+            cy.get('td').eq(tdIndex).contains(value)
+          })
+        })
+    })
+    cy.get('tr').should('have.length', correctStats.length)
+  })
+})

cypress/support/e2e.js

@@ -14,3 +14,13 @@
 // ***********************************************************
 
 import './commands'
+
+// This error is benign and seems to be only present in Cypress
+const resizeObserverLoopErrRe = /^[^(ResizeObserver loop limit exceeded)]/
+
+Cypress.on('uncaught:exception', err => {
+  /* returning false here prevents Cypress from failing the test */
+  if (resizeObserverLoopErrRe.test(err.message)) {
+    return false
+  }
+})

docker-compose.ci.yml

@@ -2,7 +2,6 @@ x-defaults: &defaults
   restart: unless-stopped
 
 services:
-
   backend:
     <<: *defaults
     container_name: backend
@@ -10,17 +9,16 @@ services:
       - kone-db
       - redis
       - sis-db
+      - user-db
     environment:
-      CI: "true"
+      CI: 'true'
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:1337
       REDIS: redis
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
     image: toska/oodikone2-backend:staging
 
   frontend:
@@ -31,11 +29,15 @@ services:
   kone-db:
     <<: *defaults
     container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
     image: registry.toska.cs.helsinki.fi/kone-db
 
   redis:
     <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
     container_name: redis
     image: redis
     ports:
@@ -45,23 +47,20 @@ services:
     <<: *defaults
     container_name: sis-db
     image: registry.toska.cs.helsinki.fi/sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    shm_size: 2gb
 
   user-db:
     <<: *defaults
     container_name: user-db
-    image: registry.toska.cs.helsinki.fi/user-db
-
-  userservice:
-    <<: *defaults
-    container_name: userservice
-    depends_on:
-      - user-db
     environment:
-      CI: "true"
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    image: toska/oodikone2-userservice:staging
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db
 
   nginx:
     image: nginx:latest
@@ -69,8 +68,71 @@ services:
     depends_on:
       - frontend
       - backend
-      - userservice
     ports:
-      -  1337:80
+      - 1337:80
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf
+
+  # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    image: registry.toska.cs.helsinki.fi/sis-importer-db
+    profiles:
+      - updater
+
+  sis-updater-worker:
+    <<: *defaults
+    build:
+      target: development
+      context: updater/sis-updater-worker
+    depends_on:
+      - redis
+      - sis-db
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_URI: //redis:6379
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+    profiles:
+      - updater
+
+  sis-updater-scheduler:
+    <<: *defaults
+    build:
+      target: development
+      context: updater/sis-updater-scheduler
+    container_name: sis-updater-scheduler
+    depends_on:
+      - redis
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      REDIS_URI: //redis:6379
+      SECRET_TOKEN: dev
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
+    ports:
+      - 8082:8082
+    profiles:
+      - updater
+
+  sis-updater-nats:
+    <<: *defaults
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_channels 1000 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    container_name: sis-updater-nats
+    image: nats-streaming:0.16.2
+    profiles:
+      - updater

docker-compose.real.yml

@@ -1,7 +1,7 @@
 services:
-
   backend:
     environment:
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db-real
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db-real
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db-real
 
@@ -18,9 +18,5 @@ services:
       DB_URL: postgres://postgres@sis-db:5432/sis-db-real
       SIS_IMPORTER_DATABASE: sis-importer-db-real
 
-  userservice:
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db-real
-
 volumes:
   redis-data-real:

docker-compose.test.yml

@@ -1,10 +0,0 @@
-services:
-
-  userservice-test:
-    build:
-      context: services/userservice
-      target: test
-    container_name: userservice-test
-    environment:
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest

docker-compose.yml

@@ -1,3 +1,5 @@
+version: '2.1'
+
 x-defaults: &defaults
   restart: unless-stopped
 
@@ -26,18 +28,17 @@ services:
       - kone-db
       - redis
       - sis-db
+      - user-db
     env_file:
       - .env
     environment:
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
       FRONT_URL: http://localhost:3000
       REDIS: redis
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
     ports:
       - 8080:8080
     profiles:
@@ -98,7 +99,7 @@ services:
 
   sis-updater-nats:
     <<: *defaults
-    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_channels 1000 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:
@@ -123,6 +124,8 @@ services:
       SIS_IMPORTER_USER: postgres
       SIS_NATS_TOKEN: dev
       SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
     ports:
       - 8082:8082
     profiles:
@@ -165,26 +168,6 @@ services:
     volumes:
       - user-db-data:/data
 
-  userservice:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/userservice
-    container_name: userservice
-    depends_on:
-      - user-db
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    ports:
-      - 4567:4567
-    profiles:
-      - oodikone
-    volumes:
-      - userservice-nodemod:/usr/src/app/node_modules
-      - ./services/userservice/:/usr/src/app
-
 volumes:
   backend-nodemod:
   frontend-nodemod:
@@ -195,4 +178,3 @@ volumes:
   sis-updater-scheduler-nodemod:
   sis-updater-worker-nodemod:
   user-db-data:
-  userservice-nodemod:

package.json

@@ -6,6 +6,7 @@
     "concurrently": "concurrently",
     "eslint": "eslint",
     "stylelint": "stylelint",
+    "actionlint": "node-actionlint",
     "prettier": "prettier",
     "cli": "./cli.sh",
     "oodikone": "./run.sh oodikone anon up --detach",
@@ -15,7 +16,8 @@
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
     "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "rapodiff": "docker exec backend node /usr/src/app/src/rapodiff.js"
   },
   "repository": {
     "type": "git",
@@ -27,31 +29,33 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@babel/eslint-parser": "^7.15.0",
-    "@babel/plugin-syntax-jsx": "^7.14.5",
-    "concurrently": "^6.2.1",
-    "cypress": "^8.3.1",
-    "eslint": "^7.32.0",
-    "eslint-config-airbnb": "^18.2.1",
-    "eslint-config-prettier": "^8.3.0",
+    "@babel/core": "^7.18.10",
+    "@babel/eslint-parser": "^7.17.0",
+    "@babel/plugin-syntax-jsx": "^7.16.7",
+    "concurrently": "^7.3.0",
+    "cypress": "^10.4.0",
+    "eslint": "^8.21.0",
+    "eslint-config-airbnb": "^19.0.4",
+    "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-babel": "^5.3.1",
-    "eslint-plugin-cypress": "^2.11.3",
-    "eslint-plugin-import": "^2.24.2",
-    "eslint-plugin-jsx-a11y": "^6.4.1",
+    "eslint-plugin-cypress": "^2.12.1",
+    "eslint-plugin-import": "^2.25.4",
+    "eslint-plugin-jsx-a11y": "^6.5.1",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
-    "eslint-plugin-prettier": "^3.4.1",
-    "eslint-plugin-react": "^7.24.0",
-    "eslint-plugin-react-hooks": "^4.2.0",
-    "husky": "^7.0.2",
-    "lint-staged": "^11.1.2",
-    "prettier": "^2.3.2",
-    "stylelint": "^13.13.1",
-    "stylelint-config-prettier": "^8.0.2",
-    "stylelint-config-standard": "^22.0.0",
-    "stylelint-prettier": "^1.2.0"
+    "eslint-plugin-prettier": "^4.2.1",
+    "eslint-plugin-react": "^7.29.3",
+    "eslint-plugin-react-hooks": "^4.3.0",
+    "husky": "^8.0.1",
+    "lint-staged": "^13.0.3",
+    "node-actionlint": "^1.2.2",
+    "prettier": "^2.5.1",
+    "stylelint": "^14.9.1",
+    "stylelint-config-prettier": "^9.0.3",
+    "stylelint-config-standard": "^26.0.0",
+    "stylelint-prettier": "^2.0.0"
   },
   "dependencies": {
     "lodash": "^4.17.21",
-    "moment": "^2.29.1"
+    "moment": "^2.29.4"
   }
 }

scripts/functions_for_setup.sh

@@ -165,7 +165,6 @@ set_up_oodikone_from_scratch() {
   folders_to_set_up=(
     "$PROJECT_ROOT"
     "$PROJECT_ROOT/services/frontend"
-    "$PROJECT_ROOT/services/userservice"
     "$PROJECT_ROOT/services/backend"
     "$PROJECT_ROOT/updater/sis-updater-scheduler"
     "$PROJECT_ROOT/updater/sis-updater-worker"

services/backend/.dockerignore

@@ -1,6 +1,3 @@
 .dockerignore
 Dockerfile
 node_modules/
-requests/
-scripts/
-*.log

services/backend/.gitignore

@@ -1,3 +0,0 @@
-# Student diff output
-/student_diff_*
-/diff_partial_att_data.json

services/backend/Dockerfile

@@ -17,7 +17,7 @@ ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
 ARG SENTRY_RELEASE
 ENV SENTRY_RELEASE=$SENTRY_RELEASE
 COPY --chown=node:node ./package* /usr/src/app/
-RUN npm ci --production
+RUN npm ci --production && apk add --no-cache vim curl nano
 COPY --chown=node:node . /usr/src/app/
 USER node
 CMD ["npm", "start"]

services/backend/config/IAMConfig.js

@@ -0,0 +1,435 @@
+const { facultyMap } = require('./data')
+
+const joryMap = {
+  'hy-ttdk-tuk-jory': '100-K001',
+  'hy-ttdk-tum-jory': '100-M001',
+  'hy-oiktdk-on-jory': '200-K001',
+  'hy-oiktdk-otm-jory': '200-M001',
+  'hy-oiktdk-ibl-jory': '200-M002',
+  'hy-oiktdk-ggl-jory': '200-M003',
+  'hy-ltdk-psyk-jory': ['300-K001', '300-M004'],
+  'hy-ltdk-logo-jory': ['300-K002', '300-M005'],
+  'hy-ltdk-ll-jory': '300-M001',
+  'hy-ltdk-tmed-jory': '300-M002',
+  'hy-ltdk-hll-jory': '300-M003',
+  'hy-humtdk-filk-jory': '400-K001',
+  'hy-humtdk-ttk-jory': '400-K002',
+  'hy-humtdk-kik-jory': '400-K003',
+  'hy-humtdk-kok-jory': '400-K004',
+  'hy-humtdk-kuka-jory': '400-K005',
+  'hy-humtdk-hisk-jory': '400-K006',
+  'hy-humtdk-ttm-jory': '400-M001',
+  'hy-humtdk-kim-jory': '400-M002',
+  'hy-humtdk-eng-jory': '400-M003',
+  'hy-humtdk-rus-jory': '400-M004',
+  'hy-humtdk-lingdig-jory': '400-M005',
+  'hy-humtdk-tra-jory': '400-M006',
+  'hy-humtdk-suku-jory': '400-M007',
+  'hy-humtdk-nor-jory': '400-M008',
+  'hy-humtdk-kir-jory': '400-M009',
+  'hy-humtdk-kuma-jory': '400-M010',
+  'hy-humtdk-ice-jory': '400-M011',
+  'hy-humtdk-alku-jory': '400-M012',
+  'hy-humtdk-spt-jory': '400-M014',
+  'hy-humtdk-hism-jory': '400-M015',
+  'hy-mltdk-mat-jory': '500-K001',
+  'hy-mltdk-fys-jory': '500-K002',
+  'hy-mltdk-kemk-jory': '500-K003',
+  'hy-mltdk-mfkk-jory': '500-K004',
+  'hy-mltdk-tkt-jory': '500-K005',
+  'hy-mltdk-geok-jory': '500-K006',
+  'hy-mltdk-maa-jory': '500-K007',
+  'hy-mltdk-bsc-jory': '500-K008',
+  'hy-mltdk-mast-jory': '500-M001',
+  'hy-mltdk-lsi-jory': '500-M002',
+  'hy-mltdk-tcm-jory': '500-M003',
+  'hy-mltdk-paras-jory': '500-M004',
+  'hy-mltdk-matres-jory': '500-M005',
+  'hy-mltdk-atm-jory': '500-M006',
+  'hy-mltdk-kem-jory': '500-M007',
+  'hy-mltdk-mfkm-jory': '500-M008',
+  'hy-mltdk-csm-jory': '500-M009',
+  'hy-mltdk-data-jory': '500-M010',
+  'hy-mltdk-geom-jory': '500-M011',
+  'hy-mltdk-geog-jory': '500-M012',
+  'hy-mltdk-usp-jory': '500-M013',
+  'hy-ftdk-farm-jory': '550-K001',
+  'hy-ftdk-prov-jory': '550-M001',
+  'hy-ftdk-mpharm-jory': '550-M002',
+  'hy-bytdk-bio-jory': '570-K001',
+  'hy-bytdk-mole-jory': '570-K002',
+  'hy-bytdk-env-jory': '570-K003',
+  'hy-bytdk-eeb-jory': '570-M001',
+  'hy-bytdk-ips-jory': '570-M002',
+  'hy-bytdk-gmb-jory': '570-M003',
+  'hy-bytdk-neuro-jory': '570-M004',
+  'hy-bytdk-ecgs-jory': '570-M005',
+  'hy-ktdk-eduk-jory': '600-K001',
+  'hy-ktdk-edum-jory': '600-M001',
+  'hy-ktdk-ce-jory': '600-M002',
+  'hy-valttdk-pvk-jory': '700-K001',
+  'hy-valttdk-yk-jory': '700-K002',
+  'hy-valttdk-sosk-jory': '700-K003',
+  'hy-valttdk-ecok-jory': '700-K004',
+  'hy-valttdk-film-jory': '700-M001',
+  'hy-valttdk-pvm-jory': '700-M002',
+  'hy-valttdk-gpc-jory': '700-M003',
+  'hy-valttdk-ym-jory': '700-M004',
+  'hy-valttdk-cos-jory': '700-M005',
+  'hy-valttdk-ens-jory': '700-M006',
+  'hy-valttdk-msv-jory': '700-M007',
+  'hy-valttdk-sosm-jory': '700-M008',
+  'hy-valttdk-econ-jory': '700-M009',
+  'hy-valttdk-sote-jory': '700-M011',
+  'hy-sskh-ksv-jory': '740-K001',
+  'hy-mmtdk-maatk-jory': '800-K001',
+  'hy-mmtdk-metsak-jory': '800-K002',
+  'hy-mmtdk-etk-jory': '800-K003',
+  'hy-mmtdk-yet-jory': '800-K004',
+  'hy-mmtdk-agri-jory': '800-M001',
+  'hy-mmtdk-agere-jory': '800-M002',
+  'hy-mmtdk-for-jory': '800-M003',
+  'hy-mmtdk-food-jory': '800-M004',
+  'hy-mmtdk-hnfb-jory': '800-M005',
+  'hy-mmtdk-ekm-jory': '800-M006',
+  'hy-mmtdk-mmb-jory': '800-M007',
+  'hy-eltdk-elk-jory': '900-K001',
+  'hy-eltdk-ell-jory': '900-M001',
+  'hy-dp-dptheol-jory': 'T920101',
+  'hy-dp-dplaw-jory': 'T920102',
+  'hy-dp-philartsoc-jory': 'T920103',
+  'hy-dp-dphistcult-jory': 'T920104',
+  'hy-dp-dplang-jory': 'T920105',
+  'hy-dp-sky-jory': 'T920106',
+  'hy-dp-dpsocs-jory': 'T920107',
+  'hy-dp-pyam-jory': 'T920108',
+  'hy-dp-dpe-jory': 'T920109',
+  'hy-dp-seduce-jory': 'T920110',
+  'hy-dp-clic-jory': 'T920111',
+  'hy-dp-dpbm-jory': 'T921101',
+  'hy-dp-klto-jory': 'T921102',
+  'hy-dp-docpop-jory': 'T921103',
+  'hy-dp-findos-jory': 'T921104',
+  'hy-dp-dpdr-jory': 'T921105',
+  'hy-dp-ils-jory': 'T921106',
+  'hy-dp-bandm-jory': 'T921107',
+  'hy-dp-cvm-jory': 'T921108',
+  'hy-dp-dphub-jory': 'T921109',
+  'hy-dp-luova-jory': 'T922101',
+  'hy-dp-dpps-jory': 'T922102',
+  'hy-dp-denvi-jory': 'T922103',
+  'hy-dp-agforee-jory': 'T922104',
+  'hy-dp-mbdp-jory': 'T922105',
+  'hy-dp-foodhealth-jory': 'T922106',
+  'hy-dp-papu-jory': 'T923101',
+  'hy-dp-geodoc-jory': 'T923102',
+  'hy-dp-atm-dp-jory': 'T923103',
+  'hy-dp-chems-jory': 'T923104',
+  'hy-dp-domast-jory': 'T923105',
+  'hy-dp-matrena-jory': 'T923106',
+  'hy-dp-docs-jory': 'T923107',
+}
+
+const kojoMap = {
+  'hy-ttdk-tuk-jory': 'hy-ttdk-kandi-kojot',
+  'hy-ttdk-tum-jory': 'hy-ttdk-maisteri-kojot',
+  'hy-oiktdk-on-jory': 'hy-oiktdk-kandi-kojot',
+  'hy-oiktdk-otm-jory': 'hy-oiktdk-maisteri-kojot',
+  'hy-oiktdk-ibl-jory': 'hy-oiktdk-maisteri-kojot',
+  'hy-oiktdk-ggl-jory': 'hy-oiktdk-maisteri-kojot',
+  'hy-ltdk-psyk-jory': 'hy-ltdk-maisteri-kojot',
+  'hy-ltdk-logo-jory': 'hy-ltdk-maisteri-kojot',
+  'hy-ltdk-ll-jory': 'hy-ltdk-maisteri-kojot',
+  'hy-ltdk-tmed-jory': 'hy-ltdk-maisteri-kojot',
+  'hy-ltdk-hll-jory': 'hy-ltdk-maisteri-kojot',
+  'hy-humtdk-filk-jory': 'hy-humtdk-kandi-kojot',
+  'hy-humtdk-ttk-jory': 'hy-humtdk-kandi-kojot',
+  'hy-humtdk-kik-jory': 'hy-humtdk-kandi-kojot',
+  'hy-humtdk-kok-jory': 'hy-humtdk-kandi-kojot',
+  'hy-humtdk-kuka-jory': 'hy-humtdk-kandi-kojot',
+  'hy-humtdk-hisk-jory': 'hy-humtdk-kandi-kojot',
+  'hy-humtdk-ttm-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-kim-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-eng-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-rus-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-lingdig-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-tra-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-suku-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-nor-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-kir-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-kuma-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-ice-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-alku-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-spt-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-humtdk-hism-jory': 'hy-humtdk-maisteri-kojot',
+  'hy-mltdk-mat-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-fys-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-kemk-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-mfkk-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-tkt-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-geok-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-maa-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-bsc-jory': 'hy-mltdk-kandi-kojot',
+  'hy-mltdk-mast-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-lsi-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-tcm-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-paras-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-matres-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-atm-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-kem-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-mfkm-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-csm-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-data-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-geom-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-geog-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-mltdk-usp-jory': 'hy-mltdk-maisteri-kojot',
+  'hy-ftdk-farm-jory': 'hy-ftdk-kandi-kojot',
+  'hy-ftdk-prov-jory': 'hy-ftdk-maisteri-kojot',
+  'hy-ftdk-mpharm-jory': 'hy-ftdk-maisteri-kojot',
+  'hy-bytdk-bio-jory': 'hy-bytdk-kandi-kojot',
+  'hy-bytdk-mole-jory': 'hy-bytdk-kandi-kojot',
+  'hy-bytdk-env-jory': 'hy-bytdk-kandi-kojot',
+  'hy-bytdk-eeb-jory': 'hy-bytdk-maisteri-kojot',
+  'hy-bytdk-ips-jory': 'hy-bytdk-maisteri-kojot',
+  'hy-bytdk-gmb-jory': 'hy-bytdk-maisteri-kojot',
+  'hy-bytdk-neuro-jory': 'hy-bytdk-maisteri-kojot',
+  'hy-bytdk-ecgs-jory': 'hy-bytdk-maisteri-kojot',
+  'hy-ktdk-eduk-jory': 'hy-ktdk-kandi-kojot',
+  'hy-ktdk-edum-jory': 'hy-ktdk-maisteri-kojot',
+  'hy-ktdk-ce-jory': 'hy-ktdk-maisteri-kojot',
+  'hy-valttdk-pvk-jory': 'hy-valttdk-kandi-kojot',
+  'hy-valttdk-yk-jory': 'hy-valttdk-kandi-kojot',
+  'hy-valttdk-sosk-jory': 'hy-valttdk-kandi-kojot',
+  'hy-valttdk-ecok-jory': 'hy-valttdk-kandi-kojot',
+  'hy-valttdk-film-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-pvm-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-gpc-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-ym-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-cos-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-ens-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-msv-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-sosm-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-econ-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-valttdk-sote-jory': 'hy-valttdk-maisteri-kojot',
+  'hy-sskh-ksv-jory': 'hy-sskh-kandi-kojot',
+  'hy-mmtdk-maatk-jory': 'hy-mmtdk-kandi-kojot',
+  'hy-mmtdk-metsak-jory': 'hy-mmtdk-kandi-kojot',
+  'hy-mmtdk-etk-jory': 'hy-mmtdk-kandi-kojot',
+  'hy-mmtdk-yet-jory': 'hy-mmtdk-kandi-kojot',
+  'hy-mmtdk-agri-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-mmtdk-agere-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-mmtdk-for-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-mmtdk-food-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-mmtdk-hnfb-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-mmtdk-ekm-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-mmtdk-mmb-jory': 'hy-mmtdk-maisteri-kojot',
+  'hy-eltdk-elk-jory': 'hy-eltdk-kandi-kojot',
+  'hy-eltdk-ell-jory': 'hy-eltdk-maisteri-kojot',
+  'hy-dp-dptheol-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dplaw-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-philartsoc-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dphistcult-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dplang-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-sky-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dpsocs-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-pyam-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dpe-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-seduce-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-clic-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dpbm-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-klto-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-docpop-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-findos-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dpdr-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-ils-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-bandm-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-cvm-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dphub-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-luova-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-dpps-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-denvi-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-agforee-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-mbdp-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-foodhealth-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-papu-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-geodoc-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-atm-dp-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-chems-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-domast-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-matrena-jory': 'hy-tohtoriohjelma-johtajat',
+  'hy-dp-docs-jory': 'hy-tohtoriohjelma-johtajat',
+}
+
+/**
+ * Maps kosu IAM to the name of TDKs, defined in data.js facultyMap
+ */
+const kosuFacultyMap = {
+  'hy-ypa-opa-kosu-kumpula': 'matemaattis-luonnontieteellinen',
+  'hy-ypa-opa-kosu-meilahti': ['lääketieteellinen'],
+  'hy-ypa-opa-kosu-kruununhaka': [
+    'oikeustieteellinen',
+    'valtiotieteellinen',
+    'kasvatustieteellinen',
+    'humanistinen',
+    'teologinen',
+    'svenska',
+  ],
+  'hy-ypa-opa-kosu-metsatalo': [
+    'oikeustieteellinen',
+    'valtiotieteellinen',
+    'kasvatustieteellinen',
+    'humanistinen',
+    'teologinen',
+    'svenska',
+  ],
+  'hy-ypa-opa-kosu-porthania': [
+    'oikeustieteellinen',
+    'valtiotieteellinen',
+    'kasvatustieteellinen',
+    'humanistinen',
+    'teologinen',
+    'svenska',
+  ],
+  'hy-ypa-opa-kosu-siltavuori': [
+    'oikeustieteellinen',
+    'valtiotieteellinen',
+    'kasvatustieteellinen',
+    'humanistinen',
+    'teologinen',
+    'svenska',
+  ],
+  'hy-ypa-opa-kosu-viikki': [
+    'bio- ja ympäristötieteellinen',
+    'farmasia',
+    'maatalous -metsätieteellinen',
+    'eläinlääketieteellinen',
+  ],
+}
+
+const doctoralIams = ['hy-tohtorikoulutus-johtoryhma', 'hy-tine']
+
+const doctoralSchoolMap = {
+  'hy-tutkijakoulut-hymy-jory': [
+    'T920101',
+    'T920102',
+    'T920103',
+    'T920104',
+    'T920105',
+    'T920106',
+    'T920107',
+    'T920108',
+    'T920109',
+    'T920110',
+    'T920111',
+  ],
+  'hy-tutkijakoulut-dshealth-jory': [
+    'T921101',
+    'T921102',
+    'T921103',
+    'T921104',
+    'T921105',
+    'T921106',
+    'T921107',
+    'T921108',
+    'T921109',
+  ],
+  'hy-tutkijakoulut-yeb-jory': ['T922101', 'T922102', 'T922103', 'T922104', 'T922105', 'T922106'],
+  'hy-tutkijakoulut-donasci-jory': ['T923101', 'T923102', 'T923103', 'T923104', 'T923105', 'T923106', 'T923107'],
+}
+
+const opetusVaradekaani = 'hy-varadekaanit-opetus'
+
+const dekaaniFacultyMap = {
+  'hy-ttdk-dekanaatti': 'teologinen',
+  'hy-oiktdk-dekanaatti': 'oikeustieteellinen',
+  'hy-ltdk-dekanaatti': 'lääketieteellinen',
+  'hy-humtdk-dekanaatti': 'humanistinen',
+  'hy-mltdk-dekanaatti': 'matemaattis-luonnontieteellinen',
+  'hy-ftdk-dekanaatti': 'farmasia',
+  'hy-bytdk-dekanaatti': 'bio- ja ympäristötieteellinen',
+  'hy-ktdk-dekanaatti': 'kasvatustieteellinen',
+  'hy-valttdk-dekanaatti': 'valtiotieteellinen',
+  'hy-sskh-rehtoraatti': 'svenska',
+  'hy-mmtdk-dekanaatti': 'maatalous -metsätieteellinen',
+  'hy-eltdk-dekanaatti': 'eläinlääketieteellinen',
+}
+
+const oodikoneIams = [
+  'grp-oodikone-staging-users',
+  'grp-oodikone-basic-staging-users',
+  'grp-oodikone-users',
+  'grp-oodikone-basic-users',
+]
+
+const universityWideGroups = ['hy-ypa-opa-opintoasiainpaallikot', 'hy-rehtoraatti']
+
+const superAdminGroups = ['grp-toska']
+
+const adminGroups = ['grp-ospa']
+
+const employeeGroups = ['hy-employees']
+
+const isSuperAdminIam = iam => superAdminGroups.includes(iam)
+
+const isAdminIam = iam => adminGroups.includes(iam)
+
+const isUniversityWideIam = iam => universityWideGroups.includes(iam)
+
+const isDoctoralIam = iam => doctoralIams.includes(iam)
+
+const isEmployeeIam = iam => employeeGroups.includes(iam)
+
+const iamToDoctoralSchool = iam => doctoralSchoolMap[iam]
+
+const getStudyLeaderGroup = iam => kojoMap[iam]
+
+const iamToOrganisationCode = iam => {
+  const organisationCodes = joryMap[iam]
+  if (Array.isArray(organisationCodes)) {
+    return organisationCodes
+  }
+  return [organisationCodes]
+}
+
+const kosuIamToFaculties = iam => {
+  const faculties = kosuFacultyMap[iam]
+  if (!Array.isArray(faculties) || !faculties.length) return []
+
+  return faculties.map(f => facultyMap[f])
+}
+
+const dekaaniIamToFaculty = iam => {
+  const faculty = dekaaniFacultyMap[iam]
+  if (!faculty) return null
+  return facultyMap[faculty]
+}
+
+const relevantIAMs = []
+  .concat(Object.keys(joryMap))
+  .concat(Object.keys(kojoMap))
+  .concat(Object.values(kojoMap))
+  .concat(Object.keys(kosuFacultyMap))
+  .concat(Object.keys(dekaaniFacultyMap))
+  .concat(opetusVaradekaani)
+  .concat(doctoralIams)
+  .concat(Object.keys(doctoralSchoolMap))
+  .concat(universityWideGroups)
+  .concat(superAdminGroups)
+  .concat(adminGroups)
+  .concat(employeeGroups)
+  .concat(oodikoneIams)
+
+module.exports = {
+  isSuperAdminIam,
+  isAdminIam,
+  isUniversityWideIam,
+  isDoctoralIam,
+  isEmployeeIam,
+  iamToDoctoralSchool,
+  iamToOrganisationCode,
+  kosuIamToFaculties,
+  dekaaniIamToFaculty,
+  opetusVaradekaani,
+  getStudyLeaderGroup,
+  relevantIAMs,
+}

services/backend/config/common.js

@@ -0,0 +1,19 @@
+const mapToDegreeCode = organisationCode => {
+  if (!organisationCode) return ''
+  if (organisationCode.length < 7) return ''
+  const doctoral = organisationCode[0] === 'T'
+  if (doctoral) {
+    return organisationCode
+  }
+
+  const [start, end] = organisationCode.split('-')
+  if (end && end.length < 3) return ''
+  if (start.length < 2) return ''
+  const masters = end[0] === 'M'
+  const code = `${masters ? 'M' : 'K'}H${start.substr(0, 2)}_${end.substr(-3)}`
+  return code
+}
+
+module.exports = {
+  mapToDegreeCode,
+}

services/backend/package.json

@@ -4,36 +4,31 @@
   "scripts": {
     "start": "node --max_old_space_size=4096 index.js",
     "dev": "nodemon --max_old_space_size=4096 index.js",
-    "diff:students": "node /usr/src/app/src/sisoodi_diff/students/index.js",
-    "diff:populations": "node /usr/src/app/src/sisoodi_diff/populations.js",
-    "diff:courses": "node /usr/src/app/src/sisoodi_diff/courses/index.js",
-    "diff:transfers": "node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "node /usr/src/app/src/sisoodi_diff/trends.js"
+    "rapodiff": "node /usr/src/app/src/rapodiff.js"
   },
   "dependencies": {
     "@sentry/node": "^5.30.0",
-    "@sentry/tracing": "^6.11.0",
-    "async": "^3.2.1",
-    "async-lock": "^1.3.0",
-    "axios": "^0.21.2",
+    "@sentry/tracing": "^6.18.2",
+    "async": "^3.2.3",
+    "async-lock": "^1.3.1",
+    "axios": "^0.21.4",
     "bluebird": "^3.7.2",
     "cors": "^2.8.5",
     "cron": "^1.8.2",
-    "express": "^4.17.1",
+    "express": "^4.17.3",
     "express-async-errors": "^3.1.1",
-    "jsonwebtoken": "^8.5.1",
     "lodash": "^4.17.21",
     "lru-cache": "^5.1.1",
     "moment": "^2.29.1",
-    "moment-timezone": "^0.5.33",
+    "moment-timezone": "^0.5.34",
     "morgan": "^1.10.0",
-    "pg": "^8.7.1",
+    "pg": "^8.7.3",
     "redis": "^3.1.2",
-    "sequelize": "^6.6.5",
+    "sequelize": "^6.17.0",
     "umzug": "^2.3.0",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "uuid": "^3.4.0",
-    "winston": "^3.3.3"
+    "winston": "^3.6.0"
   },
   "devDependencies": {
     "nodemon": "^1.19.4"

services/backend/src/app.js

@@ -1,13 +1,9 @@
 require('express-async-errors')
 const express = require('express')
-const cors = require('cors')
-const Sentry = require('@sentry/node')
-const { baseUrl, backendPort, frontUrl } = require('./conf-backend')
+const { baseUrl, backendPort } = require('./conf-backend')
 const routes = require('./routes')
 const { startCron } = require('./events')
 const { initializeDatabaseConnection, dbConnections } = require('./database/connection')
-const initializeSentry = require('./util/sentry')
-const errorMiddleware = require('./middleware/errormiddleware')
 const logger = require('./util/logger')
 
 initializeDatabaseConnection()
@@ -21,28 +17,11 @@ initializeDatabaseConnection()
     dbConnections.on('error', () => {
       logger.error('Failed to connect to sis db!')
     })
-
-    const app = express()
-    initializeSentry(app)
-    app.use(Sentry.Handlers.requestHandler())
-    app.use(Sentry.Handlers.tracingHandler())
-
     startCron()
 
-    app.use(cors({ credentials: true, origin: frontUrl }))
-    app.use(express.json())
-
+    const app = express()
     routes(app, baseUrl)
 
-    app.get('*', async (_, res) => {
-      const results = { error: 'unknown endpoint' }
-      res.status(404).json(results)
-    })
-
-    app.use(Sentry.Handlers.errorHandler())
-
-    app.use(errorMiddleware)
-
     const server = app.listen(backendPort, () => logger.info(`Backend listening on port ${backendPort}!`))
     process.on('SIGTERM', () => {
       server.close(() => {

services/backend/src/conf-backend.js

@@ -36,6 +36,7 @@ const MATOMO_URL = process.env.MATOMO_URL
 
 // Other stuff
 const DB_URL_KONE = process.env.DB_URL_KONE
+const DB_URL_USER = process.env.DB_URL_USER
 
 let DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS, 10)
 if (isNaN(DB_MAX_CONNECTIONS)) {
@@ -47,10 +48,8 @@ if (DB_MAX_CRON_CONNECTIONS < 1) {
 }
 
 const redis = process.env.REDIS
-const TOKEN_SECRET = process.env.TOKEN_SECRET
 const SECRET_TOKEN = process.env.SECRET_TOKEN
 const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
-const USERSERVICE_URL = process.env.USERSERVICE_URL
 const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL
 
 const formatURL = url => {
@@ -59,18 +58,14 @@ const formatURL = url => {
 
 const addSlashToEnd = url => (url.endsWith('/') ? url : url + '/')
 
-const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
-
 module.exports = {
+  DB_URL_USER,
   DB_URL_KONE,
   DB_MAX_CONNECTIONS,
   DB_MAX_CRON_CONNECTIONS,
   redis,
-  TOKEN_SECRET,
   SECRET_TOKEN,
   DB_SCHEMA_KONE,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL),
-  ACCESS_TOKEN_HEADER_KEY,
   requiredGroup,
   SIS_UPDATER_URL,
   ANALYTICS_INFLUXDB_URL: formatURL(ANALYTICS_INFLUXDB_URL),

services/backend/src/database/connection.js

@@ -16,7 +16,7 @@ class DbConnection extends EventEmitter {
       pool: {
         max: 25,
         min: 0,
-        acquire: 10000,
+        acquire: 30000,
         idle: 300000000,
       },
       logging: false,
@@ -39,54 +39,73 @@ class DbConnection extends EventEmitter {
   }
 }
 
+// Old-style kone + user db connections
 const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
   schema: conf.DB_SCHEMA_KONE,
   searchPath: conf.DB_SCHEMA_KONE,
   logging: false,
 })
+
 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)
 
+const sequelizeUser = new Sequelize(conf.DB_URL_USER, {
+  logging: false,
+})
+
 const initializeDatabaseConnection = async () => {
   const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-  const waitSeconds = 60
-  for (let i = 1; i <= waitSeconds; i++) {
-    try {
-      await sequelizeKone.authenticate()
-      break
-    } catch (e) {
-      if (i === waitSeconds) {
-        logger.error(`Could not connect to database in ${waitSeconds} seconds`)
-        throw e
+  const rounds = 60
+
+  for (const [seq, dbName] of [
+    [sequelizeKone, 'kone-db'],
+    [sequelizeUser, 'user-db'],
+  ]) {
+    logger.info(`Connecting to ${dbName}...`)
+    for (let round = 1; round <= rounds; round++) {
+      try {
+        await seq.authenticate()
+        break
+      } catch (error) {
+        if (round === rounds) {
+          logger.error(`${dbName} database connection failed!`)
+          throw error
+        }
+        await sleep(1000)
       }
-      await sleep(1000)
     }
-  }
-  try {
+    logger.info(`${dbName} database connection established`)
+
+    const schema = dbName === 'kone-db' ? conf.DB_SCHEMA_KONE : undefined
+    const migrationsFolder = dbName === 'kone-db' ? 'migrations_kone' : 'migrations_user'
     const migrator = new Umzug({
       storage: 'sequelize',
       storageOptions: {
-        sequelize: sequelizeKone,
+        sequelize: seq,
         tableName: 'migrations',
-        schema: conf.DB_SCHEMA_KONE,
+        schema,
       },
       migrations: {
-        params: [sequelizeKone.getQueryInterface(), Sequelize],
-        path: `${process.cwd()}/src/database/migrations_kone`,
+        params: [seq.getQueryInterface(), Sequelize],
+        path: `${process.cwd()}/src/database/${migrationsFolder}`,
         pattern: /\.js$/,
-        schema: conf.DB_SCHEMA_KONE,
+        schema,
       },
     })
-    const migrations = await migrator.up()
-    logger.info({ message: 'Kone Migrations up to date', meta: migrations })
-  } catch (e) {
-    logger.error({ message: 'Kone Migration error: ', meta: e })
-    throw e
+    try {
+      const migrations = await migrator.up()
+      logger.info({ message: `${dbName} migrations up to date: `, meta: migrations.map(m => m.file) })
+    } catch (error) {
+      logger.error({ message: `${dbName} migrations failed`, meta: error })
+      throw error
+    }
   }
 }
 
 const dbConnections = new DbConnection()
+
 module.exports = {
   dbConnections,
   sequelizeKone,
+  sequelizeUser,
   initializeDatabaseConnection,
 }

services/backend/src/database/migrations_kone/20220307_00_remove_thesis_courses.js

@@ -0,0 +1,6 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('thesis_courses')
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_user/20211227_00_drop_affiliations.js

@@ -0,0 +1,9 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('user_affiliation')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS user_affilation_id_seq')
+    await queryInterface.dropTable('affiliations')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS affiliations_id_seq')
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_user/20220112_00_drop_hy_groups.js

@@ -0,0 +1,9 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('user_hy_group')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS user_hy_group_id_seq')
+    await queryInterface.dropTable('hy_groups')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS hy_groups_id_seq')
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_user/20220712_00_add_iam_groups.js

@@ -0,0 +1,14 @@
+const { ARRAY, STRING } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.addColumn('users', 'iam_groups', {
+      type: ARRAY(STRING),
+      allowNull: false,
+      defaultValue: [],
+    })
+  },
+  down: async queryInterface => {
+    await queryInterface.removeColumn('users', 'iam_groups')
+  },
+}

services/backend/src/events.js

@@ -1,18 +1,10 @@
 const { CronJob } = require('cron')
 const moment = require('moment')
-const {
-  refreshProtoC,
-  refreshStatus,
-  refreshStatusGraduated,
-  refreshUber,
-  refreshProtoCProgramme,
-  getStartYears,
-} = require('./services/trends')
+const { refreshStatus, refreshStatusGraduated, refreshUber, getStartYears } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
-const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studyprogramme')
+const { updateBasicView, updateStudytrackView } = require('./services/studyprogrammeUpdates')
 const { findAndSaveTeachers } = require('./services/topteachers')
-const { setProductivity, setThroughput, patchProductivity, patchThroughput } = require('./services/analyticsService')
 const { isProduction } = require('./conf-backend')
 const { getCurrentSemester } = require('./services/semesters')
 const logger = require('./util/logger')
@@ -24,47 +16,23 @@ const refreshStudyrightAssociations = async () => {
   logger.info('Studyright associations refreshed!')
 }
 
-const refreshOverview = async () => {
-  logger.info('Refreshing throughput and productivity for programmes...')
-  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH') || code.includes('MH'))
+const refreshNewOverviews = async () => {
+  logger.info('Refreshing studyprogramme and studytrack overview statistics for all programmes')
+  const codes = (await getAllProgrammes())
+    .map(p => p.code)
+    .filter(code => code.includes('KH') || code.includes('MH') || /^(T)[0-9]{6}$/.test(code))
   let ready = 0
   for (const code of codes) {
-    let programmeStatsSince = new Date('2017-07-31')
-    if (code.includes('MH') || code.includes('KH')) {
-      programmeStatsSince = new Date('2017-07-31')
-    } else {
-      programmeStatsSince = new Date('2000-07-31')
-    }
     try {
-      await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-      const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
-      await setThroughput(data)
+      await updateBasicView(code)
+      await updateStudytrackView(code)
+      ready += 1
     } catch (e) {
-      logger.error({ message: `failed to recalculate throughput for ${code}`, meta: e })
-      try {
-        await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-      } catch (e) {
-        logger.error({ message: 'failed to update throughtput status to error', meta: e })
-      }
+      logger.error({ message: `Failed to update overview stats for programme ${code}`, meta: e })
     }
-    try {
-      await patchProductivity({ [code]: { status: 'RECALCULATING' } })
-      const data = await productivityStatsForStudytrack(code, programmeStatsSince)
-      await setProductivity(data)
-    } catch (e) {
-      logger.error({ message: `failed to recalculate productivity for ${code}`, meta: e })
-      try {
-        await patchProductivity({
-          [code]: { status: 'RECALCULATION ERRORED' },
-        })
-      } catch (e) {
-        logger.error({ message: 'failed to update productivity status to error', meta: e })
-      }
-    }
-    ready += 1
     logger.info(`${ready}/${codes.length} programmes done`)
   }
-  logger.info('Throughput and productivity for programmes refreshed')
+  logger.info('Studyprogramme and studytrack overview stats refreshed!')
 }
 
 const refreshTeacherLeaderboard = async () => {
@@ -73,19 +41,6 @@ const refreshTeacherLeaderboard = async () => {
   await findAndSaveTeachers(currentSemestersYearCode - 1, currentSemestersYearCode)
 }
 
-const refreshProtoCtoRedis = async () => {
-  logger.info('Refreshing CDS ProtoC')
-  const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false' }
-  const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false' }
-  const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true' }
-  const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true' }
-  await refreshProtoC(defaultQuery)
-  await refreshProtoC(onlyOld)
-  await refreshProtoC(onlyEnr)
-  await refreshProtoC(bothToggles)
-  logger.info('Refreshing CDS ProtoC doned')
-}
-
 const refreshStatusToRedis = async () => {
   const unixMillis = moment().valueOf()
   const date = new Date(Number(unixMillis))
@@ -122,32 +77,8 @@ const refreshUberToRedis = async () => {
   logger.info(`Refreshing CDS Uber data doned`)
 }
 
-const refreshProtoCProgrammeToRedis = async () => {
-  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH'))
-  for (const code of codes) {
-    logger.info(`Refreshing ProtoCProgramme code ${code}`)
-    const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
-    const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
-    const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
-    const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true', code }
-    const queries = [defaultQuery, onlyOld, onlyEnr, bothToggles]
-    for (const query of queries) {
-      try {
-        await refreshProtoCProgramme(query)
-      } catch (e) {
-        logger.error({
-          message: `Error when refreshing ProtocProgramme code ${code}, query ${Object.keys({ query })[0]}`,
-          meta: e,
-        }) // object.keys is hax to print query variable name
-      }
-    }
-    logger.info(`Refreshing ProtoCProgramme code ${code} doned`)
-  }
-  logger.info(`Refreshing protocprogramme doned`)
-}
-
 const refreshStatistics = async () => {
-  const statfuncs = [refreshStudyrightAssociations, refreshOverview, refreshTeacherLeaderboard]
+  const statfuncs = [refreshStudyrightAssociations, refreshTeacherLeaderboard]
   logger.info('Refreshing statistics')
   for (const func of statfuncs) {
     await func()
@@ -156,7 +87,7 @@ const refreshStatistics = async () => {
 }
 
 const refreshTrends = async () => {
-  const trendfuncs = [refreshProtoCtoRedis, refreshStatusToRedis, refreshUberToRedis, refreshProtoCProgrammeToRedis]
+  const trendfuncs = [refreshStatusToRedis, refreshUberToRedis]
   logger.info('Refreshing trends')
   for (const func of trendfuncs) {
     await func()
@@ -166,9 +97,10 @@ const refreshTrends = async () => {
 
 const startCron = () => {
   if (isProduction) {
+    logger.info('Cronjob for refreshing stats started: runs at 3am every day.')
     // refresh 3am every day
     schedule('0 3 * * *', async () => {
-      for (const func of [refreshStatistics, refreshTrends]) {
+      for (const func of [refreshStatistics, refreshTrends, refreshNewOverviews]) {
         await func()
       }
     })

services/backend/src/middleware/accessLogger.js

@@ -1,5 +1,6 @@
 const morgan = require('morgan')
 const logger = require('../util/logger')
+const _ = require('lodash')
 
 // So this appears to be a hack to get neatly formatted stats like response-time etc. from morgan
 // without actually using morgan what it's used for (LOGGING REQUESTS!).
@@ -7,7 +8,8 @@ const logger = require('../util/logger')
 // so don't e.g. make it into an async function or it'll print [object Promise] on every request
 const accessLogger = morgan((tokens, req, res) => {
   const fields = ['method', 'url', 'status', 'response-time', 'remote-addr', 'remote-user', 'user-agent', 'referrer']
-  const meta = { ...req.decodedToken }
+  const { user } = req
+  const meta = { ...user }
   fields.forEach(field => (meta[field] = tokens[field](req, res)))
   meta['time'] = tokens['date'](req, res, 'iso')
   if (req.route) {
@@ -15,7 +17,7 @@ const accessLogger = morgan((tokens, req, res) => {
   }
 
   const message = [
-    req.decodedToken.name,
+    user.name,
     ':',
     tokens['method'](req, res),
     tokens['url'](req, res),
@@ -26,7 +28,8 @@ const accessLogger = morgan((tokens, req, res) => {
   ].join(' ')
 
   logger.info(message, {
-    ...meta,
+    // don't log student list which might be huge
+    ..._.omit(meta, ['studentsUserCanAccess']),
     // pass this as a custom field so we can filter by it in graylog
     isUsageStats: true,
   })

services/backend/src/middleware/auth.js

@@ -1,43 +1,8 @@
-const jwt = require('jsonwebtoken')
-const conf = require('../conf-backend')
-const blacklist = require('../services/blacklist')
-const { getUserDataFor } = require('../services/userService')
-const { ACCESS_TOKEN_HEADER_KEY } = require('../conf-backend')
-const { hasRequiredGroup, parseHyGroups } = require('../util/utils')
-
-const TOKEN_VERSION = 1.1 // When token structure changes, increment in userservice, backend and frontend
-
-const checkAuth = async (req, res, next) => {
-  const token = req.headers[ACCESS_TOKEN_HEADER_KEY]
-  const uid = req.headers['uid']
-
-  if (!token) {
-    return res.status(403).json({ error: 'No token in headers' })
-  }
-
-  jwt.verify(token, conf.TOKEN_SECRET, async (err, decoded) => {
-    if (err) {
-      return res.status(403).json(err)
-    }
-
-    if (decoded.version !== TOKEN_VERSION) {
-      return res.status(401).json({ error: 'Token needs to be refreshed - invalid version' })
-    }
-
-    if (decoded.mockedBy ? decoded.mockedBy !== uid : decoded.userId !== uid) {
-      return res.status(403).json({ error: 'User shibboleth id and token id did not match' })
-    }
-
-    const userData = await getUserDataFor(decoded.userId)
-    req.decodedToken = decoded
-    Object.assign(req, userData)
-    next()
-  })
-}
-
 const roles = requiredRoles => async (req, res, next) => {
-  if (req.decodedToken) {
-    const { roles } = req
+  if (req.user) {
+    const {
+      user: { roles },
+    } = req
 
     if (requiredRoles.every(r => roles.indexOf(r) >= 0) || roles.includes('admin')) {
       return next()
@@ -47,37 +12,6 @@ const roles = requiredRoles => async (req, res, next) => {
   res.status(403).json({ error: 'missing required roles' })
 }
 
-const checkRequiredGroup = async (req, res, next) => {
-  const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-  const enabled = hasRequiredGroup(hyGroups)
-  const tokenOutdated = req.decodedToken.enabled !== enabled
-
-  if (tokenOutdated) {
-    return res.status(401).json({
-      error: 'Token needs to be refreshed - enabled does not match hy-group requirement',
-    })
-  }
-
-  if (!enabled) {
-    return res.status(403).json({ error: 'User is not enabled' })
-  }
-  next()
-}
-
-const checkUserBlacklisting = async (req, res, next) => {
-  const { userId, createdAt } = req.decodedToken
-  const isBlacklisted = await blacklist.isUserBlacklisted(userId, createdAt)
-
-  if (isBlacklisted) {
-    return res.status(401).json({ error: 'Token needs to be refreshed - blacklisted' })
-  }
-
-  next()
-}
-
 module.exports = {
-  checkAuth,
   roles,
-  checkRequiredGroup,
-  checkUserBlacklisting,
 }

services/backend/src/middleware/currentUserMiddleware.js

@@ -0,0 +1,58 @@
+const Sentry = require('@sentry/node')
+const { ApplicationError } = require('../util/customErrors')
+const { getUser, getMockedUser } = require('../services/userService')
+const { requiredGroup } = require('../conf-backend')
+const _ = require('lodash')
+const { relevantIAMs } = require('../../config/IAMConfig')
+const { getOrganizationAccess } = require('../util/organizationAccess')
+
+const parseIamGroups = iamGroups => iamGroups?.split(';').filter(Boolean) ?? []
+
+const hasRequiredIamGroup = (iamGroups, iamRights) =>
+  _.intersection(iamGroups, requiredGroup).length > 0 || iamRights.length > 0
+
+const currentUserMiddleware = async (req, _res, next) => {
+  const {
+    'shib-session-id': sessionId,
+    'x-show-as-user': showAsUser,
+    displayname: name,
+    hygroupcn,
+    hypersonsisuid: sisId,
+    mail: email,
+    shib_logout_url: logoutUrl,
+    uid: username,
+  } = req.headers
+
+  if (!sessionId || !username) {
+    throw new ApplicationError('Not enough data in request headers', 403, { logoutUrl })
+  }
+
+  const iamGroups = parseIamGroups(hygroupcn).filter(iam => relevantIAMs.includes(iam))
+
+  const iamRights = Object.keys(await getOrganizationAccess({ iamGroups }))
+
+  if (!hasRequiredIamGroup(iamGroups, iamRights)) {
+    throw new ApplicationError('User does not have required iam group', 403, { logoutUrl })
+  }
+
+  let user = await getUser({
+    username,
+    name,
+    email,
+    iamGroups,
+    sisId,
+  })
+
+  if (showAsUser && user.isAdmin) {
+    user = await getMockedUser({ userToMock: showAsUser, mockedBy: username })
+  }
+
+  Sentry.setUser({ username: user.mockedBy ?? username })
+
+  req.user = user
+  req.logoutUrl = logoutUrl
+
+  next()
+}
+
+module.exports = currentUserMiddleware

services/backend/src/middleware/sentryUserId.js

@@ -1,13 +0,0 @@
-const Sentry = require('@sentry/node')
-
-const sentryUserIdMiddleware = (req, res, next) => {
-  Sentry.getCurrentHub().configureScope(scope => {
-    scope.setUser({
-      username: req.decodedToken && req.decodedToken.userId,
-    })
-  })
-
-  next()
-}
-
-module.exports = sentryUserIdMiddleware

services/backend/src/middleware/shibbolethCharsetMiddleware.js

@@ -0,0 +1,5 @@
+const headersMiddleware = require('unfuck-utf8-headers-middleware')
+
+const headers = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
+
+module.exports = headersMiddleware(headers)

services/backend/src/models/course.js

@@ -47,6 +47,13 @@ Course.init(
     substitutions: {
       type: JSONB,
     },
+    course_unit_type: {
+      type: STRING,
+    },
+    mainCourseCode: {
+      field: 'main_course_code',
+      type: STRING,
+    },
   },
   {
     underscored: false,

services/backend/src/models/credit.js

@@ -75,6 +75,12 @@ Credit.init(
     updatedAt: {
       type: DATE,
     },
+    language: {
+      type: STRING,
+    },
+    is_open: {
+      type: BOOLEAN,
+    },
   },
   {
     underscored: false,

services/backend/src/models/enrollment.js

@@ -0,0 +1,69 @@
+const { Model, DATE, INTEGER, STRING, BOOLEAN } = require('sequelize')
+const { dbConnections } = require('../database/connection')
+
+class Enrollment extends Model {}
+
+Enrollment.init(
+  {
+    id: {
+      primaryKey: true,
+      type: STRING,
+    },
+    studentnumber: {
+      type: STRING,
+      references: {
+        model: 'student',
+        key: 'studentnumber',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+    },
+    course_code: {
+      type: STRING,
+    },
+    state: {
+      type: STRING,
+    },
+    enrollment_date_time: {
+      type: DATE,
+    },
+    course_id: {
+      type: STRING,
+      references: {
+        model: 'course',
+        key: 'id',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+    },
+    semester_composite: {
+      type: STRING,
+      references: {
+        model: 'semesters',
+        key: 'composite',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+    },
+    semestercode: {
+      type: INTEGER,
+    },
+    createdAt: {
+      type: DATE,
+    },
+    updatedAt: {
+      type: DATE,
+    },
+    is_open: {
+      type: BOOLEAN,
+    },
+  },
+  {
+    underscored: false,
+    sequelize: dbConnections.sequelize,
+    modelName: 'enrollment',
+    tableName: 'enrollment',
+  }
+)
+
+module.exports = Enrollment

services/backend/src/models/index.js

@@ -10,6 +10,7 @@ const CreditType = require('./creditType')
 const Credit = require('./credit')
 const CreditTeacher = require('./creditTeacher')
 const ElementDetail = require('./elementDetail')
+const Studyplan = require('./studyplan')
 const StudyrightExtent = require('./studyrightExtent')
 const Studyright = require('./studyright')
 const StudyrightElement = require('./studyrightElement')
@@ -17,6 +18,7 @@ const Transfer = require('./transfer')
 const ProgrammeModule = require('./programmeModule')
 const ProgrammeModuleChild = require('./programmeModuleChild')
 const ExcludedCourse = require('./excludedCourse')
+const Enrollment = require('./enrollment')
 
 CourseType.hasMany(Course, { foreignKey: 'coursetypecode', sourceKey: 'coursetypecode' })
 Course.belongsTo(CourseType, { foreignKey: 'coursetypecode', targetKey: 'coursetypecode' })
@@ -62,6 +64,9 @@ Teacher.belongsToMany(Credit, { through: CreditTeacher, foreignKey: 'teacher_id'
 Organization.hasMany(Studyright, { foreignKey: 'facultyCode', sourceKey: 'code' })
 Studyright.belongsTo(Organization, { foreignKey: 'facultyCode', targetKey: 'code' })
 
+Studyplan.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
+Student.hasMany(Studyplan, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
+
 Studyright.belongsTo(Student, { foreignKey: 'studentStudentnumber', targetKey: 'studentnumber' })
 Student.hasMany(Studyright, { foreignKey: 'studentStudentnumber', sourceKey: 'studentnumber' })
 
@@ -88,6 +93,12 @@ Studyright.hasMany(Transfer, { foreignKey: 'studyrightid', sourceKey: 'studyrigh
 Transfer.belongsTo(ElementDetail, { as: 'source', foreignKey: 'sourcecode' })
 Transfer.belongsTo(ElementDetail, { as: 'target', foreignKey: 'targetcode' })
 
+Enrollment.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
+Student.hasMany(Enrollment, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
+Enrollment.belongsTo(Course, { foreignKey: 'course_id' })
+Course.hasMany(Enrollment, { foreignKey: 'course_id' })
+Enrollment.belongsTo(Semester, { foreignKey: { name: 'semester_composite', allowNull: false } })
+
 ProgrammeModule.belongsToMany(ProgrammeModule, {
   as: 'parents',
   through: ProgrammeModuleChild,
@@ -114,6 +125,7 @@ module.exports = {
   Credit,
   CreditTeacher,
   ElementDetail,
+  Studyplan,
   StudyrightExtent,
   Studyright,
   StudyrightElement,
@@ -121,4 +133,5 @@ module.exports = {
   ProgrammeModule,
   ProgrammeModuleChild,
   ExcludedCourse,
+  Enrollment,
 }

services/backend/src/models/models_kone.js

@@ -104,20 +104,6 @@ const MandatoryCourseLabels = sequelizeKone.define('mandatory_course_labels', {
   },
 })
 
-const ThesisCourse = sequelizeKone.define('thesis_courses', {
-  programmeCode: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  courseCode: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  thesisType: {
-    type: Sequelize.ENUM([ThesisTypeEnums.BACHELOR, ThesisTypeEnums.MASTER]),
-  },
-})
-
 const CustomPopulationSearch = sequelizeKone.define('custom_population_searches', {
   id: {
     primaryKey: true,
@@ -173,7 +159,6 @@ module.exports = {
   MandatoryCourse,
   TagStudent,
   Tag,
-  ThesisCourse,
   ThesisTypeEnums,
   MandatoryCourseLabels,
   CustomPopulationSearch,

services/backend/src/models/models_user.js

@@ -1,7 +1,7 @@
 const Sequelize = require('sequelize')
-const { sequelize } = require('../database/connection')
+const { sequelizeUser } = require('../database/connection')
 
-const UserElementDetails = sequelize.define(
+const UserElementDetails = sequelizeUser.define(
   'user_elementdetails',
   {
     userId: {
@@ -18,7 +18,7 @@ const UserElementDetails = sequelize.define(
   }
 )
 
-const AccessGroup = sequelize.define('access_group', {
+const AccessGroup = sequelizeUser.define('access_group', {
   id: {
     primaryKey: true,
     type: Sequelize.BIGINT,
@@ -39,7 +39,7 @@ const AccessGroup = sequelize.define('access_group', {
   },
 })
 
-const UserFaculties = sequelize.define('user_faculties', {
+const UserFaculties = sequelizeUser.define('user_faculties', {
   userId: {
     primaryKey: true,
     type: Sequelize.BIGINT,
@@ -60,7 +60,7 @@ const UserFaculties = sequelize.define('user_faculties', {
   },
 })
 
-const User = sequelize.define(
+const User = sequelizeUser.define(
   'users',
   {
     id: {
@@ -76,6 +76,11 @@ const User = sequelize.define(
     email: { type: Sequelize.STRING },
     language: { type: Sequelize.STRING },
     sisu_person_id: { type: Sequelize.STRING },
+    iam_groups: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+      allowNull: false,
+      defaultValue: [],
+    },
     last_login: { type: Sequelize.DATE },
   },
   {
@@ -83,41 +88,6 @@ const User = sequelize.define(
     timestamps: false,
   }
 )
-const HyGroup = sequelize.define('hy_group', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true,
-  },
-
-  code: { type: Sequelize.STRING },
-
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})
-
-const Affiliation = sequelize.define('affiliations', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true,
-  },
-
-  code: { type: Sequelize.STRING },
-
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})
 
 User.hasMany(UserElementDetails, { as: 'programme' })
 UserElementDetails.belongsTo(User)
@@ -128,23 +98,12 @@ User.belongsToMany(AccessGroup, {
 })
 AccessGroup.belongsToMany(User, { through: 'user_accessgroup' })
 
-User.belongsToMany(HyGroup, { through: 'user_hy_group', as: 'hy_group' })
-HyGroup.belongsToMany(User, { through: 'user_hy_group' })
-
-User.belongsToMany(Affiliation, {
-  through: 'user_affiliation',
-  as: 'affiliation',
-})
-Affiliation.belongsToMany(User, { through: 'user_affiliation' })
-
 User.hasMany(UserFaculties, { as: 'faculty', foreignKey: 'userId' })
 
 module.exports = {
   User,
   UserElementDetails,
   AccessGroup,
-  HyGroup,
-  Affiliation,
   UserFaculties,
-  sequelize,
+  sequelizeUser,
 }

services/backend/src/models/student.js

@@ -17,7 +17,6 @@ Student.init(
     dateofuniversityenrollment: { type: DATE },
     email: { type: STRING },
     national_student_number: { type: STRING },
-    home_county_id: { type: INTEGER },
     country_fi: { type: STRING },
     country_sv: { type: STRING },
     country_en: { type: STRING },
@@ -25,6 +24,7 @@ Student.init(
     home_country_sv: { type: STRING },
     home_country_en: { type: STRING },
     gender_code: { type: INTEGER },
+    sis_person_id: { type: STRING },
     createdAt: {
       type: DATE,
     },

services/backend/src/models/studyplan.js

@@ -0,0 +1,48 @@
+const { Model, DATE, ARRAY, STRING, INTEGER } = require('sequelize')
+const { dbConnections } = require('../database/connection')
+
+class Studyplan extends Model {}
+
+Studyplan.init(
+  {
+    id: {
+      primaryKey: true,
+      type: STRING,
+      allowNull: false,
+    },
+    studentnumber: {
+      type: STRING,
+      references: {
+        model: 'student',
+        key: 'studentnumber',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+      allowNull: false,
+    },
+    programme_code: {
+      type: STRING,
+      allowNull: false,
+    },
+    included_courses: {
+      type: ARRAY(STRING),
+    },
+    completed_credits: {
+      type: INTEGER,
+    },
+    createdAt: {
+      type: DATE,
+    },
+    updatedAt: {
+      type: DATE,
+    },
+  },
+  {
+    underscored: false,
+    sequelize: dbConnections.sequelize,
+    modelName: 'studyplan',
+    tableName: 'studyplan',
+  }
+)
+
+module.exports = Studyplan

services/backend/src/models/studyright.js

@@ -9,9 +9,6 @@ Studyright.init(
       primaryKey: true,
       type: STRING,
     },
-    canceldate: {
-      type: DATE,
-    },
     startdate: {
       type: DATE,
     },
@@ -27,6 +24,9 @@ Studyright.init(
     graduated: {
       type: INTEGER,
     },
+    active: {
+      type: INTEGER,
+    },
     // irtisanomisperuste
     studentStudentnumber: {
       type: STRING,

services/backend/src/rapodiff.js

@@ -0,0 +1,363 @@
+/* eslint no-console: 0 */
+
+/*
+  depends on nodeproxy running in importer.cs.helsinki.fi, and env RAPO_NODEPROXY (see importer for a proper value)
+  needs also acces to importer db api, uses env IMPORTER_API
+
+  known differences in APIs
+    - rapo does not show students that have graduated (it however takes some days for rapo to notice a graduation)
+    - rapo does not handle the starting year of transfers properly (shows in old staring population)
+    - rapo show some students incorrectly passive, see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3521
+*/
+
+const axios = require('axios').default
+
+const _ = require('lodash')
+
+const { optimizedStatisticsOf } = require('./services/populations')
+const { Student, Transfer, StudyrightElement, Studyright } = require('./models')
+
+const token = process.env.RAPO_NODEPROXY
+
+const getFromRapo = async (urlStart, urlEnd) => {
+  const { data: rapoStuffLasna } = await axios.get(urlStart)
+  const { data: rapoStuffLasnaMinus } = await axios.get(urlEnd)
+
+  const rapoStudents = _.difference(
+    rapoStuffLasna.map(s => s.opiskelijanumero),
+    rapoStuffLasnaMinus.map(s => s.opiskelijanumero)
+  )
+
+  return rapoStudents
+}
+
+const known_problems_in_oodikone = {
+  KH30_001: ['013896191'],
+  KH40_003: ['013200457'],
+  KH55_001: ['013195834', '015119708', '015138336'],
+}
+
+const known_problems_in_rapo = {
+  KH55_001: ['015160142', '015135575', '015119708', '015138336'],
+}
+
+const getFromOodikone = async (programme, year) => {
+  const isNotAmongKnownProblems = student => {
+    const hasProblems = known_problems_in_oodikone[programme] ? known_problems_in_oodikone[programme] : []
+    return !hasProblems.includes(student.studentNumber)
+  }
+
+  const theStudyrights = student => {
+    const theOne = s => s.studyright_elements.find(e => e.code === programme)
+    // in some rare cases student may have more than one studyright for same programme
+    return student.studyrights.filter(theOne)
+  }
+
+  const notGraduated = student => {
+    const graduation = theStudyrights(student).filter(r => r.graduated === 1)
+    return graduation.length === 0
+  }
+
+  const notTransferred = student => {
+    const transfer = student.transfers.find(t => t.targetcode === programme)
+    return transfer === undefined
+  }
+
+  const months = 12
+
+  const oodikoneQuery = {
+    semesters: ['FALL', 'SPRING'],
+    months,
+    studyRights: { programme },
+    year,
+  }
+
+  const oodikoneStuff = (await optimizedStatisticsOf(oodikoneQuery)).students
+
+  return oodikoneStuff
+    .filter(notGraduated)
+    .filter(notTransferred)
+    .filter(isNotAmongKnownProblems)
+    .map(s => s.studentNumber)
+}
+
+const isAcuallyActive = async student => {
+  const currentSemester = 144
+  const tokenImporterApi = process.env.IMPORTER_API
+
+  const url = `https://importer.cs.helsinki.fi/api/importer/students/${student}/rapo_semester_enrollments?token=${tokenImporterApi}`
+  const { data } = await axios.get(url)
+  const terms = data.reduce((all, s) => all.concat(s.terms), []).filter(t => t.semester_code === currentSemester)
+  const active = terms.filter(t => t.type !== 'NEGLECTED')
+  const negl = terms.filter(t => t.type === 'NEGLECTED')
+  return active.length > 0 && negl.length > 0
+}
+
+const hasTransferred = async (student, programme) => {
+  const data = await Student.findByPk(student, {
+    include: [
+      {
+        model: Transfer,
+      },
+    ],
+  })
+
+  const transferTo = data.transfers.find(t => t.targetcode === programme)
+  return transferTo !== undefined
+}
+
+const graduatedVeryRecently = async (student, programme) => {
+  const data = await Student.findByPk(student, {
+    include: [
+      {
+        model: Studyright,
+        include: [
+          {
+            model: StudyrightElement,
+            required: true,
+            where: {
+              code: programme,
+            },
+          },
+        ],
+        where: {
+          graduated: 1,
+        },
+      },
+    ],
+  })
+
+  if (!data || data.studyrights.length === 0) {
+    return false
+  }
+
+  const graduated = new Date(data.studyrights[0].enddate)
+  const weekAgo = new Date(new Date().getTime() - 7 * 24 * 60 * 60 * 1000)
+  return graduated > weekAgo
+}
+
+const programme_diff_year = async (programme, facultyCode, year, brief = false) => {
+  const oodikoneStudents = await getFromOodikone(programme, year)
+
+  const studyrightStartDate = `${year}-08-01`
+  const studyrightEndDate = `${year + 1}-08-01`
+
+  const rapoUrlStart = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+    )
+  const rapoUrlEnd = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+    )
+
+  const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
+  const rapoStudentsPoissa = await getFromRapo(rapoUrlStart('Poissa'), rapoUrlEnd('Poissa'))
+  const rapoStudentPoissaLaki = await getFromRapo(
+    rapoUrlStart('Poissa ei kuluta opintoaikaa'),
+    rapoUrlEnd('Poissa ei kuluta opintoaikaa')
+  )
+
+  const isNotAmongKnownProblems = student => {
+    const hasProblems = known_problems_in_rapo[programme] ? known_problems_in_rapo[programme] : []
+    return !hasProblems.includes(student.studentNumber)
+  }
+
+  const rapoStudents = rapoStudentsLasna
+    .concat(rapoStudentsPoissa)
+    .concat(rapoStudentPoissaLaki)
+    .filter(isNotAmongKnownProblems)
+
+  const onlyInOodikone = _.difference(oodikoneStudents, rapoStudents)
+  const onlyInRapo = _.difference(rapoStudents, oodikoneStudents)
+
+  const wrongMissingInRapo = []
+  for (let student of onlyInOodikone) {
+    const status = await isAcuallyActive(student)
+    if (status) {
+      wrongMissingInRapo.push(student)
+    }
+  }
+
+  // these are in wrong RAPO-population
+  const transferred = []
+  for (let student of onlyInRapo) {
+    const isTransfer = await hasTransferred(student, programme)
+    if (isTransfer) {
+      transferred.push(student)
+    } else {
+      const isRecentGraduation = await graduatedVeryRecently(student, programme)
+      if (isRecentGraduation) {
+        transferred.push(student)
+      }
+    }
+  }
+
+  const checkOodikoneOnly = _.difference(onlyInOodikone, wrongMissingInRapo)
+  const checkRapoOnly = _.difference(onlyInRapo, transferred)
+
+  if (!brief) {
+    console.log('vuosi', year)
+    console.log('')
+    console.log('oodikone', oodikoneStudents.length, ' transferred', transferred.length)
+    console.log('rapo', rapoStudents.length, ' activeness wrong rapo', wrongMissingInRapo.length)
+    console.log('only rapo', checkRapoOnly.length)
+    console.log('only oodikone', checkOodikoneOnly.length)
+
+    console.log('')
+
+    if (checkRapoOnly.length > 0) {
+      console.log('Only in Rapo:')
+      console.log(checkRapoOnly.join('\n'))
+    }
+
+    if (checkOodikoneOnly.length > 0) {
+      console.log('Only in Oodikone:')
+      console.log(checkOodikoneOnly.join('\n'))
+    }
+  }
+
+  if (checkRapoOnly.length + checkOodikoneOnly.length) {
+    console.log('******************************', year)
+    if (checkRapoOnly.length > 0) {
+      console.log('Only in Rapo:')
+      console.log(checkRapoOnly.join('\n'))
+    }
+
+    if (checkOodikoneOnly.length > 0) {
+      console.log('Only in Oodikone:')
+      console.log(checkOodikoneOnly.join('\n'))
+    }
+  }
+
+  if (!brief) {
+    console.log('')
+  }
+}
+
+const programme_diff = async (programme, faculty, brief) => {
+  console.log()
+  console.log(programme)
+  console.log('--------', '\n')
+
+  await programme_diff_year(programme, faculty, 2017, brief)
+  await programme_diff_year(programme, faculty, 2018, brief)
+  await programme_diff_year(programme, faculty, 2019, brief)
+  await programme_diff_year(programme, faculty, 2020, brief)
+  await programme_diff_year(programme, faculty, 2021, brief)
+}
+
+const faculty_diff = async (faculty, programmes, brief) => {
+  console.log('\n')
+  console.log(faculty)
+  console.log('==')
+  for (let programme of programmes) {
+    await programme_diff(programme, faculty, brief)
+  }
+}
+
+// eslint-disable-next-line no-unused-vars
+const msc_programme_diff_year = async (programme, facultyCode, year, brief = false) => {
+  const oodikoneStudents = await getFromOodikone(programme, year)
+
+  const studyrightStartDate = `${year}-08-01`
+  const studyrightEndDate = `${year + 1}-08-01`
+
+  const rapoUrlStart = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+    )
+  const rapoUrlEnd = presence =>
+    encodeURI(
+      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+    )
+
+  const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
+  const rapoStudentsPoissa = await getFromRapo(rapoUrlStart('Poissa'), rapoUrlEnd('Poissa'))
+  const rapoStudentPoissaLaki = await getFromRapo(
+    rapoUrlStart('Poissa ei kuluta opintoaikaa'),
+    rapoUrlEnd('Poissa ei kuluta opintoaikaa')
+  )
+
+  const rapoStudents = rapoStudentsLasna.concat(rapoStudentsPoissa).concat(rapoStudentPoissaLaki)
+
+  const onlyInOodikone = _.difference(oodikoneStudents, rapoStudents)
+  const onlyInRapo = _.difference(rapoStudents, oodikoneStudents)
+
+  const wrongInRapo = []
+  for (let student of onlyInOodikone) {
+    const status = await isAcuallyActive(student)
+    if (status) {
+      //wrongInRapo.push(student)
+    }
+  }
+
+  // in wrong RAPO population
+  const transferred = []
+  for (let student of onlyInRapo) {
+    const status = await hasTransferred(student, programme)
+    if (status) {
+      transferred.push(student)
+    }
+  }
+
+  const checkOodikoneOnly = _.difference(onlyInOodikone, wrongInRapo)
+  const checkRapoOnly = _.difference(onlyInRapo, transferred)
+  const both = _.intersection(oodikoneStudents, rapoStudents)
+
+  console.log('vuosi', year)
+
+  if (!brief) {
+    console.log('')
+    console.log('oodikone', oodikoneStudents.length, ' transferred', transferred.length)
+    console.log('rapo', rapoStudents.length, ' activeness wrong rapo', wrongInRapo.length)
+    console.log('both', both.length)
+    console.log('only rapo', checkRapoOnly.length)
+    console.log('only oodikone', checkOodikoneOnly.length)
+
+    console.log('')
+
+    if (checkRapoOnly.length > 0) {
+      console.log('Only in Rapo:')
+      console.log(checkRapoOnly.join('\n'))
+    }
+
+    if (checkOodikoneOnly.length > 0) {
+      console.log('Only in Oodikone:')
+      console.log(checkOodikoneOnly.join('\n'))
+    }
+  }
+
+  if (checkRapoOnly.length + checkOodikoneOnly.length) {
+    console.log('******************************************************')
+  }
+
+  if (!brief) {
+    console.log('')
+  }
+}
+
+const bsc_priogrammes_of_faculties = {
+  H10: ['KH10_001'],
+  H20: ['KH20_001'],
+  H30: ['KH30_001', 'KH30_002'],
+  H40: ['KH40_001', 'KH40_002', 'KH40_003', 'KH40_004', 'KH40_005', 'KH40_006'],
+  H50: ['KH50_001', 'KH50_002', 'KH50_003', 'KH50_004', 'KH50_005', 'KH50_006', 'KH50_007', 'KH50_008'],
+  H55: ['KH55_001'],
+  H60: ['KH60_001'],
+  H70: ['KH70_001', 'KH70_002', 'KH70_003', 'KH70_004'],
+  H74: ['KH74_001'],
+  H80: ['KH80_001', 'KH80_002', 'KH80_003', 'KH80_004'],
+  H90: ['KH90_001'],
+}
+
+const main = async () => {
+  for (let faculty of Object.keys(bsc_priogrammes_of_faculties)) {
+    await faculty_diff(faculty, bsc_priogrammes_of_faculties[faculty], true)
+  }
+
+  //await programme_diff_year('KH55_001', 'H55', 2021, true)
+  //await programme_diff('KH60_001', 'H60', true)
+}
+
+main().then(() => process.exit(0))

services/backend/src/routes.js

@@ -1,50 +1,52 @@
-const shibbolethHeadersFix = require('unfuck-utf8-headers-middleware')
-const accessLogger = require('./middleware/accesslogger')
-const sentryUserId = require('./middleware/sentryUserId')
+const Sentry = require('@sentry/node')
+const express = require('express')
+const cors = require('cors')
+const { frontUrl } = require('./conf-backend')
+const shibbolethCharsetMiddleware = require('./middleware/shibbolethCharsetMiddleware')
+const matomoInit = require('./routes/matomo-init')
+const currentUserMiddleware = require('./middleware/currentUserMiddleware')
+const accessLogger = require('./middleware/accessLogger')
+const auth = require('./middleware/auth')
+
 const courses = require('./routes/courses')
 const students = require('./routes/students')
 const population = require('./routes/population')
 const login = require('./routes/login')
-const superlogin = require('./routes/superlogin')
 const language = require('./routes/language')
 const users = require('./routes/users')
 const elementdetails = require('./routes/elementdetails')
-const auth = require('./middleware/auth')
 const teachers = require('./routes/teachers')
 const providers = require('./routes/providers')
 const faculties = require('./routes/faculties')
 const semesters = require('./routes/semesters')
 const mandatoryCourses = require('./routes/mandatorycourses')
 const mandatoryCourseLabels = require('./routes/mandatorycourselabels')
-const ping = require('./routes/ping')
 const feedback = require('./routes/feedback')
 const tags = require('./routes/tags')
 const updater = require('./routes/updater')
 const tsaAnalytics = require('./routes/tsaAnalytics')
-const matomoInit = require('./routes/matomo-init')
 const customPopulationSearch = require('./routes/customPopulationSearch')
 const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
 const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
+const studyProgramme = require('./routes/studyProgramme')
+const initializeSentry = require('./util/sentry')
+
+const errorMiddleware = require('./middleware/errorMiddleware')
 
 module.exports = (app, url) => {
-  app.use(url, ping)
-  app.use(
-    shibbolethHeadersFix([
-      'hyGroupCn',
-      'SHIB_LOGOUT_URL',
-      'eduPersonAffiliation',
-      'uid',
-      'displayName',
-      'mail',
-      'hyPersonSisuId',
-      'preferredLanguage',
-    ])
-  )
+  initializeSentry(app)
+  app.use(Sentry.Handlers.requestHandler())
+  app.use(Sentry.Handlers.tracingHandler())
+
+  app.use(cors({ credentials: true, origin: frontUrl }))
+  app.use(express.json())
+
+  app.use(shibbolethCharsetMiddleware)
   app.use(url, matomoInit)
+  app.use(currentUserMiddleware)
+  app.use(accessLogger)
   app.use(url, login)
-  app.use(`${url}/superlogin`, superlogin)
-  app.use(auth.checkAuth, auth.checkRequiredGroup, auth.checkUserBlacklisting, accessLogger, sentryUserId)
   app.use(url, elementdetails)
   app.use(url, courses)
   app.use(url, students)
@@ -54,7 +56,8 @@ module.exports = (app, url) => {
   app.use(url, semesters)
   app.use(url, tags)
   app.use(url, programmeModules)
-  app.use(`${url}/faculties`, faculties)
+  app.use(url, studyProgramme)
+  app.use(url, faculties)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
   app.use(`${url}/users`, auth.roles(['admin']), users)
@@ -65,4 +68,10 @@ module.exports = (app, url) => {
   app.use(`${url}/custom-population-search`, customPopulationSearch)
   app.use(`${url}/cool-data-science`, trends)
   app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
+  app.get('*', async (_, res) => {
+    const results = { error: 'unknown endpoint' }
+    res.status(404).json(results)
+  })
+  app.use(Sentry.Handlers.errorHandler())
+  app.use(errorMiddleware)
 }

services/backend/src/routes/courses.js

@@ -16,7 +16,9 @@ router.get('/v2/coursesmulti', async (req, res) => {
 })
 
 router.get('/v3/courseyearlystats', async (req, res) => {
-  const { rights, roles } = req
+  const {
+    user: { rights, roles },
+  } = req
 
   const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
 
@@ -26,10 +28,9 @@ router.get('/v3/courseyearlystats', async (req, res) => {
     return res.status(403).json({ error: 'No programmes so no access to course stats' })
   }
 
-  const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
+  const { codes, separate: sep } = req.query
 
   const separate = !sep ? false : JSON.parse(sep)
-  const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
   if (!codes) {
     res.status(422).send('Missing required query parameters')
   } else {
@@ -37,11 +38,9 @@ router.get('/v3/courseyearlystats', async (req, res) => {
     // and users with rights to any specific study programmes
     const anonymize = allowedRoles !== 'admin' && rights.length < 1
     const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
-    const results = await Course.courseYearlyStats(codes, separate, unifyOpenUniCourses, anonymizationSalt)
+    const results = await Course.courseYearlyStats(codes, separate, anonymizationSalt)
     res.json(results)
   }
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/customPopulationSearch.js

@@ -8,7 +8,7 @@ const {
 
 router.get('/', async (req, res) => {
   const {
-    decodedToken: { id },
+    user: { id },
   } = req
   const customPopulationSearches = await getCustomPopulationSearchesByUser(id)
   res.json(customPopulationSearches)
@@ -16,7 +16,7 @@ router.get('/', async (req, res) => {
 
 router.post('/', async (req, res) => {
   const {
-    decodedToken: { id },
+    user: { id },
     body: { name, students },
   } = req
 
@@ -29,7 +29,7 @@ router.post('/', async (req, res) => {
 
 router.put('/:id', async (req, res) => {
   const {
-    decodedToken: { id: userId },
+    user: { id: userId },
     body: { students },
   } = req
   const { id } = req.params
@@ -46,7 +46,7 @@ router.put('/:id', async (req, res) => {
 
 router.delete('/:id', async (req, res) => {
   const {
-    decodedToken: { id: userId },
+    user: { id: userId },
   } = req
   const { id } = req.params
 

services/backend/src/routes/elementdetails.js

@@ -1,23 +1,6 @@
 const router = require('express').Router()
 const MandatoryCourses = require('../services/mandatoryCourses')
-const {
-  productivityStatsForStudytrack,
-  throughputStatsForStudytrack,
-  optionData,
-} = require('../services/studyprogramme')
-const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
-const {
-  getProductivity,
-  setProductivity,
-  getThroughput,
-  setThroughput,
-  patchProductivity,
-  patchThroughput,
-} = require('../services/analyticsService')
 const { getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
-const logger = require('../util/logger')
-
-const programmeStatsSince = new Date('2017-07-31')
 
 router.get('/elementdetails/all', async (req, res) => {
   const elementdetails = await getAllElementDetails()
@@ -29,22 +12,6 @@ router.get('/studyprogrammes', async (req, res) => {
   res.json(studyrights)
 })
 
-router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
-  const code = req.params.id
-
-  let level
-  if (code.includes('MH')) {
-    level = 'MSC'
-  } else if (code.includes('KH')) {
-    level = 'BSC'
-  } else {
-    return res.json([])
-  }
-
-  const data = await optionData(new Date('2017-07-31'), new Date(), code, level)
-  res.json(data)
-})
-
 router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
   if (req.params.id) {
     const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
@@ -54,140 +21,4 @@ router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
   }
 })
 
-router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    let data = null
-    try {
-      data = await getProductivity(code)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} productivity`)
-    }
-    if (!data) {
-      try {
-        const stats = await productivityStatsForStudytrack(code, programmeStatsSince)
-        data = await setProductivity(stats)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      }
-    }
-    return res.json(data)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-  res.status(200).end()
-
-  try {
-    if (code.includes('MH') || code.includes('KH')) {
-      const data = await productivityStatsForStudytrack(code, new Date('2017-07-31'))
-      await setProductivity(data)
-    } else {
-      const data = await productivityStatsForStudytrack(code, new Date('2000-07-31'))
-      await setProductivity(data)
-    }
-  } catch (e) {
-    try {
-      await patchProductivity({
-        [code]: { status: 'RECALCULATION ERRORED' },
-      })
-    } catch (e) {
-      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      return
-    }
-
-    logger.error(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-  }
-  logger.info(`Produdtiviy stats recalculation for studyprogramme ${code} done`)
-})
-
-router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    const thesisPromise = findProgrammeTheses(req.params.id)
-
-    let data = null
-    try {
-      data = await getThroughput(code)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} throughput`)
-    }
-    if (!data) {
-      try {
-        const result = await throughputStatsForStudytrack(req.params.id, programmeStatsSince.getFullYear())
-        data = await setThroughput(result)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      }
-    }
-    return res.json({ ...data, thesis: await thesisPromise })
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-  res.status(200).end()
-
-  try {
-    if (code.includes('MH') || code.includes('KH')) {
-      const data = await throughputStatsForStudytrack(code, 2017)
-      await setThroughput(data)
-    } else {
-      const data = await throughputStatsForStudytrack(code, 2000)
-      await setThroughput(data)
-    }
-  } catch (e) {
-    try {
-      await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-    } catch (e) {
-      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      return
-    }
-    logger.error(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
-  }
-  logger.info(`Throughput stats recalculation for studyprogramme ${code} done`)
-})
-
-router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {
-  const { id } = req.params
-  if (id) {
-    const thesis = await findProgrammeTheses(id)
-    res.json(thesis)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.post('/v2/studyprogrammes/:id/thesis', async (req, res) => {
-  const { id } = req.params
-  const { course, thesisType } = req.body
-  if (id && course && thesisType) {
-    const thesis = await createThesisCourse(id, course, thesisType)
-    res.status(201).json(thesis)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
-  const { id, course } = req.params
-  if (id && course) {
-    const deleted = await deleteThesisCourse(id, course)
-    res.status(204).json(deleted)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/faculties.js

@@ -1,9 +1,103 @@
 const router = require('express').Router()
-const { faculties } = require('../services/organisations')
+const { faculties, degreeProgrammeCodesOfFaculty } = require('../services/organisations')
+const { combineFacultyBasics } = require('../services/facultyBasics')
+const { combineFacultyCredits } = require('../services/facultyCredits')
+const { degreeProgrammesOfFaculty } = require('../services/faculty')
 
-router.get('/', async (req, res) => {
+// Faculty uses a lot of tools designed for Study programme.
+// Some of them have been copied here and slightly edited for faculty purpose.
+
+router.get('/faculties', async (req, res) => {
   const facultyList = await faculties()
   res.json(facultyList)
 })
 
+router.get('/faculties/:id/basicstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+
+  if (!code) return res.status(422).end()
+
+  let allBasics = {
+    id: code,
+    years: [],
+    programmeNames: {},
+    studentInfo: {
+      tableStats: [],
+      graphStats: [],
+      titles: ['', 'Started studying', 'Graduated', 'Transferred inside', 'Transferred away', 'Transferred to'],
+      programmeTableStats: {},
+    },
+    graduationInfo: {
+      tableStats: [],
+      graphStats: [],
+      titles: ['', 'All graduations', 'Bachelors', 'Masters', 'Doctors', 'Others'],
+      programmeTableStats: {},
+    },
+    status: 'DONE',
+    lastUpdated: '',
+  }
+
+  const programmes = await degreeProgrammeCodesOfFaculty(code)
+  if (programmes) {
+    await combineFacultyBasics(allBasics, code, programmes, yearType)
+  }
+  return res.json(allBasics)
+})
+
+router.get('/faculties/:id/creditstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+  const specialGroups = 'SPECIAL_INCLUDED' // req.query?.special_groups
+
+  if (!code) return res.status(422).end()
+
+  let counts = {}
+  let years = []
+  let allCredits = {
+    id: code,
+    years: [],
+    tableStats: [],
+    graphStats: [],
+    programmeTableStats: {},
+    programmeNames: {},
+    titles: [
+      '',
+      'Total',
+      'Major students credits',
+      'Non-major faculty students credits',
+      'Non-major other faculty students credits',
+      'Non-degree student credits',
+    ],
+    status: 'DONE',
+    lastUpdated: '',
+  }
+
+  const programmes = await degreeProgrammesOfFaculty(code)
+
+  if (programmes) {
+    await combineFacultyCredits(allCredits, programmes, yearType, specialGroups, counts, years)
+  }
+  let majors = []
+  let facultyNonMajor = []
+  let otherNonMajor = []
+  let nonDegree = []
+
+  years.forEach(year => {
+    majors.push(counts[year][1])
+    facultyNonMajor.push(counts[year][2])
+    otherNonMajor.push(counts[year][3])
+    nonDegree.push(counts[year][4])
+  })
+
+  allCredits.graphStats = [
+    { name: 'Major students credits', data: majors },
+    { name: 'Non-major faculty students credits', data: facultyNonMajor },
+    { name: 'Non-major other faculty students credits', data: otherNonMajor },
+    { name: 'Non-degree credits', data: nonDegree },
+  ]
+
+  return res.json(allCredits)
+})
+
 module.exports = router

services/backend/src/routes/feedback.js

@@ -1,28 +1,19 @@
 const router = require('express').Router()
-const { getUserDataFor } = require('../services/userService')
 const { sendFeedbackToToska } = require('../services/mailservice')
 const logger = require('../util/logger')
 
-router.post('/email', async (req, res) => {
-  const { content } = req.body
+router.post('/email', async req => {
   const {
-    decodedToken: { userId },
+    body: { content },
+    user,
   } = req
 
-  const { email, full_name } = await getUserDataFor(userId)
-
-  const result = await sendFeedbackToToska({
+  await sendFeedbackToToska({
     feedbackContent: content,
-    userId,
-    userEmail: email,
-    userFullName: full_name,
+    user,
   })
-  if (result.error) {
-    return res.status(400).json(result).end()
-  }
-  logger.info(`${userId} succesfully sent some feedback mail to toska`)
 
-  res.status(200).json('success').end()
+  logger.info(`${user.userId} succesfully sent feedback to toska`)
 })
 
 module.exports = router

services/backend/src/routes/language.js

@@ -2,8 +2,10 @@ const router = require('express').Router()
 const userService = require('../services/userService')
 
 router.post('/language', async (req, res) => {
-  const { language } = req.body
-  const { userId } = req.decodedToken
+  const {
+    body: { language },
+    user: { userId },
+  } = req
   if (!['fi', 'sv', 'en'].includes(language)) {
     return res.status(400).json('invalid language')
   }

services/backend/src/routes/login.js

@@ -1,47 +1,19 @@
 const router = require('express').Router()
-const userService = require('../services/userService')
-const { parseHyGroups } = require('../util/utils')
-const { sendNotificationAboutNewUser } = require('../services/mailservice')
-const logger = require('../util/logger')
+const { ApplicationError } = require('../util/customErrors')
+const _ = require('lodash')
 
-router.post('/login', async (req, res) => {
-  const uid = req.headers['uid']
-  if (req.headers['shib-session-id'] && uid) {
-    const full_name = req.headers.displayname || ''
-    const mail = req.headers.mail || ''
-    const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-    const affiliations = parseHyGroups(req.headers['edupersonaffiliation'])
-    const hyPersonSisuId = req.headers.hypersonsisuid || ''
-    logger.info(`${uid} trying to login, referring to userservice`)
-    let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail, hyPersonSisuId)
-    if (isNew) {
-      const result = await sendNotificationAboutNewUser({ userId: uid, userFullName: full_name })
-      if (result.error) {
-        return res.status(400).json(result).end()
-      }
-    }
-    res.status(200).json({ token })
-  } else {
-    res
-      .status(401)
-      .json({
-        message: `Not enough headers login, uid: ${req.headers.uid}
-        session-id ${req.headers['shib-session-id']}`,
-      })
-      .end()
-  }
-})
+router.get('/login', async (req, res) => {
+  const { user, logoutUrl } = req
 
-router.delete('/logout', async (req, res) => {
-  const logoutUrl = req.headers.shib_logout_url
-  const { returnUrl } = req.body
-  if (logoutUrl) {
-    return res
-      .status(200)
-      .send({ logoutUrl: `${logoutUrl}?return=${returnUrl}` })
-      .end()
+  if (!user) {
+    throw new ApplicationError('User not found', 404)
   }
-  res.status(200).send({ logoutUrl: returnUrl }).end()
+
+  res.send({
+    // don't send possibly huge list of students to frontend
+    user: _.omit(user, ['studentsUserCanAccess']),
+    logoutUrl,
+  })
 })
 
 module.exports = router

services/backend/src/routes/matomo-init.js

@@ -28,7 +28,7 @@ _paq.push(["enableLinkTracking"]);
 
 const script = config.MATOMO_URL ? acual : ''
 
-router.get('/matomo-init', (req, res) => {
+router.get('/matomo-init', (_, res) => {
   res.contentType('text/javascript; charset=UTF-8')
   res.send(script)
 })

services/backend/src/routes/ping.js

@@ -1,85 +0,0 @@
-/* eslint-disable */
-
-const { router, wrapper } = require('./router').routerWithWrapper()
-const axios = require('axios')
-
-wrapper.get('/ping', async (req, res) => {
-  res.json({
-    message: 'pong'
-  })
-})
-
-wrapper.get('/v3/mandatory_courses/:code', async (req, res) => {
-  const courses = []
-  const included = new Set()
-
-  const dumb_flatten = module => {
-    if (module.children) {
-      module.children.forEach(child => {
-        dumb_flatten(child)
-      })
-    } else if (module.code && !module.code.startsWith('KK') && !courses.map(c => c.code).includes(module.code)) {
-      courses.push({
-        name: {
-          fi: module.name,
-          en: module.name
-        },
-        code: module.code,
-        label: { id: `${courses.length}`, label: module.code.replace('-', '').slice(0, 4), orderNumber: courses.length }
-      })
-    }
-  }
-
-  const better_flatten = (module, label) => {
-    if (module.children) {
-      module.children.forEach(child => {
-        better_flatten(child, module.module ? module.module.name : label)
-      })
-    } else if (module.code && !module.code.startsWith('KK')) {
-      courses.push({
-        name: {
-          fi: module.name,
-          en: module.name
-        },
-        code: module.code,
-        label: { id: `${courses.length}`, label, orderNumber: courses.length }
-      })
-    }
-  }
-
-  const superFlattenFlatten = (module, label) => {
-    if (module.children) {
-      module.children.forEach(child => {
-        superFlattenFlatten(child, module.module ? module.module.name : label)
-      })
-    } else if (Array.isArray(module)) {
-      module.forEach(elem => superFlattenFlatten(elem, label))
-    } else if (module.code && !module.code.startsWith('KK')) {
-      if (!included.has(`${label}${module.code}`)) {
-        included.add(`${label}${module.code}`)
-        courses.push({
-          name: {
-            fi: module.name,
-            en: module.name
-          },
-          code: module.code,
-          label: { id: `${courses.length}`, label, orderNumber: courses.length }
-        })
-      }
-    }
-  }
-
-  const code = req.params.code
-  const response = await axios.get(`http://sis-updater-scheduler:8082/v1/courses/${code}?token=dev&superFlatten=true`)
-
-  // dumb_flatten(response.data)
-  // better_flatten(response.data)
-  superFlattenFlatten(response.data)
-
-  const byCode = (c1, c2) => (c1.code < c2.code ? -1 : 1)
-
-  courses.sort(byCode)
-  res.json(courses)
-})
-
-module.exports = router