[Study programme overview] Fix graduation times graph

After recent fixes every test has passed on the first try, so if many of them start failing at the same time again, it's most likely a genuine error.

.env.template

@@ -1,3 +1,6 @@
 # Populate and change this to .env
+# To get the crypt key, run `openssl rand -hex 32` or
+# `node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"`
 IMPORTER_DB_API_TOKEN=
 PATE_TOKEN=
+CRYPT_KEY=

.eslintrc.json

@@ -1,20 +1,19 @@
 {
   "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
+    "commonjs": true,
+    "es2021": true,
+    "node": true
   },
-  "parser": "@babel/eslint-parser",
   "parserOptions": {
-    "requireConfigFile": false
+    "ecmaVersion": "latest",
+    "sourceType": "commonjs"
   },
   "extends": ["eslint:recommended", "plugin:prettier/recommended", "plugin:cypress/recommended"],
-  "plugins": ["prettier", "cypress"],
   "rules": {
     "no-console": "error",
-    "react/no-unescaped-entities": "off",
     "no-async-promise-executor": "off",
     "prettier/prettier": "error",
-    "cypress/no-assigning-return-values": "off"
+    "cypress/no-assigning-return-values": "off",
+    "cypress/unsafe-to-chain-command": "off"
   }
 }

.github/actions/run-tests/action.yaml

@@ -16,7 +16,7 @@ runs:
   using: composite
   steps:
     - name: 'Login to toska Docker registry'
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
         username: ${{ inputs.username }}
@@ -27,7 +27,7 @@ runs:
       run: docker-compose --file docker-compose.ci.yml up --detach
 
     - name: 'Run cypress'
-      uses: cypress-io/github-action@v4
+      uses: cypress-io/github-action@v6
       with:
         browser: chrome
         spec: ${{ inputs.spec }}

.github/dependabot.yml

@@ -8,6 +8,8 @@ updates:
     labels:
       - 'dependencies'
       - 'infra'
+    allow:
+      - dependency-type: 'production'
     ignore:
       - dependency-name: '*'
         update-types: ['version-update:semver-major']
@@ -20,6 +22,8 @@ updates:
     labels:
       - 'dependencies'
       - 'updater'
+    allow:
+      - dependency-type: 'production'
     ignore:
       - dependency-name: '*'
         update-types: ['version-update:semver-major']
@@ -32,6 +36,8 @@ updates:
     labels:
       - 'dependencies'
       - 'updater'
+    allow:
+      - dependency-type: 'production'
     ignore:
       - dependency-name: '*'
         update-types: ['version-update:semver-major']
@@ -44,6 +50,8 @@ updates:
     labels:
       - 'dependencies'
       - 'back'
+    allow:
+      - dependency-type: 'production'
     ignore:
       - dependency-name: '*'
         update-types: ['version-update:semver-major']
@@ -56,6 +64,8 @@ updates:
     labels:
       - 'dependencies'
       - 'front'
+    allow:
+      - dependency-type: 'production'
     ignore:
       - dependency-name: '*'
         update-types: ['version-update:semver-major']

.github/workflows/production.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: 'Get current time'
         id: get_current_time
@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Extract release version
         id: extract_version
@@ -83,7 +83,7 @@ jobs:
       - release_frontend
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Extract release version
         id: extract_version
@@ -105,8 +105,8 @@ jobs:
         with:
           environment: production
           set_commits: 'skip'
-          sourcemaps: './build/static/js'
-          url_prefix: '~/static/js'
+          sourcemaps: './build/assets'
+          url_prefix: '~/assets'
           version: ${{ steps.extract_version.outputs.VERSION }}
 
   report_success:
@@ -120,7 +120,7 @@ jobs:
     if: success()
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: 'Send notification to slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
@@ -140,7 +140,7 @@ jobs:
     if: failure()
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: 'Send notification to slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:

.github/workflows/staging.yaml

@@ -11,7 +11,7 @@ on:
       - 'cypress/**'
       - 'cypress.config.json'
       - 'services/**'
-      - '!services/backend/src/rapodiff.js'
+      - '!services/backend/src/rapodiff/**'
       - 'docker-compose.ci.yml'
       - 'package*.json'
       - 'updater/sis-updater-worker/src/db/migrations/**'
@@ -29,7 +29,7 @@ jobs:
     needs: test
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Get current time
         id: get_current_time
@@ -65,7 +65,7 @@ jobs:
     needs: test
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Build backend image
         id: build-image
@@ -100,9 +100,9 @@ jobs:
     if: success()
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: 'Send notification to slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -119,9 +119,9 @@ jobs:
     if: failure()
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: 'Send notification to slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure

.github/workflows/test.yaml

@@ -16,10 +16,10 @@ jobs:
     name: 'Lint backend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3.4.1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
-          node-version: 14
+          node-version: 18
       - name: 'Install dependencies'
         run: npm ci && cd services/backend && npm ci
       - name: 'Run backend linter'
@@ -29,15 +29,15 @@ jobs:
     name: 'Lint frontend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3.4.1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
-          node-version: 14
+          node-version: 18
       - name: 'Install dependencies'
         run: npm ci && cd services/frontend && npm ci
       - name: 'Run frontend linter'
         run: npm run concurrently \
-          'eslint --ext .jsx,.js services/frontend' \
+          'eslint --ext .jsx,.js services/frontend/src --report-unused-disable-directives' \
           'stylelint services/frontend/src/**/*.css'
 
   # === Build database images ===
@@ -45,10 +45,10 @@ jobs:
     name: 'Build backend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
       - name: 'Build image and export it'
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           build-args: |
             SENTRY_RELEASE=${{ github.sha }}
@@ -66,8 +66,8 @@ jobs:
     name: 'Build anon sis-db and push to toska hub if new migrations have been added'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
         id: check_migrations
         with:
           filters: |
@@ -75,7 +75,7 @@ jobs:
               - 'updater/sis-updater-worker/src/db/migrations/**'
               - '.github/workflows/test.yaml'
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
@@ -93,8 +93,8 @@ jobs:
       - build_backend
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
         id: check_migrations
         with:
           filters: |
@@ -108,7 +108,7 @@ jobs:
       - name: 'Load built backend image'
         run: docker load --input /tmp/artifact/backend.tar
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
@@ -126,8 +126,8 @@ jobs:
       - build_backend
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
         id: check_migrations
         with:
           filters: |
@@ -141,7 +141,7 @@ jobs:
       - name: 'Load built backend image'
         run: docker load --input /tmp/artifact/backend.tar
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
@@ -154,167 +154,51 @@ jobs:
           ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
 
   # === End-to-end tests ===
-  test_course_statistics:
-    name: 'Test course statistics'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Course_statistics.js
-
-  test_custom_population:
-    name: 'Test custom population'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Custom_population.js
-
-  test_population_statistics:
-    name: 'Test population statistics'
+  test_end_to_end:
     runs-on: ubuntu-latest
     needs:
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: 'Test course statistics'
+            spec: cypress/e2e/Course_statistics.js
+          - name: 'Test custom population'
+            spec: cypress/e2e/Custom_population.js
+          - name: 'Test population statistics'
+            spec: cypress/e2e/Population_statistics.js
+          - name: 'Test filters'
+            spec: cypress/e2e/Filters.js
+          - name: 'Test students'
+            spec: cypress/e2e/Students.js
+          - name: 'Test completed courses'
+            spec: cypress/e2e/Completed_courses.js
+          - name: 'Test studyprogramme overview'
+            spec: cypress/e2e/Studyprogramme_overview.js
+          - name: 'Test teachers and users'
+            spec: |
+              cypress/e2e/Teachers.js
+              cypress/e2e/Users.js
+          - name: 'Test faculty statistics'
+            spec: cypress/e2e/Faculty_statistics.js
+          - name: 'Test open uni search'
+            spec: cypress/e2e/Openuni.js
+          - name: 'Test language center view'
+            spec: cypress/e2e/Language_center.js
+          - name: 'Test study guidance groups'
+            spec: cypress/e2e/Study_guidance_groups.js
+    name: ${{ matrix.name }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
-          spec: cypress/e2e/Population_statistics.js
-
-  test_filters:
-    name: 'Test filters'
-    runs-on: ubuntu-latest
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Filters.js
-
-  test_student_statistics:
-    name: 'Test student statistics'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Student_statistics.js
-
-  test_completed_courses:
-    name: 'Test completed courses'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Completed_courses.js
-
-  test_studyprogramme_overview:
-    name: 'Test studyprogramme overview'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Studyprogramme_overview.js
-
-  test_teachers_and_users:
-    name: 'Test teachers and users'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: |
-            cypress/e2e/Teachers.js
-            cypress/e2e/Users.js
-
-  test_faculty_statistics:
-    name: 'Test faculty statistics'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Faculty_statistics.js
-
-  test_open_uni_search:
-    name: 'Test open uni search'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Openuni.js
+          spec: ${{ matrix.spec }}
 
   # === Other tests ===
   test_service_exit_statuses:
@@ -325,9 +209,9 @@ jobs:
       - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}

.github/workflows/updater.yaml

@@ -14,10 +14,10 @@ jobs:
     name: 'Build Updater worker'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
       - name: 'Build the image'
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: updater/sis-updater-worker/
           push: false
@@ -32,10 +32,10 @@ jobs:
     name: 'Build Updater scheduler'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
       - name: 'Build the image'
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: updater/sis-updater-scheduler/
           push: false
@@ -46,62 +46,57 @@ jobs:
         with:
           path: /tmp/scheduler.tar
 
-  # test_updater:
-  #   name: 'Test Updater'
-  #   runs-on: ubuntu-latest
-  #   needs:
-  #     - build_worker
-  #     - build_scheduler
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - name: 'Login to toska docker registry'
-  #       uses: docker/login-action@v2
-  #       with:
-  #         registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
-  #         username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-  #         password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-  #     - name: 'Download built images'
-  #       uses: actions/download-artifact@v3
-  #       with:
-  #         path: /tmp/
-  #     - name: 'Load worker image'
-  #       run: docker load --input /tmp/artifact/worker.tar
-  #     - name: 'Load scheduler image'
-  #       run: docker load --input /tmp/artifact/scheduler.tar
-  #     - name: 'Start worker'
-  #       run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
-  #     - name: 'Run hourly jobs'
-  #       run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-  #       env:
-  #         SCHEDULE_IMMEDIATE: hourly
-  #         EXIT_AFTER_IMMEDIATES: yes
-  #     - name: 'Run daily jobs'
-  #       run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-  #       env:
-  #         SCHEDULE_IMMEDIATE: daily
-  #         EXIT_AFTER_IMMEDIATES: yes
-  #     - name: 'Run weekly jobs'
-  #       run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-  #       env:
-  #         SCHEDULE_IMMEDIATE: weekly
-  #         EXIT_AFTER_IMMEDIATES: yes
-  #     - name: 'Run prepurge'
-  #       run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-  #       env:
-  #         SCHEDULE_IMMEDIATE: prepurge
-  #         EXIT_AFTER_IMMEDIATES: yes
-  #     - name: 'Run purge'
-  #       run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-  #       env:
-  #         SCHEDULE_IMMEDIATE: purge
-  #         EXIT_AFTER_IMMEDIATES: yes
+  test_updater:
+    name: 'Test Updater'
+    runs-on: ubuntu-latest
+    needs:
+      - build_worker
+      - build_scheduler
+    steps:
+      - uses: actions/checkout@v4
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v3
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Download built images'
+        uses: actions/download-artifact@v3
+        with:
+          path: /tmp/
+      - name: 'Load worker image'
+        run: docker load --input /tmp/artifact/worker.tar
+      - name: 'Load scheduler image'
+        run: docker load --input /tmp/artifact/scheduler.tar
+      - name: 'Start worker'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
+      - name: 'Run hourly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: hourly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run weekly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: weekly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run prepurge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: prepurge
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run purge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: purge
+          EXIT_AFTER_IMMEDIATES: yes
 
   release_worker:
     name: 'Build and release Updater worker'
     runs-on: ubuntu-latest
-    # needs: test_updater
+    needs: test_updater
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Build worker image
         id: build-image
@@ -125,10 +120,10 @@ jobs:
   release_scheduler:
     name: 'Build and release Updater scheduler'
     runs-on: ubuntu-latest
-    # needs: test_updater
+    needs: test_updater
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Build scheduler image
         id: build-image

.lintstagedrc.js

@@ -4,10 +4,11 @@ const dockerCmdBase = `docker run --rm --volume ${cwd}:/oodikone --workdir /oodi
 const relativeFilePaths = files => [...files.map(file => path.relative(cwd, file))].join(' ')
 
 module.exports = {
-  '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
-  '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
+  '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')} --report-unused-disable-directives`,
+  '*.{js,json,md,yml,yaml,html,css}': files => `prettier --write ${files.join(' ')}`,
   '*.css': files => `stylelint --fix ${files.join(' ')}`,
-  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.12.0 hadolint ${relativeFilePaths(files)}`,
+  Dockerfile: files =>
+    `${dockerCmdBase} hadolint/hadolint:v2.12.0 hadolint --ignore DL3006 ${relativeFilePaths(files)}`,
   '*.sh': files => `${dockerCmdBase} koalaman/shellcheck:v0.7.2 ${relativeFilePaths(files)} -x`,
   '.github/{workflows,actions}/*': files => `npm run actionlint ${relativeFilePaths(files)}`,
   'docker-compose*': files => {

.prettierrc

@@ -5,5 +5,6 @@
   "printWidth": 120,
   "tabWidth": 2,
   "useTabs": false,
-  "jsxSingleQuote": false
-}
+  "jsxSingleQuote": false,
+  "trailingComma": "es5"
+}

.stylelintrc

@@ -1,22 +1,6 @@
 {
-  "extends": ["stylelint-config-standard", "stylelint-prettier/recommended"],
+  "extends": ["stylelint-config-standard"],
   "rules": {
-    "selector-type-no-unknown": [
-      true,
-      {
-        "ignoreTypes": ["/^--/", "--"]
-      }
-    ],
-    "selector-pseudo-class-no-unknown": [
-      true,
-      {
-        "ignorePseudoClasses": ["global"]
-      }
-    ],
-    "prettier/prettier": true,
-    "selector-class-pattern": null,
-    "custom-property-pattern": null,
-    "value-no-vendor-prefix": null,
-    "property-no-vendor-prefix": null
+    "selector-class-pattern": "^[a-z][a-zA-Z-]*$"
   }
 }

.vscode/extensions.json

@@ -0,0 +1,9 @@
+{
+  "recommendations": [
+    "aaron-bond.better-comments",
+    "dbaeumer.vscode-eslint",
+    "esbenp.prettier-vscode",
+    "github.vscode-github-actions",
+    "stylelint.vscode-stylelint"
+  ]
+}

README.md

@@ -61,6 +61,7 @@ graph TB
             client[React app] --> server[NodeJS server]
             server --> ok_redis(Redis)
             konedb[(kone-db)]
+            userdb[(user-db)]
         end
         server -->|Read| sisdb[(sis-db)]
         subgraph Updater
@@ -118,6 +119,7 @@ Some useful commands are defined in `package.json` and can be run with `npm run
 - `npm run docker:down`: stops the whole environment
 - `npm run testupdater`: runs tests for updater
 - `npm run flushredis`: Clears redis, which forces big calculations to be redone in e.g. studyprogramme overview and faculty views.
+- `npm run rapodiff`: runs [Rapodiff](./services/backend/src/rapodiff/)
 
 Once you have ran setup for oodikone , you can just execute the first one (`npm run oodikone`). After starting and waiting for a while for containers to compile, oodikone can be accessed at [http://localhost:3000/](http://localhost:3000/) and Adminer (database investigation tool) at [http://localhost:5050/](http://localhost:5050/). Adminer requires you to login with username `postgres` and with any password you choose (for example `p`).
 
@@ -170,6 +172,9 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 - After a successful test run, oodikone is deployed to staging
 - After creating a release, oodikone is deployed to production
 - Updater is deployed to production when pushed if tests pass
+- Test databases can be updated, instructions in [anonyymioodi README](anonyymioodi/README.md)
+
+[Rapodiff](./services/backend/src/rapodiff/) can be used to check differences between Rapo API and Oodikone data.
 
 ## ❓FAQ
 

anonyymioodi/README.md

@@ -1,65 +1,18 @@
-# Anonyymioodi - anonymous database setup for oodikone
+# Anonyymioodi - test database setup
 
-(name anonyymioodi for :culture: reasons)
+This folder contains a guide and scripts to modify or create anonymous database images for oodikone, mainly used for testing.
 
-This folder contains a guide and scripts to create anonymous database images for oodikone, mainly used for testing.
+This quick guide should be enough for most purposes: Updating schema, data or postgres version. If not, a more elaborate guide is stored in [old readme](old_readme.md) (though it may have some outdated info).
 
-## How to create / modify database images
+### Quick guide to update a test database
 
-Creating / modifying images varies a bit depending on the db. Generally, to modify images, you can either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose. 
-
-By default, previous db images from Toska Hub are used as a base. If needed, comment out Toska hub image and uncomment corresponding clean postgres image in docker-compose before running other steps.
-
-See more details on how to create / modify databases below.
-
-### Sis-importer-db
-
-The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
-
-- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
-- Move `sis-importer-db.sqz` to this folder
-- Run `./create_container_from_dump.sh sis-importer-db`
-- Enrich db with command in file `enrich_data.sql` either by piping command to container or by running command inside container.
-
-### Sis-db:
-
-The following steps are for building a clean sis-db image from scratch using oodikone's updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
-
-- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
-- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
-- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
-- Start updater, see oodikone readme for more info.
-- Populate sis-db from sis-importer-db:
-
-```
-curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
-curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
-curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
-```
-(or run these from frontend. You might also want to follow sis-updater-workers logs to ensure things are running smoothly.)
-- Clean up any extra stuff you don't want to keep in sis-db.
-- Bonus: check that cypress tests run okay
-
-### User-db / kone-db
-
-- Temporarily replace the db service in oodikone's docker-compose.yml with the same db service from this folder's docker-compose.yml
-- Ensure the newest user-db/kone-db image is pulled from the registry with `docker-compose pull`
-- Modify db to wanted state. Tips:
-  - Run oodikone if you want to upgrade something from frontend (e.g. tags in kone-db, courses in sis-db etc.).
-  - Easy way to create new users to user-db is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
-
-## Publishing the image
-
-After completing the steps above and making sure your local version has the correct data, run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the db image to Toska Hub.
-
-## How to update Postgres versions
-
-When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
-
-- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
-- Start database with docker-compose
-- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
-- Close container with docker-compose down.
-- Update the Postgres version in docker-compose.yml in this folder.
-- Uncomment row with Postgres image and comment row with Toskahub image.
-- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`
+- Make sure you have the latest test-db. Just pulling images doesn't seem to replace your already existing volume, so you may need to remove it by hand. This may work: `docker volume rm oodikone_sis-db-data`. The new image including data should be pulled next time you launch oodikone normally.
+- Do your dev stuff normally with the test database, for example with `npm run both`. Edit data however you like, run updater via frontend for example, then run tests to see if they pass.
+- When you're happy with the database, create a dump:
+- `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > anonyymioodi/<dbname>.sqz`
+- Now you can shut it down `npm run docker:down`
+- Navigate to this folder `cd anonyymioodi`
+- If you need to **update postgres version**, do it now by changing it in this directory's `docker-compose.yml`.
+- Create new container from dump `./create_container_from_dump.sh <dbname>`
+- Commit and push it to toska registry `./commit_and_push_to_toska_hub.sh <dbname>`
+- `docker compose down`

anonyymioodi/docker-compose.yml

@@ -9,8 +9,7 @@ services:
       PGDATA: /data
       POSTGRES_DB: kone-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
-    # image: postgres:13.4-alpine
+    image: postgres:15.3
     profiles:
       - oodikone
 
@@ -21,8 +20,7 @@ services:
       PGDATA: /data
       POSTGRES_DB: sis-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    # image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
-    image: postgres:13.2
+    image: postgres:15.3
     shm_size: 2gb
 
   sis-importer-db:
@@ -32,8 +30,7 @@ services:
       PGDATA: /data
       POSTGRES_DB: sis-importer-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    # image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
-    image: postgres:13.1
+    image: postgres:15.3
     profiles:
       - updater
 
@@ -44,7 +41,6 @@ services:
       PGDATA: /data
       POSTGRES_DB: user-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
-    # image: postgres:13.4-alpine
+    image: postgres:15.3
     profiles:
       - oodikone

anonyymioodi/old_readme.md

@@ -0,0 +1,71 @@
+# Old readme of anonyymioodi
+
+This is mostly obsolete, but may help with something sometimes.
+
+## Anonyymioodi - anonymous database setup for oodikone
+
+(name anonyymioodi for :culture: reasons)
+
+This folder contains a guide and scripts to create anonymous database images for oodikone, mainly used for testing.
+
+## How to create / modify database images
+
+Creating / modifying images varies a bit depending on the db. Generally, to modify images, you can either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose.
+
+By default, previous db images from Toska Hub are used as a base. If needed, comment out Toska hub image and uncomment corresponding clean postgres image in docker-compose before running other steps.
+
+See more details on how to create / modify databases below.
+
+### Sis-importer-db
+
+The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
+- Move `sis-importer-db.sqz` to this folder
+- Run `./create_container_from_dump.sh sis-importer-db`
+- Enrich db with command in file `enrich_data.sql` either by piping command to container or by running command inside container.
+
+### Sis-db:
+
+The following steps are for building a clean sis-db image from scratch using oodikone's updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
+- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
+- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
+- Start updater, see oodikone readme for more info.
+- Populate sis-db from sis-importer-db:
+
+```
+curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
+```
+
+(or run these from frontend. You might also want to follow sis-updater-workers logs to ensure things are running smoothly.)
+
+- Clean up any extra stuff you don't want to keep in sis-db.
+- Bonus: check that cypress tests run okay
+
+### User-db / kone-db
+
+- Temporarily replace the db service in oodikone's docker-compose.yml with the same db service from this folder's docker-compose.yml
+- Ensure the newest user-db/kone-db image is pulled from the registry with `docker-compose pull`
+- Modify db to wanted state. Tips:
+  - Run oodikone if you want to upgrade something from frontend (e.g. tags in kone-db, courses in sis-db etc.).
+  - Easy way to create new users to user-db is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
+
+## Publishing the image
+
+After completing the steps above and making sure your local version has the correct data, run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the db image to Toska Hub.
+
+## How to update Postgres versions
+
+When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
+
+- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
+- Start database with docker-compose
+- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
+- Close container with docker-compose down.
+- Update the Postgres version in docker-compose.yml in this folder.
+- Uncomment row with Postgres image and comment row with Toskahub image.
+- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`

babel.config.json

@@ -1,5 +0,0 @@
-{
-  "plugins": [
-    "@babel/plugin-syntax-jsx"
-  ]
-}

cypress.config.js

@@ -15,6 +15,5 @@ module.exports = defineConfig({
     },
     baseUrl: 'http://localhost:3000',
     specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}',
-    testIsolation: false,
   },
 })

cypress/e2e/Completed_courses.js

@@ -48,7 +48,7 @@ describe('When search modal is opened', () => {
     hasLanded()
     openCompletedCoursesModal()
     cy.contains('Search completed courses of students')
-    cy.contains('Insert one or more student numbers, separated by a space, a newline, or a comma.')
+    cy.contains('Insert one or more student numbers, separated by a space, a newline, a comma, or a semicolon.')
     cy.contains('Insert one or more courses, separated by a space, a newline, or a comma.')
     cy.contains('Insert name for this course list if you wish to save it')
   })
@@ -73,7 +73,7 @@ describe('When search modal is opened', () => {
   })
 
   describe('When a search with correct data is executed', () => {
-    before(() => {
+    beforeEach(() => {
       cy.init('/completedcoursessearch')
       hasLanded()
       openCompletedCoursesModal()
@@ -90,14 +90,8 @@ describe('When search modal is opened', () => {
         '/completedcoursessearch?courseList=TKT10001&courseList=TKT10002&courseList=TKT10003&courseList=TKT10004&studentList=010450938&studentList=010589388&studentList=010614509'
       )
     })
-    /*
-    TODO: FIX
-    This is skipped because it's flaky in github actions for unknown reasons.
-    Instead of the results, a text appears saying "Something went wrong, please try refreshing the page".
-    */
-    it.skip('Finds correct students and courses', () => {
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(5000)
+
+    it('Finds correct students and courses', () => {
       cy.fixture('completedCoursesData').then(({ studentSet1, coursesSet1 }) => {
         cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', studentSet1[0])
         cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', studentSet1[1])

cypress/e2e/Course_statistics.js

@@ -1,33 +1,103 @@
-/* eslint-disable cypress/no-unnecessary-waiting */
 /// <reference types="Cypress" />
 
+const checkGradeTable = gradesTableContents => {
+  cy.get('[data-cy="Grade distribution"]')
+    .parent()
+    .siblings('.content.active')
+    .get('table > tbody')
+    .first()
+    .within(() => {
+      gradesTableContents.forEach((values, trIndex) => {
+        cy.get('tr')
+          .eq(trIndex)
+          .within(() => {
+            values.forEach((value, tdIndex) => {
+              if (value === null) return
+              cy.get('td').eq(tdIndex).contains(value)
+            })
+          })
+      })
+      cy.get('tr').should('have.length', gradesTableContents.length)
+    })
+}
+
 describe('Course Statistics tests', () => {
   describe('when using basic user', () => {
     beforeEach(() => {
       cy.init('/coursestatistics')
-    })
-    it('Searching single course having substitution mappings shows course statistics', () => {
       cy.url().should('include', '/coursestatistics')
+    })
+
+    describe('It shows correct statistics for courses with other grades than 0-5', () => {
+      it('Shows correct statistics for courses with scale HT-TT', () => {
+        const gradesTableContents = [
+          [null, 'TT', 190],
+          [null, 'Hyl.', 4],
+          [null, 'HT', 177],
+        ]
+        cy.contains('Search for courses')
+        cy.get("input[placeholder='Search by a course code']").type('KK-RUKIRJ')
+        cy.contains('td', 'KK-RUKIRJ').click()
+        cy.contains('Search for courses').should('not.exist')
+        cy.contains('KK-RUKIRJ, 992912, AYKK-RUKIRJ Toisen kotimaisen kielen kirjallinen taito, ruotsi (CEFR B1)')
+        cy.contains('.tabular.menu a', 'Grade distribution').click()
+        cy.contains('svg', 'Grades')
+        cy.contains('Show population').should('be.enabled').click()
+        checkGradeTable(gradesTableContents)
+      })
+
+      it("Shows correct statistics for old master's thesis grade scales", () => {
+        const gradesTableContents = [
+          [null, 'L', 1],
+          [null, 'CL', 1],
+        ]
+        cy.contains('Search for courses')
+        cy.get("input[placeholder='Search by a course code']").type('704220')
+        cy.contains('td', '704220').click()
+        cy.contains('Search for courses').should('not.exist')
+        cy.contains('704220 Taloustiede. Pro gradu -tutkielma')
+        cy.contains('.tabular.menu a', 'Grade distribution').click()
+        cy.contains('svg', 'Grades')
+        cy.contains('Show population').should('be.enabled').click()
+        checkGradeTable(gradesTableContents)
+      })
+
+      it('Shows correct statistics for courses with scale passed-failed', () => {
+        const gradesTableContents = [[null, 'Hyv.', 258]]
+        cy.contains('Search for courses')
+        cy.get("input[placeholder='Search by a course code']").type('KK-ENMALU1')
+        cy.contains('td', 'KK-ENMALU1').click()
+        cy.contains('Search for courses').should('not.exist')
+        cy.contains(
+          'KK-ENFARM1, KK-ENOIK1, KK-ENLAAK1, KK-ENMALU1, 995010Mat-lu Academic and Professional Communication in English 1 (CEFR B2)'
+        )
+        cy.contains('.tabular.menu a', 'Grade distribution').click()
+        cy.contains('svg', 'Grades')
+        cy.contains('Show population').should('be.enabled').click()
+        checkGradeTable(gradesTableContents)
+      })
+    })
+
+    it('Searching single course having substitution mappings shows course statistics', () => {
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by a course code']").type('TKT20001')
       // Click uni course, not avoin
       cy.contains('td', /^TKT20001/).click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
 
       cy.contains('Tietorakenteet ja algoritmit')
       cy.contains('TKT20001')
       cy.contains('58131') // old mapped code
 
-      cy.contains('.tabular.menu a', 'Table').click()
+      cy.contains('.tabular.menu a', 'Students').click()
       cy.contains('All')
-      cy.cs('viewMode-Attempts').click()
-      cy.cs('viewMode-Students').click()
+      cy.contains('svg', 'Pass rate')
 
-      cy.contains('.tabular.menu a', 'Pass rate chart').click()
-      cy.contains('svg', 'Pass rate chart')
+      cy.contains('.tabular.menu a', 'Attempts').click()
+      cy.contains('All')
+      cy.contains('svg', 'Pass rate')
 
-      cy.contains('.tabular.menu a', 'Grade distribution chart').click()
+      cy.contains('.tabular.menu a', 'Grade distribution').click()
       cy.contains('svg', 'Grades')
 
       cy.contains('a', 'New query').click()
@@ -35,8 +105,9 @@ describe('Course Statistics tests', () => {
     })
 
     it('Searching multiple courses having substitution mappings shows course statistics', () => {
-      cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')
+      cy.cs('select-multiple-courses-toggle').should('not.have.class', 'checked').click()
+      cy.cs('select-multiple-courses-toggle').should('have.class', 'checked')
       cy.get("input[placeholder='Search by a course code']").type('TKT')
       cy.contains('td', /^TKT20001/).click()
       cy.contains('td', /^TKT10002/).click()
@@ -59,31 +130,47 @@ describe('Course Statistics tests', () => {
       cy.contains('Search for courses')
     })
 
+    it('On searches with multiple courses, has correct links on the Course tab', () => {
+      cy.cs('select-multiple-courses-toggle').click()
+      cy.get("input[placeholder='Search by a course code']").type('TKT')
+      cy.contains('td', /^TKT20001/).click()
+      cy.contains('td', /^TKT10002/).click()
+      cy.contains('Fetch statistics').click()
+      cy.contains('Search for courses').should('not.exist')
+
+      cy.get('.ui.contentSegment .ui.menu').contains('Course').click()
+      cy.cs('course-selector').get('.active.selected.item').contains('TKT10002')
+      cy.get('#CourseStatPanes table a.item:first').click()
+      cy.contains('Population of course Ohjelmoinnin perusteet')
+
+      cy.go('back')
+      cy.get('.ui.contentSegment .ui.menu').contains('Course').click()
+      cy.cs('course-selector').click()
+      cy.cs('course-selector').contains('TKT20001').click()
+      cy.get('#CourseStatPanes table a.item:first').click()
+      cy.contains('Population of course Tietorakenteet ja algoritmit')
+    })
+
     it('On consecutive searches should not crash and search should work', () => {
-      cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by a course code']").type('TKT20003')
       cy.contains('td', 'TKT20003').click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Show population')
       cy.contains('TKT20003')
 
-      cy.contains('Course statistics').click()
+      cy.contains('Courses').click()
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by a course code']").type('TKT20001')
       cy.contains('td', 'TKT20001').click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Show population')
       cy.contains('TKT20001')
     })
 
     it('Shows statistics when separating by semester', () => {
       const courseCode = 'DATA11002'
-      cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by a course code']").type(courseCode)
       cy.contains('td', courseCode).click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
       cy.get('[data-cy=separateToggle]', { force: true }).click({ force: true })
       cy.contains('Syksy 2019')
       cy.contains('Syksy 2017')
@@ -92,18 +179,15 @@ describe('Course Statistics tests', () => {
     })
 
     it('Searching course by name displays right courses', () => {
-      cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
 
-      cy.contains('Tietokantojen perusteet')
       cy.contains('Tietokantojen perusteet')
       cy.contains('A581328')
       cy.contains('AYTKT10004')
       cy.contains('TKT10004, 581328')
       cy.contains('td', /^TKT10004/).click()
 
-      cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
 
       cy.contains('TKT10004, 581328, AYTKT10004, A581328 Tietokantojen perusteet')
@@ -111,29 +195,50 @@ describe('Course Statistics tests', () => {
       cy.contains('No results')
     })
 
+    it('"Select all search results" button is not showing unless "Select multiple courses" toggle is on', () => {
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT')
+      cy.get('[data-cy="select-multiple-courses-toggle"]').should('not.have.class', 'checked')
+      cy.contains('TKT10004, 581328')
+      cy.contains('Select all search results').should('not.exist')
+      cy.get('[data-cy="select-multiple-courses-toggle"]').click()
+      cy.get('[data-cy="select-multiple-courses-toggle"]').should('have.class', 'checked')
+      cy.contains('Select all search results')
+    })
+
+    // Skipped for now, due to limit being removed. Subject to change after testing in production
+    it.skip('"Fetch statistics" button is disabled if over 40 courses are selected', () => {
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT')
+      cy.get('[data-cy="select-multiple-courses-toggle"]').should('not.have.class', 'checked').click()
+      cy.get('[data-cy="select-multiple-courses-toggle"]').should('have.class', 'checked')
+      cy.contains('Select all search results').click()
+      cy.contains('Fetch statistics').should('not.be.enabled').trigger('mouseover', { force: true })
+      cy.contains('Voit valita tarkasteltavaksi enintään 40 kurssia kerrallaan.')
+      cy.get("input[placeholder='Search by a course code']").type('21')
+      cy.contains('Fetch statistics').should('be.enabled')
+    })
+
     it('Provider organization toggle works', () => {
-      cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
 
       cy.contains('td', /^TKT10004/).click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
 
       cy.contains('TKT10004, 581328, AYTKT10004, A581328 Tietokantojen perusteet')
-      cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.cs('providerCheckboxUniversity').should('have.class', 'checked').click()
+      cy.cs('providerCheckboxOpenUni').should('have.class', 'checked').click()
       cy.contains('TKT10004, 581328 Tietokantojen perusteet')
     })
 
     it('Searching course by name displays right courses, 10 credit courses', () => {
-      cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
       cy.contains('Tietorakenteet ja algoritmit')
       cy.contains('AYTKT20001')
       cy.contains('TKT20001, 58131')
       cy.contains('td', /^TKT20001/).click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
 
       cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
@@ -142,20 +247,16 @@ describe('Course Statistics tests', () => {
       cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
       cy.contains('td', 'TKT20001, 58131, AYTKT20001').click()
 
-      cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
       cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
     })
 
     it('Can find course population', () => {
-      cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by a course code']").type('TKT20003')
       cy.contains('tr', 'TKT20003').click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Käyttöjärjestelmät')
       cy.contains('TKT20003')
-      cy.wait(5000)
       cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
       cy.contains('Population of course Käyttöjärjestelmät 2019-2020 (open and normal')
       cy.contains('TKT20003')
@@ -167,28 +268,20 @@ describe('Course Statistics tests', () => {
 
     it('Population of course shows grades for each student', () => {
       cy.get("input[placeholder='Search by a course code']").type('TKT20001')
-      cy.wait(2000)
-      cy.contains(/^TKT20001, AYTKT20001$/).click({ force: true })
-      cy.wait(2000)
-      cy.contains('Fetch statistics').should('be.enabled').click({ force: true })
-      cy.wait(2000)
+      cy.contains(/^TKT20001, 58131, AYTKT20001$/).click({ force: true })
       cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
       cy.get('tbody >:nth-child(3) > :nth-child(2) .level').click({ force: true })
-      cy.wait(2000)
       cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019 (open and normal)')
       cy.contains('Students (204)').click({ force: true })
-      cy.wait(2000)
       cy.contains('td', '010262566').siblings().contains('4')
       cy.contains('td', '010674989').siblings().contains('1')
     })
 
     it('Language distribution is correct', () => {
       cy.get("input[placeholder='Search by a course code']").type('TKT20003')
-      cy.contains(/^TKT20003$/).click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains(/^TKT20003, 582219$/).click()
       cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
-      cy.get('[data-cy=unify_radio_reqular]').click()
-      cy.wait(5000)
+      cy.cs('providerCheckboxOpenUni').click()
       cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
       cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
       cy.contains('Language distribution').click()
@@ -196,13 +289,85 @@ describe('Course Statistics tests', () => {
       cy.contains('td', 'english').siblings().contains('5')
     })
 
+    it('Shows correct statistics when course is not combined with its substitutions', () => {
+      cy.get('[data-cy="combine-substitutions-toggle"]').should('have.class', 'checked').click()
+      cy.get('[data-cy="combine-substitutions-toggle"]').should('not.have.class', 'checked')
+
+      cy.get("input[placeholder='Search by a course code']").type('TKT10002')
+      cy.contains('td', /^TKT10002$/).click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('TKT10002 Ohjelmoinnin perusteet')
+
+      // Statistics
+      const yearRange = { from: '2010-2011', to: '2020-2021' }
+      const attemptsTableContents = [
+        // [time, attempts, passed, failed]
+        ['Total', null, 290, 278, 12],
+      ]
+
+      const gradesTableContents = [
+        // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
+        ['Total', null, 290, 12, 8, 9, 12, 17, 223, 9],
+        ['2020-2021', null, 1, 0, 0, 0, 0, 0, 1, 0],
+        ['2019-2020', null, 37, 1, 2, 0, 3, 3, 28],
+        ['2018-2019', null, 109, 4, 2, 5, 4, 6, 88, 0],
+        ['2017-2018', null, 129, 7, 3, 3, 3, 4, 101, 8],
+        ['2016-2017', null, 9, 0, 0, 1, 2, 2, 4, 0],
+        ['2015-2016', null, 2, 0, 0, 0, 0, 1, 1, 0],
+        ['2014-2015', null, 1, 0, 0, 0, 0, 0, 0, 1],
+        ['2013-2014', null, 1, 0, 1, 0, 0, 0, 0, 0],
+        ['2010-2011', null, 1, 0, 0, 0, 0, 1, 0, 0],
+      ]
+      // Time range
+      cy.get("div[name='fromYear']").within(() => {
+        cy.get("div[role='option']").first().should('have.text', yearRange.to)
+        cy.contains("div[role='option']", yearRange.from).should('have.class', 'selected')
+        cy.get("div[role='option']").last().should('have.text', '2010-2011')
+        cy.get("div[role='option']").should('have.length', 11)
+      })
+      cy.get("div[name='toYear']").within(() => {
+        cy.get("div[role='option']").first().should('have.text', '2020-2021')
+        cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
+        cy.get("div[role='option']").last().should('have.text', yearRange.from)
+        cy.get("div[role='option']").should('have.length', 11)
+      })
+
+      cy.contains('#CourseStatPanes a.item', 'Attempts').click()
+      cy.get('#CourseStatPanes table>tbody').within(() => {
+        attemptsTableContents.forEach((values, trIndex) => {
+          cy.get('tr')
+            .eq(trIndex)
+            .within(() => {
+              values.forEach((value, tdIndex) => {
+                if (value === null) return
+                cy.get('td').eq(tdIndex).contains(value)
+              })
+            })
+        })
+        cy.get('tr').should('have.length', 10)
+      })
+      cy.get('[data-cy=gradeToggle]', { force: true }).click({ force: true })
+      cy.get('#CourseStatPanes table>tbody').within(() => {
+        gradesTableContents.forEach((values, trIndex) => {
+          cy.get('tr')
+            .eq(trIndex)
+            .within(() => {
+              values.forEach((value, tdIndex) => {
+                if (value === null) return
+                cy.get('td').eq(tdIndex).contains(value)
+              })
+            })
+        })
+        cy.get('tr').should('have.length', 10)
+      })
+    })
+
     describe('When searching unified course stats', () => {
       beforeEach(() => {
         cy.url().should('include', '/coursestatistics')
         cy.contains('Search for courses')
         cy.get("input[placeholder='Search by a course code']").type('TKT10002')
-        cy.contains('td', 'TKT10002, AYTKT10002').click()
-        cy.contains('Fetch statistics').should('be.enabled').click()
+        cy.contains('td', /^TKT10002, 581325, A581325, AYTKT10002$/).click()
         cy.contains('Search for courses').should('not.exist')
         cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
       })
@@ -214,16 +379,6 @@ describe('Course Statistics tests', () => {
         ['Total', null, 511, 486, 25],
       ]
 
-      const attemptsTableContentsOpen = [
-        // [time, passed, failed, passrate]
-        ['Total', null, 175, 175, 0],
-      ]
-
-      const attemptsTableContentsReqular = [
-        // [time, passed, failed, passrate]
-        ['Total', null, 312, 298, 14],
-      ]
-
       const gradesTableContents = [
         // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
         ['Total', null, 511, 25, 11, 17, 21, 59, 365, 13],
@@ -251,7 +406,6 @@ describe('Course Statistics tests', () => {
         })
         cy.contains('Show population').should('not.be.enabled')
 
-        cy.contains('#CourseStatPanes a.item', 'Table').click()
         cy.contains('#CourseStatPanes a.item', 'Attempts').click()
         cy.get('#CourseStatPanes table>tbody').within(() => {
           attemptsTableContents.forEach((values, trIndex) => {
@@ -282,44 +436,6 @@ describe('Course Statistics tests', () => {
         })
       })
 
-      it('Toggling course provider changes stats correctly', () => {
-        cy.get('[data-cy=unify_radio_reqular]').click()
-        cy.get("div[name='toYear']").within(() => {
-          cy.get("div[role='option']").first().should('have.text', '2020-2021')
-          cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
-          cy.get("div[role='option']").last().should('have.text', yearRange.from)
-          cy.get("div[role='option']").should('have.length', 21)
-        })
-        cy.contains('#CourseStatPanes a.item', 'Table').click()
-        cy.contains('#CourseStatPanes a.item', 'Attempts').click()
-        cy.get('#CourseStatPanes table>tbody').within(() => {
-          attemptsTableContentsReqular.forEach((values, trIndex) => {
-            cy.get('tr')
-              .eq(trIndex)
-              .within(() => {
-                values.forEach((value, tdIndex) => {
-                  if (value === null) return
-                  cy.get('td').eq(tdIndex).contains(value)
-                })
-              })
-          })
-        })
-        cy.get('[data-cy=unify_radio_open]').click()
-        cy.get('#CourseStatPanes table>tbody').within(() => {
-          attemptsTableContentsOpen.forEach((values, trIndex) => {
-            cy.get('tr')
-              .eq(trIndex)
-              .within(() => {
-                values.forEach((value, tdIndex) => {
-                  if (value === null) return
-                  cy.get('td').eq(tdIndex).contains(value)
-                })
-              })
-          })
-        })
-        cy.get('[data-cy=unify_radio_unify]').click()
-      })
-
       it('After changing time range shows same stats', () => {
         const newYearRange = { from: '2016-2017', to: '2019-2020' }
         cy.get("div[name='fromYear']")
@@ -350,16 +466,15 @@ describe('Course Statistics tests', () => {
       })
     })
 
-    it('If no data available, unify toggle is disabled', () => {
+    it('If no data available, toggle is disabled', () => {
       cy.get("input[placeholder='Search by a course code']").type('AYCSM14111')
       cy.contains('AYCSM14111').click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('AYCSM14111 Avoin yo: Full Stack -websovelluskehitys: React Native')
-      cy.get('[data-cy=unify_radio_reqular]').find('input').should('be.disabled')
-      cy.get('[data-cy=unify_radio_unify]').find('input').should('not.be.disabled')
+      cy.cs('providerCheckboxUniversity').find('input').should('be.disabled')
+      cy.cs('providerCheckboxOpenUni').find('input').should('not.be.disabled')
     })
 
-    it('Has right to see all the sudents, because course provider is TKT', () => {
+    it('Has right to see all the students, because course provider is TKT', () => {
       cy.visit('coursestatistics?courseCodes=%5B%22TKT10004%22%5D&cs_tab=0&separate=false')
       cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
       cy.contains('Students (154)').click()
@@ -372,12 +487,10 @@ describe('Course Statistics tests', () => {
     cy.get('[data-cy=navbar-courseStatistics]').click()
     cy.get('[data-cy=course-code-input]').type('TKT20003')
     cy.contains('tr', 'TKT20003').click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
     cy.contains('Filter statistics by study programmes').should('not.exist')
     cy.contains('Faculty statistics').should('not.exist')
     cy.contains('Show population').should('not.exist')
-
-    cy.cs('viewMode-Attempts').click()
+    cy.contains('.tabular.menu a', 'Attempts').click()
 
     const attemptsTableContents = [
       // [time, passed, failed, passrate, total enrollments, enrolled, rejected, aborted]

cypress/e2e/Custom_population.js

@@ -18,7 +18,7 @@ const save = () => {
 }
 
 const selectSavedPopulation = name => {
-  cy.get('[data-cy="history-search"]').children().eq(0).type(name).type('{enter}')
+  cy.get('[data-cy="history-search"]').children().eq(0).type(`${name}{enter}`)
 }
 
 const deleteAllSearches = () => {
@@ -33,7 +33,7 @@ const deleteAllSearches = () => {
       const searchItems = d.find('div[role=option] > span[class=text]')
       for (let i = 0; i < searchItems.length; i++) {
         if (searchItems[i].textContent.includes('TEST-')) {
-          cy.get('[data-cy="history-search"]').children().eq(0).type(searchItems[i].textContent).type('{enter}')
+          cy.get('[data-cy="history-search"]').children().eq(0).type(`${searchItems[i].textContent}{enter}`)
           cy.get('button').contains('Delete').click()
         }
       }

cypress/e2e/Faculty_statistics.js

@@ -1,4 +1,3 @@
-/* eslint-disable cypress/no-unnecessary-waiting */
 /// <reference types="Cypress" />
 
 describe('Faculty overview', () => {
@@ -30,7 +29,6 @@ describe('Faculty overview', () => {
   describe('Faculty basic information: basic user', () => {
     beforeEach(() => {
       cy.init('/faculties')
-      cy.wait(3000)
       cy.contains('td', 'H80').click()
     })
     it('Basic information tab show all graphs and tables', () => {
@@ -98,9 +96,6 @@ describe('Faculty overview', () => {
     it('Study programme credit information can be toggled', () => {
       cy.get('[data-cy="Table-CreditsProducedByTheFaculty"]').should('be.visible')
 
-      // I tried to fix this without cy.wait() method without effort. Sorting the programmes
-      // or something else causes rerendering of the view and yield to failing tests.
-      cy.wait(10000)
       cy.get('[data-cy="Button-Show-CreditsProducedByTheFaculty-0"]').click()
       cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
       cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
@@ -126,8 +121,6 @@ describe('Faculty overview', () => {
 
     it('Graph stays open when sorted', () => {
       cy.get('[data-cy="Table-CreditsProducedByTheFaculty"]').should('be.visible')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
       cy.get('[data-cy="Button-Show-CreditsProducedByTheFaculty-0"]').click()
       cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
       cy.get('[data-cy="Menu-CreditsProducedByTheFaculty-Total"]').click()
@@ -138,11 +131,8 @@ describe('Faculty overview', () => {
   describe('Average graduation times', () => {
     beforeEach(() => {
       cy.init('/faculties')
-      cy.wait(3000)
       cy.contains('td', 'H60').click()
-      cy.wait(3000)
       cy.contains('Graduation times').click()
-      cy.wait(3000)
     })
 
     it('User can view graduation graphs', () => {
@@ -184,19 +174,20 @@ describe('Faculty overview', () => {
 
       cy.get('[data-cy="Section-master"]').within(() => {
         cy.get('div[class="faculty-graph"]')
-        cy.contains('1 graduated').should('have.length', 1)
-        cy.contains('1 graduated').trigger('mouseover')
-        cy.contains('1 students graduated in year 2020')
-        cy.contains('median study time: 25 months')
+        cy.contains('24 graduated').should('have.length', 1)
+        cy.contains('24 graduated').trigger('mouseover')
+        cy.contains('24 students graduated in year 2020')
+        cy.contains('median study time: 22 months')
         cy.contains('0 graduated over year late')
-        cy.contains('1 graduated max year overtime')
+        cy.contains('20 graduated on time')
+        cy.contains('4 graduated max year overtime')
 
-        cy.contains('1 graduated').click()
+        cy.contains('24 graduated').click()
         cy.contains('Year 2020 by graduation year')
         cy.get('div[class="programmes-graph"]').should('be.visible')
         cy.get('div[class="programmes-graph"]').within(() => {
           cy.contains('EDUM')
-          cy.contains('1 graduated').trigger('mouseover')
+          cy.contains('24 graduated').trigger('mouseover')
           cy.contains('Kasvatustieteiden maisteriohjelma')
           cy.contains('MH60_001')
         })
@@ -211,18 +202,21 @@ describe('Faculty overview', () => {
 
       cy.get('[data-cy="Section-master"]').within(() => {
         cy.get('div[class="faculty-graph"]')
-        cy.contains('1 graduated (100 % of class)').should('have.length', 1)
-        cy.contains('1 graduated').trigger('mouseover')
-        cy.contains('From class of 2018, 1/1 students have graduated')
+        cy.contains('19 graduated (44.2 % of class)').should('have.length', 1)
+        cy.contains('19 graduated').trigger('mouseover')
+        cy.contains('From class of 2018, 19/43 students have graduated')
       })
 
       cy.get('[data-cy="GraduationTimeToggle"]').click()
       cy.get('[data-cy="Section-master"]').within(() => {
-        cy.contains('1').click()
+        cy.contains('16').click()
         cy.get('div[class="programmes-breakdown-graph"]').should('be.visible')
         cy.get('div[class="programmes-breakdown-graph"]').within(() => {
           cy.contains('Year 2018 by start year')
           cy.contains('EDUM')
+          cy.contains('16').trigger('mouseover')
+          cy.contains('Kasvatustieteiden maisteriohjelma')
+          cy.contains('MH60_001')
         })
       })
     })
@@ -232,9 +226,7 @@ describe('Faculty overview', () => {
     beforeEach(() => {
       cy.init('/faculties')
       cy.contains('td', 'H50').click()
-      cy.wait(1000)
       cy.contains('Programmes and student populations').click()
-      cy.wait(1000)
     })
 
     it('User can view studentdata table without %', () => {

cypress/e2e/Filters.js

@@ -1,8 +1,5 @@
-/* eslint-disable cypress/no-unnecessary-waiting */
 /// <reference types="Cypress" />
 
-// const baseUrl = Cypress.config().baseUrl
-
 const MOCKED_DATE = new Date(2022, 6, 12).getTime()
 
 const checkFilteringResult = studentCount => {
@@ -73,7 +70,7 @@ describe('Population Statistics with CS master', () => {
     '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22MH50_009%22}&tag&year=2019'
   const defaultAmountOfStudents = 27
   const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
-  before(() => {
+  beforeEach(() => {
     cy.init(pathToCSMast2019)
   })
 
@@ -85,6 +82,32 @@ describe('Population Statistics with CS master', () => {
       programmeDropdown.get('i.delete').click()
     })
   })
+
+  describe('Studyright Type filter', () => {
+    it('is visible', () => {
+      cy.contains('Studyright Type')
+    })
+
+    it('is set to all by default', () => {
+      runTestStepWithPreAndPostParts('studyright-type', () => {
+        const card = cy.cs('studyright-type-filter-card')
+        card.get('[data-cy="all"] input').should('be.checked')
+      })
+    })
+
+    it('works', () => {
+      runTestStepWithPreAndPostParts('studyright-type', () => {
+        const expectedBaMaStudents = 27
+        const expectedMasterOnlyStudents = 0
+        cy.cs('studyright-type-filter-card').cs('bama').click()
+        checkFilteringResult(expectedBaMaStudents)
+        cy.cs('studyright-type-filter-card').cs('master').click()
+        checkFilteringResult(expectedMasterOnlyStudents)
+        cy.cs('studyright-type-filter-card').cs('all').click()
+        checkFilteringResult(defaultAmountOfStudents)
+      })
+    })
+  })
 })
 
 describe('Population Statistics', () => {
@@ -93,7 +116,7 @@ describe('Population Statistics', () => {
   const defaultAmountOfStudents = 159
   const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
-  before(() => {
+  beforeEach(() => {
     cy.init(pathToCSBach2018)
   })
 
@@ -106,7 +129,7 @@ describe('Population Statistics', () => {
     })
   })
 
-  it('Graduation filter works', () => {
+  it('Graduation filter works', { retries: 2 }, () => {
     runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
       const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
 
@@ -117,7 +140,6 @@ describe('Population Statistics', () => {
       checkFilteringResult(defaultAmountOfStudents - graduated)
       getCard().cs('option-all').click()
     })
-    cy.cs('reset-all-filters').click()
   })
 
   it('Transfer filter works', () => {
@@ -129,7 +151,6 @@ describe('Population Statistics', () => {
       checkFilteringResult(defaultAmountOfStudents + transferred)
       cy.cs('TransferredToProgramme-filter-card').cs('option-havenot').click()
     })
-    cy.cs('reset-all-filters').click()
   })
 
   it('Enrollment filter works', () => {
@@ -145,7 +166,6 @@ describe('Population Statistics', () => {
     runTestStepWithPreAndPostParts('CreditsEarned', () => {
       testRangeFilter('CreditsEarned-filter-card', 50, 150, 118)
     })
-    cy.cs('reset-all-filters').click()
   })
 
   it('Age filter works', () => {
@@ -156,7 +176,6 @@ describe('Population Statistics', () => {
   })
 
   it('Gender filter works', () => {
-    cy.cs('reset-all-filters').click()
     runTestStepWithPreAndPostParts('Gender', () => {
       cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(42)
@@ -175,8 +194,8 @@ describe('Population Statistics', () => {
       cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.cs('startYearAtUni-dropdown').selectFromDropdown(7)
-      checkFilteringResult(152)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(14)
+      checkFilteringResult(136)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
@@ -192,24 +211,20 @@ describe('Population Statistics', () => {
     })
   })
 
-  it.skip('Courses filter works', () => {
-    // courses takes long time to load, wait for it to complete
-    cy.wait(25000)
-    cy.contains('Courses of class').click()
+  it('Courses filter works', () => {
     runTestStepWithPreAndPostParts('Courses', () => {
       const courses = ['TKT20002', 'MAT11002']
       cy.cs('courseFilter-course-dropdown').click().contains(`${courses[0]} - Ohjelmistotekniikka`).click()
-      checkFilteringResult(118)
+      checkFilteringResult(133)
       cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
-      checkFilteringResult(117)
+      checkFilteringResult(127)
       cy.cs('courseFilter-course-dropdown')
         .click()
         .contains(`${courses[1]} - Lineaarialgebra ja matriisilaskenta I`)
         .click()
-      checkFilteringResult(50)
+      checkFilteringResult(100)
       cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
       checkFilteringResult(1)
-      cy.contains('Courses of class').click()
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })
   })
@@ -225,6 +240,21 @@ describe('Population Statistics', () => {
       })
     })
   })
+
+  it('"Reset All Filters" button works', () => {
+    cy.cs('Gender-header').click()
+    cy.cs('genderFilter-dropdown').selectFromDropdown(0)
+    checkFilteringResult(42)
+    cy.cs('StartYearAtUni-header').click()
+    cy.cs('startYearAtUni-dropdown').selectFromDropdown(7)
+    checkFilteringResult(33)
+    cy.cs('reset-all-filters').click()
+    checkFilteringResult(defaultAmountOfStudents)
+  })
+
+  it('Studyright Type filter is not visible', () => {
+    cy.contains('Studyright Type').should('not.exist')
+  })
 })
 
 describe('Course Statistics', () => {
@@ -233,7 +263,7 @@ describe('Course Statistics', () => {
   const defaultAmountOfStudents = 118
   const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
-  before(() => {
+  beforeEach(() => {
     cy.init(pathToDSAndAlgoSpring2019)
   })
 
@@ -263,28 +293,14 @@ describe('Course Statistics', () => {
     })
   })
 
-  // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
-  // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
-  // So fix this
-  it.skip('Study track filter works', () => {
-    runTestStepWithPreAndPostParts('StudyTrack', () => {
-      const card = cy.cs('StudyTrack-filter-card')
-      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
-      checkFilteringResult(1)
-      programmeDropdown.get('i.delete').click()
-    })
-  })
-
   it('Age filter works', () => {
     cy.clock(MOCKED_DATE, ['Date'])
-    cy.cs('reset-all-filters').click()
     runTestStepWithPreAndPostParts('Age', () => {
       testRangeFilter('Age-filter-card', 20, 40, 33)
     })
   })
 
   it('Gender filter works', () => {
-    cy.cs('reset-all-filters').click()
     runTestStepWithPreAndPostParts('Gender', () => {
       cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(27)
@@ -303,8 +319,8 @@ describe('Course Statistics', () => {
       cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.cs('startYearAtUni-dropdown').selectFromDropdown(3)
-      checkFilteringResult(84)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(13)
+      checkFilteringResult(71)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
@@ -324,14 +340,13 @@ describe('Course Statistics', () => {
 describe('Custom Population Statistics', () => {
   let runTestStepWithPreAndPostParts
 
-  before(() => {
+  beforeEach(() => {
     cy.init('/custompopulation')
     cy.cs('custom-pop-search-button').click()
     cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
       const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
       runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
       cy.cs('student-no-input').click().type(students.join('\n'))
-      cy.wait(10000)
     })
     cy.cs('search-button').click()
   })
@@ -341,7 +356,6 @@ describe('Custom Population Statistics', () => {
     runTestStepWithPreAndPostParts('Age', () => {
       testRangeFilter('Age-filter-card', 20, 30, 1)
     })
-    cy.cs('reset-all-filters').click()
   })
 
   it('Programme defaults to "Active Study Right" mode', () => {
@@ -385,10 +399,12 @@ describe('Custom Population Statistics', () => {
     runTestStepWithPreAndPostParts('Courses', () => {
       const courses = ['MAT11001', 'TKT20004']
       cy.cs('courseFilter-course-dropdown').click().contains(`${courses[0]} - Johdatus yliopistomatematiikkaan`).click()
-      checkFilteringResult(4)
+      checkFilteringResult(5)
       cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
-      checkFilteringResult(4)
+      checkFilteringResult(5)
       cy.cs('courseFilter-course-dropdown').click().contains(`${courses[1]} - Tietoliikenteen perusteet`).click()
+      checkFilteringResult(5)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(1)
       checkFilteringResult(3)
       cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
       checkFilteringResult(1)

cypress/e2e/Language_center.js

@@ -0,0 +1,65 @@
+/// <reference types="Cypress" />
+
+const hasLanded = () => {
+  cy.contains('This view displays amounts')
+  cy.contains('Show number of')
+}
+
+/*
+  TODO FIX
+  These use dumb numbers, because actual_studyright update in db is not yet
+  in anon data, and the faculties do not get resolved.
+*/
+
+describe('When language center is opened', () => {
+  describe('as an admin user', () => {
+    beforeEach(() => {
+      cy.init('/languagecenterview', 'admin')
+      cy.get('[data-cy="completions-button"]').click()
+      cy.get('[data-cy="semester-from"]').click()
+      cy.contains('Syksy 2017').click()
+      cy.get('[data-cy="semester-to"]').click()
+      hasLanded()
+    })
+
+    it('Initial view is correct', () => {
+      cy.contains('All courses total')
+      cy.contains('Academic and Professional')
+      cy.contains('AYKK-RUKIRJ')
+    })
+
+    it('Faculties tab shows numbers', () => {
+      cy.contains('All courses total')
+      cy.contains('1363')
+    })
+
+    it('Faculties tab "exceeding" button works', () => {
+      cy.get('[data-cy="difference-button"]').click()
+      cy.contains('0')
+      cy.contains('Total')
+      cy.contains('1363').should('not.exist')
+    })
+
+    it('Faculties tab semester selector changes numbers', () => {
+      cy.get('[data-cy="semester-from"]').click()
+      cy.contains('Syksy 2020').click()
+      cy.contains('1363').should('not.exist')
+      cy.contains('All courses total')
+    })
+
+    it('Semester tab opens and contains coloring filter', () => {
+      cy.contains('By semesters').click()
+      cy.contains('1363')
+      cy.contains('Coloring mode')
+    })
+  })
+  describe('with a user with no rights', () => {
+    it('"Access denied" is shown', () => {
+      cy.init('/languagecenterview', 'norights')
+      cy.contains('Access denied')
+      cy.contains("You're currently not allowed to see this page.")
+      cy.contains('Special populations').click()
+      cy.contains('Language center view').should('not.exist')
+    })
+  })
+})

cypress/e2e/Population_statistics.js

@@ -73,20 +73,26 @@ describe('Population Statistics tests', () => {
       cy.contains('Excludes students with non-degree study right')
       cy.contains('Excludes students who have transferred out of this programme')
       cy.cs('filtered-students')
+    })
 
-      // Tests that use curriculum structure are commented out until
-      // the test database is fixed to the same structure as real database
-      // after the modules rework
-      // cy.contains('Courses of class').click()
+    it('Courses of class is displayed and link to individual course stats page works', () => {
+      cy.visit(pathToCSBach2017)
+      cy.contains('Courses of class').click()
+      cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
+      cy.get('[data-cy=toggle-group-module-TKT1]').click()
+      cy.contains('td', 'TKT10002').siblings().find('i.level.up').click()
+      cy.wait('@coursePage')
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
+    })
 
-      // cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
-      // // eslint-disable-next-line cypress/no-unnecessary-waiting
-      // cy.wait(25000) // a bit hacky way, wait until ui is ready
-      // cy.get('[data-cy=toggle-group-module-TKT1]').click()
-      // cy.contains('td', 'TKT10002').siblings().find('i.level.up').click()
-      // cy.wait('@coursePage')
-      // cy.url().should('include', '/coursestatistics')
-      // cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
+    it('Courses of class curriculum selection works', () => {
+      cy.visit(pathToCSBach2017)
+      cy.contains('Courses of class').click()
+      cy.get('Opiskelijan digitaidot - Kumpula').should('not.exist')
+      cy.get('[data-cy=curriculum-picker]').click()
+      cy.contains('2020 - 2022').click()
+      cy.contains('Opiskelijan digitaidot - Kumpula')
     })
 
     it("Empty 'tags' tab has a link to the page where tags can be created", () => {
@@ -103,14 +109,15 @@ describe('Population Statistics tests', () => {
       cy.contains('Create new tag')
     })
 
-    it('Advanced settings work', () => {
+    // This test sometimes fails on headless mode. It seems that the click on the
+    // 'Fetch class with new settings' button doesn't always trigger history.push()
+    // so the page doesn't reload. This is why waiting also doesn't help.
+    it('Advanced settings work', { retries: 2 }, () => {
       cy.visit(pathToCSBach2017)
       cy.get('[data-cy=advanced-toggle]').click()
       // only spring
       cy.cs('toggle-fall').click()
       cy.contains('Fetch class').click()
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
       cy.contains('Students (0)')
       // only fall
       cy.get('[data-cy=advanced-toggle]').click()
@@ -229,17 +236,14 @@ describe('Population Statistics tests', () => {
     })
   })
 
-  /*
   describe('when using IAM user', () => {
     beforeEach(() => {
       cy.init(pathToCSBach2017, 'onlyiamrights')
-    })
-    it('Population statistics is visible', () => {
-      cy.reload()
-
       cy.contains('Tietojenkäsittelytieteen kandiohjelma')
       cy.contains('class size 170 students')
+    })
 
+    it('Population statistics is visible', () => {
       cy.get('.card').within(() => {
         cy.contains('Excludes exchange students')
         cy.contains('Excludes students with non-degree study right')
@@ -250,10 +254,8 @@ describe('Population Statistics tests', () => {
       cy.contains('Credit statistics')
     })
 
-    it.skip('Students tab is not available', () => {
+    it('Students tab is not available', () => {
       cy.contains('Students (170)').should('not.exist')
-      // TODO: somehow check that obfuscation works
     })
   })
-  */
 })

cypress/e2e/Students.js

@@ -1,9 +1,9 @@
 /// <reference types="Cypress" />
 
-describe('Student Statistics tests', () => {
+describe('Students tests', () => {
   beforeEach(() => {
     cy.init()
-    cy.contains('Student statistics').click()
+    cy.contains('Students').click()
     cy.contains('Show student names')
   })
 
@@ -14,7 +14,7 @@ describe('Student Statistics tests', () => {
     sis_person_id: 'hy-hlo-84830887',
   }
 
-  it('Student statistics search form is usable', () => {
+  it('Students search form is usable', () => {
     cy.contains('Show student names')
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.lastname)
@@ -39,7 +39,6 @@ describe('Student Statistics tests', () => {
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.studentnumber)
     cy.contains('td a', student.studentnumber).click()
-    cy.contains('Started: 01.08.2017')
     cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 31.07.2025')
     cy.contains(student.lastname).should('not.exist')
     cy.contains(student.firstnames).should('not.exist')
@@ -104,7 +103,6 @@ describe('Testing with admin rights', () => {
   it('Does not crash if student has no studyright or courses', () => {
     const studentNumber = '011825096'
     cy.init(`/students/${studentNumber}`, 'admin') // use admin to see all students
-    cy.contains('Started: Unavailable')
     cy.contains('Credits: 0')
   })
 })

cypress/e2e/Study_guidance_groups.js

@@ -0,0 +1,163 @@
+/// <reference types="Cypress" />
+
+describe('Study guidance group tests', () => {
+  describe('Study guidance group overview page', () => {
+    describe('without study guidance groups', () => {
+      it('shows the correct notification', () => {
+        cy.init('/studyguidancegroups', 'admin')
+        cy.contains('Study guidance groups')
+        cy.contains('You do not have access to any study guidance groups.')
+      })
+    })
+
+    describe('without correct rights', () => {
+      it('shows the correct notification', () => {
+        cy.init('/studyguidancegroups')
+        cy.contains('Access denied')
+        cy.contains(
+          "You're currently not allowed to see this page. Please contant oodikone@helsinki.fi, if this is a mistake."
+        )
+      })
+    })
+
+    describe('with study guidance groups', () => {
+      beforeEach(() => {
+        cy.init('/studyguidancegroups', 'onlystudyguidancegroups')
+        cy.intercept('/api/studyguidancegroups', { fixture: 'studyGuidanceGroups.json' })
+      })
+
+      it('shows the correct data', () => {
+        cy.contains('Study guidance groups')
+
+        const studyGuidanceGroupsTableContent = [
+          ['TKT kandit 2018', 3, 'Tietojenkäsittelytieteen kandiohjelma', '2018 - 2019'],
+          ['Oma ohjausryhmä', 2, 'Add study programme', 'Add year'],
+        ]
+
+        cy.checkTableStats(studyGuidanceGroupsTableContent, 'study-guidance-group-overview')
+      })
+
+      it('has working links to single study guidance groups', () => {
+        cy.cs('study-guidance-group-link-sgg-cypress-1').click()
+        cy.url().should('eq', `${Cypress.config().baseUrl}/studyguidancegroups/sgg-cypress-1`)
+      })
+    })
+  })
+
+  describe('Page for single study guidance group', () => {
+    beforeEach(() => {
+      cy.intercept('/api/studyguidancegroups', { fixture: 'studyGuidanceGroups.json' })
+    })
+
+    it('has a working back button', () => {
+      cy.init('/studyguidancegroups/sgg-cypress-1', 'onlystudyguidancegroups')
+
+      cy.get('button.ui.button').contains('Back').click()
+      cy.url().should('eq', `${Cypress.config().baseUrl}/studyguidancegroups`)
+    })
+
+    describe('without associated study programme and year', () => {
+      beforeEach(() => {
+        cy.init('/studyguidancegroups/sgg-cypress-2', 'onlystudyguidancegroups')
+      })
+
+      it('shows the correct panes', () => {
+        cy.contains('Oma ohjausryhmä')
+        cy.cs('Credit accumulation (for 2 students)')
+        cy.cs('Age distribution')
+        cy.cs('Courses of population')
+        cy.cs('Students (2)')
+        cy.cs('Credit statistics').should('not.exist')
+      })
+
+      it('students table has the correct tabs', () => {
+        cy.cs('Students (2)').click()
+        cy.get('[data-cy="student-table-tabs"] .ui.attached.tabular.menu').then($menu => {
+          const tabNames = Array.from($menu[0].children).map(elem => elem.innerText)
+          expect(tabNames).to.deep.equal(['General'])
+        })
+      })
+
+      it('general tab of the students table has the correct columns', () => {
+        const firstRowHeadings = [
+          'Phone number',
+          'Student number',
+          'Credits',
+          'Start year at uni',
+          'Study programmes',
+          'Tags',
+        ]
+        const secondRowHeadings = ['All', 'Since 1.1.1970']
+        cy.cs('Students (2)').click()
+        cy.get('[data-cy="student-table-tabs"] table thead tr').then($tr => {
+          const firstRowTexts = Array.from($tr[0].children).map(elem => elem.innerText.replace('\n', ' '))
+          expect(firstRowTexts).to.deep.equal(firstRowHeadings)
+          const secondRowTexts = Array.from($tr[1].children).map(elem => elem.innerText)
+          expect(secondRowTexts).to.deep.equal(secondRowHeadings)
+        })
+      })
+    })
+
+    describe('with associated study programme and year', () => {
+      beforeEach(() => {
+        cy.init('/studyguidancegroups/sgg-cypress-1', 'onlystudyguidancegroups')
+      })
+
+      it('shows the correct labels', () => {
+        cy.get('.ui.medium.header').contains('TKT kandit 2018')
+        cy.get('.ui.blue.tag.label:first').contains('Tietojenkäsittelytieteen kandiohjelma')
+        cy.get('.ui.blue.tag.label:last').contains('2018 - 2019')
+      })
+
+      it('shows the correct panes', () => {
+        cy.cs('Credit accumulation (for 3 students)')
+        cy.cs('Credit statistics')
+        cy.cs('Age distribution')
+        cy.cs('Courses of population')
+        cy.cs('Students (3)')
+      })
+
+      it("clicking the 'Show starting from associated year' button activates the 'Date of Course Credits' filter", () => {
+        cy.get('[data-cy="CreditDate-filter-card"][data-open="false"]')
+        cy.get('.ui.primary.button').contains('Show starting from associated year').click()
+        cy.get('[data-cy="CreditDate-filter-card"][data-open="true"]')
+        cy.get('.ui.mini.icon.button.credit-date-filter-input').contains('01.08.2018')
+        cy.contains('Reset All Filters')
+        cy.get('.ui.primary.button').contains('Show all credits').click()
+        cy.get('[data-cy="CreditDate-filter-card"][data-open="false"]')
+      })
+
+      it('students table has the correct tabs', () => {
+        cy.cs('Students (3)').click()
+        cy.get('[data-cy="student-table-tabs"] .ui.attached.tabular.menu').then($menu => {
+          const tabNames = Array.from($menu[0].children).map(elem => elem.innerText)
+          expect(tabNames).to.deep.equal(['General', 'Courses', 'Progress'])
+        })
+      })
+
+      it('general tab of the students table has the correct columns', () => {
+        const firstRowHeadings = [
+          'Phone number',
+          'Student number',
+          'Credits',
+          'Start of studyright',
+          'Started in programme',
+          'Semesters present',
+          'Graduation date',
+          'Start year at uni',
+          'Other programmes',
+          'Semesters present amount',
+          'Tags',
+        ]
+        const secondRowHeadings = ['All', 'HOPS', 'Since 1.8.2018']
+        cy.cs('Students (3)').click()
+        cy.get('[data-cy="student-table-tabs"] table thead tr').then($tr => {
+          const firstRowTexts = Array.from($tr[0].children).map(elem => elem.innerText.replace('\n', ' '))
+          expect(firstRowTexts).to.deep.equal(firstRowHeadings)
+          const secondRowTexts = Array.from($tr[1].children).map(elem => elem.innerText)
+          expect(secondRowTexts).to.deep.equal(secondRowHeadings)
+        })
+      })
+    })
+  })
+})

cypress/e2e/Studyprogramme_overview.js

@@ -1,4 +1,3 @@
-/* eslint-disable cypress/no-unnecessary-waiting */
 /// <reference types="Cypress" />
 
 const deleteTag = name => {
@@ -24,28 +23,11 @@ const getEmptyYears = isAcademicYear => {
 }
 
 describe('Studyprogramme overview', () => {
-  before(() => {
-    cy.init('/study-programme', 'admin')
-    cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click({ force: true })
-    cy.get('.attached').contains('Update statistics').click()
-    // eslint-disable-next-line cypress/no-unnecessary-waiting
-    cy.wait(5000)
-    cy.get('button').eq(0).click({ force: true })
-    // eslint-disable-next-line cypress/no-unnecessary-waiting
-    cy.wait(10000)
-    cy.get('button').eq(1).click({ force: true })
-    // eslint-disable-next-line cypress/no-unnecessary-waiting
-    cy.wait(10000)
-  })
-
   /* Basic information overview -tests*/
   describe('Basic information -view works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
       cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click({ force: true })
-      // cy.viewport(1536, 960)
     })
 
     // If the backend breaks for one of the sections, the section header is not rendered and this will fail
@@ -63,8 +45,8 @@ describe('Studyprogramme overview', () => {
         // [Year, Started, Graduated, Transferred Away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0]),
         [2021, 0, 0, 1, 0],
-        [2020, 10, 35, 3, 2],
-        [2019, 86, 17, 0, 1],
+        [2020, 10, 35, 2, 2],
+        [2019, 86, 17, 1, 1],
         [2018, 161, 1, 0, 0],
         [2017, 171, 0, 0, 0],
       ]
@@ -73,15 +55,11 @@ describe('Studyprogramme overview', () => {
     })
 
     it('Basic information contains correct credits', () => {
-      const years = getEmptyYears()
       const tableContents = [
-        // [Year, Total, Major credits, Non-major credits, Non-degree credits, Transferred credits]
-        ...years.map(year => [year, 0, 0, 0, 0, 0]),
-        [2021, 0, 0, 0, 0, 0],
-        [2020, 5860, 5591, 102, 0, 167],
-        [2019, 8696, 7737, 587, 23, 349],
-        [2018, 7818, 6584, 837, 125, 272],
-        [2017, 3085, 2492, 153, 52, 388],
+        [2020, 4511, 4511, 0, 0, 167],
+        [2019, 7117, 7097, 0, 0, 349],
+        [2018, 5747, 5742, 0, 0, 272],
+        [2017, 1910, 1910, 0, 0, 388],
       ]
 
       cy.checkTableStats(tableContents, 'CreditsProducedByTheStudyprogramme')
@@ -104,8 +82,6 @@ describe('Studyprogramme overview', () => {
 
     it('Special studyrights can be excluded and basic data changes accordingly', () => {
       cy.get('[data-cy=StudentToggle]').click()
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(15000)
       const years = getEmptyYears()
       const studentTableContents = [
         // [Year, Started, Graduated]
@@ -121,12 +97,10 @@ describe('Studyprogramme overview', () => {
 
       const creditTableContents = [
         // [Year, Total, Major students credits, Transferred credits]
-        ...years.map(year => [year, 0, 0, 0]),
-        [2021, 0, 0, 0],
-        [2020, 5018, 4694, 167],
-        [2019, 6185, 5702, 349],
-        [2018, 4885, 4538, 272],
-        [2017, 1988, 1600, 388],
+        [2020, 4511, 4511, 0, 0, 167],
+        [2019, 7117, 7097, 0, 0, 349],
+        [2018, 5747, 5742, 0, 0, 272],
+        [2017, 1910, 1910, 0, 0, 388],
       ]
 
       cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
@@ -147,8 +121,6 @@ describe('Studyprogramme overview', () => {
 
     it('Year can be changed to academic year, and data changes accordingly', () => {
       cy.get('[data-cy=YearToggle]').click({ force: true })
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(15000)
       const isAcademicYear = true
       const years = getEmptyYears(isAcademicYear)
       const studentTableContents = [
@@ -160,22 +132,17 @@ describe('Studyprogramme overview', () => {
         ['2018 - 2019', 161, 8, 0, 1],
         ['2017 - 2018', 171, 1, 0, 0],
       ]
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
       cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
 
       const creditTableContents = [
-        // [Year, Total, Major students credits, Non-major students credits, Non-degree credits, Transferred credits]
-        ...years.map(year => [year, 0, 0, 0, 0, 0]),
-        ['2021 - 2022', 0, 0, 0, 0, 0],
-        ['2020 - 2021', 541, 471, 10, 0, 60],
-        ['2019 - 2020', 8575, 8266, 116, 0, 193],
-        ['2018 - 2019', 10013, 8851, 791, 43, 328],
-        ['2017 - 2018', 6079, 4816, 762, 157, 344],
+        // Credit table doesnt show (later) years without data
+        ['2020 - 2021', 352, 352, 0, 0, 60],
+        ['2019 - 2020', 6966, 6966, 0, 0, 193],
+        ['2018 - 2019', 7797, 7777, 0, 0, 328],
+        ['2017 - 2018', 4170, 4165, 0, 0, 344],
       ]
 
       cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
-      //cy.wait(10000)
       cy.get('[data-cy=YearToggle]').click()
     })
 
@@ -186,11 +153,10 @@ describe('Studyprogramme overview', () => {
         .should('contain', 'Transferred away')
 
       cy.get('[data-cy=Graph-CreditsProducedByTheStudyprogramme')
-        .should('contain', 'Major students credits')
-        .should('contain', 'Non-major students credits')
-        .should('contain', 'Transferred credits')
-        .should('contain', 7737)
-        .should('contain', 6584)
+        .should('contain', 'Degree students')
+        .should('contain', 'Transferred')
+        .should('contain', 388)
+        .should('contain', 7097)
 
       cy.get('[data-cy=Graph-GraduatedAndThesisWritersOfTheProgramme')
         .should('contain', 'Graduated students')
@@ -216,17 +182,15 @@ describe('Studyprogramme overview', () => {
         .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
         .should('contain', 'Datatieteen maisteriohjelma')
         .should('contain', 'Matematiikan ja tilastotieteen maisteriohjelma')
-        .should('contain', 1)
-        .should('contain', 4)
-        .should('contain', 13)
+        .should('contain', 11)
+        .should('contain', 12)
+        .should('contain', 24)
     })
   })
 
   describe('Graduation times of master programmes', () => {
     it('are split into two graphs', () => {
       cy.init('/study-programme', 'admin')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(5000)
       cy.contains('a', 'Kasvatustieteiden maisteriohjelma').click({ force: true })
 
       cy.get('[data-cy=graduation-times-graph-breakdownMaster]')
@@ -256,9 +220,7 @@ describe('Studyprogramme overview', () => {
   describe('Studytrack overview works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
-      cy.wait(10000)
       cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.wait(5000)
       cy.get('.attached').contains('Studytracks and class statistics').click()
     })
 
@@ -272,11 +234,11 @@ describe('Studyprogramme overview', () => {
     it('Students of the studyprogramme are shown correctly', () => {
       const tableContents = [
         // [Year, All, Started studying, Currently enrolled, Absent, Inactive, Graduated, Men, Women, other/unknown Finnish, other]
-        ['2020 - 2021', 12, 10, 0, 0, 0, 0, 10, 2, 0, 12, 0],
-        ['2019 - 2020', 86, 86, 0, 0, 0, 1, 65, 21, 0, 86, 0],
-        ['2018 - 2019', 162, 161, 0, 0, 0, 11, 118, 44, 0, 161, 1],
-        ['2017 - 2018', 171, 171, 0, 0, 0, 42, 135, 36, 0, 166, 5],
-        ['Total', 431, 428, 0, 0, 0, 54, 328, 103, 0, 425, 6],
+        ['2020 - 2021', 12, 10, 0, 0, 12, 0, 10, 2, 0, 12, 0],
+        ['2019 - 2020', 86, 86, 0, 0, 85, 1, 65, 21, 0, 86, 0],
+        ['2018 - 2019', 162, 161, 0, 0, 151, 11, 118, 44, 0, 161, 1],
+        ['2017 - 2018', 171, 171, 0, 0, 129, 42, 135, 36, 0, 166, 5],
+        ['Total', 431, 428, 0, 0, 377, 54, 328, 103, 0, 425, 6],
       ]
 
       cy.checkTableStats(tableContents, 'StudytrackOverview')
@@ -288,10 +250,10 @@ describe('Studyprogramme overview', () => {
         ['2022 - 2023', 0, 0, 0, 0, 0, 0, 0, 0],
         ['2021 - 2022', 0, 0, 0, 0, 0, 0, 0, 0],
         ['2020 - 2021', 12, 12, 0, 0, 0, 0, 0, 0],
-        ['2019 - 2020', 86, 86, 0, 0, 0, 0, 0, 0],
-        ['2018 - 2019', 162, 162, 0, 0, 0, 0, 0, 0],
-        ['2017 - 2018', 171, 171, 0, 0, 0, 0, 0, 0],
-        ['Total', 431, 431, 0, 0, 0, 0, 0, 0],
+        ['2019 - 2020', 86, 26, 34, 21, 5, 0, 0, 0],
+        ['2018 - 2019', 162, 10, 27, 47, 36, 26, 12, 4],
+        ['2017 - 2018', 171, 20, 21, 26, 22, 22, 33, 27],
+        ['Total', 431, 68, 82, 94, 63, 48, 45, 31],
       ]
 
       cy.checkTableStats(tableContents, 'StudytrackProgress')
@@ -321,8 +283,6 @@ describe('Studyprogramme overview', () => {
   describe('Programme courses works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
       cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('.attached').contains('Programme courses').click()
     })
@@ -343,21 +303,14 @@ describe('Studyprogramme overview', () => {
     })
 
     it('calendar year -> academic year toggle works', () => {
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
       cy.get('[data-cy=fromYear]').click().contains('2018').click()
-
       cy.get('[data-cy=toYear').click().contains('2019').click()
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(20000)
       cy.get('[data-cy=CoursesSortableTable]').within(() => {
         cy.get('tr').eq(1).contains('684')
       })
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(20000)
       cy.get('[data-cy=calendarAcademicYearToggle]').first().click()
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(20000)
+      cy.get('[data-cy=fromYear]').click().contains('2018-2019').click()
+      cy.get('[data-cy=toYear').click().contains('2019-2020').click()
       cy.get('[data-cy=CoursesSortableTable]').within(() => {
         cy.get('tr').eq(1).contains('772')
       })
@@ -399,8 +352,6 @@ describe('Studyprogramme overview', () => {
   describe('Basic information -view works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
       cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('.attached').contains('Tags').click()
     })
@@ -443,25 +394,22 @@ describe('Studyprogramme overview', () => {
       cy.get('textarea').type('010113437')
       cy.get('.positive').click()
 
-      cy.contains('Student statistics').click()
+      cy.contains('Students').click()
       cy.get('.prompt').type(student)
       cy.contains('a', student).click()
       cy.contains(name)
 
       cy.go('back')
       cy.go('back')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
       deleteTag(name)
 
-      cy.contains('Student statistics').click()
+      cy.contains('Students').click()
       cy.get('.prompt').type(student)
       cy.contains('a', student).click()
       cy.contains(name).should('not.exist')
     })
   })
 
-  /*
   describe('IAM user', () => {
     beforeEach(() => {
       cy.init('/study-programme', 'onlyiamrights')
@@ -497,5 +445,4 @@ describe('Studyprogramme overview', () => {
       cy.get('[data-cy=Section-AverageGraduationTimesStudytracks]')
     })
   })
-  */
 })

cypress/e2e/Users.js

@@ -2,14 +2,14 @@
 
 describe('Users tests', () => {
   describe('Using user with just grp-oodikone-user, no other rights', () => {
-    it('shows only frontpage, trends feedback', () => {
+    it('shows only frontpage, university, feedback', () => {
       cy.init('', 'norights') // login with norights user
-      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('University').should('exist')
       cy.get('[data-cy="navBar"]').contains('Give feedback').should('exist')
 
-      cy.get('[data-cy="navBar"]').contains('Study programme').should('not.exist')
-      cy.get('[data-cy="navBar"]').contains('Student statistics').should('not.exist')
-      cy.get('[data-cy="navBar"]').contains('Course statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Programmes').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Students').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Courses').should('not.exist')
       cy.get('[data-cy="navBar"]').contains('Teachers').should('not.exist')
       cy.get('[data-cy="navBar"]').contains('Users').should('not.exist')
       cy.get('[data-cy="navBar"]').contains('Faculty').should('not.exist')
@@ -29,12 +29,12 @@ describe('Users tests', () => {
     })
 
     it('should see more stuff than others', () => {
-      cy.get('[data-cy="navBar"]').contains('Study programme').should('exist')
-      cy.get('[data-cy="navBar"]').contains('Student statistics').should('exist')
-      cy.get('[data-cy="navBar"]').contains('Course statistics').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Programmes').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Students').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Courses').should('exist')
       // TODO: admin should probably see this as well, since can access this page: cy.get('[data-cy="navBar"]').contains('Teachers').should('exist')
       cy.get('[data-cy="navBar"]').contains('Users').should('exist')
-      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('University').should('exist')
     })
 
     it("mocking normal user shows only the mocked user's programmes", () => {
@@ -42,8 +42,7 @@ describe('Users tests', () => {
       cy.cs('user-edit-button-basic').click()
       cy.get('i.spy').click()
       cy.contains('mocking as basic')
-      cy.wait(1000)
-      cy.contains('Study programme').click().siblings().contains('Search by class').click()
+      cy.contains('Programmes').click().siblings().contains('Class statistics').click()
       cy.contains('label', 'Study programme')
       cy.contains('label', 'Study programme')
         .siblings()

cypress/fixtures/studyGuidanceGroups.json

@@ -0,0 +1,28 @@
+[
+  {
+    "id": "sgg-cypress-1",
+    "name": {
+      "fi": "TKT kandit 2018"
+    },
+    "members": [
+      { "personStudentNumber": "010450938" },
+      { "personStudentNumber": "010589388" },
+      { "personStudentNumber": "010262566" }
+    ],
+    "tags": {
+      "studyProgramme": "KH50_005",
+      "year": "2018"
+    }
+  },
+  {
+    "id": "sgg-cypress-2",
+    "name": {
+      "fi": "Oma ohjausryhmä"
+    },
+    "members": [{ "personStudentNumber": "010614509" }, { "personStudentNumber": "010307146" }],
+    "tags": {
+      "studyProgramme": null,
+      "year": null
+    }
+  }
+]

cypress/support/commands.js

@@ -1,14 +1,13 @@
 /// <reference types="Cypress" />
 //
-// Note: here we need to set keys to be all lowercase, since
-// we're replacing headers after they've left browser / frontend.
+// ! IMPORTANT: here we need to set keys to be all lowercase, since
+// ! we're replacing headers after they've left browser / frontend.
 // All of these users are available in anon user-db
 const adminUserHeaders = {
   uid: 'admin',
   displayname: 'Admin User',
   'shib-session-id': 'mock-cypress-session',
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users;grp-toska',
-  edupersonaffiliation: 'member;employee;faculty',
   mail: 'grp-toska+mockadminuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-6666666',
 }
@@ -18,17 +17,15 @@ const basicUserHeaders = {
   displayname: 'Basic User',
   'shib-session-id': 'mock-cypress-session',
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
-  edupersonaffiliation: 'member',
   mail: 'grp-toska+mockbasicuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-1234567',
 }
 
 const onlycoursestatisticsUserHeaders = {
   uid: 'onlycoursestatistics',
-  displayName: 'Onlycoursestatistics User',
-  'shib-session-id': 'mock-session',
-  hyGroupCn: 'grp-oodikone-basic-users',
-  eduPersonAffiliation: 'member',
+  displayname: 'Onlycoursestatistics User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-basic-users',
   mail: 'grp-toska+mockonlycoursestatisticsuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-7654321',
 }
@@ -38,7 +35,6 @@ const norightsUserHeaders = {
   displayname: 'Norights User',
   'shib-session-id': 'mock-cypress-session',
   hygroupcn: 'grp-oodikone-users',
-  edupersonaffiliation: 'member',
   mail: 'grp-toska+mocknorightuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-0000000',
 }
@@ -48,11 +44,20 @@ const onlyIamRightsUserHeaders = {
   displayname: 'Only IAM rights user',
   'shib-session-id': 'mock-cypress-session',
   hygroupcn: 'hy-employees;hy-mltdk-tkt-jory',
-  edupersonaffiliation: 'member',
   mail: 'grp-toska+mockonlyiamrightsuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-1111111',
 }
 
+// Also has element detail of "KH50_005" for the study guidance groups to work
+const onlyStudyGuidanceGroupsUser = {
+  uid: 'onlystudyguidancegroups',
+  displayname: 'Study Guidance Groups User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users',
+  mail: 'grp-toska+mockonlystudyguidancegroupsuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-2222222',
+}
+
 // TODO add user into anon-DB (check the need for changing hypersonid)
 // REMEMBER TO ADD ROLE openUniSearch
 // const openUniUser = {
@@ -60,7 +65,6 @@ const onlyIamRightsUserHeaders = {
 //   displayname: 'Open Uni User',
 //   'shib-session-id': 'mock-cypress-session',
 //   hygroupcn: 'grp-oodikone-basics-users;hy-ypa-opa-dojo',
-//   edupersonaffiliation: 'member',
 //   mail: 'grp-toska+mockopenuniuser@helsinki.fi',
 //   hypersonsisuid: 'hy-hlo-9999999',
 // }
@@ -71,6 +75,7 @@ const userHeaders = [
   onlycoursestatisticsUserHeaders,
   norightsUserHeaders,
   onlyIamRightsUserHeaders,
+  onlyStudyGuidanceGroupsUser,
 ]
 
 /**
@@ -134,7 +139,7 @@ Cypress.Commands.add('selectStudyProgramme', name => {
 })
 
 Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {
-  cy.get(`[data-cy=Table-${tableName}]>tbody`).within(() => {
+  cy.get(`[data-cy=Table-${tableName}] tbody`).within(() => {
     correctStats.forEach((values, trIndex) => {
       cy.get('tr')
         .eq(trIndex)

cypress/support/e2e.js

@@ -13,7 +13,7 @@
 // https://on.cypress.io/configuration
 // ***********************************************************
 
-import './commands'
+require('./commands')
 
 // This error is benign and seems to be only present in Cypress
 const resizeObserverLoopErrRe = /^[^(ResizeObserver loop limit exceeded)]/

docker-compose.ci.yml

@@ -21,6 +21,7 @@ services:
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
       IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
       JAMI_URL: 'http://jami:3003/'
+      CRYPT_KEY: 'e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'
     build:
       context: services/backend
 
@@ -70,8 +71,9 @@ services:
     <<: *defaults
     image: quay.io/toska/jami:production
     environment:
+      - npm_config_cache=/tmp
       - PORT=3003
-      - NODE_ENV=production
+      - NODE_ENV=dev
       - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
     ports:
       - 3003:3003

docker-compose.yml

@@ -23,7 +23,6 @@ services:
     build:
       <<: *build_defaults
       context: services/backend
-    container_name: backend
     depends_on:
       - kone-db
       - redis
@@ -130,8 +129,6 @@ services:
       SIS_IMPORTER_USER: postgres
       SIS_NATS_TOKEN: dev
       SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
-      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
-      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
     ports:
       - 8082:8082
     profiles:
@@ -179,6 +176,7 @@ services:
     <<: *defaults
     image: quay.io/toska/jami:production
     environment:
+      - npm_config_cache=/tmp
       - PORT=3003
       - NODE_ENV=dev
       - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres

package.json

@@ -19,7 +19,7 @@
     "testupdater": "docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start",
     "flushredis": "docker exec -it redis redis-cli flushall",
     "prepare": "husky install",
-    "rapodiff": "docker exec backend node /opt/app-root/src/src/rapodiff.js"
+    "rapodiff": "docker exec oodikone_backend_1 node /opt/app-root/src/src/rapodiff/rapodiff.js"
   },
   "repository": {
     "type": "git",
@@ -31,30 +31,23 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@babel/core": "^7.19.3",
-    "@babel/eslint-parser": "^7.19.1",
-    "@babel/plugin-syntax-jsx": "^7.16.7",
     "concurrently": "^7.4.0",
-    "cypress": "^12.17.4",
-    "eslint": "^8.25.0",
+    "cypress": "^13.6.4",
+    "eslint": "^8.56.0",
     "eslint-config-airbnb": "^19.0.4",
-    "eslint-config-prettier": "^8.5.0",
-    "eslint-plugin-babel": "^5.3.1",
-    "eslint-plugin-cypress": "^2.12.1",
-    "eslint-plugin-import": "^2.25.4",
-    "eslint-plugin-jsx-a11y": "^6.5.1",
-    "eslint-plugin-prefer-object-spread": "^1.2.1",
-    "eslint-plugin-prettier": "^4.2.1",
-    "eslint-plugin-react": "^7.31.9",
-    "eslint-plugin-react-hooks": "^4.3.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-cypress": "^2.15.1",
+    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-jsx-a11y": "^6.8.0",
+    "eslint-plugin-prettier": "^5.1.3",
+    "eslint-plugin-react": "^7.33.2",
+    "eslint-plugin-react-hooks": "^4.6.0",
     "husky": "^8.0.1",
     "lint-staged": "^13.0.3",
     "node-actionlint": "^1.2.2",
-    "prettier": "^2.5.1",
-    "stylelint": "^14.13.0",
-    "stylelint-config-prettier": "^9.0.3",
-    "stylelint-config-standard": "^26.0.0",
-    "stylelint-prettier": "^2.0.0"
+    "prettier": "3.2.5",
+    "stylelint": "^16.2.1",
+    "stylelint-config-standard": "^36.0.0"
   },
   "dependencies": {
     "lodash": "^4.17.21",

run.sh

@@ -83,8 +83,6 @@ parse_profiles
 parse_env
 if [[ "$compose_command" == "" ]]; then
   final_command="docker-compose ${env}"
-elif [[ "$compose_command" == *"down"* ]]; then
-  final_command="docker-compose ${env} ${compose_command}"
 else
   final_command="docker-compose ${env} ${profiles} ${compose_command}"
 fi

services/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/nodejs-14-minimal
+FROM registry.access.redhat.com/ubi9/nodejs-18-minimal
 
 ENV TZ="Europe/Helsinki"
 
@@ -7,6 +7,7 @@ WORKDIR /opt/app-root/src
 EXPOSE 8080
 
 ENV NODE_ENV=production
+ENV NODE_OPTIONS=--max-old-space-size=4096
 
 ARG BASE_PATH
 ENV PUBLIC_URL=$BASE_PATH
@@ -24,4 +25,4 @@ COPY ./package* ./
 RUN npm ci --omit-dev
 COPY . .
 
-CMD ["npm", "start"]
+CMD ["node", "index.js"]

services/backend/config/programmeCodes.js

@@ -14,6 +14,7 @@ const codes = {
   MH30_001: 'll',
   MH30_002: 'tmed',
   MH30_003: 'hll',
+  MH30_006: 'terv',
   KH40_001: 'filk',
   KH40_002: 'ttk',
   KH40_003: 'kik',
@@ -95,6 +96,7 @@ const codes = {
   MH80_005: 'hnfb',
   MH80_006: 'ekm',
   MH80_007: 'mmb',
+  MH80_008: 'emplant',
   KH90_001: 'elk',
   MH90_001: 'ell',
   T920101: 'dptheol',

services/backend/dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14.17-alpine3.13
+FROM node:18-alpine
 
 ENV TZ="Europe/Helsinki"
 
@@ -7,9 +7,10 @@ WORKDIR /opt/app-root/src
 EXPOSE 8080
 
 ENV NODE_ENV=development
+ENV NODE_OPTIONS=--max-old-space-size=4096
 
 COPY ./package* ./
 RUN npm ci
 COPY . .
 
-CMD ["npm", "run", "dev"]
+CMD ["node_modules/.bin/nodemon", "index.js"]

services/backend/package.json

@@ -9,10 +9,9 @@
   "dependencies": {
     "@sentry/node": "^5.30.0",
     "@sentry/tracing": "^6.18.2",
-    "async": "^3.2.3",
-    "async-lock": "^1.3.1",
     "axios": "^0.21.4",
     "bluebird": "^3.7.2",
+    "bullmq": "^4.14.2",
     "cors": "^2.8.5",
     "cron": "^1.8.2",
     "express": "^4.18.2",
@@ -22,16 +21,21 @@
     "moment": "^2.29.1",
     "moment-timezone": "^0.5.37",
     "morgan": "^1.10.0",
+    "papaparse": "^5.4.1",
     "pg": "^8.8.0",
     "redis": "^3.1.2",
     "sequelize": "^6.24.0",
     "umzug": "^2.3.0",
     "unfuck-utf8-headers-middleware": "^1.0.1",
-    "uuid": "^3.4.0",
     "winston": "^3.8.2",
     "winston-gelf-transporter": "^1.0.2"
   },
   "devDependencies": {
-    "nodemon": "^1.19.4"
+    "nodemon": "^3.0.3"
+  },
+  "overrides": {
+    "winston-gelf-transporter": {
+      "gelf-pro": "^1.3.4"
+    }
   }
 }

services/backend/src/app.js

@@ -5,6 +5,7 @@ const routes = require('./routes')
 const { startCron } = require('./events')
 const { initializeDatabaseConnection, dbConnections } = require('./database/connection')
 const logger = require('./util/logger')
+require('./worker/worker')
 
 initializeDatabaseConnection()
   .then(() => {

services/backend/src/conf-backend.js

@@ -50,8 +50,11 @@ const KONE_PASSWORD = process.env.KONE_PASSWORD || 'postgres'
 const USER_PASSWORD = process.env.USER_PASSWORD || 'postgres'
 const SIS_PASSWORD = process.env.SIS_PASSWORD || 'postgres'
 
+const CRYPT_KEY = process.env.CRYPT_KEY
+
 module.exports = {
   DB_URL_USER,
+  CRYPT_KEY,
   DB_URL_KONE,
   DB_MAX_CONNECTIONS,
   DB_MAX_CRON_CONNECTIONS,
@@ -74,4 +77,5 @@ module.exports = {
   backendPort,
   importerUrl,
   importerToken,
+  isDev,
 }

services/backend/src/events.js

@@ -1,17 +1,18 @@
 const { CronJob } = require('cron')
-const moment = require('moment')
-const { refreshStatus, refreshStatusGraduated, refreshUber, getStartYears } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
-const { updateBasicView, updateStudytrackView } = require('./services/studyprogrammeUpdates')
+const { updateBasicView, updateStudytrackView } = require('./services/studyprogramme/studyprogrammeUpdates')
 const { findAndSaveTeachers } = require('./services/topteachers')
 const { faculties } = require('./services/organisations')
-const { combinedStudyprogrammes } = require('./services/studyprogrammeHelpers')
+const { combinedStudyprogrammes } = require('./services/studyprogramme/studyprogrammeHelpers')
 const { updateFacultyOverview, updateFacultyProgressOverview } = require('./services/faculty/facultyUpdates')
-const { isProduction } = require('./conf-backend')
+const { isProduction, runningInCI } = require('./conf-backend')
 const { getCurrentSemester } = require('./services/semesters')
 const logger = require('./util/logger')
 const { getAssociations } = require('./services/studyrights')
+const { redisClient } = require('./services/redis')
+const { computeLanguageCenterData, LANGUAGE_CENTER_REDIS_KEY } = require('./services/languageCenterData')
+const { jobMaker } = require('./worker/queue')
 
 const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
 
@@ -21,24 +22,27 @@ const refreshStudyrightAssociations = async () => {
 }
 
 const refreshFaculties = async () => {
-  logger.info('Refreshing faculty overview statistics for all faculties')
+  logger.info('Adding jobs to refresh all faculties')
   const facultyList = (await faculties()).filter(f => !['Y', 'H99', 'Y01', 'H92', 'H930'].includes(f.code))
-  let ready = 0
-
   for (const faculty of facultyList) {
-    try {
-      await updateFacultyOverview(faculty.code, 'ALL')
-      await updateFacultyProgressOverview(faculty.code)
-      ready += 1
-    } catch (e) {
-      logger.error({ message: `Failed to update stats for faculty ${faculty?.code} with type ALL`, meta: `${e}` })
-    }
-    logger.info(`${ready}/${facultyList.length} faculties done; faculty ${faculty?.code}`)
+    jobMaker.faculty(faculty.code)
   }
-  logger.info('Faculty stats refreshed!')
 }
 
-const refreshNewOverviews = async () => {
+const refreshFaculty = async code => {
+  logger.info('Started updating faculty')
+  const start = new Date().getTime()
+  try {
+    await updateFacultyOverview(code, 'ALL')
+    await updateFacultyProgressOverview(code)
+    const time = new Date().getTime() - start
+    logger.info(`Updated faculty ${code} (took ${(time / 1000 / 60).toFixed(2)} minutes)`)
+  } catch (e) {
+    logger.error({ message: `Failed to update stats for faculty ${code} with type ALL`, meta: `${e}` })
+  }
+}
+
+const refreshProgrammes = async () => {
   logger.info('Refreshing studyprogramme and studytrack overview statistics for all programmes')
   // Filters out old programmes and special ones like Fitech studies. Filters also out the programmes starting with
   // 2_ or ending with _2. Those programmes are mapped to correct programme (studentProgrammeModuleFixer.js)
@@ -50,31 +54,23 @@ const refreshNewOverviews = async () => {
         (code.includes('MH') && !code.startsWith('2_MH') && !code.endsWith('_2')) ||
         /^(T)[0-9]{6}$/.test(code)
     )
-  const associations = await getAssociations()
-  let ready = 0
+
+  // Ensure that studyright associations are refreshed before launching jobs, otherwise each job will do it
+  await getAssociations()
   for (const code of codes) {
-    let combinedProgramme = ''
     // If combined programme is given, this updates only the bachelor programme
-    try {
-      await updateBasicView(code, combinedProgramme)
-      await updateStudytrackView(code, combinedProgramme, associations)
-      ready += 1
-    } catch (e) {
-      logger.error({ message: `Failed to update overview stats for programme ${code}`, meta: `${e}` })
-    }
-    combinedProgramme = combinedStudyprogrammes[code] || ''
-    // If bachelor programme is combined with master programme, then update the combination.
-    if (combinedProgramme) {
-      try {
-        await updateBasicView(code, combinedProgramme)
-        await updateStudytrackView(code, combinedProgramme, associations)
-      } catch (e) {
-        logger.error({ message: `Failed to update overview stats for combined programme ${code}`, meta: `${e}` })
-      }
-    }
-    logger.info(`${ready}/${codes.length} programmes done: code ${code}`)
+    jobMaker.programme(code)
   }
-  logger.info('Studyprogramme and studytrack overview stats refreshed!')
+}
+
+const refreshProgramme = async code => {
+  const associations = await getAssociations()
+  await updateBasicView(code, '')
+  await updateStudytrackView(code, '', associations)
+
+  let combinedProgramme = combinedStudyprogrammes[code] || ''
+  await updateBasicView(code, combinedProgramme)
+  await updateStudytrackView(code, combinedProgramme, associations)
 }
 
 const refreshTeacherLeaderboard = async () => {
@@ -83,42 +79,6 @@ const refreshTeacherLeaderboard = async () => {
   await findAndSaveTeachers(currentSemestersYearCode - 1, currentSemestersYearCode)
 }
 
-const refreshStatusToRedis = async () => {
-  const unixMillis = moment().valueOf()
-  const date = new Date(Number(unixMillis))
-
-  date.setHours(23, 59, 59, 999)
-  const showByYearOff = 'false'
-  const showByYear = 'true'
-  logger.info('Refreshing CDS Status')
-  await refreshStatus(date.getTime(), showByYearOff)
-  await refreshStatus(date.getTime(), showByYear)
-  logger.info('Refreshing CDS Status doned')
-
-  logger.info('Refreshing CDS Graduated')
-  await refreshStatusGraduated(date.getTime(), showByYearOff)
-  await refreshStatusGraduated(date.getTime(), showByYear)
-  logger.info('Refreshing CDS graduated doned')
-}
-
-const refreshUberToRedis = async () => {
-  const years = (await getStartYears()).map(({ studystartdate }) => studystartdate)
-  for (const year of years) {
-    const formattedYear = new Date(year).getFullYear()
-    logger.info(`Refreshing CDS uber year ${formattedYear}`)
-    const defaultQuery = { include_old_attainments: 'false', start_date: year }
-    const oldAttainmentsQuery = { include_old_attainments: 'true', start_date: year }
-    try {
-      await refreshUber(defaultQuery)
-      await refreshUber(oldAttainmentsQuery)
-      logger.info(`Refreshing CDS uber year ${formattedYear} doned`)
-    } catch (e) {
-      logger.error({ message: `Error when refreshing CDS uber year ${formattedYear}`, meta: `${e}` })
-    }
-  }
-  logger.info(`Refreshing CDS Uber data doned`)
-}
-
 const refreshStatistics = async () => {
   const statfuncs = [refreshStudyrightAssociations, refreshTeacherLeaderboard]
   logger.info('Refreshing statistics')
@@ -128,23 +88,34 @@ const refreshStatistics = async () => {
   logger.info('Statistics refreshed!')
 }
 
-const refreshTrends = async () => {
-  const trendfuncs = [refreshStatusToRedis, refreshUberToRedis]
-  logger.info('Refreshing trends')
-  for (const func of trendfuncs) {
-    await func()
-  }
-  logger.info('Trends refreshed!')
+const refreshLanguageCenterData = async () => {
+  logger.info('Refreshing language center data...')
+  const freshData = await computeLanguageCenterData()
+  await redisClient.setAsync(LANGUAGE_CENTER_REDIS_KEY, JSON.stringify(freshData))
+  logger.info('Language center data refreshed!')
+}
+
+const dailyJobs = () => {
+  refreshFaculties()
+  refreshProgrammes()
+  jobMaker.languagecenter()
+  jobMaker.statistics()
 }
 
 const startCron = () => {
-  if (isProduction) {
-    logger.info('Cronjob for refreshing stats started: runs at 3am every day.')
-    // refresh 3am every day
-    schedule('0 3 * * *', async () => {
-      for (const func of [refreshStatistics, refreshTrends, refreshNewOverviews, refreshFaculties]) {
-        await func()
-      }
+  if (isProduction && !runningInCI) {
+    logger.info('Cronjob for refreshing stats started: runs daily at 23:00.')
+    schedule('0 23 * * *', async () => {
+      logger.info('Running daily jobs from cron')
+      dailyJobs()
+    })
+    schedule('0 19 * * 1', async () => {
+      logger.info('Updating students whose studyplans have not been updated recently')
+      jobMaker.studyplansUpdate(4)
+    })
+    schedule('0 10 * * 2', async () => {
+      logger.info('Updating students whose studyplans have not been updated recently')
+      jobMaker.studyplansUpdate(5)
     })
   }
 }
@@ -152,7 +123,9 @@ const startCron = () => {
 module.exports = {
   startCron,
   refreshStatistics,
-  refreshTrends,
   refreshFaculties,
-  refreshNewOverviews,
+  refreshFaculty,
+  refreshProgramme,
+  refreshProgrammes,
+  refreshLanguageCenterData,
 }

services/backend/src/middleware/accessLogger.js

@@ -17,14 +17,11 @@ const accessLogger = morgan((tokens, req, res) => {
   }
 
   const message = [
-    user.name,
-    ':',
+    `${user.mockedBy ? '(mocking) ' : ''}${user.name}`,
+    `${(tokens['response-time'](req, res) || '0').split('.')[0]} ms`.padEnd(8, ' '),
+    tokens['status'](req, res),
     tokens['method'](req, res),
     tokens['url'](req, res),
-    tokens['status'](req, res),
-    '-',
-    tokens['response-time'](req, res),
-    'ms',
   ].join(' ')
 
   const usingIamRights = user.iamRights.some(programmeCode => tokens['url'](req, res).includes(programmeCode))

services/backend/src/middleware/auth.js

@@ -4,7 +4,7 @@ const roles = requiredRoles => async (req, res, next) => {
       user: { roles },
     } = req
 
-    if (requiredRoles.every(r => roles.indexOf(r) >= 0) || roles.includes('admin')) {
+    if (requiredRoles.some(r => roles.includes(r)) || roles.includes('admin')) {
       return next()
     }
   }

services/backend/src/middleware/currentUserMiddleware.js

@@ -1,9 +1,8 @@
 const Sentry = require('@sentry/node')
 const { ApplicationError } = require('../util/customErrors')
-const { getUser, getMockedUser } = require('../services/userService')
+const { getUser, getMockedUser, getOrganizationAccess } = require('../services/userService')
 const { requiredGroup } = require('../conf-backend')
 const _ = require('lodash')
-const { getOrganizationAccess } = require('../util/organizationAccess')
 const logger = require('../util/logger')
 
 const parseIamGroups = iamGroups => iamGroups?.split(';').filter(Boolean) ?? []
@@ -23,8 +22,17 @@ const currentUserMiddleware = async (req, _res, next) => {
     uid: username,
   } = req.headers
 
-  if (!sessionId || !username) {
-    throw new ApplicationError('Not enough data in request headers', 403, { logoutUrl })
+  const missingHeaders = []
+
+  if (!sessionId) missingHeaders.push('shib-session-id')
+  if (!username) missingHeaders.push('uid')
+
+  if (missingHeaders.length > 0) {
+    throw new ApplicationError(
+      `Not enough data in request headers, the following headers were missing: ${missingHeaders.join(', ')}`,
+      403,
+      { logoutUrl }
+    )
   }
 
   const iamGroups = parseIamGroups(hygroupcn)

services/backend/src/models/courseProvider.js

@@ -1,4 +1,4 @@
-const { Model, STRING, DATE } = require('sequelize')
+const { Model, STRING, DATE, JSONB } = require('sequelize')
 const { dbConnections } = require('../database/connection')
 
 class CourseProvider extends Model {}
@@ -18,6 +18,9 @@ CourseProvider.init(
       onUpdate: 'cascade',
       onDelete: 'cascade',
     },
+    shares: {
+      type: JSONB,
+    },
     organizationcode: {
       type: STRING,
       references: {

services/backend/src/models/credit.js

@@ -81,6 +81,14 @@ Credit.init(
     is_open: {
       type: BOOLEAN,
     },
+    studyright_id: {
+      type: STRING,
+      allowNull: true,
+      references: {
+        model: 'studyright',
+        key: 'studyrightid',
+      },
+    },
   },
   {
     underscored: false,

services/backend/src/models/enrollment.js

@@ -57,6 +57,14 @@ Enrollment.init(
     is_open: {
       type: BOOLEAN,
     },
+    studyright_id: {
+      type: STRING,
+      allowNull: true,
+      references: {
+        model: 'studyright',
+        key: 'studyrightid',
+      },
+    },
   },
   {
     underscored: false,

services/backend/src/models/index.js

@@ -50,6 +50,8 @@ Credit.passed = ({ credittypecode }) =>
   credittypecode === CREDIT_TYPE_CODES.PASSED || credittypecode === CREDIT_TYPE_CODES.APPROVED
 Credit.failed = credit => credit.credittypecode === CREDIT_TYPE_CODES.FAILED
 Credit.improved = credit => credit.credittypecode === CREDIT_TYPE_CODES.IMPROVED
+Credit.belongsTo(Studyright, { foreignKey: 'studyright_id', targetKey: 'studyrightid', constraints: false })
+Studyright.hasMany(Credit, { foreignKey: 'studyright_id', constraints: false })
 
 Credit.belongsTo(Student, { foreignKey: 'student_studentnumber', targetKey: 'studentnumber' })
 Student.hasMany(Credit, { foreignKey: 'student_studentnumber', sourceKey: 'studentnumber' })
@@ -97,6 +99,9 @@ Transfer.belongsTo(ElementDetail, { as: 'source', foreignKey: 'sourcecode' })
 Transfer.belongsTo(ElementDetail, { as: 'target', foreignKey: 'targetcode' })
 
 Enrollment.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
+Enrollment.belongsTo(Studyright, { foreignKey: 'studyright_id', targetKey: 'studyrightid', constraints: false })
+Studyright.hasMany(Enrollment, { foreignKey: 'studyright_id', constraints: false })
+
 Student.hasMany(Enrollment, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
 Enrollment.belongsTo(Course, { foreignKey: 'course_id' })
 Course.hasMany(Enrollment, { foreignKey: 'course_id' })

services/backend/src/models/models_kone.js

@@ -2,11 +2,6 @@ const { STRING } = require('sequelize')
 const Sequelize = require('sequelize')
 const { sequelizeKone } = require('../database/connection')
 
-const ThesisTypeEnums = {
-  MASTER: 'MASTER',
-  BACHELOR: 'BACHELOR',
-}
-
 const ExcludedCourse = sequelizeKone.define(
   'excluded_courses',
   {
@@ -197,10 +192,8 @@ Tag.hasMany(TagStudent, { foreignKey: 'tag_id', sourceKey: 'tag_id' })
 module.exports = {
   TagStudent,
   Tag,
-  ThesisTypeEnums,
   CustomPopulationSearch,
   OpenUniPopulationSearch,
-  sequelizeKone,
   ExcludedCourse,
   StudyGuidanceGroupTag,
   ProgressCriteria,

services/backend/src/models/studyright.js

@@ -1,4 +1,4 @@
-const { Model, STRING, DATE, INTEGER, BOOLEAN } = require('sequelize')
+const { Model, STRING, DATE, INTEGER, BOOLEAN, JSONB } = require('sequelize')
 const { dbConnections } = require('../database/connection')
 
 class Studyright extends Model {}
@@ -9,6 +9,9 @@ Studyright.init(
       primaryKey: true,
       type: STRING,
     },
+    actual_studyrightid: {
+      type: STRING,
+    },
     startdate: {
       type: DATE,
     },
@@ -63,6 +66,12 @@ Studyright.init(
     admissionType: {
       type: STRING,
     },
+    isBaMa: {
+      type: BOOLEAN,
+    },
+    semesterEnrollments: {
+      type: JSONB,
+    },
   },
   {
     underscored: true,

services/backend/src/rapodiff/.gitignore

@@ -0,0 +1,2 @@
+*.csv
+*.csv#

services/backend/src/rapodiff/README.md

@@ -0,0 +1,21 @@
+# Rapodiff
+
+Use this to check differences between oodikone and rapo.
+
+Before running:
+
+1. Depends on nodeproxy running in toska.cs.helsinki.fi
+2. Set these correctly in .env, notice to wrap in quotes ", otherwise strange characters may be misformatted:
+   RAPO_NODEPROXY="acual_token"
+   IMPORTER_DB_API_TOKEN="acual_token"
+
+Running:
+
+1. Have real oodikone running. Then run rapodiff: `npm run rapodiff -- args`
+2. place args with a mode. omit args to print possible modes.
+
+known differences in APIs
+
+- rapo does not show students that have graduated (it however takes some days for rapo to notice a graduation)
+- rapo does not handle the starting year of transfers properly (shows in old staring population)
+- rapo show some students incorrectly passive, see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3521

services/backend/src/rapodiff/conf.js

@@ -0,0 +1,28 @@
+// In .env, have the token wrapped in quotes! (") otherwise some signs may be mistakenly changed
+const rapoToken = process.env.RAPO_NODEPROXY
+const importerDbApiToken = process.env.IMPORTER_DB_API_TOKEN
+
+const nodeproxyUrl = `https://toska.cs.helsinki.fi`
+const importerDbApiUrl = `https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer`
+
+const bscProgrammesOfFaculties = {
+  H10: ['KH10_001'],
+  H20: ['KH20_001'],
+  H30: ['KH30_001', 'KH30_002'],
+  H40: ['KH40_001', 'KH40_002', 'KH40_003', 'KH40_004', 'KH40_005', 'KH40_006'],
+  H50: ['KH50_001', 'KH50_002', 'KH50_003', 'KH50_004', 'KH50_005', 'KH50_006', 'KH50_007', 'KH50_008'],
+  H55: ['KH55_001'],
+  H60: ['KH60_001'],
+  H70: ['KH70_001', 'KH70_002', 'KH70_003', 'KH70_004'],
+  H74: ['KH74_001'],
+  H80: ['KH80_001', 'KH80_002', 'KH80_003', 'KH80_004'],
+  H90: ['KH90_001'],
+}
+
+module.exports = {
+  rapoToken,
+  importerDbApiToken,
+  nodeproxyUrl,
+  importerDbApiUrl,
+  bscProgrammesOfFaculties,
+}

services/backend/src/rapodiff/creditsDiff.js

@@ -0,0 +1,121 @@
+/* eslint-disable no-console */
+const { getCreditStatsForRapodiff } = require('../services/studyprogramme/studyprogrammeCredits')
+const { parseCsv } = require('./helpers')
+const _ = require('lodash')
+
+const diffs = []
+let noDiffCounter = 0
+
+const diff = (rapoData, okData, code) => {
+  for (const year of Object.keys(okData)) {
+    if (year !== '2022' && year !== '2023') continue
+    const rapo = rapoData[year]
+    const ok = okData[year]
+    if (!rapo) {
+      if (!ok) console.log(`Year ${year} not in rapo for code ${code}, skipping`)
+      continue
+    }
+    for (const field of ['basic', 'incoming-exchange', 'open-uni', 'agreement', 'separate', 'other']) {
+      const okValue = Math.round(ok[field] || 0)
+      const rapoValue = Math.round(rapo[field] || 0)
+      const diff = Math.abs(okValue - rapoValue)
+      const diffStr = `${field.padEnd(10, ' ')} ${year} ${code.padEnd(
+        10,
+        ' '
+      )} Difference: ${diff} Rapo: ${rapoValue} Oodikone: ${okValue}`
+      diffs.push({ field, diff, okValue, rapoValue, code, year, diffStr })
+    }
+  }
+}
+let counter = 0
+// Acual logic in this function
+const process = async data => {
+  // Define here the fields to diff against each other. Must be an array, multiple fields will be summed
+  const rapoProgrammeData = formatData(data.slice(1))
+  const allProgrammeCodes = [...new Set(Object.keys(rapoProgrammeData))]
+  for (const programmeCode of allProgrammeCodes) {
+    const okProgrammeData = await getCreditStatsForRapodiff(programmeCode, '', 'CALENDAR_YEAR', 'SPECIAL_INCLUDED')
+    counter++
+    if (counter % 5 === 0) {
+      console.log(`Done ${Math.round((counter / allProgrammeCodes.length) * 100)} %`)
+    }
+    diff(rapoProgrammeData[programmeCode], okProgrammeData, programmeCode)
+  }
+  const orderedDiffs = _.orderBy(diffs, 'diff', 'asc')
+  orderedDiffs.forEach(d => console.log(d.diffStr))
+  console.log(`${diffs.length} diffs found with current settings. Numbers with no diff: ${noDiffCounter}`)
+  const totalDiff = orderedDiffs.reduce((sum, cur) => cur.diff + sum, 0)
+  console.log('Total difference: ', totalDiff)
+}
+
+const programmeCreditsDiff = async fileName => {
+  console.log('Running studyprogramme credits diff')
+  await parseCsv(fileName, process)
+  console.log('Diff completed.')
+}
+
+const processIds = async (rawData, code) => {
+  const data = rawData.slice(1)
+  console.log('amount of rapo credits: ', data.length)
+  const rapoData = data
+    .map(row => ({
+      id: row[4],
+      credits: row[5],
+      type: row[6],
+      incl: row[7],
+    }))
+    .filter(row => row.type === '2' && row.incl === '0')
+  const rapoIds = rapoData.map(row => row.id)
+  const stats = await getCreditStatsForRapodiff(code)
+  if (!stats.ids) {
+    console.log(stats)
+    console.log('Edit getCreditStats code so that it saves a list of the relevant ids you want to compare.')
+    return
+  }
+  const okIds = stats.ids
+  const notInOk = rapoIds.filter(id => id && !okIds[id.slice(11)]).map(str => str.slice(11))
+  const rapoIdMap = rapoIds.reduce((obj, cur) => {
+    obj[cur] = true
+    return obj
+  }, {})
+  const notInRapo = Object.keys(okIds).filter(id => !rapoIdMap[`ATTAINMENT-${id}`])
+  console.log('\n In rapo, but not in OK: ', notInOk.length)
+  console.dir(notInOk)
+  console.log('\n In OK, but not in Rapo: ', notInRapo.length)
+  console.dir(notInRapo)
+  delete stats.ids
+  console.log(stats['2022'])
+  const rapoCredits = rapoData.reduce((sum, cur) => parseInt(sum, 10) + parseInt(cur.credits, 10), 0)
+  console.log('rapoCredits: ', rapoCredits)
+}
+
+const testNewCalc = async code => {
+  await parseCsv('credits.csv', async data => processIds(data, code))
+}
+
+// Change weird rapo programmecodes to oodikone format, example: 300-M003 => MH30_003
+const transformProgrammeCode = oldCode => `${oldCode[4]}H${oldCode.slice(0, 2)}_${oldCode.slice(5, 8)}`
+
+const formatData = data =>
+  data.reduce((obj, cur) => {
+    const parseNum = str => (str === '' ? 0 : parseInt(str, 10))
+    const [year, facultyCode, programmeInfo, basic, exchange, otherUni, openUni, special, total, abroad, other] = cur
+    const programmeCode = transformProgrammeCode(programmeInfo.split(' ')[0])
+    if (!obj[programmeCode]) obj[programmeCode] = {}
+    obj[programmeCode][year] = {
+      year,
+      facultyCode,
+      programmeInfo,
+      basic: parseNum(basic),
+      'incoming-exchange': parseNum(exchange),
+      agreement: parseNum(otherUni),
+      'open-uni': parseNum(openUni),
+      separate: parseNum(special),
+      total: parseNum(total),
+      abroad: parseNum(abroad),
+      other: parseNum(other),
+    }
+    return obj
+  }, {})
+
+module.exports = { programmeCreditsDiff, testNewCalc }

services/backend/src/rapodiff/esimerkki.md

@@ -0,0 +1,50 @@
+### Esimerkki (fake data)
+
+Studycredits-route (suoritetut kreditit - kalenterivuosi?)
+
+**Route**: https://toska.cs.helsinki.fi/test/rapo/report/studyCredits?faculty=H50&education=500-K005&token=acual_token
+
+**Response**:
+
+```
+[
+  {
+    "vuosi": 2022,
+    "koulutusohjelmaKoodi": "500-K002",
+    "koulutusohjelma": "Fysikaalisten tieteiden kandiohjelma",
+    "tiedekuntaKoodi": "H50",
+    "tiedekunta": "H50 Matemaattis-luonnontieteellinen tiedekunta / Faculty of Science",
+    "perustutkinto": 12345,
+    "jatko": null,
+    "suoritetutYhteensa": 12345,
+    "eriklaakari": null,
+    "tutkAvoin": 12345,
+    "korkValYhteis": 12345,
+    "muutHyvaksiluetut": 12345,
+    "ulkomailta": null,
+    "taydennyskoul": null,
+    "avoinEiTutk": 12345,
+    "erillisoik": 12345,
+    "saapuvatVaihtoop": 12345
+  },
+  {
+    "vuosi": 2023,
+    "koulutusohjelmaKoodi": "500-M014",
+    "koulutusohjelma": "Nordic Master Programme in Environmental Changes at Higher Latitudes",
+    "tiedekuntaKoodi": "H50",
+    "tiedekunta": "H50 Matemaattis-luonnontieteellinen tiedekunta / Faculty of Science",
+    "perustutkinto": 12345,
+    "jatko": null,
+    "suoritetutYhteensa": 12345,
+    "eriklaakari": null,
+    "tutkAvoin": null,
+    "korkValYhteis": null,
+    "muutHyvaksiluetut": null,
+    "ulkomailta": 12345,
+    "taydennyskoul": null,
+    "avoinEiTutk": null,
+    "erillisoik": null,
+    "saapuvatVaihtoop": null
+  }
+]
+```

services/backend/src/rapodiff/helpers.js

@@ -0,0 +1,20 @@
+/* eslint-disable no-console */
+const Papa = require('papaparse')
+const fs = require('fs/promises')
+
+const parseCsv = async (fileName, fun) => {
+  try {
+    const file = await fs.readFile(`${__dirname}/${fileName}`, 'utf8')
+    Papa.parsePromise = file => {
+      return new Promise((complete, error) => {
+        Papa.parse(file, { complete, error })
+      })
+    }
+    const data = await Papa.parsePromise(file)
+    await fun(data?.data)
+  } catch (e) {
+    console.log(e)
+  }
+}
+
+module.exports = { parseCsv }

services/backend/src/rapodiff/rapodiff.js

@@ -0,0 +1,35 @@
+/* eslint no-console: 0 */
+
+/*
+  npm run rapodiff
+*/
+
+const { programmeCreditsDiff, testNewCalc } = require('./creditsDiff')
+
+// ** To add new mode, just type key and function here **
+const modes = {
+  credits: () => programmeCreditsDiff('data.csv'),
+  // Check credits with current local oodikone data against provided data.csv in rapodiff folder. data.csv can be downloaded from rapo by selecting all faculties, then from the bottom of the page clicking Export > data > csv.
+  check: code => testNewCalc(code),
+  // Provide programme code as argument. Parses credits from credits.csv in rapodiff folder. This must be obtained from Rapo people. This tells which credit ids are found in oodikone but not rapo, and vice versa.
+  // notice: check mode requires writing ids in to the stats object in credits calculating function.
+  // Add something like this there:
+  // if (category === 'open-uni' && credits > 0 && new Date(attainmentDate).getFullYear() === 2022) stats.ids[id] = true
+}
+
+const main = async () => {
+  console.log('**RAPODIFF**\n')
+  const mode = process.argv[2]
+  const modeFunction = modes[mode]
+  const arg = process.argv[3]
+  if (!modeFunction) {
+    console.log(
+      'Unknown or missing mode. Supply the mode as an argument. Supported modes:\n\n',
+      Object.keys(modes).join('\n\n')
+    )
+    return
+  }
+  await modeFunction(arg)
+}
+
+main().then(() => process.exit(0))

services/backend/src/rapodiff/studyrightDiff.js

@@ -1,25 +1,15 @@
-/* eslint no-console: 0 */
-
-/*
-  depends on nodeproxy running in importer.cs.helsinki.fi, and env RAPO_NODEPROXY (see importer for a proper value)
-  needs also acces to importer db api, uses env IMPORTER_API
-
-  known differences in APIs
-    - rapo does not show students that have graduated (it however takes some days for rapo to notice a graduation)
-    - rapo does not handle the starting year of transfers properly (shows in old staring population)
-    - rapo show some students incorrectly passive, see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3521
-*/
+/* eslint-disable no-console */
 
 const axios = require('axios').default
 
 const _ = require('lodash')
 
-const { optimizedStatisticsOf } = require('./services/populations')
-const { Student, Transfer, StudyrightElement, Studyright } = require('./models')
-
-const token = process.env.RAPO_NODEPROXY
+const { optimizedStatisticsOf } = require('../services/populations')
+const { Student, Transfer, StudyrightElement, Studyright } = require('../models')
+const { rapoToken, tokenImporterApi, importerDbApiUrl, nodeproxyUrl } = require('./conf.js')
 
 const getFromRapo = async (urlStart, urlEnd) => {
+  console.log({ urlStart, urlEnd })
   const { data: rapoStuffLasna } = await axios.get(urlStart)
   const { data: rapoStuffLasnaMinus } = await axios.get(urlEnd)
 
@@ -83,10 +73,12 @@ const getFromOodikone = async (programme, year) => {
 
 const isAcuallyActive = async student => {
   const currentSemester = 144
-  const tokenImporterApi = process.env.IMPORTER_API
 
-  const url = `https://importer.cs.helsinki.fi/api/importer/students/${student}/rapo_semester_enrollments?token=${tokenImporterApi}`
-  const { data } = await axios.get(url)
+  const url = `${importerDbApiUrl}/students/${student}/rapo_semester_enrollments?token=${tokenImporterApi}`
+  const result = await axios.get(url)
+  console.log('result:')
+  console.log(result)
+  const { data } = result
   const terms = data.reduce((all, s) => all.concat(s.terms), []).filter(t => t.semester_code === currentSemester)
   const active = terms.filter(t => t.type !== 'NEGLECTED')
   const negl = terms.filter(t => t.type === 'NEGLECTED')
@@ -144,11 +136,11 @@ const programme_diff_year = async (programme, facultyCode, year, brief = false)
 
   const rapoUrlStart = presence =>
     encodeURI(
-      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+      `${nodeproxyUrl}/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${rapoToken}`
     )
   const rapoUrlEnd = presence =>
     encodeURI(
-      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+      `${nodeproxyUrl}/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${rapoToken}`
     )
 
   const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
@@ -247,7 +239,7 @@ const programme_diff = async (programme, faculty, brief) => {
   await programme_diff_year(programme, faculty, 2021, brief)
 }
 
-const faculty_diff = async (faculty, programmes, brief) => {
+const facultyDiff = async (faculty, programmes, brief) => {
   console.log('\n')
   console.log(faculty)
   console.log('==')
@@ -265,11 +257,11 @@ const msc_programme_diff_year = async (programme, facultyCode, year, brief = fal
 
   const rapoUrlStart = presence =>
     encodeURI(
-      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+      `${nodeproxyUrl}/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${rapoToken}`
     )
   const rapoUrlEnd = presence =>
     encodeURI(
-      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+      `${nodeproxyUrl}/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${rapoToken}`
     )
 
   const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
@@ -337,27 +329,4 @@ const msc_programme_diff_year = async (programme, facultyCode, year, brief = fal
   }
 }
 
-const bsc_priogrammes_of_faculties = {
-  H10: ['KH10_001'],
-  H20: ['KH20_001'],
-  H30: ['KH30_001', 'KH30_002'],
-  H40: ['KH40_001', 'KH40_002', 'KH40_003', 'KH40_004', 'KH40_005', 'KH40_006'],
-  H50: ['KH50_001', 'KH50_002', 'KH50_003', 'KH50_004', 'KH50_005', 'KH50_006', 'KH50_007', 'KH50_008'],
-  H55: ['KH55_001'],
-  H60: ['KH60_001'],
-  H70: ['KH70_001', 'KH70_002', 'KH70_003', 'KH70_004'],
-  H74: ['KH74_001'],
-  H80: ['KH80_001', 'KH80_002', 'KH80_003', 'KH80_004'],
-  H90: ['KH90_001'],
-}
-
-const main = async () => {
-  for (let faculty of Object.keys(bsc_priogrammes_of_faculties)) {
-    await faculty_diff(faculty, bsc_priogrammes_of_faculties[faculty], true)
-  }
-
-  //await programme_diff_year('KH55_001', 'H55', 2021, true)
-  //await programme_diff('KH60_001', 'H60', true)
-}
-
-main().then(() => process.exit(0))
+module.exports = { facultyDiff }

services/backend/src/routes.js

@@ -17,6 +17,7 @@ const elementdetails = require('./routes/elementdetails')
 const teachers = require('./routes/teachers')
 const providers = require('./routes/providers')
 const faculties = require('./routes/faculties')
+const university = require('./routes/university')
 const courseExclusions = require('./routes/courseExclusions')
 const semesters = require('./routes/semesters')
 const feedback = require('./routes/feedback')
@@ -24,7 +25,6 @@ const tags = require('./routes/tags')
 const updater = require('./routes/updater')
 const customPopulationSearch = require('./routes/customPopulationSearch')
 const languageCenterData = require('./routes/languageCenterData')
-const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
 const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
 const studyProgramme = require('./routes/studyProgramme')
@@ -61,13 +61,13 @@ module.exports = (app, url) => {
   app.use(`${url}/changelog`, changelog)
   app.use(`${url}/completedcoursessearch`, completedCoursesSearch)
   app.use(`${url}/languagecenterdata`, languageCenterData)
-  app.use(`${url}/faculties`, auth.roles(['facultyStatistics']), faculties)
+  app.use(`${url}/faculties`, auth.roles(['facultyStatistics', 'katselmusViewer']), faculties)
+  app.use(`${url}/university`, university)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
   app.use(`${url}/users`, auth.roles(['admin']), users)
   app.use(`${url}/feedback`, feedback)
   app.use(`${url}/custom-population-search`, customPopulationSearch)
-  app.use(`${url}/cool-data-science`, trends)
   app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
   app.get('*', async (_, res) => {
     const results = { error: 'unknown endpoint' }

services/backend/src/routes/changelog.js

@@ -1,9 +1,21 @@
 const router = require('express').Router()
 const axios = require('axios')
+const { isDev } = require('../conf-backend')
+const changelog = {}
 
 router.get('/', async (req, res) => {
+  if (changelog?.data) return res.status(200).send(changelog.data)
+  if (isDev)
+    return res.status(200).json([
+      {
+        description: "### Fake release\nLet's not spam github api in development!",
+        title: 'Fake title for fake release',
+        time: new Date(),
+      },
+    ])
   const result = await axios.get('https://api.github.com/repos/UniversityOfHelsinkiCS/oodikone/releases')
   const formattedResult = result.data.map(r => ({ description: r.body, time: r.created_at, title: r.name }))
+  changelog.data = formattedResult
   res.status(200).send(formattedResult)
 })
 

services/backend/src/routes/courses.js

@@ -1,18 +1,34 @@
 const router = require('express').Router()
 const crypto = require('crypto')
-const Course = require('../services/courses')
+const courseService = require('../services/courses')
 const { faculties } = require('../services/organisations')
 const { validateParamLength } = require('../util')
 
 router.get('/v2/coursesmulti', async (req, res) => {
   let results = { courses: [] }
-  const { name, code } = req.query
+  const { name, code, combineSubstitutions } = req.query
 
   if (!(validateParamLength(name, 5) || validateParamLength(code, 2))) {
     return res.status(400).json({ error: 'name or code invalid' })
   }
 
-  results = await Course.byNameAndOrCodeLike(name, code)
+  results = await courseService.byNameAndOrCodeLike(name, code)
+
+  if (combineSubstitutions === 'false') {
+    const courseCodes = results.courses.map(course => course.code)
+
+    const substitutions = [
+      ...new Set(
+        results.courses
+          .flatMap(course => course.substitutions)
+          .filter(value => value !== null && !courseCodes.includes(value))
+      ),
+    ]
+    const substitutionDetails = await courseService.byCodes(substitutions)
+    for (const substitution of substitutionDetails) {
+      results.courses.push(substitution.dataValues)
+    }
+  }
   res.json(results)
 })
 
@@ -30,6 +46,7 @@ router.get('/v3/courseyearlystats', async (req, res) => {
   }
 
   const { codes, separate: sep } = req.query
+  const combineSubstitutions = req.query.combineSubstitutions !== 'false'
 
   const separate = !sep ? false : JSON.parse(sep)
   if (!codes) {
@@ -39,7 +56,7 @@ router.get('/v3/courseyearlystats', async (req, res) => {
     // and users with rights to any specific study programmes
     const anonymize = allowedRoles !== 'admin' && rights.length < 1
     const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
-    const results = await Course.courseYearlyStats(codes, separate, anonymizationSalt)
+    const results = await courseService.courseYearlyStats(codes, separate, anonymizationSalt, combineSubstitutions)
     res.json(results)
   }
 })

services/backend/src/routes/faculties.js

@@ -1,5 +1,4 @@
 const router = require('express').Router()
-const { faculties } = require('../services/organisations')
 const { combineFacultyBasics } = require('../services/faculty/facultyBasics')
 const { combineFacultyCredits } = require('../services/faculty/facultyCredits')
 const { combineFacultyThesisWriters } = require('../services/faculty/facultyThesisWriters')
@@ -7,7 +6,6 @@ const { countGraduationTimes } = require('../services/faculty/facultyGraduationT
 const { updateFacultyOverview, updateFacultyProgressOverview } = require('../services/faculty/facultyUpdates')
 const { combineFacultyStudentProgress } = require('../services/faculty/facultyStudentProgress')
 const { combineFacultyStudents } = require('../services/faculty/facultyStudents')
-
 const {
   getProgrammes,
   getBasicStats,
@@ -24,14 +22,13 @@ const {
   setFacultyProgressStats,
 } = require('../services/faculty/facultyService')
 const logger = require('../util/logger')
+const { getFacultyList } = require('../services/faculty/facultyHelpers')
 
 // Faculty uses a lot of tools designed for Study programme.
 // Some of them have been copied here and slightly edited for faculty purpose.
 
 router.get('/', async (req, res) => {
-  const ignore = ['Y', 'H99', 'Y01', 'H92', 'H930']
-  const facultyList = (await faculties()).filter(f => !ignore.includes(f.code))
-  facultyList.sort((a, b) => (a.name.fi > b.name.fi ? 1 : -1))
+  const facultyList = await getFacultyList()
   res.json(facultyList)
 })
 

services/backend/src/routes/population.js

@@ -2,15 +2,14 @@ const crypto = require('crypto')
 const Sentry = require('@sentry/node')
 const router = require('express').Router()
 const _ = require('lodash')
-const Population = require('../services/populations')
-const Student = require('../services/students')
+const populationService = require('../services/populations')
+const studentService = require('../services/students')
 const StudyrightService = require('../services/studyrights')
-const TagService = require('../services/tags')
 const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
 const { mapToProviders } = require('../util/utils')
 const { encrypt, decrypt } = require('../services/encrypt')
-const { mapCodesToIds } = require('../services/studyprogrammeHelpers')
+const { mapCodesToIds } = require('../services/studyprogramme/studyprogrammeHelpers')
 
 const { ApplicationError } = require('../util/customErrors')
 
@@ -59,7 +58,7 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
         }
         const newMonths = (upperYearBound - Number(year)) * 12
         const query = { ...req.body, year, months: newMonths }
-        const coursestatistics = Population.bottlenecksOf(query, null, encrypted)
+        const coursestatistics = populationService.bottlenecksOf(query, null, encrypted)
         return coursestatistics
       })
     )
@@ -74,7 +73,7 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
   } else {
     if (encrypted) req.body.selectedStudents = req.body.selectedStudents.map(decrypt)
 
-    const result = await Population.bottlenecksOf(req.body, null, encrypted)
+    const result = await populationService.bottlenecksOf(req.body, null, encrypted)
 
     if (result.error) {
       Sentry.captureException(new Error(result.error))
@@ -95,10 +94,10 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
   const {
     user: { isAdmin, studentsUserCanAccess },
   } = req
-  const studentnumbers = await Student.findByTag(tag)
+  const studentnumbers = await studentService.findByTag(tag)
   const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
 
-  const result = await Population.bottlenecksOf(
+  const result = await populationService.bottlenecksOf(
     {
       year: 1900,
       studyRights: [],
@@ -129,7 +128,7 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
   }
 
   const studentnumberlist = isAdmin ? studentnumbersInReq : _.intersection(studentnumbersInReq, studentsUserCanAccess)
-  const result = await Population.bottlenecksOf(
+  const result = await populationService.bottlenecksOf(
     {
       year: query?.year ?? 1900,
       studyRights: query?.studyRights ?? [],
@@ -186,7 +185,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
     const multipopulationstudentPromises = Promise.all(
       req.query.years.map(year => {
         const newMonths = (upperYearBound - Number(year)) * 12
-        const populationStudents = Population.optimizedStatisticsOf(
+        const populationStudents = populationService.optimizedStatisticsOf(
           {
             ...req.query,
             studyRights: { programme: studyRights.programme },
@@ -210,7 +209,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
 
     res.json(filterPersonalTags(result, user.id))
   } else {
-    const result = await Population.optimizedStatisticsOf({
+    const result = await populationService.optimizedStatisticsOf({
       ...req.query,
       studyRights: { programme: studyRights.programme },
     })
@@ -247,46 +246,6 @@ router.get('/v3/populationstatistics', async (req, res) => {
   }
 })
 
-router.get('/v3/populationstatisticsbytag', async (req, res) => {
-  const {
-    user: { id, isAdmin, studentsUserCanAccess },
-    query: { tag, studyRights: studyRightsJSON, months, year },
-  } = req
-
-  if (!tag) return res.status(400).json({ error: 'The query should have a tag defined' })
-  const foundTag = await TagService.findTagById(tag)
-  if (!foundTag) return res.status(404).json({ error: 'Tag not found' })
-
-  const semesters = ['FALL', 'SPRING']
-
-  const studentnumbers = await Student.findByTag(tag)
-
-  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
-
-  const studyRights = JSON.parse(studyRightsJSON)
-  const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
-  const yearDifference = Number(year) - Number(newStartYear)
-  const newMonths = Number(months) + 12 * yearDifference
-  const result = await Population.optimizedStatisticsOf(
-    {
-      year: newStartYear,
-      studyRights,
-      semesters,
-      months: newMonths,
-      tag: foundTag,
-    },
-    studentnumberlist
-  )
-
-  if (result.error) {
-    Sentry.captureException(new Error(result.error))
-    res.status(400).end()
-    return
-  }
-
-  res.json(filterPersonalTags(result, id))
-})
-
 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
   const { coursecodes, from, to, separate: sep, unifyCourses } = req.query
   const {
@@ -307,14 +266,14 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
   }
 
   const semesters = ['FALL', 'SPRING']
-  const studentnumbers = await Student.findByCourseAndSemesters(
+  const studentnumbers = await studentService.findByCourseAndSemesters(
     JSON.parse(coursecodes),
     from,
     to,
     separate,
     unifyCourses
   )
-  const result = await Population.optimizedStatisticsOf(
+  const result = await populationService.optimizedStatisticsOf(
     {
       // Useless, because studentnumbers are already filtered above by from & to.
       // We should probably refactor this to avoid more confusement.
@@ -374,7 +333,7 @@ router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
     tags?.studyProgramme && tags?.studyProgramme.includes('+')
       ? tags?.studyProgramme.split('+')[0]
       : tags?.studyProgramme
-  const result = await Population.optimizedStatisticsOf(
+  const result = await populationService.optimizedStatisticsOf(
     {
       year: tags?.year || 1900,
       studyRights: studyProgrammeCode ? [studyProgrammeCode] : [],
@@ -427,7 +386,7 @@ router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req
   )
   const maxYearsToCreatePopulationFromUni = await CourseService.maxYearsToCreatePopulationFrom(
     JSON.parse(courseCodes),
-    'reqularStats'
+    'regularStats'
   )
   const maxYearsToCreatePopulationFromBoth = await CourseService.maxYearsToCreatePopulationFrom(
     JSON.parse(courseCodes),

services/backend/src/routes/router.js

@@ -1,21 +0,0 @@
-const express = require('express')
-const logger = require('../util/logger')
-
-const wrapWithErrorHandler = fn => (req, res) =>
-  Promise.resolve(fn(req, res)).catch(error => {
-    logger.error(error)
-    res.status(500).send('Internal server error')
-  })
-
-const routerWithWrapper = () => {
-  const router = express.Router()
-  return {
-    router,
-    wrapper: {
-      get: (url, callback) => router.get(url, wrapWithErrorHandler(callback)),
-      post: (url, callback) => router.post(url, wrapWithErrorHandler(callback)),
-    },
-  }
-}
-
-module.exports = { routerWithWrapper }

services/backend/src/routes/studyProgramme.js

@@ -1,20 +1,21 @@
 const router = require('express').Router()
-const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics')
-const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
-const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
-const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
-const { getStudyprogrammeCoursesForStudytrack } = require('../services/studyprogrammeCourses')
+const { getBasicStatsForStudytrack } = require('../services/studyprogramme/studyprogrammeBasics')
+const { getProgrammeCreditStats } = require('../services/studyprogramme/studyprogrammeCredits')
+const { getGraduationStatsForStudytrack } = require('../services/studyprogramme/studyprogrammeGraduations')
+const {
+  getStudyprogrammeCoursesForStudytrack,
+  getStudyprogrammeStatsForColorizedCoursesTable,
+} = require('../services/studyprogramme/studyprogrammeCourses')
+const { getStudytrackStatsForStudyprogramme } = require('../services/studyprogramme/studytrackStats')
 const {
   getBasicStats,
   setBasicStats,
-  getCreditStats,
-  setCreditStats,
   getGraduationStats,
   setGraduationStats,
   getStudytrackStats,
   setStudytrackStats,
 } = require('../services/analyticsService')
-const { updateBasicView, updateStudytrackView } = require('../services/studyprogrammeUpdates')
+const { updateBasicView, updateStudytrackView } = require('../services/studyprogramme/studyprogrammeUpdates')
 const { getProgrammeName } = require('../services/studyprogramme')
 const logger = require('../util/logger')
 const { getAssociations } = require('../services/studyrights')
@@ -53,26 +54,14 @@ router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
   return res.json(updated)
 })
 
-router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
-  const code = req.params.id
-  const yearType = req.query?.year_type
-  const specialGroups = req.query?.special_groups
-  const combinedProgramme = req.query?.combined_programme
-
-  if (!code) return res.status(422).end()
-
-  const data = await getCreditStats(code, combinedProgramme, yearType, specialGroups)
-  if (data) return res.json(data)
-  const updatedStats = await getCreditStatsForStudytrack({
-    studyprogramme: req.params.id,
-    combinedProgramme,
-    settings: {
-      isAcademicYear: yearType === 'ACADEMIC_YEAR',
-      includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
-    },
-  })
-  if (updatedStats) await setCreditStats(updatedStats, yearType, specialGroups)
-  return res.json(updatedStats)
+router.get('/v2/studyprogrammes/creditstats', async (req, res) => {
+  const { codes: codesListString, isAcademicYear, includeSpecials } = req.query
+  const codes = JSON.parse(codesListString)
+  const stats = {}
+  for (const code of codes) {
+    stats[code] = await getProgrammeCreditStats(code, isAcademicYear !== 'false', includeSpecials)
+  }
+  return res.json({ stats })
 })
 
 router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
@@ -139,6 +128,17 @@ router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
   return res.json(updated)
 })
 
+router.get('/v2/studyprogrammes/:id/colorizedtablecoursestats', async (req, res) => {
+  const code = req.params.id
+
+  try {
+    const data = await getStudyprogrammeStatsForColorizedCoursesTable(code)
+    return res.json(data)
+  } catch (e) {
+    logger.error({ message: `Failed to get code ${code} colorized table course stats`, meta: `${e}` })
+  }
+})
+
 router.get('/v2/studyprogrammes/:id/update_basicview', async (req, res) => {
   const code = req.params.id
   const combinedProgramme = req.query?.combined_programme

services/backend/src/routes/studyProgrammeCriteria.js

@@ -2,7 +2,7 @@ const {
   getCriteria,
   saveYearlyCourseCriteria,
   saveYearlyCreditCriteria,
-} = require('../services/studyProgrammeCriteria')
+} = require('../services/studyprogramme/studyProgrammeCriteria')
 
 const router = require('express').Router()
 

services/backend/src/routes/tags.js

@@ -23,8 +23,10 @@ router.get('/tags/:studytrack', async (req, res) => {
     user: { rights, roles, id },
   } = req
   const programmeCodes = studytrack.includes('KH') && studytrack.includes('MH') ? studytrack.split('-') : [studytrack]
+
+  // Respond with null and 200 instead of 403 if the user isn't authorized to view the tags. This is to avoid unnecessary noise in Sentry
   if (!rights.includes(programmeCodes[0]) && !rights.includes(programmeCodes[1]) && !roles?.includes('admin'))
-    return res.status(403).end()
+    return res.json(null)
 
   const tags = await Tags.findTagsByStudytrack(studytrack)
   res.status(200).json(filterRelevantTags(tags, id))
@@ -81,8 +83,10 @@ router.get('/studenttags/:studytrack', async (req, res) => {
   } = req
 
   const programmeCodes = studytrack.includes('KH') && studytrack.includes('MH') ? studytrack.split('-') : [studytrack]
+
+  // Respond with null and 200 instead of 403 if the user isn't authorized to view the tags. This is to avoid unnecessary noise in Sentry
   if (!rights.includes(programmeCodes[0]) && !rights.includes(programmeCodes[1]) && !roles?.includes('admin'))
-    return res.status(403).end()
+    return res.json(null)
 
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
   res.status(200).json(filterRelevantStudentTags(result, id))

services/backend/src/routes/trends.js

@@ -1,79 +0,0 @@
-const router = require('express').Router()
-const {
-  withErr,
-  getProtoC,
-  getProtoCProgramme,
-  getStatus,
-  getUber,
-  getStartYears,
-  getGraduatedStatus,
-} = require('../services/trends')
-
-router.get('/start-years', async (req, res) => {
-  const years = await getStartYears()
-  res.json(years.map(({ studystartdate }) => studystartdate))
-})
-
-router.get(
-  '/uber-data',
-  withErr(async (req, res) => {
-    const data = await getUber(req.query)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/proto-c-data-programme',
-  withErr(async (req, res) => {
-    const data = await getProtoCProgramme(req.query)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/proto-c-data',
-  withErr(async (req, res) => {
-    const data = await getProtoC(req.query)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/status-graduated',
-  withErr(async (req, res) => {
-    const { date: unixMillis, showByYear } = req.query
-    const date = new Date(Number(unixMillis))
-    const endOfToday = new Date()
-    endOfToday.setHours(23, 59, 59, 999)
-
-    if (isNaN(date.getTime()) || date.getTime() > endOfToday.getTime()) {
-      return res.status(400).json({ error: 'Invalid date' })
-    }
-
-    // End of day
-    date.setHours(23, 59, 59, 999)
-    const data = await getGraduatedStatus(date.getTime(), showByYear)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/status',
-  withErr(async (req, res) => {
-    const { date: unixMillis, showByYear } = req.query
-    const date = new Date(Number(unixMillis))
-    const endOfToday = new Date()
-    endOfToday.setHours(23, 59, 59, 999)
-
-    if (isNaN(date.getTime()) || date.getTime() > endOfToday.getTime()) {
-      return res.status(400).json({ error: 'Invalid date' })
-    }
-
-    // End of day
-    date.setHours(23, 59, 59, 999)
-    const status = await getStatus(date.getTime(), showByYear)
-    res.json(status)
-  })
-)
-
-module.exports = router

services/backend/src/routes/university.js

@@ -0,0 +1,192 @@
+const router = require('express').Router()
+const { getFacultyList } = require('../services/faculty/facultyHelpers')
+const { getGraduationStats, getFacultyProgressStats } = require('../services/faculty/facultyService')
+
+const { getMedian } = require('../services/studyprogramme/studyprogrammeHelpers')
+const degreeNames = ['bachelor', 'bachelorMaster', 'master', 'licentiate', 'doctor']
+
+router.get('/allprogressstats', async (req, res) => {
+  const specialGroups = 'SPECIAL_EXCLUDED'
+  const graduated = req.query?.graduated
+  const allFaculties = await getFacultyList()
+  const facultyCodes = allFaculties.map(f => f.code)
+  const codeToData = {}
+
+  for (const facultyCode of facultyCodes) {
+    const data = await getFacultyProgressStats(facultyCode, specialGroups, graduated)
+    if (!data) {
+      return res.status(500).end()
+    }
+    codeToData[facultyCode] = data
+  }
+
+  const universityData = {
+    years: codeToData.H50.years,
+    yearlyBachelorTitles: codeToData.H50.yearlyBachelorTitles,
+    programmeNames: allFaculties.reduce((obj, fac) => {
+      const { name, ...rest } = fac.dataValues
+      obj[fac.code] = { ...rest, ...name }
+      return obj
+    }, {}),
+    bachelorsProgStats: {},
+    creditCounts: {
+      bachelor: {},
+    },
+  }
+
+  const unifyProgressStats = progStats => {
+    return progStats.reduce(
+      (all, prog) => {
+        prog.forEach((yearStats, yearIndex) =>
+          yearStats.forEach((category, categoryIndex) => {
+            all[yearIndex][categoryIndex] += prog[yearIndex][categoryIndex]
+          })
+        )
+        return all
+      },
+      // eslint-disable-next-line no-unused-vars
+      progStats[0].map(year => year.map(_num => 0))
+    )
+  }
+
+  for (const facultyCode of facultyCodes) {
+    const facultyData = codeToData[facultyCode]
+    for (const year of universityData.years.slice(1).reverse()) {
+      for (const fieldName of degreeNames) {
+        if (!facultyData.creditCounts[fieldName] || Object.keys(facultyData.creditCounts[fieldName]).length === 0)
+          continue
+        if (!universityData.creditCounts[fieldName]) universityData.creditCounts[fieldName] = {}
+        if (!universityData.creditCounts[fieldName][year]) universityData.creditCounts[fieldName][year] = []
+        universityData.creditCounts[fieldName][year].push(...facultyData.creditCounts[fieldName][year])
+      }
+      for (const fieldName of [
+        'bachelorsProgStats',
+        'bcMsProgStats',
+        'licentiateProgStats',
+        'mastersProgStats',
+        'doctoralProgStats',
+      ]) {
+        if (Object.keys(facultyData[fieldName]).length === 0) continue
+        if (!universityData[fieldName]) universityData[fieldName] = {}
+        universityData[fieldName][facultyCode] = unifyProgressStats(Object.values(facultyData[fieldName]))
+      }
+    }
+  }
+
+  // Remove ELL bachelor+master progresstats for now, because it is problematic due to different credit limits
+  // when expanding the table rows to show 'programme'(faculty) specific progress bars
+  delete universityData.bcMsProgStats['H90']
+
+  return res.status(200).json(universityData)
+})
+
+router.get('/allgraduationstats', async (req, res) => {
+  const degreeNames = ['bachelor', 'bcMsCombo', 'master', 'licentiate', 'doctor']
+  const allFaculties = await getFacultyList()
+  const facultyCodes = allFaculties.map(f => f.code)
+  const facultyData = {}
+  const timesArrays = [] // keep book of these to null them in the end, large lists not used in frontend
+  for (const facultyCode of facultyCodes) {
+    const data = await getGraduationStats(facultyCode, 'NEW_STUDY_PROGRAMMES', true)
+    if (!data) return res.status(500).json({ message: `Did not find data for ${facultyCode}` })
+    facultyData[facultyCode] = data
+  }
+
+  const unifyTotals = (facultyData, universityData, isLast) => {
+    for (const degree of degreeNames) {
+      if (!universityData[degree]) universityData[degree] = []
+      if (!facultyData[degree]) continue
+      for (const yearStats of facultyData[degree]) {
+        const universityStats = universityData[degree]
+        const universityYearStats = universityData[degree].find(stats => stats.name === yearStats.name)
+        if (!universityYearStats) {
+          universityStats.push(yearStats)
+        } else {
+          universityYearStats.times.push(...yearStats.times)
+          timesArrays.push(universityYearStats.times)
+          universityYearStats.amount += yearStats.amount
+          universityYearStats.statistics.onTime += yearStats.statistics.onTime
+          universityYearStats.statistics.yearOver += yearStats.statistics.yearOver
+          universityYearStats.statistics.wayOver += yearStats.statistics.wayOver
+          if (isLast) {
+            universityYearStats.y = getMedian(universityYearStats.times)
+          }
+        }
+      }
+    }
+  }
+
+  const unifyProgrammeStats = (universityData, facultyData, facultyCode) => {
+    for (const degree of degreeNames) {
+      if (!facultyData[degree]) continue
+      if (!universityData[degree]) universityData[degree] = {}
+      for (const yearData of facultyData[degree]) {
+        if (!universityData[degree][yearData.name]) {
+          universityData[degree][yearData.name] = { programmes: [], data: [] }
+        }
+        const uniYearStats = universityData[degree][yearData.name]
+        if (!uniYearStats.programmes.find(prog => prog === facultyCode)) {
+          uniYearStats.programmes.push(facultyCode)
+        }
+        const uniYearFacultyStats = uniYearStats.data.find(item => item.code === facultyCode)
+        const yearDataClone = { ...yearData, times: null, statistics: { ...yearData.statistics } }
+        if (!uniYearFacultyStats) {
+          uniYearStats.data.push({ ...yearDataClone, name: facultyCode, code: facultyCode })
+        } else {
+          uniYearFacultyStats.y = yearData.y
+          uniYearFacultyStats.amount += yearData.amount
+          uniYearFacultyStats.statistics.onTime += yearData.statistics.onTime
+          uniYearFacultyStats.statistics.yearOver += yearData.statistics.yearOver
+          uniYearFacultyStats.statistics.wayOver += yearData.statistics.wayOver
+        }
+      }
+    }
+
+    return universityData
+  }
+
+  const universityData = {
+    goals: {
+      bachelor: 36,
+      bcMsCombo: 60,
+      master: 24,
+      doctor: 48,
+      licentiate: 78,
+    },
+    programmeNames: allFaculties.reduce((obj, fac) => {
+      const { name, ...rest } = fac.dataValues
+      obj[fac.code] = { ...rest, ...name }
+      return obj
+    }, {}),
+    byGradYear: { medians: {}, programmes: { medians: {} } },
+    classSizes: { programmes: {} },
+  }
+
+  for (let i = 0; i < facultyCodes.length; i++) {
+    const facultyCode = facultyCodes[i]
+    const data = facultyData[facultyCode]
+    unifyTotals(data.byGradYear.medians, universityData.byGradYear.medians, i === facultyCodes.length - 1)
+    unifyProgrammeStats(universityData.byGradYear.programmes.medians, data.byGradYear.medians, facultyCode)
+    for (const degree of degreeNames) {
+      if (!universityData.classSizes[degree]) {
+        universityData.classSizes[degree] = data.classSizes[degree]
+      } else {
+        Object.entries(data.classSizes[degree]).forEach(([key, value]) => {
+          universityData.classSizes[degree][key] += value
+        })
+      }
+    }
+    // eslint-disable-next-line no-unused-vars
+    const { programmes, ...rest } = data.classSizes
+    universityData.classSizes.programmes[facultyCode] = rest
+  }
+
+  // Empty "times" arrays because that's not needed anymore.
+  timesArrays.forEach(arr => {
+    arr.length = 0
+  })
+
+  return res.status(200).json(universityData)
+})
+
+module.exports = router

services/backend/src/routes/updater.js

@@ -1,4 +1,5 @@
 const router = require('express').Router()
+const { refreshFaculties, refreshProgrammes } = require('../events')
 const {
   updateSISMetadata,
   updateSISStudents,
@@ -8,7 +9,19 @@ const {
   abort,
   updateCoursesByCourseCode,
 } = require('../services/sisUpdaterService')
-const { refreshStatistics, refreshTrends, refreshFaculties, refreshNewOverviews } = require('../events')
+
+const logger = require('../util/logger')
+const { jobMaker, getJobs } = require('../worker/queue')
+
+const refreshFacultiesByList = list => {
+  list.forEach(code => jobMaker.faculty(code))
+  return 'Added jobs for refreshing faculties'
+}
+
+const refreshProgrammesByList = list => {
+  list.forEach(code => jobMaker.programme(code))
+  return 'Added jobs for refreshing programme'
+}
 
 router.get('/update/v2/meta', async (req, res) => {
   const response = await updateSISMetadata(req)
@@ -24,10 +37,20 @@ router.get('/update/v2/students', async (req, res) => {
   }
 })
 
-router.post('/update/v2/students', async (req, res) => {
-  const response = await updateStudentsByStudentNumber(req.body)
+router.post('/update/v2/customlist/:type', async (req, res) => {
+  const type = req.params.type
+  const list = req.body
+  const typeToJob = {
+    students: updateStudentsByStudentNumber,
+    courses: updateCoursesByCourseCode,
+    faculties: refreshFacultiesByList,
+    programmes: refreshProgrammesByList,
+  }
+  if (typeof typeToJob[type] !== 'function') return res.status(400).json('Bad job type')
+  const response = await typeToJob[type](list)
+
   if (response) {
-    res.status(200).json('Update SIS students scheduled')
+    res.status(200).json('Scheduled custom list')
   }
 })
 
@@ -46,6 +69,7 @@ router.get('/update/v2/programmes', async (req, res) => {
 })
 
 router.get('/refresh_redis_cache', async (req, res) => {
+  logger.info(`${req.user.userId} requested refresh of redis cache`)
   const response = await updateSISRedisCache()
   if (response) {
     res.status(200).json('Refreshing SIS redis cache scheduled')
@@ -53,23 +77,27 @@ router.get('/refresh_redis_cache', async (req, res) => {
 })
 
 router.post('/refresh_statistic_v2', async (req, res) => {
-  await refreshStatistics()
+  logger.info(`${req.user.userId} requested refresh of statistics`)
+  jobMaker.statistics()
   res.status(200).json('Teacher and study programme statistics refreshed')
 })
 
 router.post('/refresh_study_programmes_v2', async (req, res) => {
-  await refreshNewOverviews()
-  res.status(200).json('New study programme overviews refreshed')
+  logger.info(`${req.user.userId} requested refresh of study programmes`)
+  refreshProgrammes()
+  res.status(200).json('Added job for refreshing study programme overviews')
 })
 
 router.post('/refresh_faculties_v2', async (req, res) => {
-  await refreshFaculties()
-  res.status(200).json('Faculties refreshed')
+  logger.info(`${req.user.userId} requested refresh of faculties`)
+  refreshFaculties()
+  res.status(200).json('Added job for refreshing faculties')
 })
 
-router.post('/refresh_trends', async (req, res) => {
-  await refreshTrends()
-  res.status(200).json('Trends refreshed')
+router.post('/refresh_language_center_data', async (req, res) => {
+  logger.info(`${req.user.userId} requested refresh of language center data`)
+  jobMaker.languagecenter()
+  res.status(200).json('Added job for refreshing language center data')
 })
 
 router.get('/abort', async (req, res) => {
@@ -77,4 +105,10 @@ router.get('/abort', async (req, res) => {
   res.status(200).json()
 })
 
+router.get('/jobs', async (req, res) => {
+  const waiting = await getJobs('waiting')
+  const active = await getJobs('active')
+  res.status(200).json({ waiting, active })
+})
+
 module.exports = router

services/backend/src/routes/users.js

@@ -13,6 +13,16 @@ router.get('/access_groups', async (req, res) => {
   res.json(result)
 })
 
+router.get('/:uid', async (req, res) => {
+  try {
+    const { uid } = req.params
+    const user = await userService.findOne(uid)
+    return res.json(user)
+  } catch (e) {
+    return res.status(400).json({ error: e.message })
+  }
+})
+
 router.post('/modifyaccess', async (req, res) => {
   try {
     const result = await userService.modifyAccess(req.body)

services/backend/src/services/analyticsService.js

@@ -33,18 +33,21 @@ const setBasicStats = async (data, yearType, specialGroups) => {
   return dataToRedis
 }
 
-const getCreditStats = async (id, combinedProgramme, yearType, specialGroups) => {
+const getCreditStats = async (id, isAcademicYear, specialGroups) => {
   if (!isUpdatedNewProgramme(id)) return null
-  let searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
-  const redisKey = createRedisKeyForCreditStats(searchkey, yearType, specialGroups)
+  const redisKey = createRedisKeyForCreditStats(id, isAcademicYear ? 'ACADEMIC_YEAR' : 'CALENDAR_YEAR', specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
 }
 
-const setCreditStats = async (data, yearType, specialGroups) => {
+const setCreditStats = async (data, isAcademicYear, specialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
+  const redisKey = createRedisKeyForCreditStats(
+    id,
+    isAcademicYear ? 'ACADEMIC_YEAR' : 'CALENDAR_YEAR',
+    specialGroups ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED'
+  )
   const dataToRedis = {
     ...data,
     status: 'DONE',

services/backend/src/services/completedCoursesSearch.js

@@ -1,6 +1,5 @@
 const { Credit, Enrollment, Student, Course } = require('../models')
 const { Op } = require('sequelize')
-const _ = require('lodash')
 
 const getCompletedCourses = async (studentNumbers, courseCodes) => {
   let courses = await Course.findAll({
@@ -126,6 +125,7 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
       date: enrollment.date,
       courseCode: enrollment.courseCode,
       state: enrollment.state,
+      substitution: enrollment.substitution,
     })
   })
 
@@ -136,7 +136,8 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
         studentNumber: student,
         sis_person_id: studentCredits[student].sis_person_id,
         credits: studentCredits[student].credits,
-        enrollments: studentCredits[student].enrollments,
+        enrollments: {},
+        allEnrollments: studentCredits[student].enrollments,
         firstnames: studentCredits[student].firstnames,
         lastname: studentCredits[student].lastname,
         email: studentCredits[student].email,
@@ -147,18 +148,22 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
 
   students.forEach(student => {
     courseCodes.forEach(courseCode => {
-      let [hits, rest] = _.partition(student.enrollments, e => e.courseCode === courseCode)
-      if (hits?.length === 0) return
-      let latest = hits[0]
-      for (let hit of hits) {
-        if (hit > latest) {
-          latest = hit
-        }
-      }
-      student.enrollments = [latest, ...rest]
+      student.enrollments[courseCode] = student.allEnrollments
+        .filter(e => e.courseCode === courseCode || e.substitution === courseCode)
+        .sort(e => new Date(e.date).getDate())[0]
     })
   })
-  return { students, courses }
+
+  // Omit allEnrollments, we're only supposed to show the recent, relevant enrollment,
+  // the user does not have rights to see all enrollments.
+  return {
+    students: students.map(student => {
+      // eslint-disable-next-line no-unused-vars
+      const { allEnrollments, ...rest } = student
+      return { ...rest }
+    }),
+    courses,
+  }
 }
 
 module.exports = { getCompletedCourses }

services/backend/src/services/courses.js

@@ -1,529 +0,0 @@
-const Sequelize = require('sequelize')
-const {
-  Student,
-  Credit,
-  Course,
-  ElementDetail,
-  StudyrightElement,
-  Studyright,
-  Semester,
-  Organization,
-  Enrollment,
-} = require('../models')
-
-const { parseCredit } = require('./parseCredits')
-const Op = Sequelize.Op
-const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
-const { sortMainCode, getSortRank } = require('../util/utils')
-const { parseEnrollment } = require('./parseEnrollments')
-
-const byNameOrCode = (searchTerm, language) =>
-  Course.findAll({
-    where: {
-      [Op.or]: [
-        {
-          name: {
-            [language]: {
-              [Op.iLike]: searchTerm,
-            },
-          },
-        },
-        {
-          code: {
-            [Op.like]: searchTerm,
-          },
-        },
-      ],
-    },
-  })
-
-const byName = (name, language) =>
-  Course.findAll({
-    where: {
-      name: {
-        [language]: {
-          [Op.eq]: name,
-        },
-      },
-    },
-    order: [['latest_instance_date', 'DESC']],
-    limit: 1,
-  })
-
-const byCode = code => Course.findByPk(code)
-
-const findOneByCode = code => {
-  return Course.findOne({
-    attributes: ['id', 'code', 'name'],
-    where: {
-      code: code,
-    },
-  })
-}
-
-const creditsForCourses = async (codes, anonymizationSalt, unification) => {
-  let is_open = false
-
-  if (unification === 'open') is_open = true
-
-  if (unification === 'unify') {
-    is_open = {
-      [Op.in]: [false, true],
-    }
-  }
-
-  const credits = await Credit.findAll({
-    include: [
-      {
-        model: Student,
-        attributes: ['studentnumber'],
-        include: {
-          model: StudyrightElement,
-          attributes: ['code', 'startdate'],
-          include: [
-            {
-              model: ElementDetail,
-              attributes: ['name', 'type'],
-              where: {
-                type: {
-                  [Op.eq]: 20,
-                },
-              },
-            },
-            {
-              model: Studyright,
-              attributes: ['prioritycode', 'faculty_code'],
-              where: {
-                prioritycode: {
-                  [Op.eq]: 1,
-                },
-              },
-              include: {
-                model: Organization,
-              },
-            },
-          ],
-        },
-      },
-      {
-        model: Semester,
-        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
-        where: {
-          startdate: {
-            [Op.lte]: new Date(),
-          },
-        },
-      },
-    ],
-    where: {
-      course_code: {
-        [Op.in]: codes,
-      },
-      student_studentnumber: {
-        [Op.ne]: null,
-      },
-      [Op.or]: [{ is_open }, { is_open: null }],
-    },
-    order: [['attainment_date', 'ASC']],
-  })
-
-  const parsedCredits = credits.map(credit => parseCredit(credit, anonymizationSalt))
-  return parsedCredits
-}
-
-const enrollmentsForCourses = async (codes, anonymizationSalt, unification) => {
-  let is_open = false
-
-  if (unification === 'open') is_open = true
-
-  if (unification === 'unify') {
-    is_open = {
-      [Op.in]: [false, true],
-    }
-  }
-
-  const enrollments = await Enrollment.findAll({
-    include: [
-      {
-        model: Student,
-        attributes: ['studentnumber'],
-        include: {
-          model: StudyrightElement,
-          attributes: ['code', 'startdate'],
-          include: [
-            {
-              model: ElementDetail,
-              attributes: ['name', 'type'],
-              where: {
-                type: {
-                  [Op.eq]: 20,
-                },
-              },
-            },
-            {
-              model: Studyright,
-              attributes: ['prioritycode', 'faculty_code'],
-              where: {
-                prioritycode: {
-                  [Op.eq]: 1,
-                },
-              },
-              include: {
-                model: Organization,
-              },
-            },
-          ],
-        },
-      },
-      {
-        model: Semester,
-        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
-        where: {
-          startdate: {
-            [Op.lte]: new Date(),
-          },
-        },
-      },
-    ],
-    where: {
-      course_code: {
-        [Op.in]: codes,
-      },
-      studentnumber: {
-        [Op.ne]: null,
-      },
-      enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
-      state: ['ENROLLED', 'CONFIRMED'],
-      [Op.or]: [{ is_open }, { is_open: null }],
-    },
-  })
-  return enrollments.map(enrollment => parseEnrollment(enrollment, anonymizationSalt))
-}
-
-const bySearchTerm = async (term, language) => {
-  const formatCourse = course => ({
-    name: course.name[language],
-    code: course.code,
-    date: course.latest_instance_date,
-  })
-
-  try {
-    const result = await byNameOrCode(`%${term}%`, language)
-    return result.map(formatCourse)
-  } catch (e) {
-    return {
-      error: e,
-    }
-  }
-}
-
-const createCourse = async (code, name, latest_instance_date) =>
-  Course.create({
-    code,
-    name,
-    latest_instance_date,
-  })
-
-const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
-
-const unifyOpenUniversity = code => {
-  const regexresult = isOpenUniCourseCode(code)
-  if (!regexresult) return code
-  return regexresult[1]
-}
-
-const allCodeAlternatives = async code => {
-  const course = await Course.findOne({
-    where: { code: code },
-  })
-
-  return sortMainCode([...course.substitutions, code])
-}
-
-const yearlyStatsOfNew = async (coursecode, separate, unification, anonymizationSalt) => {
-  const courseForSubs = await Course.findOne({
-    where: { code: coursecode },
-  })
-
-  let codes = courseForSubs.substitutions ? sortMainCode([...courseForSubs.substitutions, coursecode]) : [coursecode]
-
-  if (unification === 'reqular') {
-    codes = codes.filter(course => !isOpenUniCourseCode(course))
-  }
-
-  const [credits, enrollments, course] = await Promise.all([
-    creditsForCourses(codes, anonymizationSalt, unification),
-    enrollmentsForCourses(codes, anonymizationSalt, unification),
-    Course.findOne({
-      where: {
-        code: coursecode,
-      },
-    }),
-  ])
-
-  const counter = new CourseYearlyStatsCounter()
-
-  for (let credit of credits) {
-    const {
-      studentnumber,
-      grade,
-      passed,
-      semestercode,
-      semestername,
-      yearcode,
-      yearname,
-      attainment_date,
-      programmes,
-      coursecode,
-      credits,
-    } = credit
-
-    const groupcode = separate ? semestercode : yearcode
-    const groupname = separate ? semestername : yearname
-    const unknownProgramme = [
-      {
-        code: 'OTHER',
-        name: {
-          en: 'Other',
-          fi: 'Muu',
-          sv: 'Andra',
-        },
-        faculty_code: 'OTHER',
-        organization: {
-          name: {
-            en: 'Other',
-            fi: 'Muu',
-            sv: 'Andra',
-          },
-        },
-      },
-    ]
-    counter.markStudyProgrammes(
-      studentnumber,
-      programmes.length === 0 ? unknownProgramme : programmes,
-      yearcode,
-      passed,
-      credits
-    )
-    counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode)
-    counter.markCreditToStudentCategories(studentnumber, passed, attainment_date, groupcode)
-  }
-
-  enrollments.forEach(enrollment => {
-    const { studentnumber, semestercode, semestername, yearcode, yearname, coursecode, state, enrollment_date_time } =
-      enrollment
-
-    const groupcode = separate ? semestercode : yearcode
-    const groupname = separate ? semestername : yearname
-
-    counter.markEnrollmentToGroup(
-      studentnumber,
-      state,
-      enrollment_date_time,
-      groupcode,
-      groupname,
-      coursecode,
-      yearcode
-    )
-  })
-
-  const statistics = counter.getFinalStatistics(anonymizationSalt)
-
-  return {
-    ...statistics,
-    coursecode,
-    alternatives: codes,
-    name: course.name,
-  }
-}
-
-const maxYearsToCreatePopulationFrom = async (coursecodes, unifyCourses) => {
-  let is_open = false
-
-  if (unifyCourses === 'openStats') is_open = true
-
-  if (unifyCourses === 'unifyStats') {
-    is_open = {
-      [Op.in]: [false, true],
-    }
-  }
-
-  const maxAttainmentDate = new Date(
-    Math.max(
-      ...(
-        await Course.findAll({
-          where: {
-            code: {
-              [Op.in]: coursecodes,
-            },
-          },
-          attributes: ['max_attainment_date'],
-        })
-      ).map(c => new Date(c.max_attainment_date).getTime())
-    )
-  )
-
-  const attainmentThreshold = new Date(maxAttainmentDate.getFullYear(), 0, 1)
-  attainmentThreshold.setFullYear(attainmentThreshold.getFullYear() - 6)
-
-  const credits = await Credit.findAll({
-    where: {
-      course_code: {
-        [Op.in]: coursecodes,
-      },
-      attainment_date: {
-        [Op.gt]: attainmentThreshold,
-      },
-      is_open,
-    },
-    order: [['attainment_date', 'ASC']],
-  })
-
-  const yearlyStudents = Object.values(
-    credits.reduce((res, credit) => {
-      const attainmentYear = new Date(credit.attainment_date).getFullYear()
-      if (!res[attainmentYear]) res[attainmentYear] = 0
-      res[attainmentYear]++
-      return res
-    }, {})
-  )
-  const maxYearsToCreatePopulationFrom = Math.max(
-    Math.floor(
-      1200 / // Lower this value to get a smaller result if necessary
-        (yearlyStudents.reduce((acc, curr) => acc + curr, 0) / yearlyStudents.length)
-    ),
-    1
-  )
-
-  return maxYearsToCreatePopulationFrom
-}
-
-const courseYearlyStats = async (coursecodes, separate, anonymizationSalt) => {
-  const statsReqular = await Promise.all(
-    coursecodes.map(async code => {
-      const unifyStats = await yearlyStatsOfNew(code, separate, 'unify', anonymizationSalt)
-      const reqularStats = await yearlyStatsOfNew(code, separate, 'reqular', anonymizationSalt)
-      const openStats = await yearlyStatsOfNew(code, separate, 'open', anonymizationSalt)
-
-      return { unifyStats, reqularStats, openStats }
-    })
-  )
-
-  return statsReqular
-}
-
-const nameLikeTerm = name => {
-  if (!name) {
-    return undefined
-  }
-  const term = `%${name.trim()}%`
-  return {
-    name: {
-      [Op.or]: {
-        fi: {
-          [Op.iLike]: term,
-        },
-        sv: {
-          [Op.iLike]: term,
-        },
-        en: {
-          [Op.iLike]: term,
-        },
-      },
-    },
-  }
-}
-
-const byCodes = codes => {
-  return Course.findAll({
-    where: {
-      code: {
-        [Op.in]: codes,
-      },
-    },
-  })
-}
-
-const codeLikeTerm = code =>
-  !code
-    ? undefined
-    : {
-        code: {
-          // Starts with code or has AY/A in front of the code
-          [Op.iRegexp]: `^(AY|ay|A|a)?${code}`,
-        },
-      }
-
-const byNameAndOrCodeLike = async (name, code) => {
-  let rawCourses = await Course.findAll({
-    where: {
-      ...nameLikeTerm(name),
-      ...codeLikeTerm(code),
-    },
-  })
-
-  const courses = rawCourses
-    .map(course => {
-      return { ...course.dataValues }
-    })
-    .sort((x, y) => getSortRank(y.code) - getSortRank(x.code))
-
-  let substitutionGroupIndex = 0
-  const visited = []
-
-  const organizeSubgroups = course => {
-    if (visited.includes(course.code)) return
-
-    let temp = []
-    if (course.substitutions !== null) {
-      temp = courses.filter(c => course.substitutions.includes(c.code))
-    }
-
-    temp.unshift(course)
-    temp.forEach(cu => {
-      if (visited.includes(course.code)) return
-      visited.push(cu.id)
-      cu.subsId = substitutionGroupIndex
-    })
-  }
-
-  courses.forEach(course => {
-    if (!visited.includes(course.id)) {
-      substitutionGroupIndex++
-      organizeSubgroups(course)
-    }
-  })
-  return { courses }
-}
-
-const getCourseProvidersForCourses = async codes => {
-  return (
-    await Organization.findAll({
-      attributes: ['code'],
-      include: {
-        model: Course,
-        where: {
-          code: {
-            [Op.in]: codes,
-          },
-        },
-      },
-      raw: true,
-    })
-  ).map(({ code }) => code)
-}
-
-module.exports = {
-  byCode,
-  byName,
-  bySearchTerm,
-  createCourse,
-  courseYearlyStats,
-  byNameAndOrCodeLike,
-  byCodes,
-  maxYearsToCreatePopulationFrom,
-  unifyOpenUniversity,
-  allCodeAlternatives,
-  findOneByCode,
-  getCourseProvidersForCourses,
-}

services/backend/src/services/courses/courseFinders.js

@@ -0,0 +1,95 @@
+const { Op } = require('sequelize')
+const { Course } = require('../../models')
+const { getSortRank } = require('../../util/utils')
+
+const nameLikeTerm = name => {
+  if (!name) {
+    return undefined
+  }
+  const term = `%${name.trim()}%`
+  return {
+    name: {
+      [Op.or]: {
+        fi: {
+          [Op.iLike]: term,
+        },
+        sv: {
+          [Op.iLike]: term,
+        },
+        en: {
+          [Op.iLike]: term,
+        },
+      },
+    },
+  }
+}
+
+const byCodes = codes => {
+  return Course.findAll({
+    where: {
+      code: {
+        [Op.in]: codes,
+      },
+    },
+  })
+}
+
+const escapeRegExp = string => string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+
+const codeLikeTerm = code =>
+  !code
+    ? undefined
+    : {
+        code: {
+          // Starts with code or has AY/A in front of the code (case-insensitive)
+          [Op.iRegexp]: `^(AY|A)?${escapeRegExp(code)}`,
+        },
+      }
+
+const byNameAndOrCodeLike = async (name, code) => {
+  const rawCourses = await Course.findAll({
+    where: {
+      ...nameLikeTerm(name),
+      ...codeLikeTerm(code),
+    },
+    order: [['id', 'desc']],
+  })
+
+  const courses = rawCourses
+    .map(course => {
+      return { ...course.dataValues }
+    })
+    .sort((x, y) => getSortRank(y.code) - getSortRank(x.code))
+
+  let substitutionGroupIndex = 0
+  const visited = []
+
+  const organizeSubgroups = course => {
+    if (visited.includes(course.code)) return
+
+    let temp = []
+    if (course.substitutions !== null) {
+      temp = courses.filter(c => course.substitutions.includes(c.code))
+    }
+
+    temp.unshift(course)
+    temp.forEach(cu => {
+      if (visited.includes(course.code)) return
+      visited.push(cu.id)
+      cu.subsId = substitutionGroupIndex
+    })
+  }
+
+  courses.forEach(course => {
+    if (!visited.includes(course.id)) {
+      substitutionGroupIndex++
+      organizeSubgroups(course)
+    }
+  })
+  return { courses }
+}
+
+module.exports = {
+  byNameAndOrCodeLike,
+  byCodes,
+}

services/backend/src/services/courses/creditsAndEnrollmentsOfCourse.js

@@ -0,0 +1,143 @@
+const { Op } = require('sequelize')
+const {
+  Credit,
+  Student,
+  Semester,
+  Studyright,
+  Organization,
+  StudyrightElement,
+  ElementDetail,
+  Enrollment,
+} = require('../../models')
+
+const creditsForCourses = async (codes, anonymizationSalt, unification) => {
+  let is_open = false
+
+  if (unification === 'open') is_open = true
+
+  if (unification === 'unify') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
+  return await Credit.findAll({
+    include: [
+      {
+        model: Student,
+        attributes: ['studentnumber'],
+      },
+      {
+        model: Semester,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+        where: {
+          startdate: {
+            [Op.lte]: new Date(),
+          },
+        },
+      },
+    ],
+    where: {
+      course_code: {
+        [Op.in]: codes,
+      },
+      student_studentnumber: {
+        [Op.ne]: null,
+      },
+      [Op.or]: [{ is_open }, { is_open: null }],
+    },
+    order: [['attainment_date', 'ASC']],
+  })
+}
+
+const getStudentNumberToSrElementsMap = async studentNumbers => {
+  const studyrights = await Studyright.findAll({
+    attributes: ['prioritycode', 'faculty_code', 'student_studentnumber', 'studyrightid'],
+    where: {
+      prioritycode: {
+        [Op.eq]: 1,
+      },
+      student_studentnumber: { [Op.in]: studentNumbers },
+    },
+    include: {
+      model: Organization,
+    },
+  })
+
+  const studyrightMap = studyrights.reduce((obj, cur) => {
+    obj[cur.studyrightid] = cur
+    return obj
+  }, {})
+
+  const studyrightIds = Object.keys(studyrightMap)
+
+  const studyrightElements = await StudyrightElement.findAll({
+    attributes: ['code', 'startdate', 'studentnumber', 'studyrightid'],
+    include: [
+      {
+        model: ElementDetail,
+        attributes: ['name', 'type'],
+        where: {
+          type: {
+            [Op.eq]: 20,
+          },
+        },
+      },
+    ],
+    where: { studyrightid: { [Op.in]: studyrightIds } },
+  })
+
+  return studyrightElements.reduce((obj, cur) => {
+    if (!obj[cur.studentnumber]) obj[cur.studentnumber] = []
+    cur.studyright = studyrightMap[cur.studyrightid]
+    obj[cur.studentnumber].push(cur)
+    return obj
+  }, {})
+}
+
+const enrollmentsForCourses = async (codes, unification) => {
+  let is_open = false
+
+  if (unification === 'open') is_open = true
+
+  if (unification === 'unify') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
+  return await Enrollment.findAll({
+    include: [
+      {
+        model: Student,
+        attributes: ['studentnumber'],
+      },
+      {
+        model: Semester,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+        where: {
+          startdate: {
+            [Op.lte]: new Date(),
+          },
+        },
+      },
+    ],
+    where: {
+      course_code: {
+        [Op.in]: codes,
+      },
+      studentnumber: {
+        [Op.ne]: null,
+      },
+      enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
+      state: ['ENROLLED', 'CONFIRMED'],
+      [Op.or]: [{ is_open }, { is_open: null }],
+    },
+  })
+}
+
+module.exports = {
+  getStudentNumberToSrElementsMap,
+  enrollmentsForCourses,
+  creditsForCourses,
+}

services/backend/src/services/courses/index.js

@@ -0,0 +1,281 @@
+const Sequelize = require('sequelize')
+const { Credit, Course, Organization, Enrollment } = require('../../models')
+
+const { parseCredit } = require('./parseCredits')
+const { parseEnrollment } = require('./parseEnrollments')
+const Op = Sequelize.Op
+const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
+const { sortMainCode } = require('../../util/utils')
+const {
+  enrollmentsForCourses,
+  creditsForCourses,
+  getStudentNumberToSrElementsMap,
+} = require('./creditsAndEnrollmentsOfCourse')
+const { byNameAndOrCodeLike, byCodes } = require('./courseFinders')
+
+const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+
+const unifyOpenUniversity = code => {
+  const regexresult = isOpenUniCourseCode(code)
+  if (!regexresult) return code
+  return regexresult[1]
+}
+
+const allCodeAlternatives = async code => {
+  const course = await Course.findOne({
+    where: { code: code },
+  })
+
+  return sortMainCode([...course.substitutions, code])
+}
+
+const yearlyStatsOfNew = async (
+  coursecode,
+  separate,
+  unification,
+  anonymizationSalt,
+  combineSubstitutions,
+  studentNumberToSrElementsMap
+) => {
+  const courseForSubs = await Course.findOne({
+    where: { code: coursecode },
+  })
+
+  let codes =
+    combineSubstitutions && courseForSubs.substitutions
+      ? sortMainCode([...courseForSubs.substitutions, coursecode])
+      : [coursecode]
+
+  if (unification === 'regular') {
+    codes = codes.filter(course => !isOpenUniCourseCode(course))
+  }
+
+  const [credits, enrollments, course] = await Promise.all([
+    creditsForCourses(codes, anonymizationSalt, unification),
+    enrollmentsForCourses(codes, unification),
+    Course.findOne({
+      where: {
+        code: coursecode,
+      },
+    }),
+  ])
+
+  const counter = new CourseYearlyStatsCounter()
+
+  for (let credit of credits) {
+    const {
+      studentnumber,
+      grade,
+      passed,
+      semestercode,
+      semestername,
+      yearcode,
+      yearname,
+      attainment_date,
+      programmes,
+      coursecode,
+      credits,
+    } = parseCredit(credit, anonymizationSalt, studentNumberToSrElementsMap)
+
+    const groupcode = separate ? semestercode : yearcode
+    const groupname = separate ? semestername : yearname
+    const unknownProgramme = [
+      {
+        code: 'OTHER',
+        name: {
+          en: 'Other',
+          fi: 'Muu',
+          sv: 'Andra',
+        },
+        faculty_code: 'OTHER',
+        organization: {
+          name: {
+            en: 'Other',
+            fi: 'Muu',
+            sv: 'Andra',
+          },
+        },
+      },
+    ]
+    counter.markStudyProgrammes(
+      studentnumber,
+      programmes.length === 0 ? unknownProgramme : programmes,
+      yearcode,
+      passed,
+      credits
+    )
+    counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode)
+    counter.markCreditToStudentCategories(studentnumber, passed, attainment_date, groupcode)
+  }
+
+  enrollments.forEach(enrollment => {
+    const { studentnumber, semestercode, semestername, yearcode, yearname, coursecode, state, enrollment_date_time } =
+      parseEnrollment(enrollment, anonymizationSalt, studentNumberToSrElementsMap)
+
+    const groupcode = separate ? semestercode : yearcode
+    const groupname = separate ? semestername : yearname
+
+    counter.markEnrollmentToGroup(
+      studentnumber,
+      state,
+      enrollment_date_time,
+      groupcode,
+      groupname,
+      coursecode,
+      yearcode
+    )
+  })
+
+  const statistics = counter.getFinalStatistics(anonymizationSalt)
+
+  return {
+    ...statistics,
+    coursecode,
+    alternatives: codes,
+    name: course.name,
+  }
+}
+
+const maxYearsToCreatePopulationFrom = async (coursecodes, unifyCourses) => {
+  let is_open = false
+
+  if (unifyCourses === 'openStats') is_open = true
+
+  if (unifyCourses === 'unifyStats') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
+  const maxAttainmentDate = new Date(
+    Math.max(
+      ...(
+        await Course.findAll({
+          where: {
+            code: {
+              [Op.in]: coursecodes,
+            },
+          },
+          attributes: ['max_attainment_date'],
+        })
+      ).map(c => new Date(c.max_attainment_date).getTime())
+    )
+  )
+
+  const attainmentThreshold = new Date(maxAttainmentDate.getFullYear(), 0, 1)
+  attainmentThreshold.setFullYear(attainmentThreshold.getFullYear() - 6)
+
+  const credits = await Credit.findAll({
+    where: {
+      course_code: {
+        [Op.in]: coursecodes,
+      },
+      attainment_date: {
+        [Op.gt]: attainmentThreshold,
+      },
+      is_open,
+    },
+    order: [['attainment_date', 'ASC']],
+  })
+
+  const yearlyStudents = Object.values(
+    credits.reduce((res, credit) => {
+      const attainmentYear = new Date(credit.attainment_date).getFullYear()
+      if (!res[attainmentYear]) res[attainmentYear] = 0
+      res[attainmentYear]++
+      return res
+    }, {})
+  )
+  const maxYearsToCreatePopulationFrom = Math.max(
+    Math.floor(
+      1200 / // Lower this value to get a smaller result if necessary
+        (yearlyStudents.reduce((acc, curr) => acc + curr, 0) / yearlyStudents.length)
+    ),
+    1
+  )
+
+  return maxYearsToCreatePopulationFrom
+}
+
+const courseYearlyStats = async (coursecodes, separate, anonymizationSalt, combineSubstitutions) => {
+  const credits = await Credit.findAll({
+    attributes: ['student_studentnumber'],
+    where: { course_code: { [Op.in]: coursecodes } },
+  })
+
+  const enrollment = await Enrollment.findAll({
+    attributes: ['studentnumber'],
+    where: { course_code: { [Op.in]: coursecodes } },
+  })
+
+  const studentnumbersObject = {}
+
+  credits.forEach(cr => {
+    studentnumbersObject[cr.student_studentnumber] = true
+  })
+  enrollment.forEach(cr => {
+    studentnumbersObject[cr.studentnumber] = true
+  })
+
+  const studentNumberToSrElementsMap = await getStudentNumberToSrElementsMap(Object.keys(studentnumbersObject))
+
+  const statsRegular = await Promise.all(
+    coursecodes.map(async code => {
+      const unifyStats = await yearlyStatsOfNew(
+        code,
+        separate,
+        'unify',
+        anonymizationSalt,
+        combineSubstitutions,
+        studentNumberToSrElementsMap
+      )
+      const regularStats = await yearlyStatsOfNew(
+        code,
+        separate,
+        'regular',
+        anonymizationSalt,
+        combineSubstitutions,
+        studentNumberToSrElementsMap
+      )
+      const openStats = await yearlyStatsOfNew(
+        code,
+        separate,
+        'open',
+        anonymizationSalt,
+        combineSubstitutions,
+        studentNumberToSrElementsMap
+      )
+
+      return { unifyStats, regularStats, openStats }
+    })
+  )
+
+  return statsRegular
+}
+
+const getCourseProvidersForCourses = async codes => {
+  return (
+    await Organization.findAll({
+      attributes: ['code'],
+      include: {
+        model: Course,
+        where: {
+          code: {
+            [Op.in]: codes,
+          },
+        },
+      },
+      raw: true,
+    })
+  ).map(({ code }) => code)
+}
+
+module.exports = {
+  courseYearlyStats,
+  maxYearsToCreatePopulationFrom,
+  unifyOpenUniversity,
+  allCodeAlternatives,
+  getCourseProvidersForCourses,
+  byCodes,
+  byNameAndOrCodeLike,
+}

services/backend/src/services/courses/parseCredits.js

@@ -1,5 +1,5 @@
 const crypto = require('crypto')
-const { Credit } = require('../models')
+const { Credit } = require('../../models')
 
 const formatStudyrightElement = ({ code, element_detail, startdate, studyright: sr }) => {
   const studyright = sr.get({ plain: true })
@@ -17,24 +17,20 @@ const formatStudyrightElement = ({ code, element_detail, startdate, studyright:
   }
 }
 
-const parseCredit = (credit, anonymizationSalt) => {
-  const { student, semester, grade, course_code, credits, attainment_date } = credit
-  const { studentnumber, studyright_elements } = student
+const parseCredit = (credit, anonymizationSalt, studentNumberToSrElementsMap) => {
+  const { semester, grade, course_code, credits, attainment_date, student_studentnumber: studentnumber } = credit
   const { yearcode, yearname, semestercode, name: semestername } = semester
 
+  const studyrightElements = studentNumberToSrElementsMap[studentnumber] || []
+
   if (anonymizationSalt) {
     const anonymizedStudentNumber = crypto
       .createHash('sha256')
       .update(`${studentnumber}${anonymizationSalt}`)
       .digest('hex')
 
-    const anonymizedStudent = {
-      studentnumber: anonymizedStudentNumber,
-    }
-
     return {
       obfuscated: true,
-      student: anonymizedStudent,
       yearcode,
       yearname,
       semestercode,
@@ -44,13 +40,12 @@ const parseCredit = (credit, anonymizationSalt) => {
       grade,
       passed: !Credit.failed(credit) || Credit.passed(credit) || Credit.improved(credit),
       studentnumber: anonymizedStudentNumber,
-      programmes: studyright_elements.map(formatStudyrightElement),
+      programmes: studyrightElements.map(formatStudyrightElement),
       credits,
     }
   }
 
   return {
-    student,
     yearcode,
     yearname,
     semestercode,
@@ -60,7 +55,7 @@ const parseCredit = (credit, anonymizationSalt) => {
     grade,
     passed: !Credit.failed(credit) || Credit.passed(credit) || Credit.improved(credit),
     studentnumber,
-    programmes: studyright_elements.map(formatStudyrightElement),
+    programmes: studyrightElements.map(formatStudyrightElement),
     credits,
   }
 }

services/backend/src/services/courses/parseEnrollments.js

@@ -1,24 +1,20 @@
 const crypto = require('crypto')
 const { formatStudyrightElement } = require('./parseCredits')
 
-const parseEnrollment = (enrollment, anonymizationSalt) => {
-  const { student, semester, state, enrollment_date_time, course_code } = enrollment
-  const { studentnumber, studyright_elements } = student
+const parseEnrollment = (enrollment, anonymizationSalt, studentNumberToSrElementsMap) => {
+  const { studentnumber, semester, state, enrollment_date_time, course_code } = enrollment
   const { yearcode, yearname, semestercode, name: semestername } = semester
 
+  const studyrightElements = studentNumberToSrElementsMap[studentnumber] || []
+
   if (anonymizationSalt) {
     const anonymizedStudentNumber = crypto
       .createHash('sha256')
       .update(`${studentnumber}${anonymizationSalt}`)
       .digest('hex')
 
-    const anonymizedStudent = {
-      studentnumber: anonymizedStudentNumber,
-    }
-
     return {
       obfuscated: true,
-      student: anonymizedStudent,
       yearcode,
       yearname,
       semestercode,
@@ -27,12 +23,11 @@ const parseEnrollment = (enrollment, anonymizationSalt) => {
       state,
       enrollment_date_time,
       studentnumber: anonymizedStudentNumber,
-      programmes: studyright_elements.map(formatStudyrightElement),
+      programmes: studyrightElements.map(formatStudyrightElement),
     }
   }
 
   return {
-    student,
     yearcode,
     yearname,
     semestercode,
@@ -41,7 +36,7 @@ const parseEnrollment = (enrollment, anonymizationSalt) => {
     state,
     enrollment_date_time,
     studentnumber,
-    programmes: studyright_elements.map(formatStudyrightElement),
+    programmes: studyrightElements.map(formatStudyrightElement),
   }
 }
 

services/backend/src/services/diff.js

@@ -1,64 +0,0 @@
-// Returns an array containing all values that are in a but not in b taking account duplicates
-const difference = (a, b) => {
-  return a.filter(
-    function (v) {
-      return !this.get(v) || !this.set(v, this.get(v) - 1)
-    },
-    b.reduce((acc, v) => acc.set(v, (acc.get(v) || 0) + 1), new Map())
-  )
-}
-
-const mapStudentNumbers = (studentnumbers, yearcode, name) =>
-  studentnumbers.map(studentnumber => ({
-    studentnumber,
-    yearcode,
-    year: name,
-  }))
-
-const getDiffForCourse = (oodiCourseStats, sisCourseStats) => {
-  return oodiCourseStats.statistics.reduce(
-    (courseDiff, oodiYearlyStats) => {
-      const sisYearlyStats =
-        sisCourseStats && sisCourseStats.statistics.find(s => s.yearcode === oodiYearlyStats.yearcode)
-
-      if (!sisYearlyStats) {
-        return {
-          missingStudents: mapStudentNumbers(oodiYearlyStats.students.studentnumbers),
-          extraStudents: [],
-        }
-      }
-
-      const missingStudentNumbers = difference(
-        oodiYearlyStats.students.studentnumbers,
-        sisYearlyStats.students.studentnumbers
-      )
-      const extraStudentNumbers = difference(
-        sisYearlyStats.students.studentnumbers,
-        oodiYearlyStats.students.studentnumbers
-      )
-
-      const missingStudents = mapStudentNumbers(missingStudentNumbers, oodiYearlyStats.yearcode, oodiYearlyStats.name)
-      const extraStudents = mapStudentNumbers(extraStudentNumbers, oodiYearlyStats.yearcode, oodiYearlyStats.name)
-
-      const noDifferenceInCourseStats = missingStudents.length === 0 && extraStudents.length === 0
-      if (noDifferenceInCourseStats) {
-        return courseDiff
-      }
-
-      return {
-        missingStudents: courseDiff.missingStudents.concat(missingStudents),
-        extraStudents: courseDiff.extraStudents.concat(extraStudents),
-      }
-    },
-    { missingStudents: [], extraStudents: [] }
-  )
-}
-
-const getCourseYearlyStatsDiff = (sisData, oodiData) => {
-  return oodiData.map(oodiCourseStats => {
-    const sisCourseStats = sisData.find(cs => cs.coursecode === oodiCourseStats.coursecode)
-    return { coursecode: oodiCourseStats.coursecode, ...getDiffForCourse(oodiCourseStats, sisCourseStats) }
-  })
-}
-
-module.exports = { getCourseYearlyStatsDiff }

services/backend/src/services/elementdetails.js

@@ -1,22 +0,0 @@
-const { ElementDetail } = require('../models')
-const { Op } = require('sequelize')
-
-const byId = id => ElementDetail.findByPk(id)
-
-const byType = type =>
-  ElementDetail.findAll({
-    where: {
-      type,
-    },
-  })
-
-const byCodes = codes =>
-  ElementDetail.findAll({
-    where: {
-      code: {
-        [Op.in]: codes,
-      },
-    },
-  })
-
-module.exports = { byId, byType, byCodes }

services/backend/src/services/encrypt.js

@@ -1,11 +1,12 @@
-const { randomBytes, createCipheriv, createDecipheriv } = require('crypto')
+const { createCipheriv, createDecipheriv, randomBytes } = require('crypto')
+const { CRYPT_KEY } = require('../conf-backend')
 
 const algorithm = 'aes-256-cbc'
-const key = randomBytes(32)
-const iv = randomBytes(16)
 
 const encrypt = text => {
-  const cipher = createCipheriv(algorithm, Buffer.from(key), iv)
+  if (!CRYPT_KEY) throw Error('Define crypt key in environment variables')
+  const iv = randomBytes(16)
+  const cipher = createCipheriv(algorithm, Buffer.from(CRYPT_KEY, 'hex'), iv)
   let encrypted = cipher.update(text)
   encrypted = Buffer.concat([encrypted, cipher.final()])
 
@@ -16,9 +17,10 @@ const encrypt = text => {
 }
 
 const decrypt = text => {
+  if (!CRYPT_KEY) throw Error('Define crypt key in environment variables')
   const iv = Buffer.from(text.iv, 'hex')
   const encryptedText = Buffer.from(text.encryptedData, 'hex')
-  const decipher = createDecipheriv(algorithm, Buffer.from(key), iv)
+  const decipher = createDecipheriv(algorithm, Buffer.from(CRYPT_KEY, 'hex'), iv)
   let decrypted = decipher.update(encryptedText)
   decrypted = Buffer.concat([decrypted, decipher.final()])
 

services/backend/src/services/faculty/faculty.js

@@ -293,7 +293,7 @@ const degreeProgrammesOfFaculty = async facultyCode =>
     })
   ).map(facultyFormatProgramme)
 
-const facultyOgranizationId = async faculty => {
+const facultyOrganizationId = async faculty => {
   return await Organization.findOne({
     attributes: ['id'],
     where: {
@@ -359,7 +359,7 @@ const findFacultyProgrammeCodes = async (faculty, programmeFilter) => {
   allProgrammeCodes = allProgrammeCodes.concat(directAssociationProgrammes.map(prog => prog.code))
 
   // find faculty's organization id and its direct child organizations
-  const { id } = await facultyOgranizationId(faculty)
+  const { id } = await facultyOrganizationId(faculty)
   const facultyChildOrganizations = await getChildOrganizations(id)
 
   // get programme modules that have a faculty child as organization(_id)
@@ -408,10 +408,8 @@ module.exports = {
   transferredInsideFaculty,
   transferredAway,
   transferredTo,
-  degreeProgrammesOfFaculty,
   thesisWriters,
   findFacultyProgrammeCodes,
-  facultyOgranizationId,
   getTransferredToAndAway,
   getTransferredInside,
   getStudyRightsByExtent,

services/backend/src/services/faculty/facultyBasics.js

@@ -11,7 +11,7 @@ const {
   studyrightsByRightStartYear,
   getTransfersIn,
 } = require('./faculty')
-const { getStatsBasis, getYearsArray, defineYear } = require('../studyprogrammeHelpers')
+const { getStatsBasis, getYearsArray, defineYear } = require('../studyprogramme/studyprogrammeHelpers')
 const { findRightProgramme, isNewProgramme, checkTransfers, getExtentFilter } = require('./facultyHelpers')
 const { codes } = require('../../../config/programmeCodes')
 

services/backend/src/services/faculty/facultyCredits.js

@@ -7,8 +7,10 @@ const {
   isNonMajorCredit,
   getYearsArray,
   getStartDate,
-} = require('../studyprogrammeHelpers')
-const { getStudyRights, getCreditsForStudyProgramme, getCourseCodesForStudyProgramme } = require('../studyprogramme')
+} = require('../studyprogramme/studyprogrammeHelpers')
+const { getCourseCodesForStudyProgramme } = require('../studyprogramme')
+const { getCreditsForStudyProgramme } = require('../studyprogramme/creditGetters')
+const { getStudyRights } = require('../studyprogramme/studyrightFinders')
 
 const isFacultyNonMajorCredit = (studyrights, attainmentDate, facultyProgrammes, facultyCode) => {
   let right = ''
@@ -57,21 +59,31 @@ const getFacultyRegularCreditStats = async ({
   facultyProgrammes,
   facultyCode,
 }) => {
-  const providercode = mapToProviders([studyprogramme])[0]
-  const providedByProgramme = await getCourseCodesForStudyProgramme(providercode)
-  const allCredits = await getCreditsForStudyProgramme(providedByProgramme, since)
-  const credits = uniqBy(allCredits, 'id')
-  const students = [...new Set(credits.map(({ studentNumber }) => studentNumber))]
-
-  let studyRights = await getStudyRights(students)
   let majors = getStatsBasis(years) // Has studyright to programme
   let facultyNonMajors = getStatsBasis(years) // Has studyright in the same faculty but not in the programme from which credits are
   let facultyOthers = getStatsBasis(years) // Exchange students, specialization education in Medicine, Dentistry and vetenary medicine, continuous education
   let otherNonMajors = getStatsBasis(years) // Has studyright in different faculty
   let nonDegree = getStatsBasis(years) // open uni, non-degree study rights, contract studies, (other) specialisation education, credits outside of the studyright
+  if (!studyprogramme) {
+    return { majors, facultyNonMajors, otherNonMajors, facultyOthers, nonDegree }
+  }
+
+  const providercode = mapToProviders([studyprogramme])[0]
+  const providedByProgramme = await getCourseCodesForStudyProgramme(providercode)
+  const allCredits = await getCreditsForStudyProgramme(providercode, providedByProgramme, since)
+  const credits = uniqBy(allCredits, 'id')
+  const students = [...new Set(credits.map(({ studentNumber }) => studentNumber))]
+
+  let studyrights = await getStudyRights(students)
+
+  const studentNumberToStudyrightsMap = studyrights.reduce((obj, cur) => {
+    if (!obj[cur.studentNumber]) obj[cur.studentNumber] = []
+    obj[cur.studentNumber].push(cur)
+    return obj
+  }, {})
 
   credits.forEach(({ studentNumber, attainmentDate, credits }) => {
-    const studentStudyrights = studyRights.filter(studyright => studyright.studentNumber === studentNumber)
+    const studentStudyrights = studentNumberToStudyrightsMap[studentNumber] || []
     const attainmentYear = defineYear(attainmentDate, isAcademicYear)
 
     if (isMajorStudentCredit(studentStudyrights, attainmentDate, studyprogramme)) {
@@ -124,9 +136,8 @@ const getFacultyCreditStatsForStudytrack = async ({
   const years = getYearsArray(since.getFullYear(), isAcademicYear)
 
   const queryParameters = { studyprogramme, since, years, isAcademicYear, facultyProgrammes, facultyCode }
-  const { majors, facultyNonMajors, otherNonMajors, facultyOthers, nonDegree } = await getFacultyRegularCreditStats(
-    queryParameters
-  )
+  const { majors, facultyNonMajors, otherNonMajors, facultyOthers, nonDegree } =
+    await getFacultyRegularCreditStats(queryParameters)
 
   const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
   const dataFound = [majors, facultyNonMajors, otherNonMajors].some(d => d.graphStats.length) // , transferred
@@ -267,4 +278,4 @@ const combineFacultyCredits = async (faculty, programmes, allProgrammes, yearTyp
   return allCredits
 }
 
-module.exports = { combineFacultyCredits, getFacultyCreditStatsForStudytrack }
+module.exports = { combineFacultyCredits }

services/backend/src/services/faculty/facultyGraduationTimes.js

@@ -1,7 +1,7 @@
 const moment = require('moment')
 const { graduatedStudyrights, studyrightsByRightStartYear, hasMasterRight } = require('./faculty')
 const { findRightProgramme, isNewProgramme } = require('./facultyHelpers')
-const { getYearsArray, getYearsObject, getMedian, defineYear } = require('../studyprogrammeHelpers')
+const { getYearsArray, getYearsObject, getMedian, defineYear } = require('../studyprogramme/studyprogrammeHelpers')
 const { codes } = require('../../../config/programmeCodes')
 const { bachelorStudyright, countTimeCategories, getStatutoryAbsences } = require('../graduationHelpers')
 const { getProgrammes } = require('./facultyService')
@@ -285,7 +285,16 @@ const count = async (
       }
       data.medians[level] = [
         ...data.medians[level],
-        { y: median, amount: graduationAmounts[level][year], statistics, name: year },
+        {
+          y: median,
+          amount: graduationAmounts[level][year],
+          /* Include invidivual graduation times for the sake of university-level evaluation overview (UniversityView) 
+            which has to count medians by itself 
+          */
+          times: graduationTimes[level][year],
+          statistics,
+          name: year,
+        },
       ]
     })
   })

services/backend/src/services/faculty/facultyHelpers.js

@@ -1,6 +1,14 @@
 const Sequelize = require('sequelize')
 const { Op } = Sequelize
 const { codes } = require('../../../config/programmeCodes')
+const { faculties } = require('../organisations')
+
+const getFacultyList = async () => {
+  const ignore = ['Y', 'H99', 'Y01', 'H92', 'H930']
+  const facultyList = (await faculties()).filter(f => !ignore.includes(f.code))
+  facultyList.sort((a, b) => (a.name.fi > b.name.fi ? 1 : -1))
+  return facultyList
+}
 
 const findRightProgramme = (studyrightElements, code) => {
   let programme = ''
@@ -140,6 +148,7 @@ const mapCodesToIds = data => {
 }
 
 module.exports = {
+  getFacultyList,
   findRightProgramme,
   facultyFormatStudyright,
   facultyFormatProgramme,

services/backend/src/services/faculty/facultyService.js

@@ -16,6 +16,21 @@ const createRediskeyForFacultyProgress = (id, special_groups, graduated) =>
 const createRediskeyForFacultyStudents = (id, special_groups, graduated) =>
   `FACULTY_STUDENTS_STATS_${id}_${special_groups}_${graduated}`
 
+/*
+  Faculty data objects have graduation times left in as arrays, so that
+  university-level evaluation overview can count median-times by itself.
+  Faculties, however, don't need this, and it isn't needed in frontend
+  for universityview either. This removes them from the object.
+*/
+const removeGraduationTimes = data => {
+  Object.values(data.byGradYear.medians).forEach(array =>
+    array.forEach(yearStat => {
+      yearStat.times = null
+    })
+  )
+  data.byGradYear.medians
+}
+
 const setFacultyProgrammes = async (id, data, programmeFilter) => {
   const redisKey = createRedisKeyForFacultyProgrammes(id, programmeFilter)
   const dataToRedis = {
@@ -116,11 +131,15 @@ const setGraduationStats = async (data, programmeFilter) => {
   return dataToRedis
 }
 
-const getGraduationStats = async (id, programmeFilter) => {
+const getGraduationStats = async (id, programmeFilter, keepGraduationTimes = false) => {
   const redisKey = createRedisKeyForGraduationTimeStats(id, programmeFilter)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
-  return JSON.parse(dataFromRedis)
+  const data = JSON.parse(dataFromRedis)
+  if (!keepGraduationTimes) {
+    removeGraduationTimes(data)
+  }
+  return data
 }
 
 const getFacultyProgressStats = async (id, specialGroups, graduated) => {

services/backend/src/services/faculty/facultyStudentProgress.js

@@ -1,8 +1,8 @@
-const { indexOf } = require('lodash')
+const { indexOf, inRange } = require('lodash')
 const moment = require('moment')
 const { getAcademicYearDates } = require('../../util/semester')
-const { getCreditThresholds, getYearsArray } = require('../studyprogrammeHelpers')
-const { studytrackStudents } = require('../studyprogramme')
+const { getCreditThresholds, getYearsArray } = require('../studyprogramme/studyprogrammeHelpers')
+const { studytrackStudents } = require('../studyprogramme/studentGetters')
 const { getStudyRightsByExtent, getStudyRightsByBachelorStart, getTransfersIn, getTransfersOut } = require('./faculty')
 const { checkTransfers } = require('./facultyHelpers')
 
@@ -17,44 +17,39 @@ const getStudentData = (
 ) => {
   let data = {}
   let programmeData = {}
-  const creditCounts = []
+  const creditCounts = students.map(({ credits }) =>
+    credits
+      .filter(credit => moment(credit.attainment_date).isSameOrAfter(startDate))
+      .reduce((total, credit) => total + credit.credits, 0)
+  )
   thresholdKeys.forEach(t => (data[t] = 0))
   limitKeys.forEach(t => (programmeData[t] = 0))
-  students.forEach(({ credits }) => {
-    const creditcount = credits
-      .filter(credit => moment(credit.attainment_date).isSameOrAfter(startDate))
-      .reduce((prev, curr) => prev + curr.credits, 0)
-    creditCounts.push(creditcount)
-    // Data is only used for doctoral programmes, otherwise only programmeData is needed
-    if (prog === 'T') {
-      data[thresholdKeys[0]] += creditcount < thresholdAmounts[0] ? 1 : 0
-      data[thresholdKeys[1]] += creditcount >= thresholdAmounts[0] && creditcount < thresholdAmounts[1] ? 1 : 0
-      data[thresholdKeys[2]] += creditcount >= thresholdAmounts[1] && creditcount < thresholdAmounts[2] ? 1 : 0
-      data[thresholdKeys[3]] += creditcount >= thresholdAmounts[2] && creditcount < thresholdAmounts[3] ? 1 : 0
-      if (thresholdKeys.length === 8) {
-        data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
-        data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
-        data[thresholdKeys[6]] += creditcount >= thresholdAmounts[5] && creditcount < thresholdAmounts[6] ? 1 : 0
-        data[thresholdKeys[7]] += creditcount >= thresholdAmounts[6] ? 1 : 0
-      } else if (thresholdKeys.length === 7) {
-        data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
-        data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
-        data[thresholdKeys[6]] += creditcount >= thresholdAmounts[5] ? 1 : 0
-      } else if (thresholdKeys.length === 6) {
-        data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
-        data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] ? 1 : 0
-      } else {
-        data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] ? 1 : 0
-      }
-    } else {
-      programmeData[limitKeys[0]] += creditcount <= limits[5][0] ? 1 : 0
-      programmeData[limitKeys[1]] += limits[4][0] <= creditcount && limits[4][1] > creditcount ? 1 : 0
-      programmeData[limitKeys[2]] += limits[3][0] <= creditcount && limits[3][1] > creditcount ? 1 : 0
-      programmeData[limitKeys[3]] += limits[2][0] <= creditcount && limits[2][1] > creditcount ? 1 : 0
-      programmeData[limitKeys[4]] += limits[1][0] <= creditcount && limits[1][1] > creditcount ? 1 : 0
-      programmeData[limitKeys[5]] += creditcount >= limits[0][0] ? 1 : 0
-    }
-  })
+
+  // Data is only used for doctoral programmes, otherwise only programmeData is needed
+  if (prog === 'T') {
+    creditCounts.forEach(creditCount => {
+      thresholdKeys.some((thresholdKey, index) => {
+        if (
+          (index === 0 && creditCount < thresholdAmounts[0]) ||
+          (index === thresholdKeys.length - 1 && creditCount >= thresholdAmounts[index - 1]) ||
+          (index > 0 && inRange(creditCount, thresholdAmounts[index - 1], thresholdAmounts[index]))
+        ) {
+          data[thresholdKey] += 1
+          return true
+        }
+        return false
+      })
+    })
+  } else {
+    creditCounts.forEach(creditCount => {
+      programmeData[limitKeys[0]] += creditCount <= limits[5][0] ? 1 : 0
+      programmeData[limitKeys[1]] += inRange(creditCount, limits[4][0], limits[4][1]) ? 1 : 0
+      programmeData[limitKeys[2]] += inRange(creditCount, limits[3][0], limits[3][1]) ? 1 : 0
+      programmeData[limitKeys[3]] += inRange(creditCount, limits[2][0], limits[2][1]) ? 1 : 0
+      programmeData[limitKeys[4]] += inRange(creditCount, limits[1][0], limits[1][1]) ? 1 : 0
+      programmeData[limitKeys[5]] += creditCount >= limits[0][0] ? 1 : 0
+    })
+  }
 
   return { data, progData: programmeData, creditCounts }
 }

services/backend/src/services/faculty/facultyStudents.js

@@ -1,9 +1,15 @@
-const { getStartDate, getYearsArray, getPercentage, getYearsObject, tableTitles } = require('../studyprogrammeHelpers')
+const {
+  getStartDate,
+  getYearsArray,
+  getPercentage,
+  getYearsObject,
+  tableTitles,
+} = require('../studyprogramme/studyprogrammeHelpers')
 const { getAcademicYearDates } = require('../../util/semester')
 const { getStudyRightsByExtent, getStudentsByStudentnumbers, getTransfersIn } = require('./faculty')
 const { checkTransfers } = require('./facultyHelpers')
-const { inactiveStudyrights, absentStudents, enrolledStudents } = require('../studyprogramme')
-const { graduatedStudyRights } = require('../studyprogramme')
+const { inactiveStudyrights, graduatedStudyRights } = require('../studyprogramme/studyrightFinders')
+const { enrolledStudents, absentStudents } = require('../studyprogramme/studentGetters')
 const emptyTotals = () => {
   return {
     total: 0,
@@ -19,7 +25,7 @@ const emptyTotals = () => {
     allOtherCountries: 0,
   }
 }
-const toltalsCalculation = totals => {
+const totalsCalculation = totals => {
   return [
     totals.total,
     totals.allStarted,
@@ -45,10 +51,10 @@ const toltalsCalculation = totals => {
   ]
 }
 const addTotals = (facultyTableStats, year, totals) => {
-  facultyTableStats[year] = [year, ...toltalsCalculation(totals)]
+  facultyTableStats[year] = [year, ...totalsCalculation(totals)]
 }
 const addTotalsProgramme = (programmeTableStats, progId, year, totals) => {
-  programmeTableStats[progId][year] = toltalsCalculation(totals)
+  programmeTableStats[progId][year] = totalsCalculation(totals)
 }
 
 const getStudentData = (students, facultyExtra, year, code) => {

services/backend/src/services/faculty/facultyThesisWriters.js

@@ -8,7 +8,7 @@ const {
   getCorrectStudentnumbers,
   alltimeEndDate,
   alltimeStartDate,
-} = require('../studyprogrammeHelpers')
+} = require('../studyprogramme/studyprogrammeHelpers')
 
 const getFacultyThesisWriters = async ({ since, years, isAcademicYear, facultyProgrammes, includeAllSpecials }) => {
   const thesisTypes = ['urn:code:course-unit-type:bachelors-thesis', 'urn:code:course-unit-type:masters-thesis']

services/backend/src/services/faculty/facultyUpdates.js

@@ -84,6 +84,7 @@ const updateFacultyOverview = async (faculty, statsType) => {
     allProgrammes?.data.forEach(prog => allProgrammeCodes.push(prog.code))
   } catch (e) {
     logger.error(`Faculty updates: programme stats failed with error: ${e}`)
+    logger.error(`Stack: ${e.stack}`)
   }
 
   for (const option of options) {
@@ -120,6 +121,7 @@ const updateFacultyOverview = async (faculty, statsType) => {
       }
     } catch (e) {
       logger.error(`Faculty updates: basic/thesis-writers/credits stats failed with error ${e}`)
+      logger.error(`Stack: ${e.stack}`)
     }
   }
 
@@ -155,6 +157,7 @@ const updateFacultyProgressOverview = async faculty => {
     newProgrammes = await setFacultyProgrammes(faculty, onlyNew, 'NEW_STUDY_PROGRAMMES')
   } catch (e) {
     logger.error(`Faculty stats: all programme stats failed with error ${e}`)
+    logger.error(`Stack: ${e.stack}`)
   }
 
   for (const option of options) {
@@ -176,6 +179,7 @@ const updateFacultyProgressOverview = async faculty => {
       await setFacultyProgressStats(updateFacultyProgressStats, specialGroups, graduated)
     } catch (e) {
       logger.error(`Faculty stats: progress stats failed with error ${e}`)
+      logger.error(`Stack: ${e.stack}`)
     }
   }
   return 'OK'

services/backend/src/services/languageCenterData.js

@@ -1,36 +1,88 @@
-const { Course, Credit, Enrollment, StudyrightElement } = require('../models')
+const { Course, Credit, Enrollment, StudyrightElement, Studyright } = require('../models')
 const { Op } = require('sequelize')
 const { redisClient } = require('./redis')
-const REDIS_KEY = 'LANGUAGE_CENTER_DATA'
+const { orderBy } = require('lodash')
+const LANGUAGE_CENTER_REDIS_KEY = 'LANGUAGE_CENTER_DATA'
 
-const getLanguageCenterData = async () => {
-  const dataOnRedis = await redisClient.getAsync(REDIS_KEY)
-  if (dataOnRedis) return JSON.parse(dataOnRedis)
-  const freshData = await computeLanguageCenterData()
-  redisClient.setAsync(REDIS_KEY, JSON.stringify(freshData))
-  return freshData
+const isBetween = (start, date, end) =>
+  new Date(start).getTime() <= new Date(date).getTime() && new Date(date).getTime() <= new Date(end).getTime()
+
+const findStudyright = (studyrights, date) => studyrights?.find(sr => isBetween(sr.startdate, date, sr.enddate))
+
+const getDifference = stats => {
+  const value = stats.enrollments - stats.completions
+  return value < 0 ? 0 : value
 }
 
-const computeLanguageCenterData = async () => {
+const getLanguageCenterCourses = async () => {
   const courses = await Course.findAll({
     attributes: ['code', 'name'],
     where: {
       [Op.or]: [{ code: { [Op.like]: 'KK%' } }, { code: { [Op.like]: 'AYKK%' } }],
     },
+    raw: true,
   })
+  return courses
+}
+
+const getLanguageCenterData = async () => {
+  const dataOnRedis = await redisClient.getAsync(LANGUAGE_CENTER_REDIS_KEY)
+  if (dataOnRedis) return JSON.parse(dataOnRedis)
+  const freshData = await computeLanguageCenterData()
+  redisClient.setAsync(LANGUAGE_CENTER_REDIS_KEY, JSON.stringify(freshData))
+  return freshData
+}
+
+const computeLanguageCenterData = async () => {
+  const courses = await getLanguageCenterCourses()
 
   const credits = await Credit.findAll({
-    attributes: ['course_code', 'credittypecode', 'student_studentnumber', 'semestercode', 'createdate'],
+    attributes: ['course_code', 'student_studentnumber', 'semestercode', 'attainment_date', 'studyright_id'],
     where: {
       [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
+      // 135 = autumn 2017
+      semestercode: { [Op.gte]: 135 },
+      credittypecode: 4,
     },
+    raw: true,
   })
 
   const enrollments = await Enrollment.findAll({
-    attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time'],
+    attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'studyright_id', 'state'],
     where: {
       [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
+      semestercode: { [Op.gte]: 135 },
+      state: { [Op.in]: ['ENROLLED', 'REJECTED'] },
     },
+    raw: true,
+  })
+
+  const studyrights = await Studyright.findAll({ raw: true })
+
+  const attemptStudyrightToFacultyMap = studyrights.reduce((obj, cur) => {
+    if (!cur.actual_studyrightid) return obj
+    // ^The studyright id should exist, but there was once a faulty row temporarily in studyright-table,
+    // which screwed up the faculties due to null getting a value to this map
+    obj[cur.actual_studyrightid] = cur.facultyCode
+    return obj
+  }, {})
+
+  const studentnumberToStudyrightsMap = studyrights.reduce((obj, cur) => {
+    if (!obj[cur.studentStudentnumber]) obj[cur.studentStudentnumber] = []
+    obj[cur.studentStudentnumber].push(cur)
+    return obj
+  }, {})
+
+  const elementStudyrightIdToStudyrightMap = studyrights.reduce((obj, cur) => {
+    obj[cur.studyrightid] = cur
+    return obj
+  }, {})
+
+  credits.forEach(c => {
+    c.faculty = attemptStudyrightToFacultyMap[c.studyright_id]
+  })
+  enrollments.forEach(c => {
+    c.faculty = attemptStudyrightToFacultyMap[c.studyright_id]
   })
 
   const studentList = new Set()
@@ -38,61 +90,166 @@ const computeLanguageCenterData = async () => {
 
   credits.forEach(c => {
     const sn = c.student_studentnumber
-
-    // ignoring 7 (improved), 9 (transferred) and 10 (failed)
-    if (c.credittypecode !== 4) return
-    studentList.add(sn)
-    if (!attemptsByStudents[sn]) {
-      attemptsByStudents[sn] = []
-    }
-    attemptsByStudents[sn].push({ studentNumber: sn, courseCode: c.course_code, completed: true, date: c.createdate })
-  })
-
-  enrollments.forEach(e => {
-    const sn = e.studentnumber
-    if (attemptsByStudents[sn] && attemptsByStudents[sn].find(att => att.courseCode === e.course_code)) return
     studentList.add(sn)
     if (!attemptsByStudents[sn]) {
       attemptsByStudents[sn] = []
     }
+    attemptsByStudents[sn].push({
+      studentNumber: sn,
+      courseCode: c.course_code,
+      completed: true,
+      date: c.attainment_date,
+      faculty: attemptStudyrightToFacultyMap[c.studyright_id],
+      semestercode: c.semestercode,
+    })
+  })
+
+  enrollments.forEach(e => {
+    const sn = e.studentnumber
+    if (!attemptsByStudents[sn]) {
+      attemptsByStudents[sn] = []
+    }
+    studentList.add(sn)
+    if (
+      attemptsByStudents[sn].find(
+        att => !att.completed && att.semestercode === e.semestercode && att.courseCode === e.course_code
+      )
+    )
+      return
     attemptsByStudents[sn].push({
       studentNumber: sn,
       courseCode: e.course_code,
       completed: false,
       date: e.enrollment_date_time,
+      faculty: attemptStudyrightToFacultyMap[e.studyright_id],
+      semestercode: e.semestercode,
+      enrolled: e.state === 'ENROLLED',
     })
   })
 
   const studyrightElements = await StudyrightElement.findAll({
-    attributes: ['studentnumber', 'startdate', 'enddate', 'code'],
+    attributes: ['studentnumber', 'startdate', 'enddate', 'code', 'studyrightid'],
     where: {
       studentnumber: { [Op.in]: Array.from(studentList.values()) },
     },
+    raw: true,
   })
-  const studyrightMap = studyrightElements.reduce((obj, cur) => {
+
+  const studyrightElementsMap = studyrightElements.reduce((obj, cur) => {
     if (!obj[cur.studentnumber]) {
-      obj[cur.studentnumber] = [{ startdate: cur.startdate, enddate: cur.enddate, code: cur.code }]
-    } else {
-      obj[cur.studentnumber].push({ startdate: cur.startdate, enddate: cur.enddate, code: cur.code })
+      obj[cur.studentnumber] = []
     }
+    obj[cur.studentnumber].push({
+      startdate: cur.startdate,
+      enddate: cur.enddate,
+      code: elementStudyrightIdToStudyrightMap[cur.studyrightid]?.facultyCode,
+    })
     return obj
   }, {})
 
   const getFaculty = (studentNumber, date) =>
-    studyrightMap[studentNumber]?.find(sr => sr.startdate <= date && sr.enddate >= date)?.code
+    findStudyright(studyrightElementsMap[studentNumber], date)?.code ||
+    findStudyright(studentnumberToStudyrightsMap[studentNumber], date)?.facultyCode
 
   const attemptsArray = []
   studentList.forEach(sn => attemptsArray.push(...attemptsByStudents[sn]))
+  // 93033 Avoin yliopisto, yhteistyöoppilaitokset"
+  // 93034 Kesäyliopistot
+  const isOpenUni = code => ['9301', 'H930', '93033', '93034'].includes(code)
+  const isMisc = code => ['H906', 'H401'].includes(code) // Language center, Alexander institute (very few numbers)
 
   attemptsArray.forEach(item => {
+    if (item.faculty) {
+      if (isOpenUni(item.faculty)) {
+        item.faculty = 'OPEN'
+      } else if (isMisc(item.faculty) || !item.faculty.substring(0, 3).match(`^H\\d`)) {
+        item.faculty = 'OTHER'
+      }
+      return
+    }
+
+    // Credit or enrollment did not have studyright_id. Find based on date,
+    // and if studyrightElement is not found, check studyrights because open uni rights
+    // do not have studyrightelements
     const faculty = getFaculty(item.studentNumber, item.date)
-    if (faculty?.startsWith('MH') || faculty?.startsWith('KH')) {
-      item.faculty = faculty.substring(1, 4)
+    if (isOpenUni(faculty)) {
+      item.faculty = 'OPEN'
+    } else if (faculty?.substring(0, 3).match(`^H\\d`)) {
+      item.faculty = !isMisc(faculty) ? faculty : 'OTHER'
     } else {
-      item.faculty = faculty
+      item.faculty = 'OTHER'
     }
   })
-  return { attempts: attemptsArray, courses }
+
+  const faculties = [...new Set(attemptsArray.map(({ faculty }) => faculty))]
+
+  const unorderedTableData = await createArrayOfCourses(attemptsArray, courses)
+
+  const tableData = orderBy(unorderedTableData, 'code')
+
+  return { tableData, faculties }
 }
 
-module.exports = { getLanguageCenterData }
+const createArrayOfCourses = async (attempts, courses) => {
+  const fields = { completions: 0, enrollments: 0, difference: 0, rejected: 0 }
+  const semestersObject = {}
+  const facultyObject = {}
+  const semesterStatsMap = attempts.reduce((obj, cur) => {
+    const semester = cur.semestercode
+    facultyObject[cur.faculty] = true
+    semestersObject[semester] = true
+    if (!obj[cur.courseCode]) {
+      obj[cur.courseCode] = {}
+    }
+    if (!obj[cur.courseCode][semester]) {
+      obj[cur.courseCode][semester] = { ...fields }
+    }
+    if (cur.faculty && !obj[cur.courseCode][semester][cur.faculty]) {
+      obj[cur.courseCode][semester][cur.faculty] = { ...fields }
+    }
+    const allFacultiesTotal = obj[cur.courseCode][semester]
+    const allStats = [allFacultiesTotal]
+    if (cur.faculty) {
+      const semesterFacultyStats = obj[cur.courseCode][semester][cur.faculty]
+      allStats.push(semesterFacultyStats)
+    }
+    for (const stats of allStats) {
+      if (cur.completed) {
+        stats.completions += 1
+      } else {
+        if (cur.enrolled) {
+          stats.enrollments += 1
+        } else {
+          stats.rejected += 1
+        }
+      }
+    }
+    return obj
+  }, {})
+
+  const courseList = courses
+    .map(c => ({ ...c, bySemesters: { ...semesterStatsMap[c.code], cellStats: {} } }))
+    .filter(course => course.bySemesters)
+
+  courseList.forEach(course => {
+    Object.keys(semestersObject).forEach(sem => {
+      const stats = course.bySemesters[sem]
+      if (!stats) return
+      stats.difference = getDifference(stats)
+      Object.keys(facultyObject).forEach(fac => {
+        const stats = course.bySemesters[sem]?.[fac]
+        if (!stats) return
+        stats.difference = getDifference(stats)
+      })
+    })
+  })
+
+  return courseList
+}
+
+module.exports = {
+  getLanguageCenterData,
+  LANGUAGE_CENTER_REDIS_KEY,
+  computeLanguageCenterData,
+  createArrayOfCourses,
+}

services/backend/src/services/mailservice.js

@@ -64,8 +64,15 @@ const sendNotificationAboutNewUser = ({ userId, userFullName }) =>
     },
   })
 
-const accessMessageSubject = 'Authorized in Oodikone'
-const accessMessageText = `<p>You've been authorized to use <a href="https://oodikone.cs.helsinki.fi">Oodikone</a> or your privileges in <a href="https://oodikone.cs.helsinki.fi">Oodikone</a> have changed.</p>
+const accessMessageSubject =
+  'Käyttöoikeutesi Oodikoneeseen on päivitetty – Your privileges in Oodikone have been updated'
+const accessMessageText = `<p>Sinulle on annettu oikeudet käyttää <a href="https://oodikone.helsinki.fi">Oodikonetta</a> tai käyttöoikeutesi Oodikoneeseen on päivitetty.</p>
+<p>Jotta muutokset astuvat voimaan, saatat joutua kirjautumaan ulos ja takaisin sisään.</p>
+<p>HUOM! Jos automaattinen uloskirjautuminen kaikista palveluista ei onnistunut kirjautuessasi ulos Oodikoneesta, voit kirjautua ulos jokaisesta palvelusta yksitellen tai sulkea kaikki selainikkunasi.</p>
+<p>Jos olet määritellyt selaimen palauttamaan edellisen istuntosi käynnistyksen yhteydessä, saatat joutua selainikkunoiden sulkemisen lisäksi poistamaan myös listattujen palvelujen evästeet.</p>
+<p>Katso ohjeet selaimen välimuistin tyhjentämiseksi: <a href="https://helpdesk.it.helsinki.fi/help/1002">https://helpdesk.it.helsinki.fi/help/1002</a></p>
+<br/>
+<p>You've been authorized to use <a href="https://oodikone.helsinki.fi">Oodikone</a> or your privileges in Oodikone have changed.</p>
 <p>You may need to log out and back in before your changes take place.</p>
 <p>NOTE! If the automatic logging out from all services was not successful when logging out of oodikone, you can log out one by one or close all of your browser windows.</p>
 <p>If you have defined your browser to restore your previous session in connection with starting the browser, logging out may also require deleting the cookies of the listed services in addition to closing the browser windows.</p>