[Users] Refactor users page

After recent fixes every test has passed on the first try, so if many of them start failing at the same time again, it's most likely a genuine error.

.env.template

@@ -1,3 +1,6 @@
 # Populate and change this to .env
+# To get the crypt key, run `openssl rand -hex 32` or
+# `node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"`
 IMPORTER_DB_API_TOKEN=
 PATE_TOKEN=
+CRYPT_KEY=

.eslintrc.json

@@ -12,7 +12,6 @@
   "plugins": ["prettier", "cypress"],
   "rules": {
     "no-console": "error",
-    "react/no-unescaped-entities": "off",
     "no-async-promise-executor": "off",
     "prettier/prettier": "error",
     "cypress/no-assigning-return-values": "off"

.github/actions/run-tests/action.yaml

@@ -16,7 +16,7 @@ runs:
   using: composite
   steps:
     - name: 'Login to toska Docker registry'
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
         username: ${{ inputs.username }}
@@ -27,7 +27,7 @@ runs:
       run: docker-compose --file docker-compose.ci.yml up --detach
 
     - name: 'Run cypress'
-      uses: cypress-io/github-action@v4
+      uses: cypress-io/github-action@v6
       with:
         browser: chrome
         spec: ${{ inputs.spec }}

.github/dependabot.yml

@@ -8,6 +8,8 @@ updates:
     labels:
       - 'dependencies'
       - 'infra'
+    allow:
+      - dependency-type: 'production'
     ignore:
       - dependency-name: '*'
         update-types: ['version-update:semver-major']
@@ -20,6 +22,8 @@ updates:
     labels:
       - 'dependencies'
       - 'updater'
+    allow:
+      - dependency-type: 'production'
     ignore:
       - dependency-name: '*'
         update-types: ['version-update:semver-major']
@@ -32,6 +36,8 @@ updates:
     labels:
       - 'dependencies'
       - 'updater'
+    allow:
+      - dependency-type: 'production'
     ignore:
       - dependency-name: '*'
         update-types: ['version-update:semver-major']
@@ -44,6 +50,8 @@ updates:
     labels:
       - 'dependencies'
       - 'back'
+    allow:
+      - dependency-type: 'production'
     ignore:
       - dependency-name: '*'
         update-types: ['version-update:semver-major']
@@ -56,6 +64,8 @@ updates:
     labels:
       - 'dependencies'
       - 'front'
+    allow:
+      - dependency-type: 'production'
     ignore:
       - dependency-name: '*'
         update-types: ['version-update:semver-major']

.github/workflows/production.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: 'Get current time'
         id: get_current_time
@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Extract release version
         id: extract_version
@@ -83,7 +83,7 @@ jobs:
       - release_frontend
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Extract release version
         id: extract_version
@@ -120,7 +120,7 @@ jobs:
     if: success()
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: 'Send notification to slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
@@ -140,7 +140,7 @@ jobs:
     if: failure()
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: 'Send notification to slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:

.github/workflows/staging.yaml

@@ -11,7 +11,7 @@ on:
       - 'cypress/**'
       - 'cypress.config.json'
       - 'services/**'
-      - '!services/backend/src/rapodiff.js'
+      - '!services/backend/src/rapodiff/**'
       - 'docker-compose.ci.yml'
       - 'package*.json'
       - 'updater/sis-updater-worker/src/db/migrations/**'
@@ -29,7 +29,7 @@ jobs:
     needs: test
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Get current time
         id: get_current_time
@@ -65,7 +65,7 @@ jobs:
     needs: test
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Build backend image
         id: build-image
@@ -100,9 +100,9 @@ jobs:
     if: success()
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: 'Send notification to slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -119,9 +119,9 @@ jobs:
     if: failure()
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: 'Send notification to slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure

.github/workflows/test.yaml

@@ -16,10 +16,10 @@ jobs:
     name: 'Lint backend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3.4.1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
-          node-version: 14
+          node-version: 18
       - name: 'Install dependencies'
         run: npm ci && cd services/backend && npm ci
       - name: 'Run backend linter'
@@ -29,10 +29,10 @@ jobs:
     name: 'Lint frontend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3.4.1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
-          node-version: 14
+          node-version: 18
       - name: 'Install dependencies'
         run: npm ci && cd services/frontend && npm ci
       - name: 'Run frontend linter'
@@ -45,10 +45,10 @@ jobs:
     name: 'Build backend'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
       - name: 'Build image and export it'
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           build-args: |
             SENTRY_RELEASE=${{ github.sha }}
@@ -66,8 +66,8 @@ jobs:
     name: 'Build anon sis-db and push to toska hub if new migrations have been added'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
         id: check_migrations
         with:
           filters: |
@@ -75,7 +75,7 @@ jobs:
               - 'updater/sis-updater-worker/src/db/migrations/**'
               - '.github/workflows/test.yaml'
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
@@ -93,8 +93,8 @@ jobs:
       - build_backend
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
         id: check_migrations
         with:
           filters: |
@@ -108,7 +108,7 @@ jobs:
       - name: 'Load built backend image'
         run: docker load --input /tmp/artifact/backend.tar
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
@@ -126,8 +126,8 @@ jobs:
       - build_backend
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
         id: check_migrations
         with:
           filters: |
@@ -141,7 +141,7 @@ jobs:
       - name: 'Load built backend image'
         run: docker load --input /tmp/artifact/backend.tar
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}
@@ -154,167 +154,51 @@ jobs:
           ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
 
   # === End-to-end tests ===
-  test_course_statistics:
-    name: 'Test course statistics'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Course_statistics.js
-
-  test_custom_population:
-    name: 'Test custom population'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Custom_population.js
-
-  test_population_statistics:
-    name: 'Test population statistics'
+  test_end_to_end:
     runs-on: ubuntu-latest
     needs:
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: 'Test course statistics'
+            spec: cypress/e2e/Course_statistics.js
+          - name: 'Test custom population'
+            spec: cypress/e2e/Custom_population.js
+          - name: 'Test population statistics'
+            spec: cypress/e2e/Population_statistics.js
+          - name: 'Test filters'
+            spec: cypress/e2e/Filters.js
+          - name: 'Test students'
+            spec: cypress/e2e/Students.js
+          - name: 'Test completed courses'
+            spec: cypress/e2e/Completed_courses.js
+          - name: 'Test studyprogramme overview'
+            spec: cypress/e2e/Studyprogramme_overview.js
+          - name: 'Test teachers and users'
+            spec: |
+              cypress/e2e/Teachers.js
+              cypress/e2e/Users.js
+          - name: 'Test faculty statistics'
+            spec: cypress/e2e/Faculty_statistics.js
+          - name: 'Test open uni search'
+            spec: cypress/e2e/Openuni.js
+          - name: 'Test language center view'
+            spec: cypress/e2e/Language_center.js
+          - name: 'Test study guidance groups'
+            spec: cypress/e2e/Study_guidance_groups.js
+    name: ${{ matrix.name }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
-          spec: cypress/e2e/Population_statistics.js
-
-  test_filters:
-    name: 'Test filters'
-    runs-on: ubuntu-latest
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Filters.js
-
-  test_student_statistics:
-    name: 'Test student statistics'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Student_statistics.js
-
-  test_completed_courses:
-    name: 'Test completed courses'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Completed_courses.js
-
-  test_studyprogramme_overview:
-    name: 'Test studyprogramme overview'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Studyprogramme_overview.js
-
-  test_teachers_and_users:
-    name: 'Test teachers and users'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: |
-            cypress/e2e/Teachers.js
-            cypress/e2e/Users.js
-
-  test_faculty_statistics:
-    name: 'Test faculty statistics'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Faculty_statistics.js
-
-  test_open_uni_search:
-    name: 'Test open uni search'
-    needs:
-      - build_and_push_anon_sis_db
-      - build_and_push_anon_kone_db
-      - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Run tests'
-        uses: ./.github/actions/run-tests
-        with:
-          username: ${{ secrets.username }}
-          password: ${{ secrets.password }}
-          spec: cypress/e2e/Openuni.js
+          spec: ${{ matrix.spec }}
 
   # === Other tests ===
   test_service_exit_statuses:
@@ -325,9 +209,9 @@ jobs:
       - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: 'Login to toska docker registry'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.username }}

.github/workflows/updater.yaml

@@ -14,10 +14,10 @@ jobs:
     name: 'Build Updater worker'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
       - name: 'Build the image'
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: updater/sis-updater-worker/
           push: false
@@ -32,10 +32,10 @@ jobs:
     name: 'Build Updater scheduler'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
       - name: 'Build the image'
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: updater/sis-updater-scheduler/
           push: false
@@ -46,62 +46,57 @@ jobs:
         with:
           path: /tmp/scheduler.tar
 
-  # test_updater:
-  #   name: 'Test Updater'
-  #   runs-on: ubuntu-latest
-  #   needs:
-  #     - build_worker
-  #     - build_scheduler
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - name: 'Login to toska docker registry'
-  #       uses: docker/login-action@v2
-  #       with:
-  #         registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
-  #         username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-  #         password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-  #     - name: 'Download built images'
-  #       uses: actions/download-artifact@v3
-  #       with:
-  #         path: /tmp/
-  #     - name: 'Load worker image'
-  #       run: docker load --input /tmp/artifact/worker.tar
-  #     - name: 'Load scheduler image'
-  #       run: docker load --input /tmp/artifact/scheduler.tar
-  #     - name: 'Start worker'
-  #       run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
-  #     - name: 'Run hourly jobs'
-  #       run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-  #       env:
-  #         SCHEDULE_IMMEDIATE: hourly
-  #         EXIT_AFTER_IMMEDIATES: yes
-  #     - name: 'Run daily jobs'
-  #       run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-  #       env:
-  #         SCHEDULE_IMMEDIATE: daily
-  #         EXIT_AFTER_IMMEDIATES: yes
-  #     - name: 'Run weekly jobs'
-  #       run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-  #       env:
-  #         SCHEDULE_IMMEDIATE: weekly
-  #         EXIT_AFTER_IMMEDIATES: yes
-  #     - name: 'Run prepurge'
-  #       run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-  #       env:
-  #         SCHEDULE_IMMEDIATE: prepurge
-  #         EXIT_AFTER_IMMEDIATES: yes
-  #     - name: 'Run purge'
-  #       run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
-  #       env:
-  #         SCHEDULE_IMMEDIATE: purge
-  #         EXIT_AFTER_IMMEDIATES: yes
+  test_updater:
+    name: 'Test Updater'
+    runs-on: ubuntu-latest
+    needs:
+      - build_worker
+      - build_scheduler
+    steps:
+      - uses: actions/checkout@v4
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v3
+        with:
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Download built images'
+        uses: actions/download-artifact@v3
+        with:
+          path: /tmp/
+      - name: 'Load worker image'
+        run: docker load --input /tmp/artifact/worker.tar
+      - name: 'Load scheduler image'
+        run: docker load --input /tmp/artifact/scheduler.tar
+      - name: 'Start worker'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
+      - name: 'Run hourly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: hourly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run weekly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: weekly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run prepurge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: prepurge
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run purge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: purge
+          EXIT_AFTER_IMMEDIATES: yes
 
   release_worker:
     name: 'Build and release Updater worker'
     runs-on: ubuntu-latest
-    # needs: test_updater
+    needs: test_updater
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Build worker image
         id: build-image
@@ -125,10 +120,10 @@ jobs:
   release_scheduler:
     name: 'Build and release Updater scheduler'
     runs-on: ubuntu-latest
-    # needs: test_updater
+    needs: test_updater
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Build scheduler image
         id: build-image

README.md

@@ -61,6 +61,7 @@ graph TB
             client[React app] --> server[NodeJS server]
             server --> ok_redis(Redis)
             konedb[(kone-db)]
+            userdb[(user-db)]
         end
         server -->|Read| sisdb[(sis-db)]
         subgraph Updater
@@ -118,6 +119,7 @@ Some useful commands are defined in `package.json` and can be run with `npm run
 - `npm run docker:down`: stops the whole environment
 - `npm run testupdater`: runs tests for updater
 - `npm run flushredis`: Clears redis, which forces big calculations to be redone in e.g. studyprogramme overview and faculty views.
+- `npm run rapodiff`: runs [Rapodiff](./services/backend/src/rapodiff/)
 
 Once you have ran setup for oodikone , you can just execute the first one (`npm run oodikone`). After starting and waiting for a while for containers to compile, oodikone can be accessed at [http://localhost:3000/](http://localhost:3000/) and Adminer (database investigation tool) at [http://localhost:5050/](http://localhost:5050/). Adminer requires you to login with username `postgres` and with any password you choose (for example `p`).
 
@@ -170,6 +172,9 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 - After a successful test run, oodikone is deployed to staging
 - After creating a release, oodikone is deployed to production
 - Updater is deployed to production when pushed if tests pass
+- Test databases can be updated, instructions in [anonyymioodi README](anonyymioodi/README.md)
+
+[Rapodiff](./services/backend/src/rapodiff/) can be used to check differences between Rapo API and Oodikone data.
 
 ## ❓FAQ
 

anonyymioodi/README.md

@@ -1,65 +1,18 @@
-# Anonyymioodi - anonymous database setup for oodikone
+# Anonyymioodi - test database setup
 
-(name anonyymioodi for :culture: reasons)
+This folder contains a guide and scripts to modify or create anonymous database images for oodikone, mainly used for testing.
 
-This folder contains a guide and scripts to create anonymous database images for oodikone, mainly used for testing.
+This quick guide should be enough for most purposes: Updating schema, data or postgres version. If not, a more elaborate guide is stored in [old readme](old_readme.md) (though it may have some outdated info).
 
-## How to create / modify database images
+### Quick guide to update a test database
 
-Creating / modifying images varies a bit depending on the db. Generally, to modify images, you can either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose. 
-
-By default, previous db images from Toska Hub are used as a base. If needed, comment out Toska hub image and uncomment corresponding clean postgres image in docker-compose before running other steps.
-
-See more details on how to create / modify databases below.
-
-### Sis-importer-db
-
-The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
-
-- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
-- Move `sis-importer-db.sqz` to this folder
-- Run `./create_container_from_dump.sh sis-importer-db`
-- Enrich db with command in file `enrich_data.sql` either by piping command to container or by running command inside container.
-
-### Sis-db:
-
-The following steps are for building a clean sis-db image from scratch using oodikone's updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
-
-- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
-- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
-- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
-- Start updater, see oodikone readme for more info.
-- Populate sis-db from sis-importer-db:
-
-```
-curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
-curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
-curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
-```
-(or run these from frontend. You might also want to follow sis-updater-workers logs to ensure things are running smoothly.)
-- Clean up any extra stuff you don't want to keep in sis-db.
-- Bonus: check that cypress tests run okay
-
-### User-db / kone-db
-
-- Temporarily replace the db service in oodikone's docker-compose.yml with the same db service from this folder's docker-compose.yml
-- Ensure the newest user-db/kone-db image is pulled from the registry with `docker-compose pull`
-- Modify db to wanted state. Tips:
-  - Run oodikone if you want to upgrade something from frontend (e.g. tags in kone-db, courses in sis-db etc.).
-  - Easy way to create new users to user-db is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
-
-## Publishing the image
-
-After completing the steps above and making sure your local version has the correct data, run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the db image to Toska Hub.
-
-## How to update Postgres versions
-
-When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
-
-- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
-- Start database with docker-compose
-- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
-- Close container with docker-compose down.
-- Update the Postgres version in docker-compose.yml in this folder.
-- Uncomment row with Postgres image and comment row with Toskahub image.
-- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`
+- Make sure you have the latest test-db. Just pulling images doesn't seem to replace your already existing volume, so you may need to remove it by hand. This may work: `docker volume rm oodikone_sis-db-data`. The new image including data should be pulled next time you launch oodikone normally.
+- Do your dev stuff normally with the test database, for example with `npm run both`. Edit data however you like, run updater via frontend for example, then run tests to see if they pass.
+- When you're happy with the database, create a dump:
+- `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > anonyymioodi/<dbname>.sqz`
+- Now you can shut it down `npm run docker:down`
+- Navigate to this folder `cd anonyymioodi`
+- If you need to **update postgres version**, do it now by changing it in this directory's `docker-compose.yml`.
+- Create new container from dump `./create_container_from_dump.sh <dbname>`
+- Commit and push it to toska registry `./commit_and_push_to_toska_hub.sh <dbname>`
+- `docker compose down`

anonyymioodi/docker-compose.yml

@@ -9,8 +9,7 @@ services:
       PGDATA: /data
       POSTGRES_DB: kone-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
-    # image: postgres:13.4-alpine
+    image: postgres:15.3
     profiles:
       - oodikone
 
@@ -21,8 +20,7 @@ services:
       PGDATA: /data
       POSTGRES_DB: sis-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    # image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
-    image: postgres:13.2
+    image: postgres:15.3
     shm_size: 2gb
 
   sis-importer-db:
@@ -32,8 +30,7 @@ services:
       PGDATA: /data
       POSTGRES_DB: sis-importer-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    # image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
-    image: postgres:13.1
+    image: postgres:15.3
     profiles:
       - updater
 
@@ -44,7 +41,6 @@ services:
       PGDATA: /data
       POSTGRES_DB: user-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
-    # image: postgres:13.4-alpine
+    image: postgres:15.3
     profiles:
       - oodikone

anonyymioodi/old_readme.md

@@ -0,0 +1,71 @@
+# Old readme of anonyymioodi
+
+This is mostly obsolete, but may help with something sometimes.
+
+## Anonyymioodi - anonymous database setup for oodikone
+
+(name anonyymioodi for :culture: reasons)
+
+This folder contains a guide and scripts to create anonymous database images for oodikone, mainly used for testing.
+
+## How to create / modify database images
+
+Creating / modifying images varies a bit depending on the db. Generally, to modify images, you can either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose.
+
+By default, previous db images from Toska Hub are used as a base. If needed, comment out Toska hub image and uncomment corresponding clean postgres image in docker-compose before running other steps.
+
+See more details on how to create / modify databases below.
+
+### Sis-importer-db
+
+The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
+- Move `sis-importer-db.sqz` to this folder
+- Run `./create_container_from_dump.sh sis-importer-db`
+- Enrich db with command in file `enrich_data.sql` either by piping command to container or by running command inside container.
+
+### Sis-db:
+
+The following steps are for building a clean sis-db image from scratch using oodikone's updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
+- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
+- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
+- Start updater, see oodikone readme for more info.
+- Populate sis-db from sis-importer-db:
+
+```
+curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
+```
+
+(or run these from frontend. You might also want to follow sis-updater-workers logs to ensure things are running smoothly.)
+
+- Clean up any extra stuff you don't want to keep in sis-db.
+- Bonus: check that cypress tests run okay
+
+### User-db / kone-db
+
+- Temporarily replace the db service in oodikone's docker-compose.yml with the same db service from this folder's docker-compose.yml
+- Ensure the newest user-db/kone-db image is pulled from the registry with `docker-compose pull`
+- Modify db to wanted state. Tips:
+  - Run oodikone if you want to upgrade something from frontend (e.g. tags in kone-db, courses in sis-db etc.).
+  - Easy way to create new users to user-db is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
+
+## Publishing the image
+
+After completing the steps above and making sure your local version has the correct data, run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the db image to Toska Hub.
+
+## How to update Postgres versions
+
+When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
+
+- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
+- Start database with docker-compose
+- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
+- Close container with docker-compose down.
+- Update the Postgres version in docker-compose.yml in this folder.
+- Uncomment row with Postgres image and comment row with Toskahub image.
+- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`

cypress/e2e/Completed_courses.js

@@ -48,7 +48,7 @@ describe('When search modal is opened', () => {
     hasLanded()
     openCompletedCoursesModal()
     cy.contains('Search completed courses of students')
-    cy.contains('Insert one or more student numbers, separated by a space, a newline, or a comma.')
+    cy.contains('Insert one or more student numbers, separated by a space, a newline, a comma, or a semicolon.')
     cy.contains('Insert one or more courses, separated by a space, a newline, or a comma.')
     cy.contains('Insert name for this course list if you wish to save it')
   })
@@ -73,7 +73,7 @@ describe('When search modal is opened', () => {
   })
 
   describe('When a search with correct data is executed', () => {
-    before(() => {
+    beforeEach(() => {
       cy.init('/completedcoursessearch')
       hasLanded()
       openCompletedCoursesModal()
@@ -90,14 +90,8 @@ describe('When search modal is opened', () => {
         '/completedcoursessearch?courseList=TKT10001&courseList=TKT10002&courseList=TKT10003&courseList=TKT10004&studentList=010450938&studentList=010589388&studentList=010614509'
       )
     })
-    /*
-    TODO: FIX
-    This is skipped because it's flaky in github actions for unknown reasons.
-    Instead of the results, a text appears saying "Something went wrong, please try refreshing the page".
-    */
-    it.skip('Finds correct students and courses', () => {
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(5000)
+
+    it('Finds correct students and courses', () => {
       cy.fixture('completedCoursesData').then(({ studentSet1, coursesSet1 }) => {
         cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', studentSet1[0])
         cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', studentSet1[1])

cypress/e2e/Course_statistics.js

@@ -1,5 +1,26 @@
 /// <reference types="Cypress" />
 
+const checkGradeTable = gradesTableContents => {
+  cy.get('[data-cy="Grade distribution"]')
+    .parent()
+    .siblings('.content.active')
+    .get('table > tbody')
+    .first()
+    .within(() => {
+      gradesTableContents.forEach((values, trIndex) => {
+        cy.get('tr')
+          .eq(trIndex)
+          .within(() => {
+            values.forEach((value, tdIndex) => {
+              if (value === null) return
+              cy.get('td').eq(tdIndex).contains(value)
+            })
+          })
+      })
+      cy.get('tr').should('have.length', gradesTableContents.length)
+    })
+}
+
 describe('Course Statistics tests', () => {
   describe('when using basic user', () => {
     beforeEach(() => {
@@ -7,6 +28,56 @@ describe('Course Statistics tests', () => {
       cy.url().should('include', '/coursestatistics')
     })
 
+    describe('It shows correct statistics for courses with other grades than 0-5', () => {
+      it('Shows correct statistics for courses with scale HT-TT', () => {
+        const gradesTableContents = [
+          [null, 'TT', 190],
+          [null, 'Hyl.', 4],
+          [null, 'HT', 177],
+        ]
+        cy.contains('Search for courses')
+        cy.get("input[placeholder='Search by a course code']").type('KK-RUKIRJ')
+        cy.contains('td', 'KK-RUKIRJ').click()
+        cy.contains('Search for courses').should('not.exist')
+        cy.contains('KK-RUKIRJ, 992912, AYKK-RUKIRJ Toisen kotimaisen kielen kirjallinen taito, ruotsi (CEFR B1)')
+        cy.contains('.tabular.menu a', 'Grade distribution').click()
+        cy.contains('svg', 'Grades')
+        cy.contains('Show population').should('be.enabled').click()
+        checkGradeTable(gradesTableContents)
+      })
+
+      it("Shows correct statistics for old master's thesis grade scales", () => {
+        const gradesTableContents = [
+          [null, 'L', 1],
+          [null, 'CL', 1],
+        ]
+        cy.contains('Search for courses')
+        cy.get("input[placeholder='Search by a course code']").type('704220')
+        cy.contains('td', '704220').click()
+        cy.contains('Search for courses').should('not.exist')
+        cy.contains('704220 Taloustiede. Pro gradu -tutkielma')
+        cy.contains('.tabular.menu a', 'Grade distribution').click()
+        cy.contains('svg', 'Grades')
+        cy.contains('Show population').should('be.enabled').click()
+        checkGradeTable(gradesTableContents)
+      })
+
+      it('Shows correct statistics for courses with scale passed-failed', () => {
+        const gradesTableContents = [[null, 'Hyv.', 258]]
+        cy.contains('Search for courses')
+        cy.get("input[placeholder='Search by a course code']").type('KK-ENMALU1')
+        cy.contains('td', 'KK-ENMALU1').click()
+        cy.contains('Search for courses').should('not.exist')
+        cy.contains(
+          'KK-ENFARM1, KK-ENOIK1, KK-ENLAAK1, KK-ENMALU1, 995010Mat-lu Academic and Professional Communication in English 1 (CEFR B2)'
+        )
+        cy.contains('.tabular.menu a', 'Grade distribution').click()
+        cy.contains('svg', 'Grades')
+        cy.contains('Show population').should('be.enabled').click()
+        checkGradeTable(gradesTableContents)
+      })
+    })
+
     it('Searching single course having substitution mappings shows course statistics', () => {
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by a course code']").type('TKT20001')
@@ -18,15 +89,15 @@ describe('Course Statistics tests', () => {
       cy.contains('TKT20001')
       cy.contains('58131') // old mapped code
 
-      cy.contains('.tabular.menu a', 'Table').click()
+      cy.contains('.tabular.menu a', 'Students').click()
       cy.contains('All')
-      cy.cs('viewMode-Attempts').click()
-      cy.cs('viewMode-Students').click()
+      cy.contains('svg', 'Pass rate')
 
-      cy.contains('.tabular.menu a', 'Pass rate chart').click()
-      cy.contains('svg', 'Pass rate chart')
+      cy.contains('.tabular.menu a', 'Attempts').click()
+      cy.contains('All')
+      cy.contains('svg', 'Pass rate')
 
-      cy.contains('.tabular.menu a', 'Grade distribution chart').click()
+      cy.contains('.tabular.menu a', 'Grade distribution').click()
       cy.contains('svg', 'Grades')
 
       cy.contains('a', 'New query').click()
@@ -59,6 +130,27 @@ describe('Course Statistics tests', () => {
       cy.contains('Search for courses')
     })
 
+    it('On searches with multiple courses, has correct links on the Course tab', () => {
+      cy.cs('select-multiple-courses-toggle').click()
+      cy.get("input[placeholder='Search by a course code']").type('TKT')
+      cy.contains('td', /^TKT20001/).click()
+      cy.contains('td', /^TKT10002/).click()
+      cy.contains('Fetch statistics').click()
+      cy.contains('Search for courses').should('not.exist')
+
+      cy.get('.ui.contentSegment .ui.menu').contains('Course').click()
+      cy.cs('course-selector').get('.active.selected.item').contains('TKT10002')
+      cy.get('#CourseStatPanes table a.item:first').click()
+      cy.contains('Population of course Ohjelmoinnin perusteet')
+
+      cy.go('back')
+      cy.get('.ui.contentSegment .ui.menu').contains('Course').click()
+      cy.cs('course-selector').click()
+      cy.cs('course-selector').contains('TKT20001').click()
+      cy.get('#CourseStatPanes table a.item:first').click()
+      cy.contains('Population of course Tietorakenteet ja algoritmit')
+    })
+
     it('On consecutive searches should not crash and search should work', () => {
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by a course code']").type('TKT20003')
@@ -66,7 +158,7 @@ describe('Course Statistics tests', () => {
       cy.contains('Show population')
       cy.contains('TKT20003')
 
-      cy.contains('Course statistics').click()
+      cy.contains('Courses').click()
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by a course code']").type('TKT20001')
       cy.contains('td', 'TKT20001').click()
@@ -103,6 +195,30 @@ describe('Course Statistics tests', () => {
       cy.contains('No results')
     })
 
+    it('"Select all search results" button is not showing unless "Select multiple courses" toggle is on', () => {
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT')
+      cy.get('[data-cy="select-multiple-courses-toggle"]').should('not.have.class', 'checked')
+      cy.contains('TKT10004, 581328')
+      cy.contains('Select all search results').should('not.exist')
+      cy.get('[data-cy="select-multiple-courses-toggle"]').click()
+      cy.get('[data-cy="select-multiple-courses-toggle"]').should('have.class', 'checked')
+      cy.contains('Select all search results')
+    })
+
+    // Skipped for now, due to limit being removed. Subject to change after testing in production
+    it.skip('"Fetch statistics" button is disabled if over 40 courses are selected', () => {
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT')
+      cy.get('[data-cy="select-multiple-courses-toggle"]').should('not.have.class', 'checked').click()
+      cy.get('[data-cy="select-multiple-courses-toggle"]').should('have.class', 'checked')
+      cy.contains('Select all search results').click()
+      cy.contains('Fetch statistics').should('not.be.enabled').trigger('mouseover', { force: true })
+      cy.contains('Voit valita tarkasteltavaksi enintään 40 kurssia kerrallaan.')
+      cy.get("input[placeholder='Search by a course code']").type('21')
+      cy.contains('Fetch statistics').should('be.enabled')
+    })
+
     it('Provider organization toggle works', () => {
       cy.contains('Search for courses')
       cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
@@ -111,7 +227,7 @@ describe('Course Statistics tests', () => {
       cy.contains('Search for courses').should('not.exist')
 
       cy.contains('TKT10004, 581328, AYTKT10004, A581328 Tietokantojen perusteet')
-      cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.get('[data-cy=unify_radio_regular]').click()
       cy.contains('TKT10004, 581328 Tietokantojen perusteet')
     })
 
@@ -164,7 +280,7 @@ describe('Course Statistics tests', () => {
       cy.get("input[placeholder='Search by a course code']").type('TKT20003')
       cy.contains(/^TKT20003, 582219$/).click()
       cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
-      cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.get('[data-cy=unify_radio_regular]').click()
       cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
       cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
       cy.contains('Language distribution').click()
@@ -215,7 +331,6 @@ describe('Course Statistics tests', () => {
         cy.get("div[role='option']").should('have.length', 11)
       })
 
-      cy.contains('#CourseStatPanes a.item', 'Table').click()
       cy.contains('#CourseStatPanes a.item', 'Attempts').click()
       cy.get('#CourseStatPanes table>tbody').within(() => {
         attemptsTableContents.forEach((values, trIndex) => {
@@ -263,16 +378,6 @@ describe('Course Statistics tests', () => {
         ['Total', null, 511, 486, 25],
       ]
 
-      const attemptsTableContentsOpen = [
-        // [time, passed, failed, passrate]
-        ['Total', null, 175, 175, 0],
-      ]
-
-      const attemptsTableContentsReqular = [
-        // [time, passed, failed, passrate]
-        ['Total', null, 312, 298, 14],
-      ]
-
       const gradesTableContents = [
         // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
         ['Total', null, 511, 25, 11, 17, 21, 59, 365, 13],
@@ -300,7 +405,6 @@ describe('Course Statistics tests', () => {
         })
         cy.contains('Show population').should('not.be.enabled')
 
-        cy.contains('#CourseStatPanes a.item', 'Table').click()
         cy.contains('#CourseStatPanes a.item', 'Attempts').click()
         cy.get('#CourseStatPanes table>tbody').within(() => {
           attemptsTableContents.forEach((values, trIndex) => {
@@ -331,44 +435,6 @@ describe('Course Statistics tests', () => {
         })
       })
 
-      it('Toggling course provider changes stats correctly', () => {
-        cy.get('[data-cy=unify_radio_reqular]').click()
-        cy.get("div[name='toYear']").within(() => {
-          cy.get("div[role='option']").first().should('have.text', '2020-2021')
-          cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
-          cy.get("div[role='option']").last().should('have.text', yearRange.from)
-          cy.get("div[role='option']").should('have.length', 21)
-        })
-        cy.contains('#CourseStatPanes a.item', 'Table').click()
-        cy.contains('#CourseStatPanes a.item', 'Attempts').click()
-        cy.get('#CourseStatPanes table>tbody').within(() => {
-          attemptsTableContentsReqular.forEach((values, trIndex) => {
-            cy.get('tr')
-              .eq(trIndex)
-              .within(() => {
-                values.forEach((value, tdIndex) => {
-                  if (value === null) return
-                  cy.get('td').eq(tdIndex).contains(value)
-                })
-              })
-          })
-        })
-        cy.get('[data-cy=unify_radio_open]').click()
-        cy.get('#CourseStatPanes table>tbody').within(() => {
-          attemptsTableContentsOpen.forEach((values, trIndex) => {
-            cy.get('tr')
-              .eq(trIndex)
-              .within(() => {
-                values.forEach((value, tdIndex) => {
-                  if (value === null) return
-                  cy.get('td').eq(tdIndex).contains(value)
-                })
-              })
-          })
-        })
-        cy.get('[data-cy=unify_radio_unify]').click()
-      })
-
       it('After changing time range shows same stats', () => {
         const newYearRange = { from: '2016-2017', to: '2019-2020' }
         cy.get("div[name='fromYear']")
@@ -403,11 +469,11 @@ describe('Course Statistics tests', () => {
       cy.get("input[placeholder='Search by a course code']").type('AYCSM14111')
       cy.contains('AYCSM14111').click()
       cy.contains('AYCSM14111 Avoin yo: Full Stack -websovelluskehitys: React Native')
-      cy.get('[data-cy=unify_radio_reqular]').find('input').should('be.disabled')
+      cy.get('[data-cy=unify_radio_regular]').find('input').should('be.disabled')
       cy.get('[data-cy=unify_radio_unify]').find('input').should('not.be.disabled')
     })
 
-    it('Has right to see all the sudents, because course provider is TKT', () => {
+    it('Has right to see all the students, because course provider is TKT', () => {
       cy.visit('coursestatistics?courseCodes=%5B%22TKT10004%22%5D&cs_tab=0&separate=false')
       cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
       cy.contains('Students (154)').click()
@@ -415,7 +481,6 @@ describe('Course Statistics tests', () => {
     })
   })
 
-  // ! This test does not pass locally but it does pass on GitHub Actions (see issue #4317)
   it('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
     cy.init('/coursestatistics', 'onlycoursestatistics')
     cy.get('[data-cy=navbar-courseStatistics]').click()
@@ -424,8 +489,7 @@ describe('Course Statistics tests', () => {
     cy.contains('Filter statistics by study programmes').should('not.exist')
     cy.contains('Faculty statistics').should('not.exist')
     cy.contains('Show population').should('not.exist')
-
-    cy.cs('viewMode-Attempts').click()
+    cy.contains('.tabular.menu a', 'Attempts').click()
 
     const attemptsTableContents = [
       // [time, passed, failed, passrate, total enrollments, enrolled, rejected, aborted]

cypress/e2e/Custom_population.js

@@ -18,7 +18,7 @@ const save = () => {
 }
 
 const selectSavedPopulation = name => {
-  cy.get('[data-cy="history-search"]').children().eq(0).type(name).type('{enter}')
+  cy.get('[data-cy="history-search"]').children().eq(0).type(`${name}{enter}`)
 }
 
 const deleteAllSearches = () => {
@@ -33,7 +33,7 @@ const deleteAllSearches = () => {
       const searchItems = d.find('div[role=option] > span[class=text]')
       for (let i = 0; i < searchItems.length; i++) {
         if (searchItems[i].textContent.includes('TEST-')) {
-          cy.get('[data-cy="history-search"]').children().eq(0).type(searchItems[i].textContent).type('{enter}')
+          cy.get('[data-cy="history-search"]').children().eq(0).type(`${searchItems[i].textContent}{enter}`)
           cy.get('button').contains('Delete').click()
         }
       }

cypress/e2e/Faculty_statistics.js

@@ -1,4 +1,3 @@
-/* eslint-disable cypress/no-unnecessary-waiting */
 /// <reference types="Cypress" />
 
 describe('Faculty overview', () => {
@@ -30,7 +29,6 @@ describe('Faculty overview', () => {
   describe('Faculty basic information: basic user', () => {
     beforeEach(() => {
       cy.init('/faculties')
-      cy.wait(3000)
       cy.contains('td', 'H80').click()
     })
     it('Basic information tab show all graphs and tables', () => {
@@ -98,9 +96,6 @@ describe('Faculty overview', () => {
     it('Study programme credit information can be toggled', () => {
       cy.get('[data-cy="Table-CreditsProducedByTheFaculty"]').should('be.visible')
 
-      // I tried to fix this without cy.wait() method without effort. Sorting the programmes
-      // or something else causes rerendering of the view and yield to failing tests.
-      cy.wait(10000)
       cy.get('[data-cy="Button-Show-CreditsProducedByTheFaculty-0"]').click()
       cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
       cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
@@ -126,8 +121,6 @@ describe('Faculty overview', () => {
 
     it('Graph stays open when sorted', () => {
       cy.get('[data-cy="Table-CreditsProducedByTheFaculty"]').should('be.visible')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
       cy.get('[data-cy="Button-Show-CreditsProducedByTheFaculty-0"]').click()
       cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
       cy.get('[data-cy="Menu-CreditsProducedByTheFaculty-Total"]').click()
@@ -138,11 +131,8 @@ describe('Faculty overview', () => {
   describe('Average graduation times', () => {
     beforeEach(() => {
       cy.init('/faculties')
-      cy.wait(3000)
       cy.contains('td', 'H60').click()
-      cy.wait(3000)
       cy.contains('Graduation times').click()
-      cy.wait(3000)
     })
 
     it('User can view graduation graphs', () => {
@@ -184,19 +174,20 @@ describe('Faculty overview', () => {
 
       cy.get('[data-cy="Section-master"]').within(() => {
         cy.get('div[class="faculty-graph"]')
-        cy.contains('1 graduated').should('have.length', 1)
-        cy.contains('1 graduated').trigger('mouseover')
-        cy.contains('1 students graduated in year 2020')
-        cy.contains('median study time: 25 months')
+        cy.contains('24 graduated').should('have.length', 1)
+        cy.contains('24 graduated').trigger('mouseover')
+        cy.contains('24 students graduated in year 2020')
+        cy.contains('median study time: 22 months')
         cy.contains('0 graduated over year late')
-        cy.contains('1 graduated max year overtime')
+        cy.contains('20 graduated on time')
+        cy.contains('4 graduated max year overtime')
 
-        cy.contains('1 graduated').click()
+        cy.contains('24 graduated').click()
         cy.contains('Year 2020 by graduation year')
         cy.get('div[class="programmes-graph"]').should('be.visible')
         cy.get('div[class="programmes-graph"]').within(() => {
           cy.contains('EDUM')
-          cy.contains('1 graduated').trigger('mouseover')
+          cy.contains('24 graduated').trigger('mouseover')
           cy.contains('Kasvatustieteiden maisteriohjelma')
           cy.contains('MH60_001')
         })
@@ -211,18 +202,21 @@ describe('Faculty overview', () => {
 
       cy.get('[data-cy="Section-master"]').within(() => {
         cy.get('div[class="faculty-graph"]')
-        cy.contains('1 graduated (100 % of class)').should('have.length', 1)
-        cy.contains('1 graduated').trigger('mouseover')
-        cy.contains('From class of 2018, 1/1 students have graduated')
+        cy.contains('19 graduated (44.2 % of class)').should('have.length', 1)
+        cy.contains('19 graduated').trigger('mouseover')
+        cy.contains('From class of 2018, 19/43 students have graduated')
       })
 
       cy.get('[data-cy="GraduationTimeToggle"]').click()
       cy.get('[data-cy="Section-master"]').within(() => {
-        cy.contains('1').click()
+        cy.contains('16').click()
         cy.get('div[class="programmes-breakdown-graph"]').should('be.visible')
         cy.get('div[class="programmes-breakdown-graph"]').within(() => {
           cy.contains('Year 2018 by start year')
           cy.contains('EDUM')
+          cy.contains('16').trigger('mouseover')
+          cy.contains('Kasvatustieteiden maisteriohjelma')
+          cy.contains('MH60_001')
         })
       })
     })
@@ -232,9 +226,7 @@ describe('Faculty overview', () => {
     beforeEach(() => {
       cy.init('/faculties')
       cy.contains('td', 'H50').click()
-      cy.wait(1000)
       cy.contains('Programmes and student populations').click()
-      cy.wait(1000)
     })
 
     it('User can view studentdata table without %', () => {

cypress/e2e/Filters.js

@@ -1,4 +1,3 @@
-/* eslint-disable cypress/no-unnecessary-waiting */
 /// <reference types="Cypress" />
 
 const MOCKED_DATE = new Date(2022, 6, 12).getTime()
@@ -71,7 +70,7 @@ describe('Population Statistics with CS master', () => {
     '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22MH50_009%22}&tag&year=2019'
   const defaultAmountOfStudents = 27
   const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
-  before(() => {
+  beforeEach(() => {
     cy.init(pathToCSMast2019)
   })
 
@@ -83,6 +82,32 @@ describe('Population Statistics with CS master', () => {
       programmeDropdown.get('i.delete').click()
     })
   })
+
+  describe('Studyright Type filter', () => {
+    it('is visible', () => {
+      cy.contains('Studyright Type')
+    })
+
+    it('is set to all by default', () => {
+      runTestStepWithPreAndPostParts('studyright-type', () => {
+        const card = cy.cs('studyright-type-filter-card')
+        card.get('[data-cy="all"] input').should('be.checked')
+      })
+    })
+
+    it('works', () => {
+      runTestStepWithPreAndPostParts('studyright-type', () => {
+        const expectedBaMaStudents = 27
+        const expectedMasterOnlyStudents = 0
+        cy.cs('studyright-type-filter-card').cs('bama').click()
+        checkFilteringResult(expectedBaMaStudents)
+        cy.cs('studyright-type-filter-card').cs('master').click()
+        checkFilteringResult(expectedMasterOnlyStudents)
+        cy.cs('studyright-type-filter-card').cs('all').click()
+        checkFilteringResult(defaultAmountOfStudents)
+      })
+    })
+  })
 })
 
 describe('Population Statistics', () => {
@@ -104,7 +129,7 @@ describe('Population Statistics', () => {
     })
   })
 
-  it('Graduation filter works', () => {
+  it('Graduation filter works', { retries: 2 }, () => {
     runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
       const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
 
@@ -169,8 +194,8 @@ describe('Population Statistics', () => {
       cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.cs('startYearAtUni-dropdown').selectFromDropdown(7)
-      checkFilteringResult(152)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(14)
+      checkFilteringResult(136)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
@@ -186,26 +211,20 @@ describe('Population Statistics', () => {
     })
   })
 
-  // This tests needs the curriculum structure and the test db doesn't yet have the same structure as the real db
-  // Although there was something wrong with the test before the change in the db (see issue #3111), must be investigated
-  it.skip('Courses filter works', () => {
-    // courses takes long time to load, wait for it to complete
-    cy.wait(25000)
-    cy.contains('Courses of class').click()
+  it('Courses filter works', () => {
     runTestStepWithPreAndPostParts('Courses', () => {
       const courses = ['TKT20002', 'MAT11002']
       cy.cs('courseFilter-course-dropdown').click().contains(`${courses[0]} - Ohjelmistotekniikka`).click()
-      checkFilteringResult(118)
+      checkFilteringResult(133)
       cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
-      checkFilteringResult(117)
+      checkFilteringResult(127)
       cy.cs('courseFilter-course-dropdown')
         .click()
         .contains(`${courses[1]} - Lineaarialgebra ja matriisilaskenta I`)
         .click()
-      checkFilteringResult(50)
+      checkFilteringResult(100)
       cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
       checkFilteringResult(1)
-      cy.contains('Courses of class').click()
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })
   })
@@ -227,11 +246,15 @@ describe('Population Statistics', () => {
     cy.cs('genderFilter-dropdown').selectFromDropdown(0)
     checkFilteringResult(42)
     cy.cs('StartYearAtUni-header').click()
-    cy.cs('startYearAtUni-dropdown').selectFromDropdown(3)
-    checkFilteringResult(39)
+    cy.cs('startYearAtUni-dropdown').selectFromDropdown(7)
+    checkFilteringResult(33)
     cy.cs('reset-all-filters').click()
     checkFilteringResult(defaultAmountOfStudents)
   })
+
+  it('Studyright Type filter is not visible', () => {
+    cy.contains('Studyright Type').should('not.exist')
+  })
 })
 
 describe('Course Statistics', () => {
@@ -296,8 +319,8 @@ describe('Course Statistics', () => {
       cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.cs('startYearAtUni-dropdown').selectFromDropdown(3)
-      checkFilteringResult(84)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(13)
+      checkFilteringResult(71)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
@@ -376,10 +399,12 @@ describe('Custom Population Statistics', () => {
     runTestStepWithPreAndPostParts('Courses', () => {
       const courses = ['MAT11001', 'TKT20004']
       cy.cs('courseFilter-course-dropdown').click().contains(`${courses[0]} - Johdatus yliopistomatematiikkaan`).click()
-      checkFilteringResult(4)
+      checkFilteringResult(5)
       cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
-      checkFilteringResult(4)
+      checkFilteringResult(5)
       cy.cs('courseFilter-course-dropdown').click().contains(`${courses[1]} - Tietoliikenteen perusteet`).click()
+      checkFilteringResult(5)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(1)
       checkFilteringResult(3)
       cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
       checkFilteringResult(1)

cypress/e2e/Language_center.js

@@ -5,18 +5,24 @@ const hasLanded = () => {
   cy.contains('Show number of')
 }
 
+/*
+  TODO FIX
+  These use dumb numbers, because actual_studyright update in db is not yet
+  in anon data, and the faculties do not get resolved.
+*/
+
 describe('When language center is opened', () => {
-  describe('with an admin user', () => {
+  describe('as an admin user', () => {
     beforeEach(() => {
       cy.init('/languagecenterview', 'admin')
       cy.get('[data-cy="completions-button"]').click()
       cy.get('[data-cy="semester-from"]').click()
       cy.contains('Syksy 2017').click()
       cy.get('[data-cy="semester-to"]').click()
+      hasLanded()
     })
 
     it('Initial view is correct', () => {
-      hasLanded()
       cy.contains('All courses total')
       cy.contains('Academic and Professional')
       cy.contains('AYKK-RUKIRJ')
@@ -24,34 +30,26 @@ describe('When language center is opened', () => {
 
     it('Faculties tab shows numbers', () => {
       cy.contains('All courses total')
-      cy.contains('1339')
-      cy.contains('832')
-      cy.contains('505')
+      cy.contains('1363')
     })
 
-    it('Faculties tab ratio button works', () => {
-      cy.get('[data-cy="ratio-button"]').click()
+    it('Faculties tab "exceeding" button works', () => {
+      cy.get('[data-cy="difference-button"]').click()
       cy.contains('0')
-      cy.contains('Total ratio')
+      cy.contains('Total')
+      cy.contains('1363').should('not.exist')
     })
 
     it('Faculties tab semester selector changes numbers', () => {
       cy.get('[data-cy="semester-from"]').click()
       cy.contains('Syksy 2020').click()
-      cy.contains('1339').should('not.exist')
-      cy.contains('3')
-      cy.contains('1')
-      cy.contains('4')
+      cy.contains('1363').should('not.exist')
       cy.contains('All courses total')
     })
 
     it('Semester tab opens and contains coloring filter', () => {
       cy.contains('By semesters').click()
-      cy.contains('50')
-      cy.contains('191')
-      cy.contains('210')
-      cy.contains('308')
-      cy.contains('1339')
+      cy.contains('1363')
       cy.contains('Coloring mode')
     })
   })
@@ -60,7 +58,7 @@ describe('When language center is opened', () => {
       cy.init('/languagecenterview', 'norights')
       cy.contains('Access denied')
       cy.contains("You're currently not allowed to see this page.")
-      cy.contains('Custom populations').click()
+      cy.contains('Special populations').click()
       cy.contains('Language center view').should('not.exist')
     })
   })

cypress/e2e/Population_statistics.js

@@ -73,20 +73,26 @@ describe('Population Statistics tests', () => {
       cy.contains('Excludes students with non-degree study right')
       cy.contains('Excludes students who have transferred out of this programme')
       cy.cs('filtered-students')
+    })
 
-      // Tests that use curriculum structure are commented out until
-      // the test database is fixed to the same structure as real database
-      // after the modules rework
-      // cy.contains('Courses of class').click()
+    it('Courses of class is displayed and link to individual course stats page works', () => {
+      cy.visit(pathToCSBach2017)
+      cy.contains('Courses of class').click()
+      cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
+      cy.get('[data-cy=toggle-group-module-TKT1]').click()
+      cy.contains('td', 'TKT10002').siblings().find('i.level.up').click()
+      cy.wait('@coursePage')
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
+    })
 
-      // cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
-      // // eslint-disable-next-line cypress/no-unnecessary-waiting
-      // cy.wait(25000) // a bit hacky way, wait until ui is ready
-      // cy.get('[data-cy=toggle-group-module-TKT1]').click()
-      // cy.contains('td', 'TKT10002').siblings().find('i.level.up').click()
-      // cy.wait('@coursePage')
-      // cy.url().should('include', '/coursestatistics')
-      // cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
+    it('Courses of class curriculum selection works', () => {
+      cy.visit(pathToCSBach2017)
+      cy.contains('Courses of class').click()
+      cy.get('Opiskelijan digitaidot - Kumpula').should('not.exist')
+      cy.get('[data-cy=curriculum-picker]').click()
+      cy.contains('2020 - 2022').click()
+      cy.contains('Opiskelijan digitaidot - Kumpula')
     })
 
     it("Empty 'tags' tab has a link to the page where tags can be created", () => {
@@ -103,14 +109,15 @@ describe('Population Statistics tests', () => {
       cy.contains('Create new tag')
     })
 
+    // This test sometimes fails on headless mode. It seems that the click on the
+    // 'Fetch class with new settings' button doesn't always trigger history.push()
+    // so the page doesn't reload. This is why waiting also doesn't help.
     it('Advanced settings work', () => {
       cy.visit(pathToCSBach2017)
       cy.get('[data-cy=advanced-toggle]').click()
       // only spring
       cy.cs('toggle-fall').click()
       cy.contains('Fetch class').click()
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000)
       cy.contains('Students (0)')
       // only fall
       cy.get('[data-cy=advanced-toggle]').click()

cypress/e2e/Students.js

@@ -1,9 +1,9 @@
 /// <reference types="Cypress" />
 
-describe('Student Statistics tests', () => {
+describe('Students tests', () => {
   beforeEach(() => {
     cy.init()
-    cy.contains('Student statistics').click()
+    cy.contains('Students').click()
     cy.contains('Show student names')
   })
 
@@ -14,7 +14,7 @@ describe('Student Statistics tests', () => {
     sis_person_id: 'hy-hlo-84830887',
   }
 
-  it('Student statistics search form is usable', () => {
+  it('Students search form is usable', () => {
     cy.contains('Show student names')
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.lastname)

cypress/e2e/Study_guidance_groups.js

@@ -0,0 +1,163 @@
+/// <reference types="Cypress" />
+
+describe('Study guidance group tests', () => {
+  describe('Study guidance group overview page', () => {
+    describe('without study guidance groups', () => {
+      it('shows the correct notification', () => {
+        cy.init('/studyguidancegroups', 'admin')
+        cy.contains('Study guidance groups')
+        cy.contains('You do not have access to any study guidance groups.')
+      })
+    })
+
+    describe('without correct rights', () => {
+      it('shows the correct notification', () => {
+        cy.init('/studyguidancegroups')
+        cy.contains('Access denied')
+        cy.contains(
+          "You're currently not allowed to see this page. Please contant oodikone@helsinki.fi, if this is a mistake."
+        )
+      })
+    })
+
+    describe('with study guidance groups', () => {
+      beforeEach(() => {
+        cy.init('/studyguidancegroups', 'onlystudyguidancegroups')
+        cy.intercept('/api/studyguidancegroups', { fixture: 'studyGuidanceGroups.json' })
+      })
+
+      it('shows the correct data', () => {
+        cy.contains('Study guidance groups')
+
+        const studyGuidanceGroupsTableContent = [
+          ['TKT kandit 2018', 3, 'Tietojenkäsittelytieteen kandiohjelma', '2018 - 2019'],
+          ['Oma ohjausryhmä', 2, 'Add study programme', 'Add year'],
+        ]
+
+        cy.checkTableStats(studyGuidanceGroupsTableContent, 'study-guidance-group-overview')
+      })
+
+      it('has working links to single study guidance groups', () => {
+        cy.cs('study-guidance-group-link-sgg-cypress-1').click()
+        cy.url().should('eq', `${Cypress.config().baseUrl}/studyguidancegroups/sgg-cypress-1`)
+      })
+    })
+  })
+
+  describe('Page for single study guidance group', () => {
+    beforeEach(() => {
+      cy.intercept('/api/studyguidancegroups', { fixture: 'studyGuidanceGroups.json' })
+    })
+
+    it('has a working back button', () => {
+      cy.init('/studyguidancegroups/sgg-cypress-1', 'onlystudyguidancegroups')
+
+      cy.get('button.ui.button').contains('Back').click()
+      cy.url().should('eq', `${Cypress.config().baseUrl}/studyguidancegroups`)
+    })
+
+    describe('without associated study programme and year', () => {
+      beforeEach(() => {
+        cy.init('/studyguidancegroups/sgg-cypress-2', 'onlystudyguidancegroups')
+      })
+
+      it('shows the correct panes', () => {
+        cy.contains('Oma ohjausryhmä')
+        cy.cs('Credit accumulation (for 2 students)')
+        cy.cs('Age distribution')
+        cy.cs('Courses of population')
+        cy.cs('Students (2)')
+        cy.cs('Credit statistics').should('not.exist')
+      })
+
+      it('students table has the correct tabs', () => {
+        cy.cs('Students (2)').click()
+        cy.get('[data-cy="student-table-tabs"] .ui.attached.tabular.menu').then($menu => {
+          const tabNames = Array.from($menu[0].children).map(elem => elem.innerText)
+          expect(tabNames).to.deep.equal(['General'])
+        })
+      })
+
+      it('general tab of the students table has the correct columns', () => {
+        const firstRowHeadings = [
+          'Phone number',
+          'Student number',
+          'Credits',
+          'Start year at uni',
+          'Study programmes',
+          'Tags',
+        ]
+        const secondRowHeadings = ['All', 'Since 1.1.1970']
+        cy.cs('Students (2)').click()
+        cy.get('[data-cy="student-table-tabs"] table thead tr').then($tr => {
+          const firstRowTexts = Array.from($tr[0].children).map(elem => elem.innerText.replace('\n', ' '))
+          expect(firstRowTexts).to.deep.equal(firstRowHeadings)
+          const secondRowTexts = Array.from($tr[1].children).map(elem => elem.innerText)
+          expect(secondRowTexts).to.deep.equal(secondRowHeadings)
+        })
+      })
+    })
+
+    describe('with associated study programme and year', () => {
+      beforeEach(() => {
+        cy.init('/studyguidancegroups/sgg-cypress-1', 'onlystudyguidancegroups')
+      })
+
+      it('shows the correct labels', () => {
+        cy.get('.ui.medium.header').contains('TKT kandit 2018')
+        cy.get('.ui.blue.tag.label:first').contains('Tietojenkäsittelytieteen kandiohjelma')
+        cy.get('.ui.blue.tag.label:last').contains('2018 - 2019')
+      })
+
+      it('shows the correct panes', () => {
+        cy.cs('Credit accumulation (for 3 students)')
+        cy.cs('Credit statistics')
+        cy.cs('Age distribution')
+        cy.cs('Courses of population')
+        cy.cs('Students (3)')
+      })
+
+      it("clicking the 'Show starting from associated year' button activates the 'Date of Course Credits' filter", () => {
+        cy.get('[data-cy="CreditDate-filter-card"][data-open="false"]')
+        cy.get('.ui.primary.button').contains('Show starting from associated year').click()
+        cy.get('[data-cy="CreditDate-filter-card"][data-open="true"]')
+        cy.get('.ui.mini.icon.button.credit-date-filter-input').contains('01.08.2018')
+        cy.contains('Reset All Filters')
+        cy.get('.ui.primary.button').contains('Show all credits').click()
+        cy.get('[data-cy="CreditDate-filter-card"][data-open="false"]')
+      })
+
+      it('students table has the correct tabs', () => {
+        cy.cs('Students (3)').click()
+        cy.get('[data-cy="student-table-tabs"] .ui.attached.tabular.menu').then($menu => {
+          const tabNames = Array.from($menu[0].children).map(elem => elem.innerText)
+          expect(tabNames).to.deep.equal(['General', 'Courses', 'Progress'])
+        })
+      })
+
+      it('general tab of the students table has the correct columns', () => {
+        const firstRowHeadings = [
+          'Phone number',
+          'Student number',
+          'Credits',
+          'Start of studyright',
+          'Started in programme',
+          'Semesters present',
+          'Graduation date',
+          'Start year at uni',
+          'Other programmes',
+          'Semesters present amount',
+          'Tags',
+        ]
+        const secondRowHeadings = ['All', 'HOPS', 'Since 1.8.2018']
+        cy.cs('Students (3)').click()
+        cy.get('[data-cy="student-table-tabs"] table thead tr').then($tr => {
+          const firstRowTexts = Array.from($tr[0].children).map(elem => elem.innerText.replace('\n', ' '))
+          expect(firstRowTexts).to.deep.equal(firstRowHeadings)
+          const secondRowTexts = Array.from($tr[1].children).map(elem => elem.innerText)
+          expect(secondRowTexts).to.deep.equal(secondRowHeadings)
+        })
+      })
+    })
+  })
+})

cypress/e2e/Studyprogramme_overview.js

@@ -45,8 +45,8 @@ describe('Studyprogramme overview', () => {
         // [Year, Started, Graduated, Transferred Away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0]),
         [2021, 0, 0, 1, 0],
-        [2020, 10, 35, 3, 2],
-        [2019, 86, 17, 0, 1],
+        [2020, 10, 35, 2, 2],
+        [2019, 86, 17, 1, 1],
         [2018, 161, 1, 0, 0],
         [2017, 171, 0, 0, 0],
       ]
@@ -60,10 +60,10 @@ describe('Studyprogramme overview', () => {
         // [Year, Total, Major credits, Non-major credits, Non-degree credits, Transferred credits]
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
         [2021, 0, 0, 0, 0, 0],
-        [2020, 5860, 5591, 102, 0, 167],
-        [2019, 8696, 7737, 587, 23, 349],
-        [2018, 7818, 6584, 837, 125, 272],
-        [2017, 3085, 2492, 153, 52, 388],
+        [2020, 4928, 4736, 25, 0, 167],
+        [2019, 7737, 7266, 99, 23, 349],
+        [2018, 6730, 6289, 84, 85, 272],
+        [2017, 2900, 2452, 33, 27, 388],
       ]
 
       cy.checkTableStats(tableContents, 'CreditsProducedByTheStudyprogramme')
@@ -103,10 +103,10 @@ describe('Studyprogramme overview', () => {
         // [Year, Total, Major students credits, Transferred credits]
         ...years.map(year => [year, 0, 0, 0]),
         [2021, 0, 0, 0],
-        [2020, 5018, 4694, 167],
-        [2019, 6185, 5702, 349],
-        [2018, 4885, 4538, 272],
-        [2017, 1988, 1600, 388],
+        [2020, 4224, 3954, 167],
+        [2019, 5812, 5357, 349],
+        [2018, 4639, 4302, 272],
+        [2017, 1968, 1580, 388],
       ]
 
       cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
@@ -144,10 +144,10 @@ describe('Studyprogramme overview', () => {
         // [Year, Total, Major students credits, Non-major students credits, Non-degree credits, Transferred credits]
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
         ['2021 - 2022', 0, 0, 0, 0, 0],
-        ['2020 - 2021', 541, 471, 10, 0, 60],
-        ['2019 - 2020', 8575, 8266, 116, 0, 193],
-        ['2018 - 2019', 10013, 8851, 791, 43, 328],
-        ['2017 - 2018', 6079, 4816, 762, 157, 344],
+        ['2020 - 2021', 412, 342, 10, 0, 60],
+        ['2019 - 2020', 7546, 7319, 34, 0, 193],
+        ['2018 - 2019', 8866, 8380, 115, 43, 328],
+        ['2017 - 2018', 5220, 4702, 82, 92, 344],
       ]
 
       cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
@@ -164,8 +164,8 @@ describe('Studyprogramme overview', () => {
         .should('contain', 'Major students credits')
         .should('contain', 'Non-major students credits')
         .should('contain', 'Transferred credits')
-        .should('contain', 7737)
-        .should('contain', 6584)
+        .should('contain', 7266)
+        .should('contain', 6289)
 
       cy.get('[data-cy=Graph-GraduatedAndThesisWritersOfTheProgramme')
         .should('contain', 'Graduated students')
@@ -191,9 +191,9 @@ describe('Studyprogramme overview', () => {
         .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
         .should('contain', 'Datatieteen maisteriohjelma')
         .should('contain', 'Matematiikan ja tilastotieteen maisteriohjelma')
-        .should('contain', 1)
-        .should('contain', 4)
-        .should('contain', 13)
+        .should('contain', 11)
+        .should('contain', 12)
+        .should('contain', 24)
     })
   })
 
@@ -243,11 +243,11 @@ describe('Studyprogramme overview', () => {
     it('Students of the studyprogramme are shown correctly', () => {
       const tableContents = [
         // [Year, All, Started studying, Currently enrolled, Absent, Inactive, Graduated, Men, Women, other/unknown Finnish, other]
-        ['2020 - 2021', 12, 10, 0, 0, 0, 0, 10, 2, 0, 12, 0],
-        ['2019 - 2020', 86, 86, 0, 0, 0, 1, 65, 21, 0, 86, 0],
-        ['2018 - 2019', 162, 161, 0, 0, 0, 11, 118, 44, 0, 161, 1],
-        ['2017 - 2018', 171, 171, 0, 0, 0, 42, 135, 36, 0, 166, 5],
-        ['Total', 431, 428, 0, 0, 0, 54, 328, 103, 0, 425, 6],
+        ['2020 - 2021', 12, 10, 0, 0, 12, 0, 10, 2, 0, 12, 0],
+        ['2019 - 2020', 86, 86, 0, 0, 85, 1, 65, 21, 0, 86, 0],
+        ['2018 - 2019', 162, 161, 0, 0, 151, 11, 118, 44, 0, 161, 1],
+        ['2017 - 2018', 171, 171, 0, 0, 129, 42, 135, 36, 0, 166, 5],
+        ['Total', 431, 428, 0, 0, 377, 54, 328, 103, 0, 425, 6],
       ]
 
       cy.checkTableStats(tableContents, 'StudytrackOverview')
@@ -259,10 +259,10 @@ describe('Studyprogramme overview', () => {
         ['2022 - 2023', 0, 0, 0, 0, 0, 0, 0, 0],
         ['2021 - 2022', 0, 0, 0, 0, 0, 0, 0, 0],
         ['2020 - 2021', 12, 12, 0, 0, 0, 0, 0, 0],
-        ['2019 - 2020', 86, 86, 0, 0, 0, 0, 0, 0],
-        ['2018 - 2019', 162, 162, 0, 0, 0, 0, 0, 0],
-        ['2017 - 2018', 171, 171, 0, 0, 0, 0, 0, 0],
-        ['Total', 431, 431, 0, 0, 0, 0, 0, 0],
+        ['2019 - 2020', 86, 26, 34, 21, 5, 0, 0, 0],
+        ['2018 - 2019', 162, 10, 27, 47, 36, 26, 12, 4],
+        ['2017 - 2018', 171, 20, 21, 26, 22, 22, 33, 27],
+        ['Total', 431, 68, 82, 94, 63, 48, 45, 31],
       ]
 
       cy.checkTableStats(tableContents, 'StudytrackProgress')
@@ -318,6 +318,8 @@ describe('Studyprogramme overview', () => {
         cy.get('tr').eq(1).contains('684')
       })
       cy.get('[data-cy=calendarAcademicYearToggle]').first().click()
+      cy.get('[data-cy=fromYear]').click().contains('2018-2019').click()
+      cy.get('[data-cy=toYear').click().contains('2019-2020').click()
       cy.get('[data-cy=CoursesSortableTable]').within(() => {
         cy.get('tr').eq(1).contains('772')
       })
@@ -401,7 +403,7 @@ describe('Studyprogramme overview', () => {
       cy.get('textarea').type('010113437')
       cy.get('.positive').click()
 
-      cy.contains('Student statistics').click()
+      cy.contains('Students').click()
       cy.get('.prompt').type(student)
       cy.contains('a', student).click()
       cy.contains(name)
@@ -410,7 +412,7 @@ describe('Studyprogramme overview', () => {
       cy.go('back')
       deleteTag(name)
 
-      cy.contains('Student statistics').click()
+      cy.contains('Students').click()
       cy.get('.prompt').type(student)
       cy.contains('a', student).click()
       cy.contains(name).should('not.exist')

cypress/e2e/Users.js

@@ -2,14 +2,14 @@
 
 describe('Users tests', () => {
   describe('Using user with just grp-oodikone-user, no other rights', () => {
-    it('shows only frontpage, trends feedback', () => {
+    it('shows only frontpage, university, feedback', () => {
       cy.init('', 'norights') // login with norights user
-      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('University').should('exist')
       cy.get('[data-cy="navBar"]').contains('Give feedback').should('exist')
 
-      cy.get('[data-cy="navBar"]').contains('Study programme').should('not.exist')
-      cy.get('[data-cy="navBar"]').contains('Student statistics').should('not.exist')
-      cy.get('[data-cy="navBar"]').contains('Course statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Programmes').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Students').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Courses').should('not.exist')
       cy.get('[data-cy="navBar"]').contains('Teachers').should('not.exist')
       cy.get('[data-cy="navBar"]').contains('Users').should('not.exist')
       cy.get('[data-cy="navBar"]').contains('Faculty').should('not.exist')
@@ -29,12 +29,12 @@ describe('Users tests', () => {
     })
 
     it('should see more stuff than others', () => {
-      cy.get('[data-cy="navBar"]').contains('Study programme').should('exist')
-      cy.get('[data-cy="navBar"]').contains('Student statistics').should('exist')
-      cy.get('[data-cy="navBar"]').contains('Course statistics').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Programmes').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Students').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Courses').should('exist')
       // TODO: admin should probably see this as well, since can access this page: cy.get('[data-cy="navBar"]').contains('Teachers').should('exist')
       cy.get('[data-cy="navBar"]').contains('Users').should('exist')
-      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('University').should('exist')
     })
 
     it("mocking normal user shows only the mocked user's programmes", () => {
@@ -42,7 +42,7 @@ describe('Users tests', () => {
       cy.cs('user-edit-button-basic').click()
       cy.get('i.spy').click()
       cy.contains('mocking as basic')
-      cy.contains('Study programme').click().siblings().contains('Search by class').click()
+      cy.contains('Programmes').click().siblings().contains('Class statistics').click()
       cy.contains('label', 'Study programme')
       cy.contains('label', 'Study programme')
         .siblings()

cypress/fixtures/studyGuidanceGroups.json

@@ -0,0 +1,28 @@
+[
+  {
+    "id": "sgg-cypress-1",
+    "name": {
+      "fi": "TKT kandit 2018"
+    },
+    "members": [
+      { "personStudentNumber": "010450938" },
+      { "personStudentNumber": "010589388" },
+      { "personStudentNumber": "010262566" }
+    ],
+    "tags": {
+      "studyProgramme": "KH50_005",
+      "year": "2018"
+    }
+  },
+  {
+    "id": "sgg-cypress-2",
+    "name": {
+      "fi": "Oma ohjausryhmä"
+    },
+    "members": [{ "personStudentNumber": "010614509" }, { "personStudentNumber": "010307146" }],
+    "tags": {
+      "studyProgramme": null,
+      "year": null
+    }
+  }
+]

cypress/support/commands.js

@@ -1,14 +1,13 @@
 /// <reference types="Cypress" />
 //
-// Note: here we need to set keys to be all lowercase, since
-// we're replacing headers after they've left browser / frontend.
+// ! IMPORTANT: here we need to set keys to be all lowercase, since
+// ! we're replacing headers after they've left browser / frontend.
 // All of these users are available in anon user-db
 const adminUserHeaders = {
   uid: 'admin',
   displayname: 'Admin User',
   'shib-session-id': 'mock-cypress-session',
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users;grp-toska',
-  edupersonaffiliation: 'member;employee;faculty',
   mail: 'grp-toska+mockadminuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-6666666',
 }
@@ -18,17 +17,15 @@ const basicUserHeaders = {
   displayname: 'Basic User',
   'shib-session-id': 'mock-cypress-session',
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
-  edupersonaffiliation: 'member',
   mail: 'grp-toska+mockbasicuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-1234567',
 }
 
 const onlycoursestatisticsUserHeaders = {
   uid: 'onlycoursestatistics',
-  displayName: 'Onlycoursestatistics User',
-  'shib-session-id': 'mock-session',
-  hyGroupCn: 'grp-oodikone-basic-users',
-  eduPersonAffiliation: 'member',
+  displayname: 'Onlycoursestatistics User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-basic-users',
   mail: 'grp-toska+mockonlycoursestatisticsuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-7654321',
 }
@@ -38,7 +35,6 @@ const norightsUserHeaders = {
   displayname: 'Norights User',
   'shib-session-id': 'mock-cypress-session',
   hygroupcn: 'grp-oodikone-users',
-  edupersonaffiliation: 'member',
   mail: 'grp-toska+mocknorightuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-0000000',
 }
@@ -48,11 +44,20 @@ const onlyIamRightsUserHeaders = {
   displayname: 'Only IAM rights user',
   'shib-session-id': 'mock-cypress-session',
   hygroupcn: 'hy-employees;hy-mltdk-tkt-jory',
-  edupersonaffiliation: 'member',
   mail: 'grp-toska+mockonlyiamrightsuser@helsinki.fi',
   hypersonsisuid: 'hy-hlo-1111111',
 }
 
+// Also has element detail of "KH50_005" for the study guidance groups to work
+const onlyStudyGuidanceGroupsUser = {
+  uid: 'onlystudyguidancegroups',
+  displayname: 'Study Guidance Groups User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users',
+  mail: 'grp-toska+mockonlystudyguidancegroupsuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-2222222',
+}
+
 // TODO add user into anon-DB (check the need for changing hypersonid)
 // REMEMBER TO ADD ROLE openUniSearch
 // const openUniUser = {
@@ -60,7 +65,6 @@ const onlyIamRightsUserHeaders = {
 //   displayname: 'Open Uni User',
 //   'shib-session-id': 'mock-cypress-session',
 //   hygroupcn: 'grp-oodikone-basics-users;hy-ypa-opa-dojo',
-//   edupersonaffiliation: 'member',
 //   mail: 'grp-toska+mockopenuniuser@helsinki.fi',
 //   hypersonsisuid: 'hy-hlo-9999999',
 // }
@@ -71,6 +75,7 @@ const userHeaders = [
   onlycoursestatisticsUserHeaders,
   norightsUserHeaders,
   onlyIamRightsUserHeaders,
+  onlyStudyGuidanceGroupsUser,
 ]
 
 /**
@@ -134,7 +139,7 @@ Cypress.Commands.add('selectStudyProgramme', name => {
 })
 
 Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {
-  cy.get(`[data-cy=Table-${tableName}]>tbody`).within(() => {
+  cy.get(`[data-cy=Table-${tableName}] tbody`).within(() => {
     correctStats.forEach((values, trIndex) => {
       cy.get('tr')
         .eq(trIndex)

docker-compose.ci.yml

@@ -21,6 +21,7 @@ services:
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
       IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
       JAMI_URL: 'http://jami:3003/'
+      CRYPT_KEY: 'e16730fce263d5270ba8dcf84d068a32556542455e71331880d6c1c5e4db9b4e'
     build:
       context: services/backend
 
@@ -70,8 +71,9 @@ services:
     <<: *defaults
     image: quay.io/toska/jami:production
     environment:
+      - npm_config_cache=/tmp
       - PORT=3003
-      - NODE_ENV=production
+      - NODE_ENV=dev
       - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
     ports:
       - 3003:3003

docker-compose.yml

@@ -23,7 +23,6 @@ services:
     build:
       <<: *build_defaults
       context: services/backend
-    container_name: backend
     depends_on:
       - kone-db
       - redis
@@ -130,8 +129,6 @@ services:
       SIS_IMPORTER_USER: postgres
       SIS_NATS_TOKEN: dev
       SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
-      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
-      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
     ports:
       - 8082:8082
     profiles:
@@ -179,6 +176,7 @@ services:
     <<: *defaults
     image: quay.io/toska/jami:production
     environment:
+      - npm_config_cache=/tmp
       - PORT=3003
       - NODE_ENV=dev
       - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres

package.json

@@ -19,7 +19,7 @@
     "testupdater": "docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start",
     "flushredis": "docker exec -it redis redis-cli flushall",
     "prepare": "husky install",
-    "rapodiff": "docker exec backend node /opt/app-root/src/src/rapodiff.js"
+    "rapodiff": "docker exec oodikone_backend_1 node /opt/app-root/src/src/rapodiff/rapodiff.js"
   },
   "repository": {
     "type": "git",
@@ -33,17 +33,16 @@
   "devDependencies": {
     "@babel/core": "^7.19.3",
     "@babel/eslint-parser": "^7.19.1",
+    "@babel/eslint-plugin": "^7.22.10",
     "@babel/plugin-syntax-jsx": "^7.16.7",
     "concurrently": "^7.4.0",
     "cypress": "^12.17.4",
     "eslint": "^8.25.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-prettier": "^8.5.0",
-    "eslint-plugin-babel": "^5.3.1",
     "eslint-plugin-cypress": "^2.12.1",
     "eslint-plugin-import": "^2.25.4",
     "eslint-plugin-jsx-a11y": "^6.5.1",
-    "eslint-plugin-prefer-object-spread": "^1.2.1",
     "eslint-plugin-prettier": "^4.2.1",
     "eslint-plugin-react": "^7.31.9",
     "eslint-plugin-react-hooks": "^4.3.0",

run.sh

@@ -83,8 +83,6 @@ parse_profiles
 parse_env
 if [[ "$compose_command" == "" ]]; then
   final_command="docker-compose ${env}"
-elif [[ "$compose_command" == *"down"* ]]; then
-  final_command="docker-compose ${env} ${compose_command}"
 else
   final_command="docker-compose ${env} ${profiles} ${compose_command}"
 fi

services/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/nodejs-14-minimal
+FROM registry.access.redhat.com/ubi9/nodejs-18-minimal
 
 ENV TZ="Europe/Helsinki"
 
@@ -7,6 +7,7 @@ WORKDIR /opt/app-root/src
 EXPOSE 8080
 
 ENV NODE_ENV=production
+ENV NODE_OPTIONS=--max-old-space-size=4096
 
 ARG BASE_PATH
 ENV PUBLIC_URL=$BASE_PATH
@@ -24,4 +25,4 @@ COPY ./package* ./
 RUN npm ci --omit-dev
 COPY . .
 
-CMD ["npm", "start"]
+CMD ["node", "index.js"]

services/backend/config/programmeCodes.js

@@ -14,6 +14,7 @@ const codes = {
   MH30_001: 'll',
   MH30_002: 'tmed',
   MH30_003: 'hll',
+  MH30_006: 'terv',
   KH40_001: 'filk',
   KH40_002: 'ttk',
   KH40_003: 'kik',
@@ -95,6 +96,7 @@ const codes = {
   MH80_005: 'hnfb',
   MH80_006: 'ekm',
   MH80_007: 'mmb',
+  MH80_008: 'emplant',
   KH90_001: 'elk',
   MH90_001: 'ell',
   T920101: 'dptheol',

services/backend/dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14.17-alpine3.13
+FROM node:18-alpine
 
 ENV TZ="Europe/Helsinki"
 
@@ -7,9 +7,10 @@ WORKDIR /opt/app-root/src
 EXPOSE 8080
 
 ENV NODE_ENV=development
+ENV NODE_OPTIONS=--max-old-space-size=4096
 
 COPY ./package* ./
 RUN npm ci
 COPY . .
 
-CMD ["npm", "run", "dev"]
+CMD ["node_modules/.bin/nodemon", "index.js"]

services/backend/package.json

@@ -11,6 +11,7 @@
     "@sentry/tracing": "^6.18.2",
     "axios": "^0.21.4",
     "bluebird": "^3.7.2",
+    "bullmq": "^4.14.2",
     "cors": "^2.8.5",
     "cron": "^1.8.2",
     "express": "^4.18.2",
@@ -20,6 +21,7 @@
     "moment": "^2.29.1",
     "moment-timezone": "^0.5.37",
     "morgan": "^1.10.0",
+    "papaparse": "^5.4.1",
     "pg": "^8.8.0",
     "redis": "^3.1.2",
     "sequelize": "^6.24.0",
@@ -29,6 +31,6 @@
     "winston-gelf-transporter": "^1.0.2"
   },
   "devDependencies": {
-    "nodemon": "^1.19.4"
+    "nodemon": "^3.0.3"
   }
 }

services/backend/src/app.js

@@ -5,6 +5,7 @@ const routes = require('./routes')
 const { startCron } = require('./events')
 const { initializeDatabaseConnection, dbConnections } = require('./database/connection')
 const logger = require('./util/logger')
+require('./worker/worker')
 
 initializeDatabaseConnection()
   .then(() => {

services/backend/src/conf-backend.js

@@ -50,8 +50,11 @@ const KONE_PASSWORD = process.env.KONE_PASSWORD || 'postgres'
 const USER_PASSWORD = process.env.USER_PASSWORD || 'postgres'
 const SIS_PASSWORD = process.env.SIS_PASSWORD || 'postgres'
 
+const CRYPT_KEY = process.env.CRYPT_KEY
+
 module.exports = {
   DB_URL_USER,
+  CRYPT_KEY,
   DB_URL_KONE,
   DB_MAX_CONNECTIONS,
   DB_MAX_CRON_CONNECTIONS,

services/backend/src/events.js

@@ -1,19 +1,18 @@
 const { CronJob } = require('cron')
-const moment = require('moment')
-const { refreshStatus, refreshStatusGraduated, refreshUber, getStartYears } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
-const { updateBasicView, updateStudytrackView } = require('./services/studyprogrammeUpdates')
+const { updateBasicView, updateStudytrackView } = require('./services/studyprogramme/studyprogrammeUpdates')
 const { findAndSaveTeachers } = require('./services/topteachers')
 const { faculties } = require('./services/organisations')
-const { combinedStudyprogrammes } = require('./services/studyprogrammeHelpers')
+const { combinedStudyprogrammes } = require('./services/studyprogramme/studyprogrammeHelpers')
 const { updateFacultyOverview, updateFacultyProgressOverview } = require('./services/faculty/facultyUpdates')
-const { isProduction } = require('./conf-backend')
+const { isProduction, runningInCI } = require('./conf-backend')
 const { getCurrentSemester } = require('./services/semesters')
 const logger = require('./util/logger')
 const { getAssociations } = require('./services/studyrights')
 const { redisClient } = require('./services/redis')
 const { computeLanguageCenterData, LANGUAGE_CENTER_REDIS_KEY } = require('./services/languageCenterData')
+const { jobMaker } = require('./worker/queue')
 
 const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
 
@@ -23,24 +22,27 @@ const refreshStudyrightAssociations = async () => {
 }
 
 const refreshFaculties = async () => {
-  logger.info('Refreshing faculty overview statistics for all faculties')
+  logger.info('Adding jobs to refresh all faculties')
   const facultyList = (await faculties()).filter(f => !['Y', 'H99', 'Y01', 'H92', 'H930'].includes(f.code))
-  let ready = 0
-
   for (const faculty of facultyList) {
-    try {
-      await updateFacultyOverview(faculty.code, 'ALL')
-      await updateFacultyProgressOverview(faculty.code)
-      ready += 1
-    } catch (e) {
-      logger.error({ message: `Failed to update stats for faculty ${faculty?.code} with type ALL`, meta: `${e}` })
-    }
-    logger.info(`${ready}/${facultyList.length} faculties done; faculty ${faculty?.code}`)
+    jobMaker.faculty(faculty.code)
   }
-  logger.info('Faculty stats refreshed!')
 }
 
-const refreshNewOverviews = async () => {
+const refreshFaculty = async code => {
+  logger.info('Started updating faculty')
+  const start = new Date().getTime()
+  try {
+    await updateFacultyOverview(code, 'ALL')
+    await updateFacultyProgressOverview(code)
+    const time = new Date().getTime() - start
+    logger.info(`Updated faculty ${code} (took ${(time / 1000 / 60).toFixed(2)} minutes)`)
+  } catch (e) {
+    logger.error({ message: `Failed to update stats for faculty ${code} with type ALL`, meta: `${e}` })
+  }
+}
+
+const refreshProgrammes = async () => {
   logger.info('Refreshing studyprogramme and studytrack overview statistics for all programmes')
   // Filters out old programmes and special ones like Fitech studies. Filters also out the programmes starting with
   // 2_ or ending with _2. Those programmes are mapped to correct programme (studentProgrammeModuleFixer.js)
@@ -52,31 +54,23 @@ const refreshNewOverviews = async () => {
         (code.includes('MH') && !code.startsWith('2_MH') && !code.endsWith('_2')) ||
         /^(T)[0-9]{6}$/.test(code)
     )
-  const associations = await getAssociations()
-  let ready = 0
+
+  // Ensure that studyright associations are refreshed before launching jobs, otherwise each job will do it
+  await getAssociations()
   for (const code of codes) {
-    let combinedProgramme = ''
     // If combined programme is given, this updates only the bachelor programme
-    try {
-      await updateBasicView(code, combinedProgramme)
-      await updateStudytrackView(code, combinedProgramme, associations)
-      ready += 1
-    } catch (e) {
-      logger.error({ message: `Failed to update overview stats for programme ${code}`, meta: `${e}` })
-    }
-    combinedProgramme = combinedStudyprogrammes[code] || ''
-    // If bachelor programme is combined with master programme, then update the combination.
-    if (combinedProgramme) {
-      try {
-        await updateBasicView(code, combinedProgramme)
-        await updateStudytrackView(code, combinedProgramme, associations)
-      } catch (e) {
-        logger.error({ message: `Failed to update overview stats for combined programme ${code}`, meta: `${e}` })
-      }
-    }
-    logger.info(`${ready}/${codes.length} programmes done: code ${code}`)
+    jobMaker.programme(code)
   }
-  logger.info('Studyprogramme and studytrack overview stats refreshed!')
+}
+
+const refreshProgramme = async code => {
+  const associations = await getAssociations()
+  await updateBasicView(code, '')
+  await updateStudytrackView(code, '', associations)
+
+  let combinedProgramme = combinedStudyprogrammes[code] || ''
+  await updateBasicView(code, combinedProgramme)
+  await updateStudytrackView(code, combinedProgramme, associations)
 }
 
 const refreshTeacherLeaderboard = async () => {
@@ -85,42 +79,6 @@ const refreshTeacherLeaderboard = async () => {
   await findAndSaveTeachers(currentSemestersYearCode - 1, currentSemestersYearCode)
 }
 
-const refreshStatusToRedis = async () => {
-  const unixMillis = moment().valueOf()
-  const date = new Date(Number(unixMillis))
-
-  date.setHours(23, 59, 59, 999)
-  const showByYearOff = 'false'
-  const showByYear = 'true'
-  logger.info('Refreshing CDS Status')
-  await refreshStatus(date.getTime(), showByYearOff)
-  await refreshStatus(date.getTime(), showByYear)
-  logger.info('Refreshing CDS Status doned')
-
-  logger.info('Refreshing CDS Graduated')
-  await refreshStatusGraduated(date.getTime(), showByYearOff)
-  await refreshStatusGraduated(date.getTime(), showByYear)
-  logger.info('Refreshing CDS graduated doned')
-}
-
-const refreshUberToRedis = async () => {
-  const years = (await getStartYears()).map(({ studystartdate }) => studystartdate)
-  for (const year of years) {
-    const formattedYear = new Date(year).getFullYear()
-    logger.info(`Refreshing CDS uber year ${formattedYear}`)
-    const defaultQuery = { include_old_attainments: 'false', start_date: year }
-    const oldAttainmentsQuery = { include_old_attainments: 'true', start_date: year }
-    try {
-      await refreshUber(defaultQuery)
-      await refreshUber(oldAttainmentsQuery)
-      logger.info(`Refreshing CDS uber year ${formattedYear} doned`)
-    } catch (e) {
-      logger.error({ message: `Error when refreshing CDS uber year ${formattedYear}`, meta: `${e}` })
-    }
-  }
-  logger.info(`Refreshing CDS Uber data doned`)
-}
-
 const refreshStatistics = async () => {
   const statfuncs = [refreshStudyrightAssociations, refreshTeacherLeaderboard]
   logger.info('Refreshing statistics')
@@ -130,30 +88,34 @@ const refreshStatistics = async () => {
   logger.info('Statistics refreshed!')
 }
 
-const refreshTrends = async () => {
-  const trendfuncs = [refreshStatusToRedis, refreshUberToRedis]
-  logger.info('Refreshing trends')
-  for (const func of trendfuncs) {
-    await func()
-  }
-  logger.info('Trends refreshed!')
-}
-
-// eslint-disable-next-line no-unused-vars
 const refreshLanguageCenterData = async () => {
+  logger.info('Refreshing language center data...')
   const freshData = await computeLanguageCenterData()
   await redisClient.setAsync(LANGUAGE_CENTER_REDIS_KEY, JSON.stringify(freshData))
   logger.info('Language center data refreshed!')
 }
 
+const dailyJobs = () => {
+  refreshFaculties()
+  refreshProgrammes()
+  jobMaker.languagecenter()
+  jobMaker.statistics()
+}
+
 const startCron = () => {
-  if (isProduction) {
-    logger.info('Cronjob for refreshing stats started: runs at 4am saturday.')
-    // refresh on saturday only because was causing lag. Investigate & fix, then return to 3am daily
-    schedule('0 4 * * 6', async () => {
-      for (const func of [refreshStatistics, refreshTrends, refreshNewOverviews, refreshFaculties]) {
-        await func()
-      }
+  if (isProduction && !runningInCI) {
+    logger.info('Cronjob for refreshing stats started: runs daily at 23:00.')
+    schedule('0 23 * * *', async () => {
+      logger.info('Running daily jobs from cron')
+      dailyJobs()
+    })
+    schedule('0 19 * * 1', async () => {
+      logger.info('Updating students whose studyplans have not been updated recently')
+      jobMaker.studyplansUpdate(4)
+    })
+    schedule('0 10 * * 2', async () => {
+      logger.info('Updating students whose studyplans have not been updated recently')
+      jobMaker.studyplansUpdate(5)
     })
   }
 }
@@ -161,8 +123,9 @@ const startCron = () => {
 module.exports = {
   startCron,
   refreshStatistics,
-  refreshTrends,
   refreshFaculties,
-  refreshNewOverviews,
+  refreshFaculty,
+  refreshProgramme,
+  refreshProgrammes,
   refreshLanguageCenterData,
 }

services/backend/src/middleware/accessLogger.js

@@ -17,13 +17,11 @@ const accessLogger = morgan((tokens, req, res) => {
   }
 
   const message = [
-    user.name,
+    `${user.mockedBy ? '(mocking) ' : ''}${user.name}`,
+    `${(tokens['response-time'](req, res) || '0').split('.')[0]} ms`.padEnd(8, ' '),
+    tokens['status'](req, res),
     tokens['method'](req, res),
     tokens['url'](req, res),
-    tokens['status'](req, res),
-    '-',
-    tokens['response-time'](req, res),
-    'ms',
   ].join(' ')
 
   const usingIamRights = user.iamRights.some(programmeCode => tokens['url'](req, res).includes(programmeCode))

services/backend/src/middleware/auth.js

@@ -4,7 +4,7 @@ const roles = requiredRoles => async (req, res, next) => {
       user: { roles },
     } = req
 
-    if (requiredRoles.every(r => roles.indexOf(r) >= 0) || roles.includes('admin')) {
+    if (requiredRoles.some(r => roles.includes(r)) || roles.includes('admin')) {
       return next()
     }
   }

services/backend/src/middleware/currentUserMiddleware.js

@@ -1,9 +1,8 @@
 const Sentry = require('@sentry/node')
 const { ApplicationError } = require('../util/customErrors')
-const { getUser, getMockedUser } = require('../services/userService')
+const { getUser, getMockedUser, getOrganizationAccess } = require('../services/userService')
 const { requiredGroup } = require('../conf-backend')
 const _ = require('lodash')
-const { getOrganizationAccess } = require('../util/organizationAccess')
 const logger = require('../util/logger')
 
 const parseIamGroups = iamGroups => iamGroups?.split(';').filter(Boolean) ?? []
@@ -23,8 +22,17 @@ const currentUserMiddleware = async (req, _res, next) => {
     uid: username,
   } = req.headers
 
-  if (!sessionId || !username) {
-    throw new ApplicationError('Not enough data in request headers', 403, { logoutUrl })
+  const missingHeaders = []
+
+  if (!sessionId) missingHeaders.push('shib-session-id')
+  if (!username) missingHeaders.push('uid')
+
+  if (missingHeaders.length > 0) {
+    throw new ApplicationError(
+      `Not enough data in request headers, the following headers were missing: ${missingHeaders.join(', ')}`,
+      403,
+      { logoutUrl }
+    )
   }
 
   const iamGroups = parseIamGroups(hygroupcn)

services/backend/src/models/courseProvider.js

@@ -1,4 +1,4 @@
-const { Model, STRING, DATE } = require('sequelize')
+const { Model, STRING, DATE, JSONB } = require('sequelize')
 const { dbConnections } = require('../database/connection')
 
 class CourseProvider extends Model {}
@@ -18,6 +18,9 @@ CourseProvider.init(
       onUpdate: 'cascade',
       onDelete: 'cascade',
     },
+    shares: {
+      type: JSONB,
+    },
     organizationcode: {
       type: STRING,
       references: {

services/backend/src/models/models_kone.js

@@ -2,11 +2,6 @@ const { STRING } = require('sequelize')
 const Sequelize = require('sequelize')
 const { sequelizeKone } = require('../database/connection')
 
-const ThesisTypeEnums = {
-  MASTER: 'MASTER',
-  BACHELOR: 'BACHELOR',
-}
-
 const ExcludedCourse = sequelizeKone.define(
   'excluded_courses',
   {
@@ -197,10 +192,8 @@ Tag.hasMany(TagStudent, { foreignKey: 'tag_id', sourceKey: 'tag_id' })
 module.exports = {
   TagStudent,
   Tag,
-  ThesisTypeEnums,
   CustomPopulationSearch,
   OpenUniPopulationSearch,
-  sequelizeKone,
   ExcludedCourse,
   StudyGuidanceGroupTag,
   ProgressCriteria,

services/backend/src/models/studyright.js

@@ -1,4 +1,4 @@
-const { Model, STRING, DATE, INTEGER, BOOLEAN } = require('sequelize')
+const { Model, STRING, DATE, INTEGER, BOOLEAN, JSONB } = require('sequelize')
 const { dbConnections } = require('../database/connection')
 
 class Studyright extends Model {}
@@ -66,6 +66,12 @@ Studyright.init(
     admissionType: {
       type: STRING,
     },
+    isBaMa: {
+      type: BOOLEAN,
+    },
+    semesterEnrollments: {
+      type: JSONB,
+    },
   },
   {
     underscored: true,

services/backend/src/rapodiff/.gitignore

@@ -0,0 +1,2 @@
+*.csv
+*.csv#

services/backend/src/rapodiff/README.md

@@ -0,0 +1,21 @@
+# Rapodiff
+
+Use this to check differences between oodikone and rapo.
+
+Before running:
+
+1. Depends on nodeproxy running in toska.cs.helsinki.fi
+2. Set these correctly in .env, notice to wrap in quotes ", otherwise strange characters may be misformatted:
+   RAPO_NODEPROXY="acual_token"
+   IMPORTER_DB_API_TOKEN="acual_token"
+
+Running:
+
+1. Have real oodikone running. Then run rapodiff: `npm run rapodiff -- args`
+2. place args with a mode. omit args to print possible modes.
+
+known differences in APIs
+
+- rapo does not show students that have graduated (it however takes some days for rapo to notice a graduation)
+- rapo does not handle the starting year of transfers properly (shows in old staring population)
+- rapo show some students incorrectly passive, see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3521

services/backend/src/rapodiff/conf.js

@@ -0,0 +1,28 @@
+// In .env, have the token wrapped in quotes! (") otherwise some signs may be mistakenly changed
+const rapoToken = process.env.RAPO_NODEPROXY
+const importerDbApiToken = process.env.IMPORTER_DB_API_TOKEN
+
+const nodeproxyUrl = `https://toska.cs.helsinki.fi`
+const importerDbApiUrl = `https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer`
+
+const bscProgrammesOfFaculties = {
+  H10: ['KH10_001'],
+  H20: ['KH20_001'],
+  H30: ['KH30_001', 'KH30_002'],
+  H40: ['KH40_001', 'KH40_002', 'KH40_003', 'KH40_004', 'KH40_005', 'KH40_006'],
+  H50: ['KH50_001', 'KH50_002', 'KH50_003', 'KH50_004', 'KH50_005', 'KH50_006', 'KH50_007', 'KH50_008'],
+  H55: ['KH55_001'],
+  H60: ['KH60_001'],
+  H70: ['KH70_001', 'KH70_002', 'KH70_003', 'KH70_004'],
+  H74: ['KH74_001'],
+  H80: ['KH80_001', 'KH80_002', 'KH80_003', 'KH80_004'],
+  H90: ['KH90_001'],
+}
+
+module.exports = {
+  rapoToken,
+  importerDbApiToken,
+  nodeproxyUrl,
+  importerDbApiUrl,
+  bscProgrammesOfFaculties,
+}

services/backend/src/rapodiff/creditsDiff.js

@@ -0,0 +1,121 @@
+/* eslint-disable no-console */
+const { getCreditStatsForRapodiff } = require('../services/studyprogramme/rapoCredits')
+const { parseCsv } = require('./helpers')
+const _ = require('lodash')
+
+const diffs = []
+let noDiffCounter = 0
+
+const diff = (rapoData, okData, code) => {
+  for (const year of Object.keys(okData)) {
+    if (year !== '2022' && year !== '2023') continue
+    const rapo = rapoData[year]
+    const ok = okData[year]
+    if (!rapo) {
+      if (!ok) console.log(`Year ${year} not in rapo for code ${code}, skipping`)
+      continue
+    }
+    for (const field of ['basic', 'incoming-exchange', 'open-uni', 'agreement', 'separate', 'other']) {
+      const okValue = Math.round(ok[field] || 0)
+      const rapoValue = Math.round(rapo[field] || 0)
+      const diff = Math.abs(okValue - rapoValue)
+      const diffStr = `${field.padEnd(10, ' ')} ${year} ${code.padEnd(
+        10,
+        ' '
+      )} Difference: ${diff} Rapo: ${rapoValue} Oodikone: ${okValue}`
+      diffs.push({ field, diff, okValue, rapoValue, code, year, diffStr })
+    }
+  }
+}
+let counter = 0
+// Acual logic in this function
+const process = async data => {
+  // Define here the fields to diff against each other. Must be an array, multiple fields will be summed
+  const rapoProgrammeData = formatData(data.slice(1))
+  const allProgrammeCodes = [...new Set(Object.keys(rapoProgrammeData))]
+  for (const programmeCode of allProgrammeCodes) {
+    const okProgrammeData = await getCreditStatsForRapodiff(programmeCode, '', 'CALENDAR_YEAR', 'SPECIAL_INCLUDED')
+    counter++
+    if (counter % 5 === 0) {
+      console.log(`Done ${Math.round((counter / allProgrammeCodes.length) * 100)} %`)
+    }
+    diff(rapoProgrammeData[programmeCode], okProgrammeData, programmeCode)
+  }
+  const orderedDiffs = _.orderBy(diffs, 'diff', 'asc')
+  orderedDiffs.forEach(d => console.log(d.diffStr))
+  console.log(`${diffs.length} diffs found with current settings. Numbers with no diff: ${noDiffCounter}`)
+  const totalDiff = orderedDiffs.reduce((sum, cur) => cur.diff + sum, 0)
+  console.log('Total difference: ', totalDiff)
+}
+
+const programmeCreditsDiff = async fileName => {
+  console.log('Running studyprogramme credits diff')
+  await parseCsv(fileName, process)
+  console.log('Diff completed.')
+}
+
+const processIds = async (rawData, code) => {
+  const data = rawData.slice(1)
+  console.log('amount of rapo credits: ', data.length)
+  const rapoData = data
+    .map(row => ({
+      id: row[4],
+      credits: row[5],
+      type: row[6],
+      incl: row[7],
+    }))
+    .filter(row => row.type === '2' && row.incl === '0')
+  const rapoIds = rapoData.map(row => row.id)
+  const stats = await getCreditStatsForRapodiff(code)
+  if (!stats.ids) {
+    console.log(stats)
+    console.log('Edit getCreditStats code so that it saves a list of the relevant ids you want to compare.')
+    return
+  }
+  const okIds = stats.ids
+  const notInOk = rapoIds.filter(id => id && !okIds[id.slice(11)]).map(str => str.slice(11))
+  const rapoIdMap = rapoIds.reduce((obj, cur) => {
+    obj[cur] = true
+    return obj
+  }, {})
+  const notInRapo = Object.keys(okIds).filter(id => !rapoIdMap[`ATTAINMENT-${id}`])
+  console.log('\n In rapo, but not in OK: ', notInOk.length)
+  console.dir(notInOk)
+  console.log('\n In OK, but not in Rapo: ', notInRapo.length)
+  console.dir(notInRapo)
+  delete stats.ids
+  console.log(stats['2022'])
+  const rapoCredits = rapoData.reduce((sum, cur) => parseInt(sum, 10) + parseInt(cur.credits, 10), 0)
+  console.log('rapoCredits: ', rapoCredits)
+}
+
+const testNewCalc = async code => {
+  await parseCsv('credits.csv', async data => processIds(data, code))
+}
+
+// Change weird rapo programmecodes to oodikone format, example: 300-M003 => MH30_003
+const transformProgrammeCode = oldCode => `${oldCode[4]}H${oldCode.slice(0, 2)}_${oldCode.slice(5, 8)}`
+
+const formatData = data =>
+  data.reduce((obj, cur) => {
+    const parseNum = str => (str === '' ? 0 : parseInt(str, 10))
+    const [year, facultyCode, programmeInfo, basic, exchange, otherUni, openUni, special, total, abroad, other] = cur
+    const programmeCode = transformProgrammeCode(programmeInfo.split(' ')[0])
+    if (!obj[programmeCode]) obj[programmeCode] = {}
+    obj[programmeCode][year] = {
+      year,
+      facultyCode,
+      programmeInfo,
+      basic: parseNum(basic),
+      'incoming-exchange': parseNum(exchange),
+      agreement: parseNum(otherUni),
+      'open-uni': parseNum(openUni),
+      separate: parseNum(special),
+      total: parseNum(total),
+      abroad: parseNum(abroad),
+      other: parseNum(other),
+    }
+    return obj
+  }, {})
+
+module.exports = { programmeCreditsDiff, testNewCalc }

services/backend/src/rapodiff/esimerkki.md

@@ -0,0 +1,50 @@
+### Esimerkki (fake data)
+
+Studycredits-route (suoritetut kreditit - kalenterivuosi?)
+
+**Route**: https://toska.cs.helsinki.fi/test/rapo/report/studyCredits?faculty=H50&education=500-K005&token=acual_token
+
+**Response**:
+
+```
+[
+  {
+    "vuosi": 2022,
+    "koulutusohjelmaKoodi": "500-K002",
+    "koulutusohjelma": "Fysikaalisten tieteiden kandiohjelma",
+    "tiedekuntaKoodi": "H50",
+    "tiedekunta": "H50 Matemaattis-luonnontieteellinen tiedekunta / Faculty of Science",
+    "perustutkinto": 12345,
+    "jatko": null,
+    "suoritetutYhteensa": 12345,
+    "eriklaakari": null,
+    "tutkAvoin": 12345,
+    "korkValYhteis": 12345,
+    "muutHyvaksiluetut": 12345,
+    "ulkomailta": null,
+    "taydennyskoul": null,
+    "avoinEiTutk": 12345,
+    "erillisoik": 12345,
+    "saapuvatVaihtoop": 12345
+  },
+  {
+    "vuosi": 2023,
+    "koulutusohjelmaKoodi": "500-M014",
+    "koulutusohjelma": "Nordic Master Programme in Environmental Changes at Higher Latitudes",
+    "tiedekuntaKoodi": "H50",
+    "tiedekunta": "H50 Matemaattis-luonnontieteellinen tiedekunta / Faculty of Science",
+    "perustutkinto": 12345,
+    "jatko": null,
+    "suoritetutYhteensa": 12345,
+    "eriklaakari": null,
+    "tutkAvoin": null,
+    "korkValYhteis": null,
+    "muutHyvaksiluetut": null,
+    "ulkomailta": 12345,
+    "taydennyskoul": null,
+    "avoinEiTutk": null,
+    "erillisoik": null,
+    "saapuvatVaihtoop": null
+  }
+]
+```

services/backend/src/rapodiff/helpers.js

@@ -0,0 +1,20 @@
+/* eslint-disable no-console */
+const Papa = require('papaparse')
+const fs = require('fs/promises')
+
+const parseCsv = async (fileName, fun) => {
+  try {
+    const file = await fs.readFile(`${__dirname}/${fileName}`, 'utf8')
+    Papa.parsePromise = file => {
+      return new Promise((complete, error) => {
+        Papa.parse(file, { complete, error })
+      })
+    }
+    const data = await Papa.parsePromise(file)
+    await fun(data?.data)
+  } catch (e) {
+    console.log(e)
+  }
+}
+
+module.exports = { parseCsv }

services/backend/src/rapodiff/rapodiff.js

@@ -0,0 +1,35 @@
+/* eslint no-console: 0 */
+
+/*
+  npm run rapodiff
+*/
+
+const { programmeCreditsDiff, testNewCalc } = require('./creditsDiff')
+
+// ** To add new mode, just type key and function here **
+const modes = {
+  credits: () => programmeCreditsDiff('data.csv'),
+  // Check credits with current local oodikone data against provided data.csv in rapodiff folder. data.csv can be downloaded from rapo by selecting all faculties, then from the bottom of the page clicking Export > data > csv.
+  check: code => testNewCalc(code),
+  // Provide programme code as argument. Parses credits from credits.csv in rapodiff folder. This must be obtained from Rapo people. This tells which credit ids are found in oodikone but not rapo, and vice versa.
+  // notice: check mode requires writing ids in to the stats object in credits calculating function.
+  // Add something like this there:
+  // if (category === 'open-uni' && credits > 0 && new Date(attainmentDate).getFullYear() === 2022) stats.ids[id] = true
+}
+
+const main = async () => {
+  console.log('**RAPODIFF**\n')
+  const mode = process.argv[2]
+  const modeFunction = modes[mode]
+  const arg = process.argv[3]
+  if (!modeFunction) {
+    console.log(
+      'Unknown or missing mode. Supply the mode as an argument. Supported modes:\n\n',
+      Object.keys(modes).join('\n\n')
+    )
+    return
+  }
+  await modeFunction(arg)
+}
+
+main().then(() => process.exit(0))

services/backend/src/rapodiff/studyrightDiff.js

@@ -1,25 +1,15 @@
-/* eslint no-console: 0 */
-
-/*
-  depends on nodeproxy running in importer.cs.helsinki.fi, and env RAPO_NODEPROXY (see importer for a proper value)
-  needs also acces to importer db api, uses env IMPORTER_API
-
-  known differences in APIs
-    - rapo does not show students that have graduated (it however takes some days for rapo to notice a graduation)
-    - rapo does not handle the starting year of transfers properly (shows in old staring population)
-    - rapo show some students incorrectly passive, see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3521
-*/
+/* eslint-disable no-console */
 
 const axios = require('axios').default
 
 const _ = require('lodash')
 
-const { optimizedStatisticsOf } = require('./services/populations')
-const { Student, Transfer, StudyrightElement, Studyright } = require('./models')
-
-const token = process.env.RAPO_NODEPROXY
+const { optimizedStatisticsOf } = require('../services/populations')
+const { Student, Transfer, StudyrightElement, Studyright } = require('../models')
+const { rapoToken, tokenImporterApi, importerDbApiUrl, nodeproxyUrl } = require('./conf.js')
 
 const getFromRapo = async (urlStart, urlEnd) => {
+  console.log({ urlStart, urlEnd })
   const { data: rapoStuffLasna } = await axios.get(urlStart)
   const { data: rapoStuffLasnaMinus } = await axios.get(urlEnd)
 
@@ -83,10 +73,12 @@ const getFromOodikone = async (programme, year) => {
 
 const isAcuallyActive = async student => {
   const currentSemester = 144
-  const tokenImporterApi = process.env.IMPORTER_API
 
-  const url = `https://importer.cs.helsinki.fi/api/importer/students/${student}/rapo_semester_enrollments?token=${tokenImporterApi}`
-  const { data } = await axios.get(url)
+  const url = `${importerDbApiUrl}/students/${student}/rapo_semester_enrollments?token=${tokenImporterApi}`
+  const result = await axios.get(url)
+  console.log('result:')
+  console.log(result)
+  const { data } = result
   const terms = data.reduce((all, s) => all.concat(s.terms), []).filter(t => t.semester_code === currentSemester)
   const active = terms.filter(t => t.type !== 'NEGLECTED')
   const negl = terms.filter(t => t.type === 'NEGLECTED')
@@ -144,11 +136,11 @@ const programme_diff_year = async (programme, facultyCode, year, brief = false)
 
   const rapoUrlStart = presence =>
     encodeURI(
-      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+      `${nodeproxyUrl}/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${rapoToken}`
     )
   const rapoUrlEnd = presence =>
     encodeURI(
-      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+      `${nodeproxyUrl}/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${rapoToken}`
     )
 
   const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
@@ -247,7 +239,7 @@ const programme_diff = async (programme, faculty, brief) => {
   await programme_diff_year(programme, faculty, 2021, brief)
 }
 
-const faculty_diff = async (faculty, programmes, brief) => {
+const facultyDiff = async (faculty, programmes, brief) => {
   console.log('\n')
   console.log(faculty)
   console.log('==')
@@ -265,11 +257,11 @@ const msc_programme_diff_year = async (programme, facultyCode, year, brief = fal
 
   const rapoUrlStart = presence =>
     encodeURI(
-      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${token}`
+      `${nodeproxyUrl}/test/rapo/${facultyCode}/${programme}/${studyrightStartDate}/${presence}?token=${rapoToken}`
     )
   const rapoUrlEnd = presence =>
     encodeURI(
-      `https://importer.cs.helsinki.fi/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${token}`
+      `${nodeproxyUrl}/test/rapo/${facultyCode}/${programme}/${studyrightEndDate}/${presence}?token=${rapoToken}`
     )
 
   const rapoStudentsLasna = await getFromRapo(rapoUrlStart('Läsnä'), rapoUrlEnd('Läsnä'))
@@ -337,27 +329,4 @@ const msc_programme_diff_year = async (programme, facultyCode, year, brief = fal
   }
 }
 
-const bsc_priogrammes_of_faculties = {
-  H10: ['KH10_001'],
-  H20: ['KH20_001'],
-  H30: ['KH30_001', 'KH30_002'],
-  H40: ['KH40_001', 'KH40_002', 'KH40_003', 'KH40_004', 'KH40_005', 'KH40_006'],
-  H50: ['KH50_001', 'KH50_002', 'KH50_003', 'KH50_004', 'KH50_005', 'KH50_006', 'KH50_007', 'KH50_008'],
-  H55: ['KH55_001'],
-  H60: ['KH60_001'],
-  H70: ['KH70_001', 'KH70_002', 'KH70_003', 'KH70_004'],
-  H74: ['KH74_001'],
-  H80: ['KH80_001', 'KH80_002', 'KH80_003', 'KH80_004'],
-  H90: ['KH90_001'],
-}
-
-const main = async () => {
-  for (let faculty of Object.keys(bsc_priogrammes_of_faculties)) {
-    await faculty_diff(faculty, bsc_priogrammes_of_faculties[faculty], true)
-  }
-
-  //await programme_diff_year('KH55_001', 'H55', 2021, true)
-  //await programme_diff('KH60_001', 'H60', true)
-}
-
-main().then(() => process.exit(0))
+module.exports = { facultyDiff }

services/backend/src/routes.js

@@ -17,6 +17,7 @@ const elementdetails = require('./routes/elementdetails')
 const teachers = require('./routes/teachers')
 const providers = require('./routes/providers')
 const faculties = require('./routes/faculties')
+const university = require('./routes/university')
 const courseExclusions = require('./routes/courseExclusions')
 const semesters = require('./routes/semesters')
 const feedback = require('./routes/feedback')
@@ -24,7 +25,6 @@ const tags = require('./routes/tags')
 const updater = require('./routes/updater')
 const customPopulationSearch = require('./routes/customPopulationSearch')
 const languageCenterData = require('./routes/languageCenterData')
-const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
 const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
 const studyProgramme = require('./routes/studyProgramme')
@@ -61,13 +61,13 @@ module.exports = (app, url) => {
   app.use(`${url}/changelog`, changelog)
   app.use(`${url}/completedcoursessearch`, completedCoursesSearch)
   app.use(`${url}/languagecenterdata`, languageCenterData)
-  app.use(`${url}/faculties`, auth.roles(['facultyStatistics']), faculties)
+  app.use(`${url}/faculties`, auth.roles(['facultyStatistics', 'katselmusViewer']), faculties)
+  app.use(`${url}/university`, university)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
   app.use(`${url}/users`, auth.roles(['admin']), users)
   app.use(`${url}/feedback`, feedback)
   app.use(`${url}/custom-population-search`, customPopulationSearch)
-  app.use(`${url}/cool-data-science`, trends)
   app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
   app.get('*', async (_, res) => {
     const results = { error: 'unknown endpoint' }

services/backend/src/routes/courses.js

@@ -1,6 +1,6 @@
 const router = require('express').Router()
 const crypto = require('crypto')
-const Course = require('../services/courses')
+const courseService = require('../services/courses')
 const { faculties } = require('../services/organisations')
 const { validateParamLength } = require('../util')
 
@@ -12,7 +12,7 @@ router.get('/v2/coursesmulti', async (req, res) => {
     return res.status(400).json({ error: 'name or code invalid' })
   }
 
-  results = await Course.byNameAndOrCodeLike(name, code)
+  results = await courseService.byNameAndOrCodeLike(name, code)
 
   if (combineSubstitutions === 'false') {
     const courseCodes = results.courses.map(course => course.code)
@@ -24,7 +24,7 @@ router.get('/v2/coursesmulti', async (req, res) => {
           .filter(value => value !== null && !courseCodes.includes(value))
       ),
     ]
-    const substitutionDetails = await Course.byCodes(substitutions)
+    const substitutionDetails = await courseService.byCodes(substitutions)
     for (const substitution of substitutionDetails) {
       results.courses.push(substitution.dataValues)
     }
@@ -56,7 +56,7 @@ router.get('/v3/courseyearlystats', async (req, res) => {
     // and users with rights to any specific study programmes
     const anonymize = allowedRoles !== 'admin' && rights.length < 1
     const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
-    const results = await Course.courseYearlyStats(codes, separate, anonymizationSalt, combineSubstitutions)
+    const results = await courseService.courseYearlyStats(codes, separate, anonymizationSalt, combineSubstitutions)
     res.json(results)
   }
 })

services/backend/src/routes/faculties.js

@@ -1,5 +1,4 @@
 const router = require('express').Router()
-const { faculties } = require('../services/organisations')
 const { combineFacultyBasics } = require('../services/faculty/facultyBasics')
 const { combineFacultyCredits } = require('../services/faculty/facultyCredits')
 const { combineFacultyThesisWriters } = require('../services/faculty/facultyThesisWriters')
@@ -7,7 +6,6 @@ const { countGraduationTimes } = require('../services/faculty/facultyGraduationT
 const { updateFacultyOverview, updateFacultyProgressOverview } = require('../services/faculty/facultyUpdates')
 const { combineFacultyStudentProgress } = require('../services/faculty/facultyStudentProgress')
 const { combineFacultyStudents } = require('../services/faculty/facultyStudents')
-
 const {
   getProgrammes,
   getBasicStats,
@@ -24,14 +22,13 @@ const {
   setFacultyProgressStats,
 } = require('../services/faculty/facultyService')
 const logger = require('../util/logger')
+const { getFacultyList } = require('../services/faculty/facultyHelpers')
 
 // Faculty uses a lot of tools designed for Study programme.
 // Some of them have been copied here and slightly edited for faculty purpose.
 
 router.get('/', async (req, res) => {
-  const ignore = ['Y', 'H99', 'Y01', 'H92', 'H930']
-  const facultyList = (await faculties()).filter(f => !ignore.includes(f.code))
-  facultyList.sort((a, b) => (a.name.fi > b.name.fi ? 1 : -1))
+  const facultyList = await getFacultyList()
   res.json(facultyList)
 })
 

services/backend/src/routes/population.js

@@ -2,15 +2,14 @@ const crypto = require('crypto')
 const Sentry = require('@sentry/node')
 const router = require('express').Router()
 const _ = require('lodash')
-const Population = require('../services/populations')
-const Student = require('../services/students')
+const populationService = require('../services/populations')
+const studentService = require('../services/students')
 const StudyrightService = require('../services/studyrights')
-const TagService = require('../services/tags')
 const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
 const { mapToProviders } = require('../util/utils')
 const { encrypt, decrypt } = require('../services/encrypt')
-const { mapCodesToIds } = require('../services/studyprogrammeHelpers')
+const { mapCodesToIds } = require('../services/studyprogramme/studyprogrammeHelpers')
 
 const { ApplicationError } = require('../util/customErrors')
 
@@ -59,7 +58,7 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
         }
         const newMonths = (upperYearBound - Number(year)) * 12
         const query = { ...req.body, year, months: newMonths }
-        const coursestatistics = Population.bottlenecksOf(query, null, encrypted)
+        const coursestatistics = populationService.bottlenecksOf(query, null, encrypted)
         return coursestatistics
       })
     )
@@ -74,7 +73,7 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
   } else {
     if (encrypted) req.body.selectedStudents = req.body.selectedStudents.map(decrypt)
 
-    const result = await Population.bottlenecksOf(req.body, null, encrypted)
+    const result = await populationService.bottlenecksOf(req.body, null, encrypted)
 
     if (result.error) {
       Sentry.captureException(new Error(result.error))
@@ -95,10 +94,10 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
   const {
     user: { isAdmin, studentsUserCanAccess },
   } = req
-  const studentnumbers = await Student.findByTag(tag)
+  const studentnumbers = await studentService.findByTag(tag)
   const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
 
-  const result = await Population.bottlenecksOf(
+  const result = await populationService.bottlenecksOf(
     {
       year: 1900,
       studyRights: [],
@@ -129,7 +128,7 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
   }
 
   const studentnumberlist = isAdmin ? studentnumbersInReq : _.intersection(studentnumbersInReq, studentsUserCanAccess)
-  const result = await Population.bottlenecksOf(
+  const result = await populationService.bottlenecksOf(
     {
       year: query?.year ?? 1900,
       studyRights: query?.studyRights ?? [],
@@ -186,7 +185,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
     const multipopulationstudentPromises = Promise.all(
       req.query.years.map(year => {
         const newMonths = (upperYearBound - Number(year)) * 12
-        const populationStudents = Population.optimizedStatisticsOf(
+        const populationStudents = populationService.optimizedStatisticsOf(
           {
             ...req.query,
             studyRights: { programme: studyRights.programme },
@@ -210,7 +209,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
 
     res.json(filterPersonalTags(result, user.id))
   } else {
-    const result = await Population.optimizedStatisticsOf({
+    const result = await populationService.optimizedStatisticsOf({
       ...req.query,
       studyRights: { programme: studyRights.programme },
     })
@@ -247,46 +246,6 @@ router.get('/v3/populationstatistics', async (req, res) => {
   }
 })
 
-router.get('/v3/populationstatisticsbytag', async (req, res) => {
-  const {
-    user: { id, isAdmin, studentsUserCanAccess },
-    query: { tag, studyRights: studyRightsJSON, months, year },
-  } = req
-
-  if (!tag) return res.status(400).json({ error: 'The query should have a tag defined' })
-  const foundTag = await TagService.findTagById(tag)
-  if (!foundTag) return res.status(404).json({ error: 'Tag not found' })
-
-  const semesters = ['FALL', 'SPRING']
-
-  const studentnumbers = await Student.findByTag(tag)
-
-  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
-
-  const studyRights = JSON.parse(studyRightsJSON)
-  const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
-  const yearDifference = Number(year) - Number(newStartYear)
-  const newMonths = Number(months) + 12 * yearDifference
-  const result = await Population.optimizedStatisticsOf(
-    {
-      year: newStartYear,
-      studyRights,
-      semesters,
-      months: newMonths,
-      tag: foundTag,
-    },
-    studentnumberlist
-  )
-
-  if (result.error) {
-    Sentry.captureException(new Error(result.error))
-    res.status(400).end()
-    return
-  }
-
-  res.json(filterPersonalTags(result, id))
-})
-
 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
   const { coursecodes, from, to, separate: sep, unifyCourses } = req.query
   const {
@@ -307,14 +266,14 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
   }
 
   const semesters = ['FALL', 'SPRING']
-  const studentnumbers = await Student.findByCourseAndSemesters(
+  const studentnumbers = await studentService.findByCourseAndSemesters(
     JSON.parse(coursecodes),
     from,
     to,
     separate,
     unifyCourses
   )
-  const result = await Population.optimizedStatisticsOf(
+  const result = await populationService.optimizedStatisticsOf(
     {
       // Useless, because studentnumbers are already filtered above by from & to.
       // We should probably refactor this to avoid more confusement.
@@ -374,7 +333,7 @@ router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
     tags?.studyProgramme && tags?.studyProgramme.includes('+')
       ? tags?.studyProgramme.split('+')[0]
       : tags?.studyProgramme
-  const result = await Population.optimizedStatisticsOf(
+  const result = await populationService.optimizedStatisticsOf(
     {
       year: tags?.year || 1900,
       studyRights: studyProgrammeCode ? [studyProgrammeCode] : [],
@@ -427,7 +386,7 @@ router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req
   )
   const maxYearsToCreatePopulationFromUni = await CourseService.maxYearsToCreatePopulationFrom(
     JSON.parse(courseCodes),
-    'reqularStats'
+    'regularStats'
   )
   const maxYearsToCreatePopulationFromBoth = await CourseService.maxYearsToCreatePopulationFrom(
     JSON.parse(courseCodes),

services/backend/src/routes/router.js

@@ -1,21 +0,0 @@
-const express = require('express')
-const logger = require('../util/logger')
-
-const wrapWithErrorHandler = fn => (req, res) =>
-  Promise.resolve(fn(req, res)).catch(error => {
-    logger.error(error)
-    res.status(500).send('Internal server error')
-  })
-
-const routerWithWrapper = () => {
-  const router = express.Router()
-  return {
-    router,
-    wrapper: {
-      get: (url, callback) => router.get(url, wrapWithErrorHandler(callback)),
-      post: (url, callback) => router.post(url, wrapWithErrorHandler(callback)),
-    },
-  }
-}
-
-module.exports = { routerWithWrapper }

services/backend/src/routes/studyProgramme.js

@@ -1,9 +1,12 @@
 const router = require('express').Router()
-const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics')
-const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
-const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
-const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
-const { getStudyprogrammeCoursesForStudytrack } = require('../services/studyprogrammeCourses')
+const { getBasicStatsForStudytrack } = require('../services/studyprogramme/studyprogrammeBasics')
+const { getCreditStatsForStudytrack } = require('../services/studyprogramme/studyprogrammeCredits')
+const { getGraduationStatsForStudytrack } = require('../services/studyprogramme/studyprogrammeGraduations')
+const {
+  getStudyprogrammeCoursesForStudytrack,
+  getStudyprogrammeStatsForColorizedCoursesTable,
+} = require('../services/studyprogramme/studyprogrammeCourses')
+const { getStudytrackStatsForStudyprogramme } = require('../services/studyprogramme/studytrackStats')
 const {
   getBasicStats,
   setBasicStats,
@@ -14,7 +17,7 @@ const {
   getStudytrackStats,
   setStudytrackStats,
 } = require('../services/analyticsService')
-const { updateBasicView, updateStudytrackView } = require('../services/studyprogrammeUpdates')
+const { updateBasicView, updateStudytrackView } = require('../services/studyprogramme/studyprogrammeUpdates')
 const { getProgrammeName } = require('../services/studyprogramme')
 const logger = require('../util/logger')
 const { getAssociations } = require('../services/studyrights')
@@ -139,6 +142,17 @@ router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
   return res.json(updated)
 })
 
+router.get('/v2/studyprogrammes/:id/colorizedtablecoursestats', async (req, res) => {
+  const code = req.params.id
+
+  try {
+    const data = await getStudyprogrammeStatsForColorizedCoursesTable(code)
+    return res.json(data)
+  } catch (e) {
+    logger.error({ message: `Failed to get code ${code} colorized table course stats`, meta: `${e}` })
+  }
+})
+
 router.get('/v2/studyprogrammes/:id/update_basicview', async (req, res) => {
   const code = req.params.id
   const combinedProgramme = req.query?.combined_programme

services/backend/src/routes/studyProgrammeCriteria.js

@@ -2,7 +2,7 @@ const {
   getCriteria,
   saveYearlyCourseCriteria,
   saveYearlyCreditCriteria,
-} = require('../services/studyProgrammeCriteria')
+} = require('../services/studyprogramme/studyProgrammeCriteria')
 
 const router = require('express').Router()
 

services/backend/src/routes/trends.js

@@ -1,79 +0,0 @@
-const router = require('express').Router()
-const {
-  withErr,
-  getProtoC,
-  getProtoCProgramme,
-  getStatus,
-  getUber,
-  getStartYears,
-  getGraduatedStatus,
-} = require('../services/trends')
-
-router.get('/start-years', async (req, res) => {
-  const years = await getStartYears()
-  res.json(years.map(({ studystartdate }) => studystartdate))
-})
-
-router.get(
-  '/uber-data',
-  withErr(async (req, res) => {
-    const data = await getUber(req.query)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/proto-c-data-programme',
-  withErr(async (req, res) => {
-    const data = await getProtoCProgramme(req.query)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/proto-c-data',
-  withErr(async (req, res) => {
-    const data = await getProtoC(req.query)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/status-graduated',
-  withErr(async (req, res) => {
-    const { date: unixMillis, showByYear } = req.query
-    const date = new Date(Number(unixMillis))
-    const endOfToday = new Date()
-    endOfToday.setHours(23, 59, 59, 999)
-
-    if (isNaN(date.getTime()) || date.getTime() > endOfToday.getTime()) {
-      return res.status(400).json({ error: 'Invalid date' })
-    }
-
-    // End of day
-    date.setHours(23, 59, 59, 999)
-    const data = await getGraduatedStatus(date.getTime(), showByYear)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/status',
-  withErr(async (req, res) => {
-    const { date: unixMillis, showByYear } = req.query
-    const date = new Date(Number(unixMillis))
-    const endOfToday = new Date()
-    endOfToday.setHours(23, 59, 59, 999)
-
-    if (isNaN(date.getTime()) || date.getTime() > endOfToday.getTime()) {
-      return res.status(400).json({ error: 'Invalid date' })
-    }
-
-    // End of day
-    date.setHours(23, 59, 59, 999)
-    const status = await getStatus(date.getTime(), showByYear)
-    res.json(status)
-  })
-)
-
-module.exports = router

services/backend/src/routes/university.js

@@ -0,0 +1,192 @@
+const router = require('express').Router()
+const { getFacultyList } = require('../services/faculty/facultyHelpers')
+const { getGraduationStats, getFacultyProgressStats } = require('../services/faculty/facultyService')
+
+const { getMedian } = require('../services/studyprogramme/studyprogrammeHelpers')
+const degreeNames = ['bachelor', 'bachelorMaster', 'master', 'licentiate', 'doctor']
+
+router.get('/allprogressstats', async (req, res) => {
+  const specialGroups = 'SPECIAL_EXCLUDED'
+  const graduated = req.query?.graduated
+  const allFaculties = await getFacultyList()
+  const facultyCodes = allFaculties.map(f => f.code)
+  const codeToData = {}
+
+  for (const facultyCode of facultyCodes) {
+    const data = await getFacultyProgressStats(facultyCode, specialGroups, graduated)
+    if (!data) {
+      return res.status(500).end()
+    }
+    codeToData[facultyCode] = data
+  }
+
+  const universityData = {
+    years: codeToData.H50.years,
+    yearlyBachelorTitles: codeToData.H50.yearlyBachelorTitles,
+    programmeNames: allFaculties.reduce((obj, fac) => {
+      const { name, ...rest } = fac.dataValues
+      obj[fac.code] = { ...rest, ...name }
+      return obj
+    }, {}),
+    bachelorsProgStats: {},
+    creditCounts: {
+      bachelor: {},
+    },
+  }
+
+  const unifyProgressStats = progStats => {
+    return progStats.reduce(
+      (all, prog) => {
+        prog.forEach((yearStats, yearIndex) =>
+          yearStats.forEach((category, categoryIndex) => {
+            all[yearIndex][categoryIndex] += prog[yearIndex][categoryIndex]
+          })
+        )
+        return all
+      },
+      // eslint-disable-next-line no-unused-vars
+      progStats[0].map(year => year.map(_num => 0))
+    )
+  }
+
+  for (const facultyCode of facultyCodes) {
+    const facultyData = codeToData[facultyCode]
+    for (const year of universityData.years.slice(1).reverse()) {
+      for (const fieldName of degreeNames) {
+        if (!facultyData.creditCounts[fieldName] || Object.keys(facultyData.creditCounts[fieldName]).length === 0)
+          continue
+        if (!universityData.creditCounts[fieldName]) universityData.creditCounts[fieldName] = {}
+        if (!universityData.creditCounts[fieldName][year]) universityData.creditCounts[fieldName][year] = []
+        universityData.creditCounts[fieldName][year].push(...facultyData.creditCounts[fieldName][year])
+      }
+      for (const fieldName of [
+        'bachelorsProgStats',
+        'bcMsProgStats',
+        'licentiateProgStats',
+        'mastersProgStats',
+        'doctoralProgStats',
+      ]) {
+        if (Object.keys(facultyData[fieldName]).length === 0) continue
+        if (!universityData[fieldName]) universityData[fieldName] = {}
+        universityData[fieldName][facultyCode] = unifyProgressStats(Object.values(facultyData[fieldName]))
+      }
+    }
+  }
+
+  // Remove ELL bachelor+master progresstats for now, because it is problematic due to different credit limits
+  // when expanding the table rows to show 'programme'(faculty) specific progress bars
+  delete universityData.bcMsProgStats['H90']
+
+  return res.status(200).json(universityData)
+})
+
+router.get('/allgraduationstats', async (req, res) => {
+  const degreeNames = ['bachelor', 'bcMsCombo', 'master', 'licentiate', 'doctor']
+  const allFaculties = await getFacultyList()
+  const facultyCodes = allFaculties.map(f => f.code)
+  const facultyData = {}
+  const timesArrays = [] // keep book of these to null them in the end, large lists not used in frontend
+  for (const facultyCode of facultyCodes) {
+    const data = await getGraduationStats(facultyCode, 'NEW_STUDY_PROGRAMMES', true)
+    if (!data) return res.status(500).json({ message: `Did not find data for ${facultyCode}` })
+    facultyData[facultyCode] = data
+  }
+
+  const unifyTotals = (facultyData, universityData, isLast) => {
+    for (const degree of degreeNames) {
+      if (!universityData[degree]) universityData[degree] = []
+      if (!facultyData[degree]) continue
+      for (const yearStats of facultyData[degree]) {
+        const universityStats = universityData[degree]
+        const universityYearStats = universityData[degree].find(stats => stats.name === yearStats.name)
+        if (!universityYearStats) {
+          universityStats.push(yearStats)
+        } else {
+          universityYearStats.times.push(...yearStats.times)
+          timesArrays.push(universityYearStats.times)
+          universityYearStats.amount += yearStats.amount
+          universityYearStats.statistics.onTime += yearStats.statistics.onTime
+          universityYearStats.statistics.yearOver += yearStats.statistics.yearOver
+          universityYearStats.statistics.wayOver += yearStats.statistics.wayOver
+          if (isLast) {
+            universityYearStats.y = getMedian(universityYearStats.times)
+          }
+        }
+      }
+    }
+  }
+
+  const unifyProgrammeStats = (universityData, facultyData, facultyCode) => {
+    for (const degree of degreeNames) {
+      if (!facultyData[degree]) continue
+      if (!universityData[degree]) universityData[degree] = {}
+      for (const yearData of facultyData[degree]) {
+        if (!universityData[degree][yearData.name]) {
+          universityData[degree][yearData.name] = { programmes: [], data: [] }
+        }
+        const uniYearStats = universityData[degree][yearData.name]
+        if (!uniYearStats.programmes.find(prog => prog === facultyCode)) {
+          uniYearStats.programmes.push(facultyCode)
+        }
+        const uniYearFacultyStats = uniYearStats.data.find(item => item.code === facultyCode)
+        const yearDataClone = { ...yearData, times: null, statistics: { ...yearData.statistics } }
+        if (!uniYearFacultyStats) {
+          uniYearStats.data.push({ ...yearDataClone, name: facultyCode, code: facultyCode })
+        } else {
+          uniYearFacultyStats.y = yearData.y
+          uniYearFacultyStats.amount += yearData.amount
+          uniYearFacultyStats.statistics.onTime += yearData.statistics.onTime
+          uniYearFacultyStats.statistics.yearOver += yearData.statistics.yearOver
+          uniYearFacultyStats.statistics.wayOver += yearData.statistics.wayOver
+        }
+      }
+    }
+
+    return universityData
+  }
+
+  const universityData = {
+    goals: {
+      bachelor: 36,
+      bcMsCombo: 60,
+      master: 24,
+      doctor: 48,
+      licentiate: 78,
+    },
+    programmeNames: allFaculties.reduce((obj, fac) => {
+      const { name, ...rest } = fac.dataValues
+      obj[fac.code] = { ...rest, ...name }
+      return obj
+    }, {}),
+    byGradYear: { medians: {}, programmes: { medians: {} } },
+    classSizes: { programmes: {} },
+  }
+
+  for (let i = 0; i < facultyCodes.length; i++) {
+    const facultyCode = facultyCodes[i]
+    const data = facultyData[facultyCode]
+    unifyTotals(data.byGradYear.medians, universityData.byGradYear.medians, i === facultyCodes.length - 1)
+    unifyProgrammeStats(universityData.byGradYear.programmes.medians, data.byGradYear.medians, facultyCode)
+    for (const degree of degreeNames) {
+      if (!universityData.classSizes[degree]) {
+        universityData.classSizes[degree] = data.classSizes[degree]
+      } else {
+        Object.entries(data.classSizes[degree]).forEach(([key, value]) => {
+          universityData.classSizes[degree][key] += value
+        })
+      }
+    }
+    // eslint-disable-next-line no-unused-vars
+    const { programmes, ...rest } = data.classSizes
+    universityData.classSizes.programmes[facultyCode] = rest
+  }
+
+  // Empty "times" arrays because that's not needed anymore.
+  timesArrays.forEach(arr => {
+    arr.length = 0
+  })
+
+  return res.status(200).json(universityData)
+})
+
+module.exports = router

services/backend/src/routes/updater.js

@@ -1,4 +1,5 @@
 const router = require('express').Router()
+const { refreshFaculties, refreshProgrammes } = require('../events')
 const {
   updateSISMetadata,
   updateSISStudents,
@@ -8,14 +9,19 @@ const {
   abort,
   updateCoursesByCourseCode,
 } = require('../services/sisUpdaterService')
-const {
-  refreshStatistics,
-  refreshTrends,
-  refreshFaculties,
-  refreshNewOverviews,
-  refreshLanguageCenterData,
-} = require('../events')
+
 const logger = require('../util/logger')
+const { jobMaker, getJobs } = require('../worker/queue')
+
+const refreshFacultiesByList = list => {
+  list.forEach(code => jobMaker.faculty(code))
+  return 'Added jobs for refreshing faculties'
+}
+
+const refreshProgrammesByList = list => {
+  list.forEach(code => jobMaker.programme(code))
+  return 'Added jobs for refreshing programme'
+}
 
 router.get('/update/v2/meta', async (req, res) => {
   const response = await updateSISMetadata(req)
@@ -31,10 +37,20 @@ router.get('/update/v2/students', async (req, res) => {
   }
 })
 
-router.post('/update/v2/students', async (req, res) => {
-  const response = await updateStudentsByStudentNumber(req.body)
+router.post('/update/v2/customlist/:type', async (req, res) => {
+  const type = req.params.type
+  const list = req.body
+  const typeToJob = {
+    students: updateStudentsByStudentNumber,
+    courses: updateCoursesByCourseCode,
+    faculties: refreshFacultiesByList,
+    programmes: refreshProgrammesByList,
+  }
+  if (typeof typeToJob[type] !== 'function') return res.status(400).json('Bad job type')
+  const response = await typeToJob[type](list)
+
   if (response) {
-    res.status(200).json('Update SIS students scheduled')
+    res.status(200).json('Scheduled custom list')
   }
 })
 
@@ -62,32 +78,26 @@ router.get('/refresh_redis_cache', async (req, res) => {
 
 router.post('/refresh_statistic_v2', async (req, res) => {
   logger.info(`${req.user.userId} requested refresh of statistics`)
-  await refreshStatistics()
+  jobMaker.statistics()
   res.status(200).json('Teacher and study programme statistics refreshed')
 })
 
 router.post('/refresh_study_programmes_v2', async (req, res) => {
   logger.info(`${req.user.userId} requested refresh of study programmes`)
-  await refreshNewOverviews()
-  res.status(200).json('New study programme overviews refreshed')
+  refreshProgrammes()
+  res.status(200).json('Added job for refreshing study programme overviews')
 })
 
 router.post('/refresh_faculties_v2', async (req, res) => {
   logger.info(`${req.user.userId} requested refresh of faculties`)
-  await refreshFaculties()
-  res.status(200).json('Faculties refreshed')
-})
-
-router.post('/refresh_trends', async (req, res) => {
-  logger.info(`${req.user.userId} requested refresh of trends`)
-  await refreshTrends()
-  res.status(200).json('Trends refreshed')
+  refreshFaculties()
+  res.status(200).json('Added job for refreshing faculties')
 })
 
 router.post('/refresh_language_center_data', async (req, res) => {
   logger.info(`${req.user.userId} requested refresh of language center data`)
-  await refreshLanguageCenterData()
-  res.status(200).json('Language center data refreshed')
+  jobMaker.languagecenter()
+  res.status(200).json('Added job for refreshing language center data')
 })
 
 router.get('/abort', async (req, res) => {
@@ -95,4 +105,10 @@ router.get('/abort', async (req, res) => {
   res.status(200).json()
 })
 
+router.get('/jobs', async (req, res) => {
+  const waiting = await getJobs('waiting')
+  const active = await getJobs('active')
+  res.status(200).json({ waiting, active })
+})
+
 module.exports = router

services/backend/src/routes/users.js

@@ -13,6 +13,16 @@ router.get('/access_groups', async (req, res) => {
   res.json(result)
 })
 
+router.get('/:uid', async (req, res) => {
+  try {
+    const { uid } = req.params
+    const user = await userService.findOne(uid)
+    return res.json(user)
+  } catch (e) {
+    return res.status(400).json({ error: e.message })
+  }
+})
+
 router.post('/modifyaccess', async (req, res) => {
   try {
     const result = await userService.modifyAccess(req.body)

services/backend/src/services/completedCoursesSearch.js

@@ -1,6 +1,5 @@
 const { Credit, Enrollment, Student, Course } = require('../models')
 const { Op } = require('sequelize')
-const _ = require('lodash')
 
 const getCompletedCourses = async (studentNumbers, courseCodes) => {
   let courses = await Course.findAll({
@@ -126,6 +125,7 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
       date: enrollment.date,
       courseCode: enrollment.courseCode,
       state: enrollment.state,
+      substitution: enrollment.substitution,
     })
   })
 
@@ -136,7 +136,8 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
         studentNumber: student,
         sis_person_id: studentCredits[student].sis_person_id,
         credits: studentCredits[student].credits,
-        enrollments: studentCredits[student].enrollments,
+        enrollments: {},
+        allEnrollments: studentCredits[student].enrollments,
         firstnames: studentCredits[student].firstnames,
         lastname: studentCredits[student].lastname,
         email: studentCredits[student].email,
@@ -147,18 +148,22 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
 
   students.forEach(student => {
     courseCodes.forEach(courseCode => {
-      let [hits, rest] = _.partition(student.enrollments, e => e.courseCode === courseCode)
-      if (hits?.length === 0) return
-      let latest = hits[0]
-      for (let hit of hits) {
-        if (hit > latest) {
-          latest = hit
-        }
-      }
-      student.enrollments = [latest, ...rest]
+      student.enrollments[courseCode] = student.allEnrollments
+        .filter(e => e.courseCode === courseCode || e.substitution === courseCode)
+        .sort(e => new Date(e.date).getDate())[0]
     })
   })
-  return { students, courses }
+
+  // Omit allEnrollments, we're only supposed to show the recent, relevant enrollment,
+  // the user does not have rights to see all enrollments.
+  return {
+    students: students.map(student => {
+      // eslint-disable-next-line no-unused-vars
+      const { allEnrollments, ...rest } = student
+      return { ...rest }
+    }),
+    courses,
+  }
 }
 
 module.exports = { getCompletedCourses }

services/backend/src/services/courses.js

@@ -1,532 +0,0 @@
-const Sequelize = require('sequelize')
-const {
-  Student,
-  Credit,
-  Course,
-  ElementDetail,
-  StudyrightElement,
-  Studyright,
-  Semester,
-  Organization,
-  Enrollment,
-} = require('../models')
-
-const { parseCredit } = require('./parseCredits')
-const Op = Sequelize.Op
-const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
-const { sortMainCode, getSortRank } = require('../util/utils')
-const { parseEnrollment } = require('./parseEnrollments')
-
-const byNameOrCode = (searchTerm, language) =>
-  Course.findAll({
-    where: {
-      [Op.or]: [
-        {
-          name: {
-            [language]: {
-              [Op.iLike]: searchTerm,
-            },
-          },
-        },
-        {
-          code: {
-            [Op.like]: searchTerm,
-          },
-        },
-      ],
-    },
-  })
-
-const byName = (name, language) =>
-  Course.findAll({
-    where: {
-      name: {
-        [language]: {
-          [Op.eq]: name,
-        },
-      },
-    },
-    order: [['latest_instance_date', 'DESC']],
-    limit: 1,
-  })
-
-const byCode = code => Course.findByPk(code)
-
-const findOneByCode = code => {
-  return Course.findOne({
-    attributes: ['id', 'code', 'name'],
-    where: {
-      code: code,
-    },
-  })
-}
-
-const creditsForCourses = async (codes, anonymizationSalt, unification) => {
-  let is_open = false
-
-  if (unification === 'open') is_open = true
-
-  if (unification === 'unify') {
-    is_open = {
-      [Op.in]: [false, true],
-    }
-  }
-
-  const credits = await Credit.findAll({
-    include: [
-      {
-        model: Student,
-        attributes: ['studentnumber'],
-        include: {
-          model: StudyrightElement,
-          attributes: ['code', 'startdate'],
-          include: [
-            {
-              model: ElementDetail,
-              attributes: ['name', 'type'],
-              where: {
-                type: {
-                  [Op.eq]: 20,
-                },
-              },
-            },
-            {
-              model: Studyright,
-              attributes: ['prioritycode', 'faculty_code'],
-              where: {
-                prioritycode: {
-                  [Op.eq]: 1,
-                },
-              },
-              include: {
-                model: Organization,
-              },
-            },
-          ],
-        },
-      },
-      {
-        model: Semester,
-        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
-        where: {
-          startdate: {
-            [Op.lte]: new Date(),
-          },
-        },
-      },
-    ],
-    where: {
-      course_code: {
-        [Op.in]: codes,
-      },
-      student_studentnumber: {
-        [Op.ne]: null,
-      },
-      [Op.or]: [{ is_open }, { is_open: null }],
-    },
-    order: [['attainment_date', 'ASC']],
-  })
-
-  const parsedCredits = credits.map(credit => parseCredit(credit, anonymizationSalt))
-  return parsedCredits
-}
-
-const enrollmentsForCourses = async (codes, anonymizationSalt, unification) => {
-  let is_open = false
-
-  if (unification === 'open') is_open = true
-
-  if (unification === 'unify') {
-    is_open = {
-      [Op.in]: [false, true],
-    }
-  }
-
-  const enrollments = await Enrollment.findAll({
-    include: [
-      {
-        model: Student,
-        attributes: ['studentnumber'],
-        include: {
-          model: StudyrightElement,
-          attributes: ['code', 'startdate'],
-          include: [
-            {
-              model: ElementDetail,
-              attributes: ['name', 'type'],
-              where: {
-                type: {
-                  [Op.eq]: 20,
-                },
-              },
-            },
-            {
-              model: Studyright,
-              attributes: ['prioritycode', 'faculty_code'],
-              where: {
-                prioritycode: {
-                  [Op.eq]: 1,
-                },
-              },
-              include: {
-                model: Organization,
-              },
-            },
-          ],
-        },
-      },
-      {
-        model: Semester,
-        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
-        where: {
-          startdate: {
-            [Op.lte]: new Date(),
-          },
-        },
-      },
-    ],
-    where: {
-      course_code: {
-        [Op.in]: codes,
-      },
-      studentnumber: {
-        [Op.ne]: null,
-      },
-      enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
-      state: ['ENROLLED', 'CONFIRMED'],
-      [Op.or]: [{ is_open }, { is_open: null }],
-    },
-  })
-  return enrollments.map(enrollment => parseEnrollment(enrollment, anonymizationSalt))
-}
-
-const bySearchTerm = async (term, language) => {
-  const formatCourse = course => ({
-    name: course.name[language],
-    code: course.code,
-    date: course.latest_instance_date,
-  })
-
-  try {
-    const result = await byNameOrCode(`%${term}%`, language)
-    return result.map(formatCourse)
-  } catch (e) {
-    return {
-      error: e,
-    }
-  }
-}
-
-const createCourse = async (code, name, latest_instance_date) =>
-  Course.create({
-    code,
-    name,
-    latest_instance_date,
-  })
-
-const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
-
-const unifyOpenUniversity = code => {
-  const regexresult = isOpenUniCourseCode(code)
-  if (!regexresult) return code
-  return regexresult[1]
-}
-
-const allCodeAlternatives = async code => {
-  const course = await Course.findOne({
-    where: { code: code },
-  })
-
-  return sortMainCode([...course.substitutions, code])
-}
-
-const yearlyStatsOfNew = async (coursecode, separate, unification, anonymizationSalt, combineSubstitutions) => {
-  const courseForSubs = await Course.findOne({
-    where: { code: coursecode },
-  })
-
-  let codes =
-    combineSubstitutions && courseForSubs.substitutions
-      ? sortMainCode([...courseForSubs.substitutions, coursecode])
-      : [coursecode]
-
-  if (unification === 'reqular') {
-    codes = codes.filter(course => !isOpenUniCourseCode(course))
-  }
-
-  const [credits, enrollments, course] = await Promise.all([
-    creditsForCourses(codes, anonymizationSalt, unification),
-    enrollmentsForCourses(codes, anonymizationSalt, unification),
-    Course.findOne({
-      where: {
-        code: coursecode,
-      },
-    }),
-  ])
-
-  const counter = new CourseYearlyStatsCounter()
-
-  for (let credit of credits) {
-    const {
-      studentnumber,
-      grade,
-      passed,
-      semestercode,
-      semestername,
-      yearcode,
-      yearname,
-      attainment_date,
-      programmes,
-      coursecode,
-      credits,
-    } = credit
-
-    const groupcode = separate ? semestercode : yearcode
-    const groupname = separate ? semestername : yearname
-    const unknownProgramme = [
-      {
-        code: 'OTHER',
-        name: {
-          en: 'Other',
-          fi: 'Muu',
-          sv: 'Andra',
-        },
-        faculty_code: 'OTHER',
-        organization: {
-          name: {
-            en: 'Other',
-            fi: 'Muu',
-            sv: 'Andra',
-          },
-        },
-      },
-    ]
-    counter.markStudyProgrammes(
-      studentnumber,
-      programmes.length === 0 ? unknownProgramme : programmes,
-      yearcode,
-      passed,
-      credits
-    )
-    counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode)
-    counter.markCreditToStudentCategories(studentnumber, passed, attainment_date, groupcode)
-  }
-
-  enrollments.forEach(enrollment => {
-    const { studentnumber, semestercode, semestername, yearcode, yearname, coursecode, state, enrollment_date_time } =
-      enrollment
-
-    const groupcode = separate ? semestercode : yearcode
-    const groupname = separate ? semestername : yearname
-
-    counter.markEnrollmentToGroup(
-      studentnumber,
-      state,
-      enrollment_date_time,
-      groupcode,
-      groupname,
-      coursecode,
-      yearcode
-    )
-  })
-
-  const statistics = counter.getFinalStatistics(anonymizationSalt)
-
-  return {
-    ...statistics,
-    coursecode,
-    alternatives: codes,
-    name: course.name,
-  }
-}
-
-const maxYearsToCreatePopulationFrom = async (coursecodes, unifyCourses) => {
-  let is_open = false
-
-  if (unifyCourses === 'openStats') is_open = true
-
-  if (unifyCourses === 'unifyStats') {
-    is_open = {
-      [Op.in]: [false, true],
-    }
-  }
-
-  const maxAttainmentDate = new Date(
-    Math.max(
-      ...(
-        await Course.findAll({
-          where: {
-            code: {
-              [Op.in]: coursecodes,
-            },
-          },
-          attributes: ['max_attainment_date'],
-        })
-      ).map(c => new Date(c.max_attainment_date).getTime())
-    )
-  )
-
-  const attainmentThreshold = new Date(maxAttainmentDate.getFullYear(), 0, 1)
-  attainmentThreshold.setFullYear(attainmentThreshold.getFullYear() - 6)
-
-  const credits = await Credit.findAll({
-    where: {
-      course_code: {
-        [Op.in]: coursecodes,
-      },
-      attainment_date: {
-        [Op.gt]: attainmentThreshold,
-      },
-      is_open,
-    },
-    order: [['attainment_date', 'ASC']],
-  })
-
-  const yearlyStudents = Object.values(
-    credits.reduce((res, credit) => {
-      const attainmentYear = new Date(credit.attainment_date).getFullYear()
-      if (!res[attainmentYear]) res[attainmentYear] = 0
-      res[attainmentYear]++
-      return res
-    }, {})
-  )
-  const maxYearsToCreatePopulationFrom = Math.max(
-    Math.floor(
-      1200 / // Lower this value to get a smaller result if necessary
-        (yearlyStudents.reduce((acc, curr) => acc + curr, 0) / yearlyStudents.length)
-    ),
-    1
-  )
-
-  return maxYearsToCreatePopulationFrom
-}
-
-const courseYearlyStats = async (coursecodes, separate, anonymizationSalt, combineSubstitutions) => {
-  const statsReqular = await Promise.all(
-    coursecodes.map(async code => {
-      const unifyStats = await yearlyStatsOfNew(code, separate, 'unify', anonymizationSalt, combineSubstitutions)
-      const reqularStats = await yearlyStatsOfNew(code, separate, 'reqular', anonymizationSalt, combineSubstitutions)
-      const openStats = await yearlyStatsOfNew(code, separate, 'open', anonymizationSalt, combineSubstitutions)
-
-      return { unifyStats, reqularStats, openStats }
-    })
-  )
-
-  return statsReqular
-}
-
-const nameLikeTerm = name => {
-  if (!name) {
-    return undefined
-  }
-  const term = `%${name.trim()}%`
-  return {
-    name: {
-      [Op.or]: {
-        fi: {
-          [Op.iLike]: term,
-        },
-        sv: {
-          [Op.iLike]: term,
-        },
-        en: {
-          [Op.iLike]: term,
-        },
-      },
-    },
-  }
-}
-
-const byCodes = codes => {
-  return Course.findAll({
-    where: {
-      code: {
-        [Op.in]: codes,
-      },
-    },
-  })
-}
-
-const codeLikeTerm = code =>
-  !code
-    ? undefined
-    : {
-        code: {
-          // Starts with code or has AY/A in front of the code
-          [Op.iRegexp]: `^(AY|ay|A|a)?${code}`,
-        },
-      }
-
-const byNameAndOrCodeLike = async (name, code) => {
-  let rawCourses = await Course.findAll({
-    where: {
-      ...nameLikeTerm(name),
-      ...codeLikeTerm(code),
-    },
-  })
-
-  const courses = rawCourses
-    .map(course => {
-      return { ...course.dataValues }
-    })
-    .sort((x, y) => getSortRank(y.code) - getSortRank(x.code))
-
-  let substitutionGroupIndex = 0
-  const visited = []
-
-  const organizeSubgroups = course => {
-    if (visited.includes(course.code)) return
-
-    let temp = []
-    if (course.substitutions !== null) {
-      temp = courses.filter(c => course.substitutions.includes(c.code))
-    }
-
-    temp.unshift(course)
-    temp.forEach(cu => {
-      if (visited.includes(course.code)) return
-      visited.push(cu.id)
-      cu.subsId = substitutionGroupIndex
-    })
-  }
-
-  courses.forEach(course => {
-    if (!visited.includes(course.id)) {
-      substitutionGroupIndex++
-      organizeSubgroups(course)
-    }
-  })
-  return { courses }
-}
-
-const getCourseProvidersForCourses = async codes => {
-  return (
-    await Organization.findAll({
-      attributes: ['code'],
-      include: {
-        model: Course,
-        where: {
-          code: {
-            [Op.in]: codes,
-          },
-        },
-      },
-      raw: true,
-    })
-  ).map(({ code }) => code)
-}
-
-module.exports = {
-  byCode,
-  byName,
-  bySearchTerm,
-  createCourse,
-  courseYearlyStats,
-  byNameAndOrCodeLike,
-  byCodes,
-  maxYearsToCreatePopulationFrom,
-  unifyOpenUniversity,
-  allCodeAlternatives,
-  findOneByCode,
-  getCourseProvidersForCourses,
-}

services/backend/src/services/courses/courseFinders.js

@@ -0,0 +1,95 @@
+const { Op } = require('sequelize')
+const { Course } = require('../../models')
+const { getSortRank } = require('../../util/utils')
+
+const nameLikeTerm = name => {
+  if (!name) {
+    return undefined
+  }
+  const term = `%${name.trim()}%`
+  return {
+    name: {
+      [Op.or]: {
+        fi: {
+          [Op.iLike]: term,
+        },
+        sv: {
+          [Op.iLike]: term,
+        },
+        en: {
+          [Op.iLike]: term,
+        },
+      },
+    },
+  }
+}
+
+const byCodes = codes => {
+  return Course.findAll({
+    where: {
+      code: {
+        [Op.in]: codes,
+      },
+    },
+  })
+}
+
+const escapeRegExp = string => string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+
+const codeLikeTerm = code =>
+  !code
+    ? undefined
+    : {
+        code: {
+          // Starts with code or has AY/A in front of the code (case-insensitive)
+          [Op.iRegexp]: `^(AY|A)?${escapeRegExp(code)}`,
+        },
+      }
+
+const byNameAndOrCodeLike = async (name, code) => {
+  const rawCourses = await Course.findAll({
+    where: {
+      ...nameLikeTerm(name),
+      ...codeLikeTerm(code),
+    },
+    order: [['id', 'desc']],
+  })
+
+  const courses = rawCourses
+    .map(course => {
+      return { ...course.dataValues }
+    })
+    .sort((x, y) => getSortRank(y.code) - getSortRank(x.code))
+
+  let substitutionGroupIndex = 0
+  const visited = []
+
+  const organizeSubgroups = course => {
+    if (visited.includes(course.code)) return
+
+    let temp = []
+    if (course.substitutions !== null) {
+      temp = courses.filter(c => course.substitutions.includes(c.code))
+    }
+
+    temp.unshift(course)
+    temp.forEach(cu => {
+      if (visited.includes(course.code)) return
+      visited.push(cu.id)
+      cu.subsId = substitutionGroupIndex
+    })
+  }
+
+  courses.forEach(course => {
+    if (!visited.includes(course.id)) {
+      substitutionGroupIndex++
+      organizeSubgroups(course)
+    }
+  })
+  return { courses }
+}
+
+module.exports = {
+  byNameAndOrCodeLike,
+  byCodes,
+}

services/backend/src/services/courses/creditsAndEnrollmentsOfCourse.js

@@ -0,0 +1,143 @@
+const { Op } = require('sequelize')
+const {
+  Credit,
+  Student,
+  Semester,
+  Studyright,
+  Organization,
+  StudyrightElement,
+  ElementDetail,
+  Enrollment,
+} = require('../../models')
+
+const creditsForCourses = async (codes, anonymizationSalt, unification) => {
+  let is_open = false
+
+  if (unification === 'open') is_open = true
+
+  if (unification === 'unify') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
+  return await Credit.findAll({
+    include: [
+      {
+        model: Student,
+        attributes: ['studentnumber'],
+      },
+      {
+        model: Semester,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+        where: {
+          startdate: {
+            [Op.lte]: new Date(),
+          },
+        },
+      },
+    ],
+    where: {
+      course_code: {
+        [Op.in]: codes,
+      },
+      student_studentnumber: {
+        [Op.ne]: null,
+      },
+      [Op.or]: [{ is_open }, { is_open: null }],
+    },
+    order: [['attainment_date', 'ASC']],
+  })
+}
+
+const getStudentNumberToSrElementsMap = async studentNumbers => {
+  const studyrights = await Studyright.findAll({
+    attributes: ['prioritycode', 'faculty_code', 'student_studentnumber', 'studyrightid'],
+    where: {
+      prioritycode: {
+        [Op.eq]: 1,
+      },
+      student_studentnumber: { [Op.in]: studentNumbers },
+    },
+    include: {
+      model: Organization,
+    },
+  })
+
+  const studyrightMap = studyrights.reduce((obj, cur) => {
+    obj[cur.studyrightid] = cur
+    return obj
+  }, {})
+
+  const studyrightIds = Object.keys(studyrightMap)
+
+  const studyrightElements = await StudyrightElement.findAll({
+    attributes: ['code', 'startdate', 'studentnumber', 'studyrightid'],
+    include: [
+      {
+        model: ElementDetail,
+        attributes: ['name', 'type'],
+        where: {
+          type: {
+            [Op.eq]: 20,
+          },
+        },
+      },
+    ],
+    where: { studyrightid: { [Op.in]: studyrightIds } },
+  })
+
+  return studyrightElements.reduce((obj, cur) => {
+    if (!obj[cur.studentnumber]) obj[cur.studentnumber] = []
+    cur.studyright = studyrightMap[cur.studyrightid]
+    obj[cur.studentnumber].push(cur)
+    return obj
+  }, {})
+}
+
+const enrollmentsForCourses = async (codes, unification) => {
+  let is_open = false
+
+  if (unification === 'open') is_open = true
+
+  if (unification === 'unify') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
+  return await Enrollment.findAll({
+    include: [
+      {
+        model: Student,
+        attributes: ['studentnumber'],
+      },
+      {
+        model: Semester,
+        attributes: ['semestercode', 'name', 'yearcode', 'yearname'],
+        where: {
+          startdate: {
+            [Op.lte]: new Date(),
+          },
+        },
+      },
+    ],
+    where: {
+      course_code: {
+        [Op.in]: codes,
+      },
+      studentnumber: {
+        [Op.ne]: null,
+      },
+      enrollment_date_time: { [Op.gte]: new Date('2021-05-31') },
+      state: ['ENROLLED', 'CONFIRMED'],
+      [Op.or]: [{ is_open }, { is_open: null }],
+    },
+  })
+}
+
+module.exports = {
+  getStudentNumberToSrElementsMap,
+  enrollmentsForCourses,
+  creditsForCourses,
+}

services/backend/src/services/courses/index.js

@@ -0,0 +1,281 @@
+const Sequelize = require('sequelize')
+const { Credit, Course, Organization, Enrollment } = require('../../models')
+
+const { parseCredit } = require('./parseCredits')
+const { parseEnrollment } = require('./parseEnrollments')
+const Op = Sequelize.Op
+const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
+const { sortMainCode } = require('../../util/utils')
+const {
+  enrollmentsForCourses,
+  creditsForCourses,
+  getStudentNumberToSrElementsMap,
+} = require('./creditsAndEnrollmentsOfCourse')
+const { byNameAndOrCodeLike, byCodes } = require('./courseFinders')
+
+const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+
+const unifyOpenUniversity = code => {
+  const regexresult = isOpenUniCourseCode(code)
+  if (!regexresult) return code
+  return regexresult[1]
+}
+
+const allCodeAlternatives = async code => {
+  const course = await Course.findOne({
+    where: { code: code },
+  })
+
+  return sortMainCode([...course.substitutions, code])
+}
+
+const yearlyStatsOfNew = async (
+  coursecode,
+  separate,
+  unification,
+  anonymizationSalt,
+  combineSubstitutions,
+  studentNumberToSrElementsMap
+) => {
+  const courseForSubs = await Course.findOne({
+    where: { code: coursecode },
+  })
+
+  let codes =
+    combineSubstitutions && courseForSubs.substitutions
+      ? sortMainCode([...courseForSubs.substitutions, coursecode])
+      : [coursecode]
+
+  if (unification === 'regular') {
+    codes = codes.filter(course => !isOpenUniCourseCode(course))
+  }
+
+  const [credits, enrollments, course] = await Promise.all([
+    creditsForCourses(codes, anonymizationSalt, unification),
+    enrollmentsForCourses(codes, unification),
+    Course.findOne({
+      where: {
+        code: coursecode,
+      },
+    }),
+  ])
+
+  const counter = new CourseYearlyStatsCounter()
+
+  for (let credit of credits) {
+    const {
+      studentnumber,
+      grade,
+      passed,
+      semestercode,
+      semestername,
+      yearcode,
+      yearname,
+      attainment_date,
+      programmes,
+      coursecode,
+      credits,
+    } = parseCredit(credit, anonymizationSalt, studentNumberToSrElementsMap)
+
+    const groupcode = separate ? semestercode : yearcode
+    const groupname = separate ? semestername : yearname
+    const unknownProgramme = [
+      {
+        code: 'OTHER',
+        name: {
+          en: 'Other',
+          fi: 'Muu',
+          sv: 'Andra',
+        },
+        faculty_code: 'OTHER',
+        organization: {
+          name: {
+            en: 'Other',
+            fi: 'Muu',
+            sv: 'Andra',
+          },
+        },
+      },
+    ]
+    counter.markStudyProgrammes(
+      studentnumber,
+      programmes.length === 0 ? unknownProgramme : programmes,
+      yearcode,
+      passed,
+      credits
+    )
+    counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode)
+    counter.markCreditToStudentCategories(studentnumber, passed, attainment_date, groupcode)
+  }
+
+  enrollments.forEach(enrollment => {
+    const { studentnumber, semestercode, semestername, yearcode, yearname, coursecode, state, enrollment_date_time } =
+      parseEnrollment(enrollment, anonymizationSalt, studentNumberToSrElementsMap)
+
+    const groupcode = separate ? semestercode : yearcode
+    const groupname = separate ? semestername : yearname
+
+    counter.markEnrollmentToGroup(
+      studentnumber,
+      state,
+      enrollment_date_time,
+      groupcode,
+      groupname,
+      coursecode,
+      yearcode
+    )
+  })
+
+  const statistics = counter.getFinalStatistics(anonymizationSalt)
+
+  return {
+    ...statistics,
+    coursecode,
+    alternatives: codes,
+    name: course.name,
+  }
+}
+
+const maxYearsToCreatePopulationFrom = async (coursecodes, unifyCourses) => {
+  let is_open = false
+
+  if (unifyCourses === 'openStats') is_open = true
+
+  if (unifyCourses === 'unifyStats') {
+    is_open = {
+      [Op.in]: [false, true],
+    }
+  }
+
+  const maxAttainmentDate = new Date(
+    Math.max(
+      ...(
+        await Course.findAll({
+          where: {
+            code: {
+              [Op.in]: coursecodes,
+            },
+          },
+          attributes: ['max_attainment_date'],
+        })
+      ).map(c => new Date(c.max_attainment_date).getTime())
+    )
+  )
+
+  const attainmentThreshold = new Date(maxAttainmentDate.getFullYear(), 0, 1)
+  attainmentThreshold.setFullYear(attainmentThreshold.getFullYear() - 6)
+
+  const credits = await Credit.findAll({
+    where: {
+      course_code: {
+        [Op.in]: coursecodes,
+      },
+      attainment_date: {
+        [Op.gt]: attainmentThreshold,
+      },
+      is_open,
+    },
+    order: [['attainment_date', 'ASC']],
+  })
+
+  const yearlyStudents = Object.values(
+    credits.reduce((res, credit) => {
+      const attainmentYear = new Date(credit.attainment_date).getFullYear()
+      if (!res[attainmentYear]) res[attainmentYear] = 0
+      res[attainmentYear]++
+      return res
+    }, {})
+  )
+  const maxYearsToCreatePopulationFrom = Math.max(
+    Math.floor(
+      1200 / // Lower this value to get a smaller result if necessary
+        (yearlyStudents.reduce((acc, curr) => acc + curr, 0) / yearlyStudents.length)
+    ),
+    1
+  )
+
+  return maxYearsToCreatePopulationFrom
+}
+
+const courseYearlyStats = async (coursecodes, separate, anonymizationSalt, combineSubstitutions) => {
+  const credits = await Credit.findAll({
+    attributes: ['student_studentnumber'],
+    where: { course_code: { [Op.in]: coursecodes } },
+  })
+
+  const enrollment = await Enrollment.findAll({
+    attributes: ['studentnumber'],
+    where: { course_code: { [Op.in]: coursecodes } },
+  })
+
+  const studentnumbersObject = {}
+
+  credits.forEach(cr => {
+    studentnumbersObject[cr.student_studentnumber] = true
+  })
+  enrollment.forEach(cr => {
+    studentnumbersObject[cr.studentnumber] = true
+  })
+
+  const studentNumberToSrElementsMap = await getStudentNumberToSrElementsMap(Object.keys(studentnumbersObject))
+
+  const statsRegular = await Promise.all(
+    coursecodes.map(async code => {
+      const unifyStats = await yearlyStatsOfNew(
+        code,
+        separate,
+        'unify',
+        anonymizationSalt,
+        combineSubstitutions,
+        studentNumberToSrElementsMap
+      )
+      const regularStats = await yearlyStatsOfNew(
+        code,
+        separate,
+        'regular',
+        anonymizationSalt,
+        combineSubstitutions,
+        studentNumberToSrElementsMap
+      )
+      const openStats = await yearlyStatsOfNew(
+        code,
+        separate,
+        'open',
+        anonymizationSalt,
+        combineSubstitutions,
+        studentNumberToSrElementsMap
+      )
+
+      return { unifyStats, regularStats, openStats }
+    })
+  )
+
+  return statsRegular
+}
+
+const getCourseProvidersForCourses = async codes => {
+  return (
+    await Organization.findAll({
+      attributes: ['code'],
+      include: {
+        model: Course,
+        where: {
+          code: {
+            [Op.in]: codes,
+          },
+        },
+      },
+      raw: true,
+    })
+  ).map(({ code }) => code)
+}
+
+module.exports = {
+  courseYearlyStats,
+  maxYearsToCreatePopulationFrom,
+  unifyOpenUniversity,
+  allCodeAlternatives,
+  getCourseProvidersForCourses,
+  byCodes,
+  byNameAndOrCodeLike,
+}

services/backend/src/services/courses/parseCredits.js

@@ -1,5 +1,5 @@
 const crypto = require('crypto')
-const { Credit } = require('../models')
+const { Credit } = require('../../models')
 
 const formatStudyrightElement = ({ code, element_detail, startdate, studyright: sr }) => {
   const studyright = sr.get({ plain: true })
@@ -17,24 +17,20 @@ const formatStudyrightElement = ({ code, element_detail, startdate, studyright:
   }
 }
 
-const parseCredit = (credit, anonymizationSalt) => {
-  const { student, semester, grade, course_code, credits, attainment_date } = credit
-  const { studentnumber, studyright_elements } = student
+const parseCredit = (credit, anonymizationSalt, studentNumberToSrElementsMap) => {
+  const { semester, grade, course_code, credits, attainment_date, student_studentnumber: studentnumber } = credit
   const { yearcode, yearname, semestercode, name: semestername } = semester
 
+  const studyrightElements = studentNumberToSrElementsMap[studentnumber] || []
+
   if (anonymizationSalt) {
     const anonymizedStudentNumber = crypto
       .createHash('sha256')
       .update(`${studentnumber}${anonymizationSalt}`)
       .digest('hex')
 
-    const anonymizedStudent = {
-      studentnumber: anonymizedStudentNumber,
-    }
-
     return {
       obfuscated: true,
-      student: anonymizedStudent,
       yearcode,
       yearname,
       semestercode,
@@ -44,13 +40,12 @@ const parseCredit = (credit, anonymizationSalt) => {
       grade,
       passed: !Credit.failed(credit) || Credit.passed(credit) || Credit.improved(credit),
       studentnumber: anonymizedStudentNumber,
-      programmes: studyright_elements.map(formatStudyrightElement),
+      programmes: studyrightElements.map(formatStudyrightElement),
       credits,
     }
   }
 
   return {
-    student,
     yearcode,
     yearname,
     semestercode,
@@ -60,7 +55,7 @@ const parseCredit = (credit, anonymizationSalt) => {
     grade,
     passed: !Credit.failed(credit) || Credit.passed(credit) || Credit.improved(credit),
     studentnumber,
-    programmes: studyright_elements.map(formatStudyrightElement),
+    programmes: studyrightElements.map(formatStudyrightElement),
     credits,
   }
 }

services/backend/src/services/courses/parseEnrollments.js

@@ -1,24 +1,20 @@
 const crypto = require('crypto')
 const { formatStudyrightElement } = require('./parseCredits')
 
-const parseEnrollment = (enrollment, anonymizationSalt) => {
-  const { student, semester, state, enrollment_date_time, course_code } = enrollment
-  const { studentnumber, studyright_elements } = student
+const parseEnrollment = (enrollment, anonymizationSalt, studentNumberToSrElementsMap) => {
+  const { studentnumber, semester, state, enrollment_date_time, course_code } = enrollment
   const { yearcode, yearname, semestercode, name: semestername } = semester
 
+  const studyrightElements = studentNumberToSrElementsMap[studentnumber] || []
+
   if (anonymizationSalt) {
     const anonymizedStudentNumber = crypto
       .createHash('sha256')
       .update(`${studentnumber}${anonymizationSalt}`)
       .digest('hex')
 
-    const anonymizedStudent = {
-      studentnumber: anonymizedStudentNumber,
-    }
-
     return {
       obfuscated: true,
-      student: anonymizedStudent,
       yearcode,
       yearname,
       semestercode,
@@ -27,12 +23,11 @@ const parseEnrollment = (enrollment, anonymizationSalt) => {
       state,
       enrollment_date_time,
       studentnumber: anonymizedStudentNumber,
-      programmes: studyright_elements.map(formatStudyrightElement),
+      programmes: studyrightElements.map(formatStudyrightElement),
     }
   }
 
   return {
-    student,
     yearcode,
     yearname,
     semestercode,
@@ -41,7 +36,7 @@ const parseEnrollment = (enrollment, anonymizationSalt) => {
     state,
     enrollment_date_time,
     studentnumber,
-    programmes: studyright_elements.map(formatStudyrightElement),
+    programmes: studyrightElements.map(formatStudyrightElement),
   }
 }
 

services/backend/src/services/diff.js

@@ -1,64 +0,0 @@
-// Returns an array containing all values that are in a but not in b taking account duplicates
-const difference = (a, b) => {
-  return a.filter(
-    function (v) {
-      return !this.get(v) || !this.set(v, this.get(v) - 1)
-    },
-    b.reduce((acc, v) => acc.set(v, (acc.get(v) || 0) + 1), new Map())
-  )
-}
-
-const mapStudentNumbers = (studentnumbers, yearcode, name) =>
-  studentnumbers.map(studentnumber => ({
-    studentnumber,
-    yearcode,
-    year: name,
-  }))
-
-const getDiffForCourse = (oodiCourseStats, sisCourseStats) => {
-  return oodiCourseStats.statistics.reduce(
-    (courseDiff, oodiYearlyStats) => {
-      const sisYearlyStats =
-        sisCourseStats && sisCourseStats.statistics.find(s => s.yearcode === oodiYearlyStats.yearcode)
-
-      if (!sisYearlyStats) {
-        return {
-          missingStudents: mapStudentNumbers(oodiYearlyStats.students.studentnumbers),
-          extraStudents: [],
-        }
-      }
-
-      const missingStudentNumbers = difference(
-        oodiYearlyStats.students.studentnumbers,
-        sisYearlyStats.students.studentnumbers
-      )
-      const extraStudentNumbers = difference(
-        sisYearlyStats.students.studentnumbers,
-        oodiYearlyStats.students.studentnumbers
-      )
-
-      const missingStudents = mapStudentNumbers(missingStudentNumbers, oodiYearlyStats.yearcode, oodiYearlyStats.name)
-      const extraStudents = mapStudentNumbers(extraStudentNumbers, oodiYearlyStats.yearcode, oodiYearlyStats.name)
-
-      const noDifferenceInCourseStats = missingStudents.length === 0 && extraStudents.length === 0
-      if (noDifferenceInCourseStats) {
-        return courseDiff
-      }
-
-      return {
-        missingStudents: courseDiff.missingStudents.concat(missingStudents),
-        extraStudents: courseDiff.extraStudents.concat(extraStudents),
-      }
-    },
-    { missingStudents: [], extraStudents: [] }
-  )
-}
-
-const getCourseYearlyStatsDiff = (sisData, oodiData) => {
-  return oodiData.map(oodiCourseStats => {
-    const sisCourseStats = sisData.find(cs => cs.coursecode === oodiCourseStats.coursecode)
-    return { coursecode: oodiCourseStats.coursecode, ...getDiffForCourse(oodiCourseStats, sisCourseStats) }
-  })
-}
-
-module.exports = { getCourseYearlyStatsDiff }

services/backend/src/services/elementdetails.js

@@ -1,22 +0,0 @@
-const { ElementDetail } = require('../models')
-const { Op } = require('sequelize')
-
-const byId = id => ElementDetail.findByPk(id)
-
-const byType = type =>
-  ElementDetail.findAll({
-    where: {
-      type,
-    },
-  })
-
-const byCodes = codes =>
-  ElementDetail.findAll({
-    where: {
-      code: {
-        [Op.in]: codes,
-      },
-    },
-  })
-
-module.exports = { byId, byType, byCodes }

services/backend/src/services/encrypt.js

@@ -1,11 +1,12 @@
-const { randomBytes, createCipheriv, createDecipheriv } = require('crypto')
+const { createCipheriv, createDecipheriv, randomBytes } = require('crypto')
+const { CRYPT_KEY } = require('../conf-backend')
 
 const algorithm = 'aes-256-cbc'
-const key = randomBytes(32)
-const iv = randomBytes(16)
 
 const encrypt = text => {
-  const cipher = createCipheriv(algorithm, Buffer.from(key), iv)
+  if (!CRYPT_KEY) throw Error('Define crypt key in environment variables')
+  const iv = randomBytes(16)
+  const cipher = createCipheriv(algorithm, Buffer.from(CRYPT_KEY, 'hex'), iv)
   let encrypted = cipher.update(text)
   encrypted = Buffer.concat([encrypted, cipher.final()])
 
@@ -16,9 +17,10 @@ const encrypt = text => {
 }
 
 const decrypt = text => {
+  if (!CRYPT_KEY) throw Error('Define crypt key in environment variables')
   const iv = Buffer.from(text.iv, 'hex')
   const encryptedText = Buffer.from(text.encryptedData, 'hex')
-  const decipher = createDecipheriv(algorithm, Buffer.from(key), iv)
+  const decipher = createDecipheriv(algorithm, Buffer.from(CRYPT_KEY, 'hex'), iv)
   let decrypted = decipher.update(encryptedText)
   decrypted = Buffer.concat([decrypted, decipher.final()])
 

services/backend/src/services/faculty/faculty.js

@@ -293,7 +293,7 @@ const degreeProgrammesOfFaculty = async facultyCode =>
     })
   ).map(facultyFormatProgramme)
 
-const facultyOgranizationId = async faculty => {
+const facultyOrganizationId = async faculty => {
   return await Organization.findOne({
     attributes: ['id'],
     where: {
@@ -359,7 +359,7 @@ const findFacultyProgrammeCodes = async (faculty, programmeFilter) => {
   allProgrammeCodes = allProgrammeCodes.concat(directAssociationProgrammes.map(prog => prog.code))
 
   // find faculty's organization id and its direct child organizations
-  const { id } = await facultyOgranizationId(faculty)
+  const { id } = await facultyOrganizationId(faculty)
   const facultyChildOrganizations = await getChildOrganizations(id)
 
   // get programme modules that have a faculty child as organization(_id)
@@ -408,10 +408,8 @@ module.exports = {
   transferredInsideFaculty,
   transferredAway,
   transferredTo,
-  degreeProgrammesOfFaculty,
   thesisWriters,
   findFacultyProgrammeCodes,
-  facultyOgranizationId,
   getTransferredToAndAway,
   getTransferredInside,
   getStudyRightsByExtent,

services/backend/src/services/faculty/facultyBasics.js

@@ -11,7 +11,7 @@ const {
   studyrightsByRightStartYear,
   getTransfersIn,
 } = require('./faculty')
-const { getStatsBasis, getYearsArray, defineYear } = require('../studyprogrammeHelpers')
+const { getStatsBasis, getYearsArray, defineYear } = require('../studyprogramme/studyprogrammeHelpers')
 const { findRightProgramme, isNewProgramme, checkTransfers, getExtentFilter } = require('./facultyHelpers')
 const { codes } = require('../../../config/programmeCodes')
 

services/backend/src/services/faculty/facultyCredits.js

@@ -7,8 +7,10 @@ const {
   isNonMajorCredit,
   getYearsArray,
   getStartDate,
-} = require('../studyprogrammeHelpers')
-const { getStudyRights, getCreditsForStudyProgramme, getCourseCodesForStudyProgramme } = require('../studyprogramme')
+} = require('../studyprogramme/studyprogrammeHelpers')
+const { getCourseCodesForStudyProgramme } = require('../studyprogramme')
+const { getCreditsForStudyProgramme } = require('../studyprogramme/creditGetters')
+const { getStudyRights } = require('../studyprogramme/studyrightFinders')
 
 const isFacultyNonMajorCredit = (studyrights, attainmentDate, facultyProgrammes, facultyCode) => {
   let right = ''
@@ -57,21 +59,31 @@ const getFacultyRegularCreditStats = async ({
   facultyProgrammes,
   facultyCode,
 }) => {
-  const providercode = mapToProviders([studyprogramme])[0]
-  const providedByProgramme = await getCourseCodesForStudyProgramme(providercode)
-  const allCredits = await getCreditsForStudyProgramme(providedByProgramme, since)
-  const credits = uniqBy(allCredits, 'id')
-  const students = [...new Set(credits.map(({ studentNumber }) => studentNumber))]
-
-  let studyRights = await getStudyRights(students)
   let majors = getStatsBasis(years) // Has studyright to programme
   let facultyNonMajors = getStatsBasis(years) // Has studyright in the same faculty but not in the programme from which credits are
   let facultyOthers = getStatsBasis(years) // Exchange students, specialization education in Medicine, Dentistry and vetenary medicine, continuous education
   let otherNonMajors = getStatsBasis(years) // Has studyright in different faculty
   let nonDegree = getStatsBasis(years) // open uni, non-degree study rights, contract studies, (other) specialisation education, credits outside of the studyright
+  if (!studyprogramme) {
+    return { majors, facultyNonMajors, otherNonMajors, facultyOthers, nonDegree }
+  }
+
+  const providercode = mapToProviders([studyprogramme])[0]
+  const providedByProgramme = await getCourseCodesForStudyProgramme(providercode)
+  const allCredits = await getCreditsForStudyProgramme(providercode, providedByProgramme, since)
+  const credits = uniqBy(allCredits, 'id')
+  const students = [...new Set(credits.map(({ studentNumber }) => studentNumber))]
+
+  let studyrights = await getStudyRights(students)
+
+  const studentNumberToStudyrightsMap = studyrights.reduce((obj, cur) => {
+    if (!obj[cur.studentNumber]) obj[cur.studentNumber] = []
+    obj[cur.studentNumber].push(cur)
+    return obj
+  }, {})
 
   credits.forEach(({ studentNumber, attainmentDate, credits }) => {
-    const studentStudyrights = studyRights.filter(studyright => studyright.studentNumber === studentNumber)
+    const studentStudyrights = studentNumberToStudyrightsMap[studentNumber] || []
     const attainmentYear = defineYear(attainmentDate, isAcademicYear)
 
     if (isMajorStudentCredit(studentStudyrights, attainmentDate, studyprogramme)) {
@@ -267,4 +279,4 @@ const combineFacultyCredits = async (faculty, programmes, allProgrammes, yearTyp
   return allCredits
 }
 
-module.exports = { combineFacultyCredits, getFacultyCreditStatsForStudytrack }
+module.exports = { combineFacultyCredits }

services/backend/src/services/faculty/facultyGraduationTimes.js

@@ -1,7 +1,7 @@
 const moment = require('moment')
 const { graduatedStudyrights, studyrightsByRightStartYear, hasMasterRight } = require('./faculty')
 const { findRightProgramme, isNewProgramme } = require('./facultyHelpers')
-const { getYearsArray, getYearsObject, getMedian, defineYear } = require('../studyprogrammeHelpers')
+const { getYearsArray, getYearsObject, getMedian, defineYear } = require('../studyprogramme/studyprogrammeHelpers')
 const { codes } = require('../../../config/programmeCodes')
 const { bachelorStudyright, countTimeCategories, getStatutoryAbsences } = require('../graduationHelpers')
 const { getProgrammes } = require('./facultyService')
@@ -285,7 +285,16 @@ const count = async (
       }
       data.medians[level] = [
         ...data.medians[level],
-        { y: median, amount: graduationAmounts[level][year], statistics, name: year },
+        {
+          y: median,
+          amount: graduationAmounts[level][year],
+          /* Include invidivual graduation times for the sake of university-level evaluation overview (UniversityView) 
+            which has to count medians by itself 
+          */
+          times: graduationTimes[level][year],
+          statistics,
+          name: year,
+        },
       ]
     })
   })

services/backend/src/services/faculty/facultyHelpers.js

@@ -1,6 +1,14 @@
 const Sequelize = require('sequelize')
 const { Op } = Sequelize
 const { codes } = require('../../../config/programmeCodes')
+const { faculties } = require('../organisations')
+
+const getFacultyList = async () => {
+  const ignore = ['Y', 'H99', 'Y01', 'H92', 'H930']
+  const facultyList = (await faculties()).filter(f => !ignore.includes(f.code))
+  facultyList.sort((a, b) => (a.name.fi > b.name.fi ? 1 : -1))
+  return facultyList
+}
 
 const findRightProgramme = (studyrightElements, code) => {
   let programme = ''
@@ -140,6 +148,7 @@ const mapCodesToIds = data => {
 }
 
 module.exports = {
+  getFacultyList,
   findRightProgramme,
   facultyFormatStudyright,
   facultyFormatProgramme,

services/backend/src/services/faculty/facultyService.js

@@ -16,6 +16,21 @@ const createRediskeyForFacultyProgress = (id, special_groups, graduated) =>
 const createRediskeyForFacultyStudents = (id, special_groups, graduated) =>
   `FACULTY_STUDENTS_STATS_${id}_${special_groups}_${graduated}`
 
+/*
+  Faculty data objects have graduation times left in as arrays, so that
+  university-level evaluation overview can count median-times by itself.
+  Faculties, however, don't need this, and it isn't needed in frontend
+  for universityview either. This removes them from the object.
+*/
+const removeGraduationTimes = data => {
+  Object.values(data.byGradYear.medians).forEach(array =>
+    array.forEach(yearStat => {
+      yearStat.times = null
+    })
+  )
+  data.byGradYear.medians
+}
+
 const setFacultyProgrammes = async (id, data, programmeFilter) => {
   const redisKey = createRedisKeyForFacultyProgrammes(id, programmeFilter)
   const dataToRedis = {
@@ -116,11 +131,15 @@ const setGraduationStats = async (data, programmeFilter) => {
   return dataToRedis
 }
 
-const getGraduationStats = async (id, programmeFilter) => {
+const getGraduationStats = async (id, programmeFilter, keepGraduationTimes = false) => {
   const redisKey = createRedisKeyForGraduationTimeStats(id, programmeFilter)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
-  return JSON.parse(dataFromRedis)
+  const data = JSON.parse(dataFromRedis)
+  if (!keepGraduationTimes) {
+    removeGraduationTimes(data)
+  }
+  return data
 }
 
 const getFacultyProgressStats = async (id, specialGroups, graduated) => {

services/backend/src/services/faculty/facultyStudentProgress.js

@@ -1,8 +1,8 @@
-const { indexOf } = require('lodash')
+const { indexOf, inRange } = require('lodash')
 const moment = require('moment')
 const { getAcademicYearDates } = require('../../util/semester')
-const { getCreditThresholds, getYearsArray } = require('../studyprogrammeHelpers')
-const { studytrackStudents } = require('../studyprogramme')
+const { getCreditThresholds, getYearsArray } = require('../studyprogramme/studyprogrammeHelpers')
+const { studytrackStudents } = require('../studyprogramme/studentGetters')
 const { getStudyRightsByExtent, getStudyRightsByBachelorStart, getTransfersIn, getTransfersOut } = require('./faculty')
 const { checkTransfers } = require('./facultyHelpers')
 
@@ -17,44 +17,39 @@ const getStudentData = (
 ) => {
   let data = {}
   let programmeData = {}
-  const creditCounts = []
+  const creditCounts = students.map(({ credits }) =>
+    credits
+      .filter(credit => moment(credit.attainment_date).isSameOrAfter(startDate))
+      .reduce((total, credit) => total + credit.credits, 0)
+  )
   thresholdKeys.forEach(t => (data[t] = 0))
   limitKeys.forEach(t => (programmeData[t] = 0))
-  students.forEach(({ credits }) => {
-    const creditcount = credits
-      .filter(credit => moment(credit.attainment_date).isSameOrAfter(startDate))
-      .reduce((prev, curr) => prev + curr.credits, 0)
-    creditCounts.push(creditcount)
-    // Data is only used for doctoral programmes, otherwise only programmeData is needed
-    if (prog === 'T') {
-      data[thresholdKeys[0]] += creditcount < thresholdAmounts[0] ? 1 : 0
-      data[thresholdKeys[1]] += creditcount >= thresholdAmounts[0] && creditcount < thresholdAmounts[1] ? 1 : 0
-      data[thresholdKeys[2]] += creditcount >= thresholdAmounts[1] && creditcount < thresholdAmounts[2] ? 1 : 0
-      data[thresholdKeys[3]] += creditcount >= thresholdAmounts[2] && creditcount < thresholdAmounts[3] ? 1 : 0
-      if (thresholdKeys.length === 8) {
-        data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
-        data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
-        data[thresholdKeys[6]] += creditcount >= thresholdAmounts[5] && creditcount < thresholdAmounts[6] ? 1 : 0
-        data[thresholdKeys[7]] += creditcount >= thresholdAmounts[6] ? 1 : 0
-      } else if (thresholdKeys.length === 7) {
-        data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
-        data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
-        data[thresholdKeys[6]] += creditcount >= thresholdAmounts[5] ? 1 : 0
-      } else if (thresholdKeys.length === 6) {
-        data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
-        data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] ? 1 : 0
-      } else {
-        data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] ? 1 : 0
-      }
-    } else {
-      programmeData[limitKeys[0]] += creditcount <= limits[5][0] ? 1 : 0
-      programmeData[limitKeys[1]] += limits[4][0] <= creditcount && limits[4][1] > creditcount ? 1 : 0
-      programmeData[limitKeys[2]] += limits[3][0] <= creditcount && limits[3][1] > creditcount ? 1 : 0
-      programmeData[limitKeys[3]] += limits[2][0] <= creditcount && limits[2][1] > creditcount ? 1 : 0
-      programmeData[limitKeys[4]] += limits[1][0] <= creditcount && limits[1][1] > creditcount ? 1 : 0
-      programmeData[limitKeys[5]] += creditcount >= limits[0][0] ? 1 : 0
-    }
-  })
+
+  // Data is only used for doctoral programmes, otherwise only programmeData is needed
+  if (prog === 'T') {
+    creditCounts.forEach(creditCount => {
+      thresholdKeys.some((thresholdKey, index) => {
+        if (
+          (index === 0 && creditCount < thresholdAmounts[0]) ||
+          (index === thresholdKeys.length - 1 && creditCount >= thresholdAmounts[index - 1]) ||
+          (index > 0 && inRange(creditCount, thresholdAmounts[index - 1], thresholdAmounts[index]))
+        ) {
+          data[thresholdKey] += 1
+          return true
+        }
+        return false
+      })
+    })
+  } else {
+    creditCounts.forEach(creditCount => {
+      programmeData[limitKeys[0]] += creditCount <= limits[5][0] ? 1 : 0
+      programmeData[limitKeys[1]] += inRange(creditCount, limits[4][0], limits[4][1]) ? 1 : 0
+      programmeData[limitKeys[2]] += inRange(creditCount, limits[3][0], limits[3][1]) ? 1 : 0
+      programmeData[limitKeys[3]] += inRange(creditCount, limits[2][0], limits[2][1]) ? 1 : 0
+      programmeData[limitKeys[4]] += inRange(creditCount, limits[1][0], limits[1][1]) ? 1 : 0
+      programmeData[limitKeys[5]] += creditCount >= limits[0][0] ? 1 : 0
+    })
+  }
 
   return { data, progData: programmeData, creditCounts }
 }

services/backend/src/services/faculty/facultyStudents.js

@@ -1,9 +1,15 @@
-const { getStartDate, getYearsArray, getPercentage, getYearsObject, tableTitles } = require('../studyprogrammeHelpers')
+const {
+  getStartDate,
+  getYearsArray,
+  getPercentage,
+  getYearsObject,
+  tableTitles,
+} = require('../studyprogramme/studyprogrammeHelpers')
 const { getAcademicYearDates } = require('../../util/semester')
 const { getStudyRightsByExtent, getStudentsByStudentnumbers, getTransfersIn } = require('./faculty')
 const { checkTransfers } = require('./facultyHelpers')
-const { inactiveStudyrights, absentStudents, enrolledStudents } = require('../studyprogramme')
-const { graduatedStudyRights } = require('../studyprogramme')
+const { inactiveStudyrights, graduatedStudyRights } = require('../studyprogramme/studyrightFinders')
+const { enrolledStudents, absentStudents } = require('../studyprogramme/studentGetters')
 const emptyTotals = () => {
   return {
     total: 0,
@@ -19,7 +25,7 @@ const emptyTotals = () => {
     allOtherCountries: 0,
   }
 }
-const toltalsCalculation = totals => {
+const totalsCalculation = totals => {
   return [
     totals.total,
     totals.allStarted,
@@ -45,10 +51,10 @@ const toltalsCalculation = totals => {
   ]
 }
 const addTotals = (facultyTableStats, year, totals) => {
-  facultyTableStats[year] = [year, ...toltalsCalculation(totals)]
+  facultyTableStats[year] = [year, ...totalsCalculation(totals)]
 }
 const addTotalsProgramme = (programmeTableStats, progId, year, totals) => {
-  programmeTableStats[progId][year] = toltalsCalculation(totals)
+  programmeTableStats[progId][year] = totalsCalculation(totals)
 }
 
 const getStudentData = (students, facultyExtra, year, code) => {

services/backend/src/services/faculty/facultyThesisWriters.js

@@ -8,7 +8,7 @@ const {
   getCorrectStudentnumbers,
   alltimeEndDate,
   alltimeStartDate,
-} = require('../studyprogrammeHelpers')
+} = require('../studyprogramme/studyprogrammeHelpers')
 
 const getFacultyThesisWriters = async ({ since, years, isAcademicYear, facultyProgrammes, includeAllSpecials }) => {
   const thesisTypes = ['urn:code:course-unit-type:bachelors-thesis', 'urn:code:course-unit-type:masters-thesis']

services/backend/src/services/faculty/facultyUpdates.js

@@ -84,6 +84,7 @@ const updateFacultyOverview = async (faculty, statsType) => {
     allProgrammes?.data.forEach(prog => allProgrammeCodes.push(prog.code))
   } catch (e) {
     logger.error(`Faculty updates: programme stats failed with error: ${e}`)
+    logger.error(`Stack: ${e.stack}`)
   }
 
   for (const option of options) {
@@ -120,6 +121,7 @@ const updateFacultyOverview = async (faculty, statsType) => {
       }
     } catch (e) {
       logger.error(`Faculty updates: basic/thesis-writers/credits stats failed with error ${e}`)
+      logger.error(`Stack: ${e.stack}`)
     }
   }
 
@@ -155,6 +157,7 @@ const updateFacultyProgressOverview = async faculty => {
     newProgrammes = await setFacultyProgrammes(faculty, onlyNew, 'NEW_STUDY_PROGRAMMES')
   } catch (e) {
     logger.error(`Faculty stats: all programme stats failed with error ${e}`)
+    logger.error(`Stack: ${e.stack}`)
   }
 
   for (const option of options) {
@@ -176,6 +179,7 @@ const updateFacultyProgressOverview = async faculty => {
       await setFacultyProgressStats(updateFacultyProgressStats, specialGroups, graduated)
     } catch (e) {
       logger.error(`Faculty stats: progress stats failed with error ${e}`)
+      logger.error(`Stack: ${e.stack}`)
     }
   }
   return 'OK'

services/backend/src/services/languageCenterData.js

@@ -9,12 +9,18 @@ const isBetween = (start, date, end) =>
 
 const findStudyright = (studyrights, date) => studyrights?.find(sr => isBetween(sr.startdate, date, sr.enddate))
 
+const getDifference = stats => {
+  const value = stats.enrollments - stats.completions
+  return value < 0 ? 0 : value
+}
+
 const getLanguageCenterCourses = async () => {
   const courses = await Course.findAll({
     attributes: ['code', 'name'],
     where: {
       [Op.or]: [{ code: { [Op.like]: 'KK%' } }, { code: { [Op.like]: 'AYKK%' } }],
     },
+    raw: true,
   })
   return courses
 }
@@ -31,25 +37,22 @@ const computeLanguageCenterData = async () => {
   const courses = await getLanguageCenterCourses()
 
   const credits = await Credit.findAll({
-    attributes: [
-      'course_code',
-      'credittypecode',
-      'student_studentnumber',
-      'semestercode',
-      'createdate',
-      'studyright_id',
-    ],
+    attributes: ['course_code', 'student_studentnumber', 'semestercode', 'attainment_date', 'studyright_id'],
     where: {
       [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
+      // 135 = autumn 2017
+      semestercode: { [Op.gte]: 135 },
+      credittypecode: 4,
     },
     raw: true,
   })
 
   const enrollments = await Enrollment.findAll({
-    attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'studyright_id'],
+    attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time', 'studyright_id', 'state'],
     where: {
       [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
-      state: 'ENROLLED',
+      semestercode: { [Op.gte]: 135 },
+      state: { [Op.in]: ['ENROLLED', 'REJECTED'] },
     },
     raw: true,
   })
@@ -87,9 +90,6 @@ const computeLanguageCenterData = async () => {
 
   credits.forEach(c => {
     const sn = c.student_studentnumber
-
-    // ignoring 7 (improved), 9 (transferred) and 10 (failed)
-    if (c.credittypecode !== 4) return
     studentList.add(sn)
     if (!attemptsByStudents[sn]) {
       attemptsByStudents[sn] = []
@@ -98,7 +98,7 @@ const computeLanguageCenterData = async () => {
       studentNumber: sn,
       courseCode: c.course_code,
       completed: true,
-      date: c.createdate,
+      date: c.attainment_date,
       faculty: attemptStudyrightToFacultyMap[c.studyright_id],
       semestercode: c.semestercode,
     })
@@ -123,6 +123,7 @@ const computeLanguageCenterData = async () => {
       date: e.enrollment_date_time,
       faculty: attemptStudyrightToFacultyMap[e.studyright_id],
       semestercode: e.semestercode,
+      enrolled: e.state === 'ENROLLED',
     })
   })
 
@@ -166,6 +167,7 @@ const computeLanguageCenterData = async () => {
       }
       return
     }
+
     // Credit or enrollment did not have studyright_id. Find based on date,
     // and if studyrightElement is not found, check studyrights because open uni rights
     // do not have studyrightelements
@@ -189,46 +191,65 @@ const computeLanguageCenterData = async () => {
 }
 
 const createArrayOfCourses = async (attempts, courses) => {
-  const fields = { completions: 0, enrollments: 0, ratio: 0 }
+  const fields = { completions: 0, enrollments: 0, difference: 0, rejected: 0 }
+  const semestersObject = {}
+  const facultyObject = {}
   const semesterStatsMap = attempts.reduce((obj, cur) => {
     const semester = cur.semestercode
+    facultyObject[cur.faculty] = true
+    semestersObject[semester] = true
     if (!obj[cur.courseCode]) {
-      obj[cur.courseCode] = { ...fields }
+      obj[cur.courseCode] = {}
     }
     if (!obj[cur.courseCode][semester]) {
       obj[cur.courseCode][semester] = { ...fields }
     }
-    if (!obj[cur.courseCode][semester][cur.faculty]) {
+    if (cur.faculty && !obj[cur.courseCode][semester][cur.faculty]) {
       obj[cur.courseCode][semester][cur.faculty] = { ...fields }
     }
-    if (!obj[cur.courseCode].byFaculties?.[cur.faculty]) {
-      if (!obj[cur.courseCode].byFaculties) obj[cur.courseCode].byFaculties = { ...fields }
-      obj[cur.courseCode].byFaculties[cur.faculty] = { ...fields }
-    }
-    const semesterFacultyStats = obj[cur.courseCode][semester][cur.faculty]
     const allFacultiesTotal = obj[cur.courseCode][semester]
-    const allSemestersTotal = obj[cur.courseCode]
-    const facultyTotalStats = obj[cur.courseCode].byFaculties[cur.faculty]
-    const allStats = [semesterFacultyStats, allFacultiesTotal, allSemestersTotal, facultyTotalStats]
+    const allStats = [allFacultiesTotal]
+    if (cur.faculty) {
+      const semesterFacultyStats = obj[cur.courseCode][semester][cur.faculty]
+      allStats.push(semesterFacultyStats)
+    }
     for (const stats of allStats) {
       if (cur.completed) {
         stats.completions += 1
       } else {
-        stats.enrollments += 1
+        if (cur.enrolled) {
+          stats.enrollments += 1
+        } else {
+          stats.rejected += 1
+        }
       }
-      stats.ratio = stats.enrollments === 0 ? 1 : stats.completions / stats.enrollments
-      if (stats.ratio > 1) stats.ratio = 1
     }
     return obj
   }, {})
 
-  return courses
-    .map(c => ({ ...c.dataValues, bySemesters: { ...semesterStatsMap[c.code], cellStats: {} } }))
+  const courseList = courses
+    .map(c => ({ ...c, bySemesters: { ...semesterStatsMap[c.code], cellStats: {} } }))
     .filter(course => course.bySemesters)
+
+  courseList.forEach(course => {
+    Object.keys(semestersObject).forEach(sem => {
+      const stats = course.bySemesters[sem]
+      if (!stats) return
+      stats.difference = getDifference(stats)
+      Object.keys(facultyObject).forEach(fac => {
+        const stats = course.bySemesters[sem]?.[fac]
+        if (!stats) return
+        stats.difference = getDifference(stats)
+      })
+    })
+  })
+
+  return courseList
 }
 
 module.exports = {
   getLanguageCenterData,
   LANGUAGE_CENTER_REDIS_KEY,
   computeLanguageCenterData,
+  createArrayOfCourses,
 }

services/backend/src/services/mailservice.js

@@ -64,8 +64,15 @@ const sendNotificationAboutNewUser = ({ userId, userFullName }) =>
     },
   })
 
-const accessMessageSubject = 'Authorized in Oodikone'
-const accessMessageText = `<p>You've been authorized to use <a href="https://oodikone.cs.helsinki.fi">Oodikone</a> or your privileges in <a href="https://oodikone.cs.helsinki.fi">Oodikone</a> have changed.</p>
+const accessMessageSubject =
+  'Käyttöoikeutesi Oodikoneeseen on päivitetty – Your privileges in Oodikone have been updated'
+const accessMessageText = `<p>Sinulle on annettu oikeudet käyttää <a href="https://oodikone.helsinki.fi">Oodikonetta</a> tai käyttöoikeutesi Oodikoneeseen on päivitetty.</p>
+<p>Jotta muutokset astuvat voimaan, saatat joutua kirjautumaan ulos ja takaisin sisään.</p>
+<p>HUOM! Jos automaattinen uloskirjautuminen kaikista palveluista ei onnistunut kirjautuessasi ulos Oodikoneesta, voit kirjautua ulos jokaisesta palvelusta yksitellen tai sulkea kaikki selainikkunasi.</p>
+<p>Jos olet määritellyt selaimen palauttamaan edellisen istuntosi käynnistyksen yhteydessä, saatat joutua selainikkunoiden sulkemisen lisäksi poistamaan myös listattujen palvelujen evästeet.</p>
+<p>Katso ohjeet selaimen välimuistin tyhjentämiseksi: <a href="https://helpdesk.it.helsinki.fi/help/1002">https://helpdesk.it.helsinki.fi/help/1002</a></p>
+<br/>
+<p>You've been authorized to use <a href="https://oodikone.helsinki.fi">Oodikone</a> or your privileges in Oodikone have changed.</p>
 <p>You may need to log out and back in before your changes take place.</p>
 <p>NOTE! If the automatic logging out from all services was not successful when logging out of oodikone, you can log out one by one or close all of your browser windows.</p>
 <p>If you have defined your browser to restore your previous session in connection with starting the browser, logging out may also require deleting the cookies of the listed services in addition to closing the browser windows.</p>

services/backend/src/services/populations/bottlenecksOf.js

@@ -0,0 +1,211 @@
+const { keyBy } = require('lodash')
+const {
+  checkThatSelectedStudentsAreUnderRequestedStudyright,
+  findCourses,
+  dateMonthsFromNow,
+  findCourseEnrollments,
+  parseQueryParams,
+  parseCreditInfo,
+} = require('./shared')
+const { studentnumbersWithAllStudyrightElements } = require('./studentnumbersWithAllStudyrightElements')
+const { encrypt } = require('../encrypt')
+const { CourseStatsCounter } = require('../courses/course_stats_counter')
+const { getPassingSemester } = require('../../util/semester')
+
+const bottlenecksOf = async (query, studentnumberlist, encryptdata = false) => {
+  const isValidRequest = async (query, params) => {
+    const { studyRights, startDate, endDate, exchangeStudents, nondegreeStudents, transferredStudents } = params
+
+    if (!query.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
+      return { error: 'Semester should be either SPRING OR FALL' }
+    }
+    if (
+      query.studentStatuses &&
+      !query.studentStatuses.every(
+        status => status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
+      )
+    ) {
+      return { error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' }
+    }
+    if (query.selectedStudents) {
+      const allStudents = await studentnumbersWithAllStudyrightElements({
+        studyRights,
+        startDate,
+        endDate,
+        exchangeStudents,
+        nondegreeStudents,
+        transferredOutStudents: transferredStudents,
+        tag: query.tag,
+        transferredToStudents: true,
+        graduatedStudents: true,
+      })
+      const disallowedRequest = checkThatSelectedStudentsAreUnderRequestedStudyright(
+        query.selectedStudents,
+        allStudents
+      )
+      if (disallowedRequest) return { error: 'Trying to request unauthorized students data' }
+    }
+    return null
+  }
+
+  const getStudentsAndCourses = async (selectedStudents, studentnumberlist, courseCodes) => {
+    if (!studentnumberlist) {
+      const { months, studyRights, startDate, endDate, exchangeStudents, nondegreeStudents, transferredStudents, tag } =
+        params
+      const studentnumbers =
+        selectedStudents ||
+        (await studentnumbersWithAllStudyrightElements({
+          studyRights,
+          startDate,
+          endDate,
+          exchangeStudents,
+          nondegreeStudents,
+          transferredOutStudents: transferredStudents,
+          tag,
+          transferredToStudents: true,
+          graduatedStudents: true,
+        }))
+
+      const allstudents = studentnumbers.reduce((numbers, num) => {
+        numbers[num] = true
+        return numbers
+      }, {})
+      const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months), courseCodes)
+      const foundCourseCodes = Object.keys(keyBy(courses, 'code'))
+      const filteredCourseCodes = courseCodes?.filter(code => !foundCourseCodes.includes(code))
+
+      const courseEnrollements = await findCourseEnrollments(
+        studentnumbers,
+        dateMonthsFromNow(startDate, months),
+        filteredCourseCodes
+      )
+      return [allstudents, courses, courseEnrollements]
+    } else {
+      const { months, startDate } = params
+      const beforeDate = months && startDate ? dateMonthsFromNow(startDate, months) : new Date()
+      const allstudents = studentnumberlist.reduce((numbers, num) => {
+        numbers[num] = true
+        return numbers
+      }, {})
+      const courses = await findCourses(studentnumberlist, beforeDate, courseCodes)
+      const foundCourseCodes = Object.keys(keyBy(courses, 'code'))
+      const filteredCourseCodes = courseCodes?.filter(code => !foundCourseCodes.includes(code))
+
+      const courseEnrollements = await findCourseEnrollments(studentnumberlist, beforeDate, filteredCourseCodes)
+      return [allstudents, courses, courseEnrollements]
+    }
+  }
+
+  const encryptStudentnumbers = bottlenecks => {
+    for (const course in bottlenecks.coursestatistics) {
+      const encryptedStudentStats = {}
+      for (const data in bottlenecks.coursestatistics[course].students) {
+        encryptedStudentStats[data] = {}
+        const studentnumbers = Object.keys(bottlenecks.coursestatistics[course].students[data])
+        studentnumbers.forEach(studentnumber => {
+          encryptedStudentStats[data][encrypt(studentnumber).encryptedData] =
+            bottlenecks.coursestatistics[course].students[data][studentnumber]
+        })
+      }
+      bottlenecks.coursestatistics[course].students = encryptedStudentStats
+    }
+  }
+
+  const params = parseQueryParams(query)
+  const allStudentsByYears = query?.selectedStudentsByYear
+    ? Object.keys(query.selectedStudentsByYear).reduce(
+        (res, year) => [...res, ...query.selectedStudentsByYear[year]],
+        []
+      )
+    : []
+  // To fix failed and enrolled, no grade filter options some not so clean and nice solutions were added
+  // Get the data with actual 1. courses and filtered students. 2. all students by year, if provided.
+  const [[allstudents, courses, courseEnrollements], [, allCourses], error] = await Promise.all([
+    getStudentsAndCourses(query.selectedStudents, studentnumberlist, query.courses),
+    getStudentsAndCourses(allStudentsByYears, null, query.courses),
+    isValidRequest(query, params),
+  ])
+
+  if (error) return error
+  // Get the substitution codes for the fetch data by selscted students
+  const substitutionCodes = Object.entries(courses).reduce(
+    (res, [, obj]) => [...res, ...(obj?.substitutions || [])],
+    []
+  )
+  const codes = Object.keys(keyBy(courses, 'code')).map(code => code)
+  // Filter substitution courses for fetched courses -> by this we avoid the situation in which there is only
+  // courses with old course codes. Frontend NEEDS in most cases the current course.
+  const substitutionCourses = allCourses.filter(
+    obj => substitutionCodes.includes(obj.code) && !codes.includes(obj.code)
+  )
+  const bottlenecks = {
+    disciplines: {},
+    coursetypes: {},
+  }
+
+  const stats = {}
+  const startYear = parseInt(query.year, 10)
+  const allstudentslength = Object.keys(allstudents).length
+  let coursesToLoop = courses.concat(substitutionCourses)
+  const codesList = coursesToLoop.map(course => course.code)
+  // This fix problem when Enrolled no grade is chosen. The sql query for fetching
+  // credits do not fetch enrollments if no credits found for selected students.
+  // This and other sql query ensures that enrollments are added.
+  const coursesOnlyWithEnrollments = courseEnrollements.filter(
+    course => !codesList.includes(course.code) && course.enrollments
+  )
+  coursesToLoop = coursesToLoop.concat(coursesOnlyWithEnrollments)
+
+  const coursesByCode = keyBy(coursesToLoop, 'code')
+  for (const course of coursesToLoop) {
+    let { course_type } = course
+    let maincourse = course
+
+    if (course.main_course_code && course.main_course_code !== course.code) {
+      let newmain = coursesByCode[course.main_course_code]
+
+      if (newmain) {
+        maincourse = newmain
+      }
+    }
+
+    if (!stats[maincourse.code]) {
+      stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudentslength)
+    }
+    const coursestats = stats[maincourse.code]
+    coursestats.addCourseType(course_type.coursetypecode, course_type.name)
+    coursestats.addCourseSubstitutions(course.substitutions)
+    bottlenecks.coursetypes[course_type.coursetypecode] = course_type.name
+    if (course.enrollments) {
+      course.enrollments.forEach(({ studentnumber, state, enrollment_date_time }) => {
+        if ((query?.selectedStudents && query?.selectedStudents.includes(studentnumber)) || !query?.selectedStudents) {
+          const semester = getPassingSemester(startYear, enrollment_date_time)
+          coursestats.markEnrollment(studentnumber, state, semester, enrollment_date_time)
+        }
+      })
+    }
+    if (course.credits) {
+      course.credits.forEach(credit => {
+        const { studentnumber, passingGrade, improvedGrade, failingGrade, grade, date } = parseCreditInfo(credit)
+        if ((query?.selectedStudents && query?.selectedStudents.includes(studentnumber)) || !query?.selectedStudents) {
+          const semester = getPassingSemester(startYear, date)
+          coursestats.markCredit(studentnumber, grade, passingGrade, failingGrade, improvedGrade, semester)
+        }
+      })
+    }
+
+    stats[maincourse.code] = coursestats
+  }
+
+  const allStats = Object.values(stats).map(coursestatistics => coursestatistics.getFinalStats())
+  bottlenecks.coursestatistics = allStats.filter(course => course.stats.students > 0)
+  bottlenecks.allStudents = allstudentslength
+
+  if (encryptdata) encryptStudentnumbers(bottlenecks)
+
+  return bottlenecks
+}
+
+module.exports = {
+  bottlenecksOf,
+}

services/backend/src/services/populations/getStudentsIncludeCoursesBetween.js

@@ -0,0 +1,281 @@
+const { Op } = require('sequelize')
+const { Tag, TagStudent } = require('../../models/models_kone')
+const {
+  Studyplan,
+  Course,
+  Credit,
+  Enrollment,
+  Student,
+  Transfer,
+  Studyright,
+  StudyrightElement,
+  ElementDetail,
+  SemesterEnrollment,
+  Semester,
+  StudyrightExtent,
+} = require('../../models')
+const moment = require('moment')
+const {
+  dbConnections: { sequelize },
+} = require('../../database/connection')
+
+const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDate, studyright, tag) => {
+  const studentTags = await TagStudent.findAll({
+    attributes: ['tag_id', 'studentnumber'],
+    include: [
+      {
+        model: Tag,
+        attributes: ['tag_id', 'tagname', 'personal_user_id'],
+      },
+    ],
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumbers,
+      },
+    },
+  })
+
+  const { studentnumbersWithTag, studentNumberToTags } = studentTags.reduce(
+    (acc, t) => {
+      acc.studentNumberToTags[t.studentnumber] = acc.studentNumberToTags[t.studentnumber] || []
+      acc.studentNumberToTags[t.studentnumber].push(t)
+      if (tag && t.tag_id === tag.tag_id) {
+        acc.studentnumbersWithTag.push(t.studentnumber)
+      }
+      return acc
+    },
+    { studentnumbersWithTag: [], studentNumberToTags: {} }
+  )
+  if (tag) studentnumbers = studentnumbersWithTag
+
+  const attainmentDateFrom = tag ? moment(startDate).year(tag.year) : startDate
+  const studyPlans = await Studyplan.findAll({
+    where: { studentnumber: studentnumbers },
+    attributes: ['included_courses'],
+    raw: true,
+  })
+  const studyPlanCourses = Array.from(new Set([...studyPlans.map(plan => plan.included_courses)].flat()))
+
+  const creditsOfStudentOther = {
+    [Op.or]: [
+      {
+        attainment_date: {
+          [Op.between]: [attainmentDateFrom, endDate],
+        },
+      },
+      {
+        course_code: studyPlanCourses,
+      },
+    ],
+    student_studentnumber: {
+      [Op.in]: studentnumbers,
+    },
+  }
+
+  // takes into accout possible progress tests taken earlier than the start date
+  const creditsOfStudentLaakis = {
+    student_studentnumber: {
+      [Op.in]: studentnumbers,
+    },
+    [Op.or]: [
+      {
+        attainment_date: {
+          [Op.between]: [attainmentDateFrom, endDate],
+        },
+      },
+      {
+        course_code: {
+          [Op.in]: ['375063', '339101'].concat(studyPlanCourses),
+        },
+      },
+    ],
+  }
+
+  const creditsOfStudent = ['320001', 'MH30_001'].includes(studyright[0])
+    ? creditsOfStudentLaakis
+    : creditsOfStudentOther
+
+  if (studentnumbers.length === 0)
+    return { students: [], enrollments: [], credits: [], extents: [], semesters: [], elementdetails: [], courses: [] }
+
+  const [courses, enrollments, students, credits, extents, semesters, elementdetails] = await Promise.all([
+    Course.findAll({
+      attributes: [sequelize.literal('DISTINCT ON("code") code'), 'name', 'coursetypecode'],
+      include: [
+        {
+          model: Credit,
+          attributes: [],
+          where: creditsOfStudent,
+        },
+      ],
+      raw: true,
+    }),
+    Enrollment.findAll({
+      attributes: ['course_code', 'state', 'enrollment_date_time', 'studentnumber', 'semestercode'],
+      where: {
+        enrollment_date_time: {
+          [Op.between]: [attainmentDateFrom, endDate],
+        },
+        studentnumber: {
+          [Op.in]: studentnumbers,
+        },
+        state: ['ENROLLED', 'CONFIRMED'],
+      },
+      raw: true,
+    }),
+    Student.findAll({
+      attributes: [
+        'firstnames',
+        'lastname',
+        'studentnumber',
+        'home_country_en',
+        'dateofuniversityenrollment',
+        'creditcount',
+        'abbreviatedname',
+        'email',
+        'secondary_email',
+        'phone_number',
+        'updatedAt',
+        'gender_code',
+        'birthdate',
+        'sis_person_id',
+      ],
+      include: [
+        {
+          model: Transfer,
+          attributes: ['transferdate', 'sourcecode', 'targetcode'],
+          separate: true,
+        },
+        {
+          model: Studyright,
+          attributes: [
+            'studyrightid',
+            'startdate',
+            'enddate',
+            'extentcode',
+            'graduated',
+            'active',
+            'prioritycode',
+            'faculty_code',
+            'studystartdate',
+            'admission_type',
+            'cancelled',
+            'is_ba_ma',
+            'semester_enrollments',
+          ],
+          separate: true,
+          include: [
+            {
+              model: StudyrightElement,
+              required: true,
+              attributes: ['id', 'startdate', 'enddate', 'studyrightid', 'code'],
+              include: {
+                model: ElementDetail,
+              },
+            },
+          ],
+        },
+        {
+          model: SemesterEnrollment,
+          attributes: ['enrollmenttype', 'semestercode', 'enrollment_date'],
+          separate: true,
+          include: {
+            model: Semester,
+            attributes: [],
+            required: true,
+          },
+        },
+        {
+          model: Studyplan,
+          attributes: ['included_courses', 'programme_code', 'completed_credits', 'studyrightid'],
+          separate: true,
+        },
+      ],
+      where: {
+        studentnumber: {
+          [Op.in]: studentnumbers,
+        },
+      },
+    }),
+    Credit.findAll({
+      attributes: [
+        'grade',
+        'credits',
+        'credittypecode',
+        'attainment_date',
+        'isStudyModule',
+        'student_studentnumber',
+        'course_code',
+        'language',
+      ],
+      where: creditsOfStudent,
+      raw: true,
+    }),
+    StudyrightExtent.findAll({
+      attributes: [
+        sequelize.literal('DISTINCT ON("studyright_extent"."extentcode") "studyright_extent"."extentcode"'),
+        'name',
+      ],
+      include: [
+        {
+          model: Studyright,
+          attributes: [],
+          required: true,
+          where: {
+            student_studentnumber: {
+              [Op.in]: studentnumbers,
+            },
+            prioritycode: {
+              [Op.not]: 6,
+            },
+          },
+        },
+      ],
+      raw: true,
+    }),
+    Semester.findAll({
+      attributes: [
+        sequelize.literal('DISTINCT ON("semester"."semestercode") "semester"."semestercode"'),
+        'name',
+        'startdate',
+        'enddate',
+      ],
+      include: {
+        model: SemesterEnrollment,
+        attributes: [],
+        required: true,
+        where: {
+          studentnumber: {
+            [Op.in]: studentnumbers,
+          },
+        },
+      },
+      where: {
+        startdate: {
+          [Op.between]: [startDate, endDate],
+        },
+      },
+      raw: true,
+    }),
+    sequelize.query(
+      `
+SELECT DISTINCT ON (code) code, name, type FROM element_details WHERE
+EXISTS (SELECT 1 FROM transfers WHERE studentnumber IN (:studentnumbers) AND (code = sourcecode OR code = targetcode)) OR
+EXISTS (SELECT 1 FROM studyright_elements WHERE studentnumber IN (:studentnumbers))`,
+      {
+        replacements: { studentnumbers },
+        type: sequelize.QueryTypes.SELECT,
+      }
+    ),
+  ])
+
+  students.forEach(student => {
+    student.tags = studentNumberToTags[student.studentnumber] || []
+  })
+
+  return { students, enrollments, credits, extents, semesters, elementdetails, courses }
+}
+
+module.exports = {
+  getStudentsIncludeCoursesBetween,
+}

services/backend/src/services/populations/index.js

@@ -0,0 +1,13 @@
+const { bottlenecksOf } = require('./bottlenecksOf')
+const { getStudentsIncludeCoursesBetween } = require('./getStudentsIncludeCoursesBetween')
+const { optimizedStatisticsOf } = require('./optimizedStatisticsOf')
+const { studentnumbersWithAllStudyrightElements } = require('./studentnumbersWithAllStudyrightElements')
+const { getEarliestYear } = require('./shared')
+
+module.exports = {
+  bottlenecksOf,
+  getStudentsIncludeCoursesBetween,
+  optimizedStatisticsOf,
+  studentnumbersWithAllStudyrightElements,
+  getEarliestYear,
+}

services/backend/src/services/populations/optimizedStatisticsOf.js

@@ -0,0 +1,83 @@
+const { getCriteria } = require('../studyprogramme/studyProgrammeCriteria')
+const { getStudentsIncludeCoursesBetween } = require('./getStudentsIncludeCoursesBetween')
+const {
+  parseQueryParams,
+  formatQueryParamArrays,
+  getOptionsForStudents,
+  dateMonthsFromNow,
+  formatStudentsForApi,
+} = require('./shared')
+const { studentnumbersWithAllStudyrightElements } = require('./studentnumbersWithAllStudyrightElements')
+const moment = require('moment')
+
+const optimizedStatisticsOf = async (query, studentnumberlist) => {
+  const formattedQueryParams = formatQueryParamArrays(query, ['semesters', 'studentStatuses'])
+
+  if (!formattedQueryParams.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
+    return { error: 'Semester should be either SPRING OR FALL' }
+  }
+
+  if (
+    formattedQueryParams.studentStatuses &&
+    !formattedQueryParams.studentStatuses.every(
+      status => status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
+    )
+  ) {
+    return { error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' }
+  }
+  const { studyRights, startDate, months, endDate, exchangeStudents, nondegreeStudents, transferredStudents, tag } =
+    parseQueryParams(formattedQueryParams)
+
+  // db startdate is formatted to utc so need to change it when querying
+  const formattedStartDate = new Date(moment.tz(startDate, 'Europe/Helsinki').format()).toUTCString()
+
+  const studentnumbers = studentnumberlist
+    ? studentnumberlist
+    : await studentnumbersWithAllStudyrightElements({
+        studyRights,
+        startDate: formattedStartDate,
+        endDate,
+        exchangeStudents,
+        nondegreeStudents,
+        transferredOutStudents: transferredStudents,
+        tag: null,
+        transferredToStudents: true,
+        graduatedStudents: true,
+      })
+
+  const code = studyRights[0] || ''
+  let optionData = {}
+  let criteria = {}
+  if (code.includes('MH')) {
+    optionData = await getOptionsForStudents(studentnumbers, code, 'MSC')
+  } else if (code.includes('KH')) {
+    optionData = await getOptionsForStudents(studentnumbers, code, 'BSC')
+  }
+  if (code.includes('KH') || ['MH30_001', 'MH30_003'].includes(code)) {
+    criteria = await getCriteria(code)
+  }
+  const { students, enrollments, credits, extents, semesters, elementdetails, courses } =
+    await getStudentsIncludeCoursesBetween(
+      studentnumbers,
+      startDate,
+      dateMonthsFromNow(startDate, months),
+      studyRights,
+      tag
+    )
+
+  const formattedStudents = await formatStudentsForApi(
+    { students, enrollments, credits, extents, semesters, elementdetails, courses },
+    startDate,
+    endDate,
+    formattedQueryParams,
+    optionData,
+    criteria,
+    code
+  )
+
+  return formattedStudents
+}
+
+module.exports = {
+  optimizedStatisticsOf,
+}

services/backend/src/services/populations/shared.js

@@ -0,0 +1,579 @@
+const Sequelize = require('sequelize')
+const { Op } = Sequelize
+const moment = require('moment')
+const { Credit, Course, Studyright, ElementDetail, StudyrightElement } = require('../../models')
+const {
+  dbConnections: { sequelize },
+} = require('../../database/connection')
+const { semesterEnd, semesterStart } = require('../../util/semester')
+const { getAllProgrammes } = require('../studyrights')
+const { getCurrentSemester } = require('../semesters')
+
+// Progress tab related helper functions.
+const createEmptyCriteriaYear = (criteria, year) => {
+  return {
+    credits: false,
+    totalSatisfied: 0,
+    coursesSatisfied:
+      criteria?.courses && criteria?.courses[year]
+        ? criteria.courses[year].reduce((acc, course) => {
+            acc[course] = null
+            return acc
+          }, {})
+        : {},
+  }
+}
+
+const getCreditAmount = (course, hops, courseCode, startDate, addition) => {
+  return moment(course.date).isBetween(moment(startDate), moment(startDate).add(addition, 'year')) &&
+    hops.length > 0 &&
+    (hops[0].included_courses.includes(courseCode) || hops[0].included_courses.includes(course.course_code))
+    ? course.credits
+    : 0
+}
+
+const updateCourseByYear = (criteria, criteriaYear, course, criteriaChecked, yearToAdd, correctCode) => {
+  if (
+    criteria?.courses &&
+    criteria?.courses[criteriaYear] &&
+    (criteria.courses[criteriaYear].includes(course.course_code) ||
+      criteria.courses[criteriaYear].some(
+        c => criteria.allCourses[c] && criteria.allCourses[c].includes(course.course_code)
+      ))
+  ) {
+    if (
+      !criteriaChecked[yearToAdd].coursesSatisfied[correctCode] ||
+      new Date(criteriaChecked[yearToAdd].coursesSatisfied[correctCode]) > new Date(course.date)
+    ) {
+      criteriaChecked[yearToAdd].coursesSatisfied[correctCode] = course.date
+    }
+  }
+}
+
+const updateCreditcriteriaInfo = (criteria, criteriaYear, criteriaChecked, yearToAdd, academicYears, academicYear) => {
+  if (criteria.courses) {
+    if (criteriaChecked?.[yearToAdd]) {
+      criteriaChecked[yearToAdd].totalSatisfied +=
+        Object.keys(criteriaChecked?.[yearToAdd].coursesSatisfied).filter(
+          course => criteriaChecked?.[yearToAdd].coursesSatisfied[course] !== null
+        ).length || 0
+    }
+    if (academicYears[academicYear] >= criteria?.credits[criteriaYear] && criteria?.credits[criteriaYear] > 0) {
+      criteriaChecked[yearToAdd].credits = true
+      criteriaChecked[yearToAdd].totalSatisfied += 1
+    }
+  }
+}
+
+const formatStudentForPopulationStatistics = (
+  {
+    firstnames,
+    lastname,
+    studentnumber,
+    dateofuniversityenrollment,
+    creditcount,
+    abbreviatedname,
+    email,
+    secondary_email,
+    phone_number,
+    studyrights,
+    studyplans,
+    semester_enrollments,
+    transfers,
+    updatedAt,
+    createdAt,
+    gender_code,
+    tags,
+    option,
+    birthdate,
+    sis_person_id,
+  },
+  enrollments,
+  credits,
+  startDate,
+  startDateMoment,
+  endDateMoment,
+  criteria,
+  code,
+  currentSemester
+) => {
+  const toCourse = ({ grade, attainment_date, credits, course_code, credittypecode, isStudyModule, language }) => {
+    const attainment_date_normailized =
+      attainment_date < startDate ? startDateMoment.clone().add(1, 'day').toISOString() : attainment_date
+    const passed = Credit.passed({ credittypecode })
+
+    return {
+      course_code,
+      date: attainment_date_normailized,
+      passed,
+      grade: passed ? grade : 'Hyl.',
+      credits,
+      isStudyModuleCredit: isStudyModule,
+      credittypecode,
+      language,
+    }
+  }
+  const criteriaCoursesBySubstitutions = criteria?.allCourses
+    ? Object.keys(criteria.allCourses).reduce((acc, code) => {
+        acc[code] = code
+        criteria.allCourses[code].map(subst => (acc[subst] = code))
+        return acc
+      }, {})
+    : {}
+
+  const toProgressCriteria = () => {
+    const criteriaChecked = {
+      year1: createEmptyCriteriaYear(criteria, 'yearOne'),
+      year2: createEmptyCriteriaYear(criteria, 'yearTwo'),
+      year3: createEmptyCriteriaYear(criteria, 'yearThree'),
+      year4: createEmptyCriteriaYear(criteria, 'yearFour'),
+      year5: createEmptyCriteriaYear(criteria, 'yearFive'),
+      year6: createEmptyCriteriaYear(criteria, 'yearSix'),
+    }
+    const correctStudyplan = studyplans ? studyplans.filter(plan => plan.programme_code === code) : []
+    const academicYears = { first: 0, second: 0, third: 0, fourth: 0, fifth: 0, sixth: 0 }
+    if (criteria.courses || criteria.credits) {
+      const courses = credits[studentnumber] ? credits[studentnumber].map(toCourse) : []
+      courses.forEach(course => {
+        if (course.passed) {
+          const correctCode = criteriaCoursesBySubstitutions[course.course_code]
+          updateCourseByYear(criteria, 'yearOne', course, criteriaChecked, 'year1', correctCode)
+          updateCourseByYear(criteria, 'yearTwo', course, criteriaChecked, 'year2', correctCode)
+          updateCourseByYear(criteria, 'yearThree', course, criteriaChecked, 'year3', correctCode)
+          updateCourseByYear(criteria, 'yearFour', course, criteriaChecked, 'year4', correctCode)
+          updateCourseByYear(criteria, 'yearFive', course, criteriaChecked, 'year5', correctCode)
+          updateCourseByYear(criteria, 'yearSix', course, criteriaChecked, 'year6', correctCode)
+          academicYears.first += getCreditAmount(course, correctStudyplan, correctCode, startDate, 1)
+          academicYears.second += getCreditAmount(course, correctStudyplan, correctCode, startDate, 2)
+          academicYears.third += getCreditAmount(course, correctStudyplan, correctCode, startDate, 3)
+          academicYears.fourth += getCreditAmount(course, correctStudyplan, correctCode, startDate, 4)
+          academicYears.fifth += getCreditAmount(course, correctStudyplan, correctCode, startDate, 5)
+          academicYears.sixth += getCreditAmount(course, correctStudyplan, correctCode, startDate, 6)
+        }
+      })
+    }
+
+    updateCreditcriteriaInfo(criteria, 'yearOne', criteriaChecked, 'year1', academicYears, 'first')
+    updateCreditcriteriaInfo(criteria, 'yearTwo', criteriaChecked, 'year2', academicYears, 'second')
+    updateCreditcriteriaInfo(criteria, 'yearThree', criteriaChecked, 'year3', academicYears, 'third')
+    updateCreditcriteriaInfo(criteria, 'yearFour', criteriaChecked, 'year4', academicYears, 'fourth')
+    updateCreditcriteriaInfo(criteria, 'yearFive', criteriaChecked, 'year5', academicYears, 'fifth')
+    updateCreditcriteriaInfo(criteria, 'yearSix', criteriaChecked, 'year6', academicYears, 'sixth')
+    return criteriaChecked
+  }
+
+  studyrights = studyrights || []
+  const started = dateofuniversityenrollment
+
+  return {
+    firstnames,
+    lastname,
+    studyrights,
+    started,
+    studentNumber: studentnumber,
+    credits: creditcount || 0,
+    courses: credits[studentnumber] ? credits[studentnumber].map(toCourse) : [],
+    enrollments: enrollments[studentnumber],
+    name: abbreviatedname,
+    transfers: transfers || [],
+    gender_code,
+    email,
+    secondaryEmail: secondary_email,
+    phoneNumber: phone_number,
+    semesterenrollments: semester_enrollments
+      ? semester_enrollments
+          .sort((a, b) => a.semestercode - b.semestercode)
+          .filter(enrollment => enrollment.semestercode <= currentSemester + (currentSemester % 2))
+      : [],
+    updatedAt: updatedAt || createdAt,
+    tags: tags || [],
+    studyrightStart: startDate,
+    starting: moment(started).isBetween(startDateMoment, endDateMoment, null, '[]'),
+    option,
+    birthdate,
+    studyplans,
+    sis_person_id,
+    criteriaProgress: toProgressCriteria(),
+  }
+}
+
+const dateMonthsFromNow = (date, months) => moment(date).add(months, 'months').format('YYYY-MM-DD')
+
+const count = (column, count, distinct = false) => {
+  const countable = !distinct ? sequelize.col(column) : sequelize.fn('DISTINCT', sequelize.col(column))
+  return sequelize.where(sequelize.fn('COUNT', countable), {
+    [Op.eq]: count,
+  })
+}
+
+const getEarliestYear = async (studentnumberlist, studyRights) => {
+  const startdates = await StudyrightElement.findAll({
+    attributes: ['startdate'],
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumberlist,
+      },
+      code: {
+        [Op.eq]: studyRights.programme,
+      },
+    },
+  })
+  const startyears = startdates.map(l => Number(new Date(l.startdate).getFullYear()))
+  return Math.min(...startyears)
+}
+
+const parseQueryParams = query => {
+  const { semesters, studentStatuses, studyRights, months, year, tag } = query
+  const startDate = semesters.includes('FALL')
+    ? `${year}-${semesterStart[semesters.find(s => s === 'FALL')]}`
+    : `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterStart[semesters.find(s => s === 'SPRING')]}`
+  const endDate = semesters.includes('SPRING')
+    ? `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd[semesters.find(s => s === 'SPRING')]}`
+    : `${year}-${semesterEnd[semesters.find(s => s === 'FALL')]}`
+  const exchangeStudents = studentStatuses && studentStatuses.includes('EXCHANGE')
+  const nondegreeStudents = studentStatuses && studentStatuses.includes('NONDEGREE')
+  const transferredStudents = studentStatuses && studentStatuses.includes('TRANSFERRED')
+
+  return {
+    exchangeStudents,
+    nondegreeStudents,
+    transferredStudents,
+    // if someone passes something falsy like null as the studyright, remove it so it doesn't break
+    // the sequelize query
+    studyRights: (Array.isArray(studyRights) ? studyRights : Object.values(studyRights)).filter(Boolean),
+    months,
+    startDate,
+    endDate,
+    tag,
+  }
+}
+
+const getOptionsForStudents = async (students, code, level) => {
+  if (!code || !students.length) return {}
+
+  let graduated
+  let currentExtent
+  let optionExtent
+
+  if (level === 'BSC') {
+    graduated = { graduated: 1 }
+    currentExtent = 1
+    optionExtent = 2
+  } else if (level === 'MSC') {
+    graduated = {}
+    currentExtent = 2
+    optionExtent = 1
+  } else {
+    throw new Error('Invalid study level ' + level)
+  }
+
+  const programmes = await getAllProgrammes()
+
+  const currentStudyrights = await Studyright.findAll({
+    include: [
+      {
+        model: StudyrightElement,
+        where: {
+          studentnumber: {
+            [Op.in]: students,
+          },
+          code: code,
+        },
+      },
+    ],
+    where: {
+      ...graduated,
+      extentcode: currentExtent,
+      student_studentnumber: {
+        [Op.in]: students,
+      },
+    },
+    attributes: ['studentStudentnumber', 'givendate'],
+  })
+
+  const currentStudyrightsMap = currentStudyrights.reduce((obj, studyright) => {
+    obj[studyright.studentStudentnumber] = studyright.givendate
+    return obj
+  }, {})
+
+  const options = await Studyright.findAll({
+    include: [
+      {
+        model: StudyrightElement,
+        where: {
+          studentnumber: {
+            [Op.in]: students,
+          },
+          code: {
+            [Op.in]: programmes.map(p => p.code),
+          },
+        },
+        include: [
+          {
+            model: ElementDetail,
+            attributes: ['name'],
+          },
+        ],
+        attributes: ['code', 'startdate'],
+      },
+    ],
+    where: {
+      extentcode: optionExtent,
+      student_studentnumber: {
+        [Op.in]: students,
+      },
+    },
+    order: [[StudyrightElement, 'startdate', 'DESC']],
+    attributes: ['studentStudentnumber', 'givendate'],
+  })
+
+  return options
+    .filter(m => m.studentStudentnumber in currentStudyrightsMap)
+    .filter(m => m.givendate.getTime() === currentStudyrightsMap[m.studentStudentnumber].getTime())
+    .reduce((obj, element) => {
+      obj[element.studentStudentnumber] = {
+        code: element.studyright_elements[0].code,
+        name: element.studyright_elements[0].element_detail.name,
+      }
+      return obj
+    }, {})
+}
+
+const formatStudentsForApi = async (
+  { students, enrollments, credits, extents, semesters, elementdetails, courses },
+  startDate,
+  endDate,
+  { studyRights },
+  optionData,
+  criteria,
+  code
+) => {
+  const startDateMoment = moment(startDate)
+  const endDateMoment = moment(endDate)
+  const currentSemester = (await getCurrentSemester()).semestercode
+
+  elementdetails = elementdetails.reduce(
+    (acc, e) => {
+      acc.data[e.code] = e
+      if (e.type === 20) acc.programmes.push(e.code)
+      return acc
+    },
+    { programmes: [], data: {} }
+  )
+  credits = credits.reduce((acc, e) => {
+    acc[e.student_studentnumber] = acc[e.student_studentnumber] || []
+    acc[e.student_studentnumber].push(e)
+    return acc
+  }, {})
+  enrollments = enrollments.reduce((acc, e) => {
+    acc[e.studentnumber] = acc[e.studentnumber] || []
+    acc[e.studentnumber].push(e)
+    return acc
+  }, {})
+  const result = students.reduce(
+    (stats, student) => {
+      student.transfers.forEach(transfer => {
+        const target = stats.transfers.targets[transfer.targetcode] || { sources: {} }
+        const source = stats.transfers.sources[transfer.sourcecode] || { targets: {} }
+        target.sources[transfer.sourcecode] = true
+        source.targets[transfer.targetcode] = true
+        stats.transfers.targets[transfer.targetcode] = target
+        stats.transfers.sources[transfer.sourcecode] = source
+      })
+      if (student.studentnumber in optionData) student.option = optionData[student.studentnumber]
+      else student.option = null
+
+      stats.students.push(
+        formatStudentForPopulationStatistics(
+          student,
+          enrollments,
+          credits,
+          startDate,
+          startDateMoment,
+          endDateMoment,
+          criteria,
+          code,
+          currentSemester
+        )
+      )
+      return stats
+    },
+    {
+      students: [],
+      transfers: {
+        targets: {},
+        sources: {},
+      },
+      studyrights: {
+        programmes: [],
+      },
+    }
+  )
+
+  const transferredStudyright = s => {
+    const transferred_from = s.transfers.find(
+      t =>
+        t.targetcode === studyRights.programme &&
+        // add bit of flex for students that transferred just before the startdate
+        moment(t.transferdate).isBetween(startDateMoment.subtract(1, 'd'), endDateMoment.add(1, 'd'))
+    )
+    if (transferred_from) {
+      s.transferredStudyright = true
+      s.transferSource = transferred_from.sourcecode
+    } else {
+      s.transferredStudyright = false
+      s.transferSource = null
+    }
+    return s
+  }
+
+  const returnvalue = {
+    students: result.students.map(transferredStudyright),
+    transfers: result.transfers,
+    extents,
+    semesters,
+    courses,
+    elementdetails,
+  }
+
+  return returnvalue
+}
+
+const formatQueryParamArrays = (query, params) => {
+  let res = { ...query }
+  params.forEach(p => {
+    if (!res[p]) return
+    res[p] = Array.isArray(res[p]) ? res[p] : [res[p]]
+  })
+  return res
+}
+
+const getSubstitutions = async codes => {
+  const courses = await Course.findAll({ where: { code: codes }, attributes: ['code', 'substitutions'], raw: true })
+  return [...new Set(courses.map(({ code, substitutions }) => [code, ...substitutions]).flat())]
+}
+// This duplicate code is added here to ensure that we get the enrollments in cases no credits found for the selected students.
+const findCourseEnrollments = async (studentnumbers, beforeDate, courses = []) => {
+  const courseCodes = courses.length > 0 ? await getSubstitutions(courses) : ['DUMMY']
+  const res = await sequelize.query(
+    `
+      SELECT DISTINCT ON (course.id)
+        course.code,
+        course.name,
+        course.coursetypecode,
+        course.substitutions,
+        course.main_course_code,
+        course_types.name AS course_type,
+        enrollment.data AS enrollments
+      FROM course
+      INNER JOIN course_types ON course_types.coursetypecode = course.coursetypecode
+      LEFT JOIN (
+        SELECT
+          course_id,
+          ARRAY_AGG(JSON_BUILD_OBJECT(
+            'studentnumber', studentnumber,
+            'state', state,
+            'enrollment_date_time', enrollment_date_time
+          )) AS data
+        FROM enrollment
+        WHERE enrollment.studentnumber IN (:studentnumbers) AND enrollment.enrollment_date_time < :beforeDate
+        GROUP BY enrollment.course_id
+      ) enrollment ON enrollment.course_id = course.id 
+      WHERE :skipCourseCodeFilter OR course.code IN (:courseCodes)
+      -- GROUP BY 1, 2, 3, 4, 5, 6
+    `,
+    {
+      replacements: {
+        studentnumbers: studentnumbers.length > 0 ? studentnumbers : ['DUMMY'],
+        beforeDate,
+        courseCodes,
+        skipCourseCodeFilter: courses.length === 0,
+      },
+      type: sequelize.QueryTypes.SELECT,
+    }
+  )
+  return res
+}
+const findCourses = async (studentnumbers, beforeDate, courses = []) => {
+  const courseCodes = courses.length > 0 ? await getSubstitutions(courses) : ['DUMMY']
+  const res = await sequelize.query(
+    `
+      SELECT DISTINCT ON (course.id)
+        course.code,
+        course.name,
+        course.coursetypecode,
+        course.substitutions,
+        course.main_course_code,
+        course_types.name AS course_type,
+        credit.data AS credits,
+        enrollment.data AS enrollments
+      FROM course
+      INNER JOIN (
+        SELECT
+          course_code,
+          ARRAY_AGG(JSON_BUILD_OBJECT(
+            'grade', grade,
+            'student_studentnumber', student_studentnumber,
+            'attainment_date', attainment_date,
+            'credittypecode', credittypecode,
+            'course_code', course_code
+          )) AS data
+        FROM credit
+        WHERE student_studentnumber IN (:studentnumbers) AND attainment_date < :beforeDate
+        GROUP BY credit.course_code
+      ) credit ON credit.course_code = course.code
+      INNER JOIN course_types ON course_types.coursetypecode = course.coursetypecode
+      LEFT JOIN (
+        SELECT
+          course_id,
+          ARRAY_AGG(JSON_BUILD_OBJECT(
+            'studentnumber', studentnumber,
+            'state', state,
+            'enrollment_date_time', enrollment_date_time
+          )) AS data
+        FROM enrollment
+        WHERE enrollment.studentnumber IN (:studentnumbers) AND enrollment.enrollment_date_time < :beforeDate
+        GROUP BY enrollment.course_id
+      ) enrollment ON enrollment.course_id = course.id 
+      WHERE :skipCourseCodeFilter OR course.code IN (:courseCodes)
+      -- GROUP BY 1, 2, 3, 4, 5, 6
+    `,
+    {
+      replacements: {
+        studentnumbers: studentnumbers.length > 0 ? studentnumbers : ['DUMMY'],
+        beforeDate,
+        courseCodes,
+        skipCourseCodeFilter: courses.length === 0,
+      },
+      type: sequelize.QueryTypes.SELECT,
+    }
+  )
+
+  return res
+}
+
+const checkThatSelectedStudentsAreUnderRequestedStudyright = (selectedStudents, allStudents) =>
+  !selectedStudents.every(s => allStudents.includes(s))
+
+const parseCreditInfo = credit => ({
+  studentnumber: credit.student_studentnumber,
+  grade: credit.grade,
+  passingGrade: Credit.passed(credit),
+  failingGrade: Credit.failed(credit),
+  improvedGrade: Credit.improved(credit),
+  date: credit.attainment_date,
+})
+
+module.exports = {
+  count,
+  parseCreditInfo,
+  checkThatSelectedStudentsAreUnderRequestedStudyright,
+  findCourses,
+  findCourseEnrollments,
+  formatQueryParamArrays,
+  formatStudentsForApi,
+  parseQueryParams,
+  getOptionsForStudents,
+  dateMonthsFromNow,
+  getEarliestYear,
+}

services/backend/src/services/populations/studentnumbersWithAllStudyrightElements.js

@@ -0,0 +1,243 @@
+const { Op } = require('sequelize')
+const { TagStudent } = require('../../models/models_kone')
+const { StudyrightElement, ElementDetail, Studyright, Transfer, Student } = require('../../models')
+const {
+  dbConnections: { sequelize },
+} = require('../../database/connection')
+const { count } = require('./shared')
+const { sortBy } = require('lodash')
+
+const studentnumbersWithAllStudyrightElements = async ({
+  studyRights,
+  startDate,
+  endDate,
+  exchangeStudents,
+  nondegreeStudents,
+  transferredOutStudents,
+  tag,
+  transferredToStudents,
+  graduatedStudents,
+}) => {
+  const filteredExtents = [16] // always filter out secondary subject students
+  if (!exchangeStudents) {
+    filteredExtents.push(7, 34)
+  }
+  if (!nondegreeStudents) {
+    filteredExtents.push(6, 9, 13, 14, 18, 22, 23, 99)
+  }
+
+  let studyrightWhere = {
+    extentcode: {
+      [Op.notIn]: filteredExtents,
+    },
+    prioritycode: {
+      [Op.not]: 6,
+    },
+  }
+
+  let studentWhere = {}
+  if (tag) {
+    const taggedStudentnumbers = await TagStudent.findAll({
+      attributes: ['studentnumber'],
+      where: {
+        tag_id: tag,
+      },
+    })
+    studentWhere.where = {
+      student_studentnumber: {
+        [Op.in]: taggedStudentnumbers.map(sn => sn.studentnumber),
+      },
+    }
+  }
+
+  const students = await Studyright.findAll({
+    attributes: ['student_studentnumber', 'graduated', 'enddate'],
+    include: {
+      model: StudyrightElement,
+      attributes: [],
+      required: true,
+      where: {
+        code: {
+          [Op.in]: studyRights,
+        },
+      },
+      include: {
+        model: ElementDetail,
+        attributes: [],
+      },
+    },
+    group: [sequelize.col('studyright.studyrightid')],
+    where: {
+      [Op.or]: [
+        {
+          ['$studyright_elements->element_detail.type$']: {
+            [Op.ne]: 20,
+          },
+        },
+        sequelize.where(
+          sequelize.fn(
+            'GREATEST',
+            sequelize.col('studyright_elements.startdate'),
+            sequelize.col('studyright.startdate')
+          ),
+          {
+            [Op.between]: [startDate, endDate],
+          }
+        ),
+      ],
+      ...studyrightWhere,
+    },
+    ...studentWhere,
+    having: count('studyright_elements.code', studyRights.length, true),
+    raw: true,
+  })
+
+  let studentnumbers = [...new Set(students.map(s => s.student_studentnumber))]
+  // bit hacky solution, but this is used to filter out studentnumbers who have since changed studytracks
+  const rights = await Studyright.findAll({
+    attributes: ['studyrightid'],
+    where: {
+      studentStudentnumber: {
+        [Op.in]: studentnumbers,
+      },
+    },
+    include: {
+      attributes: [],
+      model: StudyrightElement,
+      where: {
+        code: {
+          [Op.in]: studyRights,
+        },
+      },
+    },
+    group: ['studyright.studyrightid'],
+    having: count('studyright_elements.id', studyRights.length, true),
+    raw: true,
+  })
+
+  // bit hacky solution, but this is used to filter out studentnumbers who have since changed studytracks
+  const allStudytracksForStudents = await StudyrightElement.findAll({
+    where: {
+      studyrightid: {
+        [Op.in]: rights.map(r => r.studyrightid),
+      },
+    },
+    include: {
+      model: ElementDetail,
+      where: {
+        type: {
+          [Op.eq]: 30,
+        },
+      },
+    },
+    raw: true,
+  })
+
+  const studentNumberToSrElementMap = allStudytracksForStudents.reduce((obj, cur) => {
+    if (!obj[cur.studentnumber]) obj[cur.studentnumber] = []
+    obj[cur.studentnumber].push(cur)
+    return obj
+  }, {})
+
+  const formattedStudytracks = studentnumbers.reduce((acc, curr) => {
+    acc[curr] = studentNumberToSrElementMap[curr]
+    return acc
+  }, {})
+
+  // Take the newest studytrack primarily by latest starting date in the track, secondarily by the latest enddate
+  const filteredStudentnumbers = studentnumbers.filter(studentnumber => {
+    const newestStudytrack = sortBy(formattedStudytracks[studentnumber], ['startdate', 'enddate']).reverse()[0]
+    if (!newestStudytrack) return false
+    return studyRights.includes(newestStudytrack.code)
+  })
+
+  // Use the filtered list, if the search includes studytracks
+  // Then the studyrights length is > 1, which means that there is [studyright, studytrack].
+  // When searching only for studyprogramme, there is [studyright]
+  let studentnumberlist = studyRights.length > 1 ? filteredStudentnumbers : studentnumbers
+
+  // fetch students that have transferred out of the programme and filter out these studentnumbers
+  if (!transferredOutStudents) {
+    const transfersOut = (
+      await Transfer.findAll({
+        attributes: ['studentnumber'],
+        where: {
+          sourcecode: {
+            [Op.in]: studyRights,
+          },
+          transferdate: {
+            [Op.gt]: startDate,
+          },
+          studentnumber: {
+            [Op.in]: studentnumberlist,
+          },
+        },
+        raw: true,
+      })
+    ).map(s => s.studentnumber)
+
+    studentnumberlist = studentnumberlist.filter(sn => !transfersOut.includes(sn))
+  }
+
+  // fetch students that have transferred to the programme and filter out these studentnumbers
+  if (!transferredToStudents) {
+    const transfersTo = (
+      await Transfer.findAll({
+        attributes: ['studentnumber'],
+        where: {
+          targetcode: {
+            [Op.in]: studyRights,
+          },
+          transferdate: {
+            [Op.gt]: startDate,
+          },
+          studentnumber: {
+            [Op.in]: studentnumberlist,
+          },
+        },
+        raw: true,
+      })
+    ).map(s => s.studentnumber)
+    studentnumberlist = studentnumberlist.filter(sn => !transfersTo.includes(sn))
+  }
+
+  // fetch students that have graduated from the programme and filter out these studentnumbers
+  if (!graduatedStudents) {
+    const graduated = (
+      await Student.findAll({
+        attributes: ['studentnumber'],
+        include: [
+          {
+            model: Studyright,
+            include: [
+              {
+                model: StudyrightElement,
+                required: true,
+                where: {
+                  code: {
+                    [Op.in]: studyRights,
+                  },
+                },
+              },
+            ],
+            where: {
+              graduated: 1,
+            },
+          },
+        ],
+        where: {
+          studentnumber: {
+            [Op.in]: studentnumbers,
+          },
+        },
+      })
+    ).map(s => s.studentnumber)
+    studentnumberlist = studentnumberlist.filter(sn => !graduated.includes(sn))
+  }
+
+  return studentnumberlist
+}
+
+module.exports = {
+  studentnumbersWithAllStudyrightElements,
+}

services/backend/src/services/programmeModules.js

@@ -2,7 +2,7 @@ const Sequelize = require('sequelize')
 const { ExcludedCourse } = require('../models/models_kone')
 const { Op } = Sequelize
 const logger = require('../util/logger')
-const { combinedStudyprogrammes } = require('./studyprogrammeHelpers.js')
+const { combinedStudyprogrammes } = require('./studyprogramme/studyprogrammeHelpers.js')
 const { dbConnections } = require('../database/connection')
 const { ProgrammeModule } = require('../models')
 

services/backend/src/services/sisUpdaterService.js

@@ -1,42 +1,60 @@
 const axios = require('axios')
 const { SIS_UPDATER_URL, SECRET_TOKEN } = require('../conf-backend')
+const { Studyplan } = require('../models')
+const { Op } = require('sequelize')
 
 const client = axios.create({ baseURL: SIS_UPDATER_URL })
 
+const params = { params: { token: SECRET_TOKEN } }
+
 const updateSISStudents = async () => {
-  const response = await client.get('/v1/students', { params: { token: SECRET_TOKEN } })
+  const response = await client.get('/v1/students', params)
   return response.data
 }
 
 const updateStudentsByStudentNumber = async studentnumbers => {
   const data = { studentnumbers }
-  const response = await client.post('/v1/students', data, { params: { token: SECRET_TOKEN } })
+  const response = await client.post('/v1/students', data, params)
   return response.data
 }
 
 const updateCoursesByCourseCode = async coursecodes => {
   const data = { coursecodes }
-  const response = await client.post('/v1/courses', data, { params: { token: SECRET_TOKEN } })
+  const response = await client.post('/v1/courses', data, params)
   return response.data
 }
 
 const updateSISMetadata = async () => {
-  const response = await client.get('/v1/meta', { params: { token: SECRET_TOKEN } })
+  const response = await client.get('/v1/meta', params)
   return response.data
 }
 
 const updateSISProgrammes = async () => {
-  const response = await client.get('/v1/programmes', { params: { token: SECRET_TOKEN } })
+  const response = await client.get('/v1/programmes', params)
   return response.data
 }
 
 const updateSISRedisCache = async () => {
-  const response = await client.get('/v1/rediscache', { params: { token: SECRET_TOKEN } })
+  const response = await client.get('/v1/rediscache', params)
+  return response.data
+}
+
+const studyplansUpdate = async days => {
+  const limitDate = new Date()
+  limitDate.setDate(limitDate.getDate() - days)
+  const result = await Studyplan.findAll({
+    where: { updatedAt: { [Op.lte]: limitDate } },
+    attributes: ['studentnumber'],
+    raw: true,
+    limit: 1000,
+  })
+  const studentnumbers = result.map(r => r.studentnumber)
+  const response = await client.post('v1/studyplans', { studentnumbers }, params)
   return response.data
 }
 
 const abort = async () => {
-  const response = await client.get('/v1/abort', { params: { token: SECRET_TOKEN } })
+  const response = await client.get('/v1/abort', params)
   return response.data
 }
 
@@ -48,4 +66,5 @@ module.exports = {
   updateSISRedisCache,
   abort,
   updateCoursesByCourseCode,
+  studyplansUpdate,
 }

services/backend/src/services/students.js

@@ -117,7 +117,7 @@ const getUnifyStatus = unifyCourses => {
     case 'openStats':
       return [true]
 
-    case 'reqularStats':
+    case 'regularStats':
       return [false]
 
     default:

services/backend/src/services/studyprogramme.js

@@ -1,934 +0,0 @@
-const _ = require('lodash')
-const Sequelize = require('sequelize')
-const {
-  dbConnections: { sequelize },
-} = require('../database/connection')
-const { Op } = Sequelize
-const {
-  Credit,
-  Course,
-  Organization,
-  Studyright,
-  StudyrightElement,
-  ElementDetail,
-  ProgrammeModule,
-  Transfer,
-  Student,
-  SemesterEnrollment,
-  Semester,
-  Enrollment,
-} = require('../models')
-const { formatStudyright, formatStudent, formatTransfer, formatCredit } = require('./studyprogrammeHelpers')
-const { getCurrentSemester } = require('./semesters')
-const logger = require('../util/logger')
-
-const whereStudents = studentnumbers => {
-  return studentnumbers ? studentnumbers : { [Op.not]: null }
-}
-
-const sinceDate = since => {
-  return since ? { [Op.gte]: since } : { [Op.not]: null }
-}
-
-const studytrackStudents = async studentnumbers =>
-  (
-    await Student.findAll({
-      include: {
-        model: Credit,
-        separate: true,
-        attributes: ['credits', 'attainment_date'],
-        where: {
-          isStudyModule: false,
-          credittypecode: {
-            [Op.in]: [4, 9],
-          },
-        },
-      },
-      where: {
-        studentnumber: {
-          [Op.in]: studentnumbers,
-        },
-      },
-    })
-  ).map(formatStudent)
-
-const enrolledStudents = async (studytrack, studentnumbers) => {
-  const currentSemester = await getCurrentSemester()
-
-  const students = await Student.findAll({
-    attributes: ['studentnumber'],
-    include: [
-      {
-        model: Studyright,
-        include: [
-          {
-            model: StudyrightElement,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        ],
-        attributes: ['studyrightid'],
-        where: {
-          graduated: 0,
-          active: 1,
-        },
-      },
-      {
-        model: SemesterEnrollment,
-        attributes: ['semestercode', 'enrollmenttype'],
-        include: [
-          {
-            model: Semester,
-            where: {
-              semestercode: currentSemester.semestercode,
-            },
-          },
-        ],
-        where: {
-          enrollmenttype: 1,
-        },
-      },
-    ],
-    where: {
-      studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-    },
-  })
-
-  return students.filter(s => s.semester_enrollments?.length)
-}
-
-const absentStudents = async (studytrack, studentnumbers) => {
-  const currentSemester = await getCurrentSemester()
-  const students = await Student.findAll({
-    attributes: ['studentnumber'],
-    include: [
-      {
-        model: Studyright,
-        include: [
-          {
-            model: StudyrightElement,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        ],
-        attributes: ['studyrightid'],
-        where: {
-          graduated: 0,
-          active: 1,
-        },
-      },
-      {
-        model: SemesterEnrollment,
-        attributes: ['semestercode'],
-        where: {
-          enrollmenttype: 2,
-          semestercode: currentSemester.semestercode,
-        },
-      },
-    ],
-    where: {
-      studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-    },
-  })
-
-  return students
-}
-
-const allStudyrights = async (studytrack, studentnumbers) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        student_studentnumber: whereStudents(studentnumbers),
-      },
-    })
-  ).map(formatStudyright)
-
-const startedStudyrights = async (studytrack, since, studentnumbers) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        studystartdate: {
-          [Op.gte]: since,
-        },
-        student_studentnumber: whereStudents(studentnumbers),
-      },
-    })
-  ).map(formatStudyright)
-
-const graduatedStudyRightsByStartDate = async (studytrack, since, academicEnddate) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
-            where: {
-              code: studytrack,
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        graduated: 1,
-        startdate: {
-          [Op.between]: [since, academicEnddate],
-        },
-        student_studentnumber: {
-          [Op.not]: null,
-        },
-      },
-    })
-  ).map(formatStudyright)
-
-const graduatedStudyRights = async (studytrack, since, studentnumbers) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        graduated: 1,
-        enddate: sinceDate(since),
-        student_studentnumber: whereStudents(studentnumbers),
-      },
-    })
-  ).map(formatStudyright)
-
-const inactiveStudyrights = async (studytrack, studentnumbers) => {
-  const currentSemester = await getCurrentSemester()
-  const students = await Student.findAll({
-    attributes: ['studentnumber'],
-    include: [
-      {
-        model: Studyright,
-        required: true,
-        include: [
-          {
-            model: StudyrightElement,
-            required: true,
-            where: {
-              code: studytrack,
-            },
-          },
-        ],
-        attributes: ['studyrightid', 'enddate'],
-        where: {
-          graduated: 0,
-          active: 0,
-        },
-      },
-      {
-        model: SemesterEnrollment,
-        attributes: ['semestercode', 'enrollmenttype'],
-      },
-    ],
-    where: {
-      studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-    },
-  })
-
-  return students.filter(
-    student =>
-      student.studyrights[0].enddate <= new Date() ||
-      !student.semester_enrollments.find(enrollment => enrollment.semestercode === currentSemester) ||
-      student.semester_enrollments.find(enrollment => enrollment.semestercode === currentSemester).enrollmenttype === 3
-  )
-}
-
-const followingStudyrights = async (since, programmes, studentnumbers) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          where: {
-            code: {
-              [Op.in]: programmes.map(p => p.code),
-            },
-          },
-          include: [
-            {
-              model: ElementDetail,
-              attributes: ['name', 'code', 'type'],
-            },
-          ],
-          attributes: ['code'],
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        studystartdate: {
-          [Op.gte]: since,
-        },
-        extentcode: 2,
-        student_studentnumber: whereStudents(studentnumbers),
-      },
-    })
-  ).map(formatStudyright)
-
-const previousStudyrights = async (programmes, studentnumbers) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          where: {
-            code: {
-              [Op.in]: programmes.map(p => p.code),
-            },
-          },
-          include: [
-            {
-              model: ElementDetail,
-              attributes: ['name', 'code', 'type'],
-            },
-          ],
-          attributes: ['code'],
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        extentcode: 1,
-        student_studentnumber: whereStudents(studentnumbers),
-      },
-      order: [[StudyrightElement, 'startdate', 'DESC']],
-    })
-  ).map(formatStudyright)
-
-const transfersAway = async (studytrack, since) =>
-  (
-    await Transfer.findAll({
-      where: {
-        transferdate: {
-          [Op.gte]: since,
-        },
-        sourcecode: studytrack,
-      },
-    })
-  ).map(formatTransfer)
-
-const transfersTo = async (studytrack, since) =>
-  (
-    await Transfer.findAll({
-      where: {
-        transferdate: {
-          [Op.gte]: since,
-        },
-        targetcode: studytrack,
-      },
-    })
-  ).map(formatTransfer)
-
-const allTransfers = async (studytrack, since) => {
-  const transferredTo = await transfersTo(studytrack, since)
-  const transferredAway = await transfersAway(studytrack, since)
-  return [...transferredTo, ...transferredAway]
-}
-
-const getStudyRights = async students =>
-  (
-    await Studyright.findAll({
-      attributes: [
-        'studyrightid',
-        'startdate',
-        'studystartdate',
-        'enddate',
-        'graduated',
-        'prioritycode',
-        'extentcode',
-        'cancelled',
-        'facultyCode',
-      ],
-      where: {
-        studentStudentnumber: students,
-      },
-      include: [
-        {
-          model: StudyrightElement,
-          include: {
-            model: ElementDetail,
-            where: {
-              type: 20,
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-    })
-  ).map(formatStudyright)
-
-const getCreditsForStudyProgramme = async (codes, since) =>
-  (
-    await Credit.findAll({
-      attributes: ['course_code', 'credits', 'attainment_date', 'student_studentnumber'],
-      include: {
-        model: Course,
-        attributes: ['code'],
-        required: true,
-        where: {
-          code: codes,
-        },
-      },
-      where: {
-        credittypecode: 4,
-        isStudyModule: {
-          [Op.not]: true,
-        },
-        attainment_date: {
-          [Op.gte]: since,
-        },
-      },
-    })
-  ).map(formatCredit)
-
-const getCourseCodesForStudyProgramme = async provider => {
-  const coursesByProvider = await Course.findAll({
-    attributes: ['id', 'code', 'substitutions'],
-    include: {
-      model: Organization,
-      required: true,
-      where: {
-        code: provider,
-      },
-      through: {
-        attributes: [],
-      },
-    },
-  })
-  // here is work in progress
-  // const courseIds = coursesByProvider.map(course => course.id)
-  // const coursesWithManyProviders = await CourseProvider.findAll({
-  //   attributes: ['coursecode', [sequelize.fn('COUNT', sequelize.col('organizationcode')), 'n_courses']],
-  //   where: {
-  //     coursecode: {
-  //       [Op.in]: courseIds,
-  //     },
-  //   },
-  //   group: 'coursecode',
-  // })
-
-  // const coursesProvidedByManyProgrammes = coursesWithManyProviders
-  //   .filter(course => course.n_courses >= 2)
-  //   .map(course => course.coursecode)
-  // // We are basically not interested other substitutions than those with open uni ones.
-  // const coursesToHave = coursesByProvider.filter(course => !coursesProvidedByManyProgrammes.includes(course.id))
-  const coursesWithOpenUniSubstitutions = coursesByProvider.map(({ code, substitutions }) => {
-    if (!substitutions || !substitutions.length) return [code]
-    const alternatives = [`AY-${code}`, `AY${code}`, `A-${code}`]
-    return [code].concat(substitutions.filter(sub => alternatives.includes(sub)))
-  })
-  // const providedByManyProgrammes = coursesByProvider
-  //   .filter(course => coursesProvidedByManyProgrammes.includes(course.id))
-  //   .map(({ code, substitutions }) => {
-  //     if (!substitutions || !substitutions.length) return [code]
-  //     const alternatives = [`AY-${code}`, `AY${code}`, `A-${code}`]
-  //     return [code].concat(substitutions.filter(sub => alternatives.includes(sub)))
-  //   })
-  return coursesWithOpenUniSubstitutions.flat()
-}
-
-const getAllProgrammeCourses = async providerCode => {
-  const res = await Course.findAll({
-    attributes: ['id', 'code', 'substitutions'],
-    include: [
-      {
-        model: Organization,
-        attributes: ['id'],
-        required: true,
-        where: {
-          code: providerCode,
-        },
-        through: {
-          attributes: [],
-        },
-      },
-    ],
-    raw: true,
-  })
-  return res
-}
-
-const getStudentsForProgrammeCourses = async (from, to, programmeCourses) => {
-  if (!programmeCourses.length) return []
-  try {
-    const res = await sequelize.query(
-      `
-      WITH Dist AS (
-        SELECT DISTINCT cr.student_studentnumber AS student, cr.credits AS credits, 
-         co.code AS code, co.name AS course_name, cr."isStudyModule" FROM credit cr
-         INNER JOIN course co ON cr.course_code = co.code
-         WHERE cr.attainment_date BETWEEN :from AND :to
-         AND cr.course_code IN (:programmeCourses)
-         AND cr.credittypecode = 4
-         )
-         SELECT COUNT(student) AS total_students, SUM(credits) AS total_credits, code, course_name, "isStudyModule"
-         FROM Dist
-         GROUP BY dist.code, dist.course_name, dist."isStudyModule";
-      `,
-      {
-        type: sequelize.QueryTypes.SELECT,
-        replacements: { from, to, programmeCourses },
-      }
-    )
-    const result = res.map(course => ({
-      code: course.code,
-      name: course.course_name,
-      totalPassed: parseInt(course.total_students),
-      totalAllcredits: parseInt(course.total_credits),
-      type: 'passed',
-      isStudyModule: course.isStudyModule,
-    }))
-    return result
-  } catch (e) {
-    logger.error(`getStudentsForProgrammeCourses() function failed ${e}`)
-  }
-}
-
-const getNotCompletedForProgrammeCourses = async (from, to, programmeCourses) => {
-  try {
-    const enrollmentsCourses = await Enrollment.findAll({
-      attributes: ['studentnumber', 'course_code'],
-      where: {
-        course_code: {
-          [Op.in]: programmeCourses,
-        },
-        studentnumber: {
-          [Op.ne]: null,
-        },
-        enrollment_date_time: {
-          [Op.between]: [from, to],
-        },
-        state: 'ENROLLED',
-      },
-    })
-    const getCourseCode = code => {
-      if (code.startsWith('AY')) return code.replace('AY', '')
-      if (code.startsWith('A')) return code.replace('AY', '')
-      return code
-    }
-    const allEnrollments = {}
-    for (const { studentnumber, course_code } of enrollmentsCourses) {
-      const code = getCourseCode(course_code)
-      if (!(code in allEnrollments)) {
-        allEnrollments[code] = []
-      }
-      if (!allEnrollments[code].includes(studentnumber)) allEnrollments[code].push(studentnumber)
-    }
-
-    const credits = await Credit.findAll({
-      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'isStudyModule'],
-      include: {
-        model: Course,
-        attributes: ['code', 'name'],
-        required: true,
-        where: {
-          code: programmeCourses,
-        },
-      },
-      where: {
-        attainment_date: {
-          [Op.between]: [from, to],
-        },
-      },
-    })
-
-    const creditsObj = credits.map(credit => {
-      return {
-        code: getCourseCode(credit.course_code),
-        studentnumber: credit.student_studentnumber,
-        credittype: credit.credittypecode,
-        courseName: credit.course.name,
-        isStudyModule: credit.isStudyModule,
-      }
-    })
-
-    const passedByCourseCodes = {}
-    const notCompletedByCourseCodes = {}
-    const courses = {}
-    for (const course of creditsObj) {
-      if (!(course.code in courses)) {
-        courses[course.code] = {
-          code: course.code,
-          name: course.courseName,
-          isStudyModule: course.isStudyModule,
-        }
-
-        passedByCourseCodes[course.code] = []
-        notCompletedByCourseCodes[course.code] = []
-      }
-      if ([4, 7, 9].includes(course.credittype)) {
-        passedByCourseCodes[course.code].push(course.studentnumber)
-      }
-      if (course.credittype === 10 && !passedByCourseCodes[course.code].includes(course.studentnumber)) {
-        notCompletedByCourseCodes[course.code].push(course.studentnumber)
-      }
-    }
-    // If student has enrollments, but no attainment for a particular course, they have no credit info.
-    programmeCourses.forEach(courseCode => {
-      if (allEnrollments[courseCode]) {
-        allEnrollments[courseCode].forEach(studentnumber => {
-          if (passedByCourseCodes[courseCode] && !passedByCourseCodes[courseCode].includes(studentnumber)) {
-            notCompletedByCourseCodes[courseCode].push(studentnumber)
-          }
-        })
-      }
-    })
-
-    return Object.keys(courses)
-      .reduce((acc, val) => [...acc, { ...courses[val] }], [])
-      .map(course => ({
-        code: course.code,
-        name: course.name,
-        totalNotCompleted: [...new Set(notCompletedByCourseCodes[course.code])].length,
-        type: 'notCompleted',
-        isStudyModule: course.isStudyModule,
-      }))
-  } catch (e) {
-    logger.error(`getNotCompletedForProgrammeCourses failed ${e}`)
-  }
-}
-const getOwnStudentsForProgrammeCourses = async (from, to, programmeCourses, studyprogramme) => {
-  if (!programmeCourses.length) return []
-  const res = await sequelize.query(
-    `
-    WITH Dist AS (
-      SELECT DISTINCT cr.student_studentnumber AS student, cr.credits AS credits,
-        co.code AS code, co.name AS course_name, cr."isStudyModule" FROM credit cr
-        INNER JOIN studyright_elements se ON se.studentnumber = cr.student_studentnumber
-        INNER JOIN course co ON cr.course_code = co.code
-        INNER JOIN course_providers cp ON cp.coursecode = co.id
-        INNER JOIN organization o ON o.id = cp.organizationcode
-        WHERE cr.attainment_date BETWEEN :from AND :to
-        AND cr.course_code IN (:programmeCourses)
-        AND cr.credittypecode = 4
-        AND (se.code = :studyprogramme AND cr.attainment_date BETWEEN se.startdate AND se.enddate)
-    )
-     SELECT COUNT(student) AS total_students, SUM(credits) AS total_credits, code, course_name, "isStudyModule"
-       FROM Dist
-       GROUP BY dist.code, course_name, "isStudyModule";
-      `,
-    {
-      type: sequelize.QueryTypes.SELECT,
-      replacements: { from, to, programmeCourses, studyprogramme },
-    }
-  )
-  return res.map(course => {
-    const res = {
-      code: course.code,
-      name: course.course_name,
-      totalProgrammeStudents: parseInt(course.total_students),
-      totalProgrammeCredits: parseInt(course.total_credits),
-      type: 'ownProgramme',
-    }
-    return res
-  })
-}
-
-const getOtherStudentsForProgrammeCourses = async (from, to, programmeCourses, studyprogramme) => {
-  if (!programmeCourses.length) return []
-  const res = await sequelize.query(
-    `
-    WITH Dist AS (
-      SELECT DISTINCT cr.student_studentnumber AS student, cr.credits AS credits,
-      co.code AS code, co.name AS course_name, cr."isStudyModule" FROM credit cr
-      INNER JOIN studyright_elements se ON se.studentnumber = cr.student_studentnumber
-      INNER JOIN course co ON cr.course_code = co.code
-      WHERE cr.attainment_date BETWEEN :from AND :to
-      AND cr.course_code IN (:programmeCourses)
-      AND cr.credittypecode = 4
-      AND cr.student_studentnumber IN
-        (
-        SELECT student_studentnumber FROM studyright_elements
-        WHERE studyright_elements.studentnumber = cr.student_studentnumber
-        AND (studyright_elements.code != :studyprogramme AND cr.attainment_date BETWEEN studyright_elements.startdate AND studyright_elements.enddate)
-        AND cr.student_studentnumber NOT IN
-          (
-          SELECT studentnumber FROM studyright_elements
-          WHERE studyright_elements.studentnumber = cr.student_studentnumber
-          AND (studyright_elements.code = :studyprogramme AND cr.attainment_date BETWEEN studyright_elements.startdate AND studyright_elements.enddate)) 
-          )
-      )
-      SELECT COUNT(student) AS total_students, SUM(credits) AS total_credits, code, course_name, "isStudyModule"
-      FROM Dist
-      GROUP BY dist.code, course_name, "isStudyModule";
-      `,
-    {
-      type: sequelize.QueryTypes.SELECT,
-      replacements: { from, to, programmeCourses, studyprogramme },
-    }
-  )
-  return res.map(course => ({
-    code: course.code,
-    name: course.course_name,
-    totalOtherProgrammeStudents: parseInt(course.total_students),
-    totalOtherProgrammeCredits: parseInt(course.total_credits),
-    type: 'otherProgramme',
-    isStudyModule: course.isStudyModule,
-  }))
-}
-
-const getTransferStudentsForProgrammeCourses = async (from, to, programmeCourses) => {
-  if (!programmeCourses.length) return []
-  const res = await sequelize.query(
-    `
-    WITH Dist AS (
-      SELECT DISTINCT cr.student_studentnumber AS student, cr.credits AS credits,
-      co.code AS code, co.name AS course_name, cr."isStudyModule" FROM credit cr
-      INNER JOIN course co ON cr.course_code = co.code
-      WHERE cr.attainment_date BETWEEN :from AND :to
-      AND cr.course_code IN (:programmeCourses)
-      AND cr.credittypecode = 9
-      )
-      SELECT COUNT(student) AS total_students, SUM(credits) AS total_credits, code, course_name, "isStudyModule"
-      FROM Dist
-      GROUP BY dist.code, course_name, "isStudyModule";
-      `,
-    {
-      type: sequelize.QueryTypes.SELECT,
-      replacements: { from, to, programmeCourses },
-    }
-  )
-  return res.map(course => ({
-    code: course.code,
-    name: course.course_name,
-    totalTransferStudents: parseInt(course.total_students),
-    totalTransferCredits: parseInt(course.total_credits),
-    isStudyModule: course.isStudyModule,
-    type: 'transfer',
-  }))
-}
-
-const getStudentsWithoutStudyrightForProgrammeCourses = async (from, to, programmeCourses) => {
-  if (!programmeCourses.length) return []
-  const res = await sequelize.query(
-    `
-    WITH Dist AS (
-      SELECT DISTINCT cr.student_studentnumber AS student, cr.credits AS credits,
-      co.code AS code, co.name AS course_name, cr."isStudyModule" FROM credit cr
-      INNER JOIN course co ON cr.course_code = co.code
-      WHERE cr.attainment_date BETWEEN :from AND :to
-      AND cr.course_code IN (:programmeCourses)
-      AND cr.credittypecode = 4
-      AND cr.student_studentnumber NOT IN
-        (
-        SELECT student_studentnumber FROM studyright_elements
-        WHERE studyright_elements.studentnumber = cr.student_studentnumber
-        AND cr.attainment_date BETWEEN studyright_elements.startdate AND studyright_elements.enddate)
-      )
-      SELECT COUNT(student) AS total_students, SUM(credits) AS total_credits, code, course_name, "isStudyModule"
-      FROM Dist
-      GROUP BY dist.code, course_name, "isStudyModule";
-      `,
-    {
-      type: sequelize.QueryTypes.SELECT,
-      replacements: { from, to, programmeCourses },
-    }
-  )
-  return res.map(course => ({
-    code: course.code,
-    name: course.course_name,
-    totalWithoutStudyrightStudents: parseInt(course.total_students),
-    totalWithoutStudyrightCredits: parseInt(course.total_credits),
-    type: 'noStudyright',
-  }))
-}
-
-const getTransferredCredits = async (provider, since) =>
-  await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
-      include: {
-        model: Organization,
-        required: true,
-        where: {
-          code: provider,
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.eq]: [9],
-      },
-      isStudyModule: {
-        [Op.not]: true,
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-    },
-  })
-
-const getThesisCredits = async (provider, since, thesisType, studentnumbers) =>
-  await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
-      where: {
-        course_unit_type: {
-          [Op.in]: thesisType,
-        },
-      },
-      include: {
-        model: Organization,
-        required: true,
-        where: {
-          code: provider,
-        },
-      },
-    },
-    where: {
-      credittypecode: {
-        [Op.notIn]: [10, 9, 7],
-      },
-      isStudyModule: {
-        [Op.not]: true,
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-      student_studentnumber: whereStudents(studentnumbers),
-    },
-  })
-
-const getCurrentStudyYearStartDate = _.memoize(
-  async unixMillis =>
-    new Date(
-      (
-        await sequelize.query(
-          `
-      SELECT startdate FROM SEMESTERS s WHERE yearcode = (SELECT yearcode FROM SEMESTERS WHERE startdate < :a ORDER BY startdate DESC LIMIT 1) ORDER BY startdate LIMIT 1;
-      `,
-          {
-            type: sequelize.QueryTypes.SELECT,
-            replacements: { a: new Date(unixMillis) },
-          }
-        )
-      )[0].startdate
-    )
-)
-
-const getProgrammeName = async code => {
-  return ProgrammeModule.findOne({
-    attributes: ['name'],
-    where: {
-      code,
-    },
-  })
-}
-
-module.exports = {
-  studytrackStudents,
-  enrolledStudents,
-  absentStudents,
-  allStudyrights,
-  startedStudyrights,
-  graduatedStudyRights,
-  graduatedStudyRightsByStartDate,
-  inactiveStudyrights,
-  previousStudyrights,
-  followingStudyrights,
-  transfersAway,
-  transfersTo,
-  allTransfers,
-  getCreditsForStudyProgramme,
-  getTransferredCredits,
-  getThesisCredits,
-  getStudentsForProgrammeCourses,
-  getCurrentStudyYearStartDate,
-  getOwnStudentsForProgrammeCourses,
-  getStudentsWithoutStudyrightForProgrammeCourses,
-  getOtherStudentsForProgrammeCourses,
-  getAllProgrammeCourses,
-  getStudyRights,
-  getCourseCodesForStudyProgramme,
-  getTransferStudentsForProgrammeCourses,
-  getProgrammeName,
-  getNotCompletedForProgrammeCourses,
-}

services/backend/src/services/studyprogramme/creditGetters.js

@@ -0,0 +1,117 @@
+const { Op } = require('sequelize')
+const { Credit, Course, Organization } = require('../../models')
+const { whereStudents } = require('.')
+const {
+  dbConnections: { sequelize },
+} = require('../../database/connection')
+const { formatCredit } = require('./studyprogrammeHelpers')
+
+const getCreditsForStudyProgramme = async (provider, codes, since) =>
+  (
+    await sequelize.query(
+      `
+      SELECT 
+        cr.id,
+        cr.course_code, 
+        cr.attainment_date, 
+        cr.student_studentnumber, 
+        cr.credits * COALESCE(share_element.share::numeric, 1) AS credits,
+        cr.studyright_id,
+        cr.semestercode
+      FROM 
+        credit cr
+      JOIN 
+        course_providers cp ON cr.course_id = cp.coursecode
+      JOIN 
+        organization o ON cp.organizationcode = o.id
+      LEFT JOIN LATERAL 
+        (
+          SELECT (jae ->> 'share')::numeric AS share
+          FROM jsonb_array_elements(cp.shares) AS jae
+          WHERE (cr.attainment_date >= (jae ->> 'startDate')::date OR jae ->> 'startDate' IS NULL)
+            AND (cr.attainment_date <= (jae ->> 'endDate')::date OR jae ->> 'endDate' IS NULL)
+          ORDER BY 
+            (jae ->> 'startDate')::date DESC NULLS LAST,
+            (jae ->> 'endDate')::date DESC NULLS LAST
+          LIMIT 1
+        ) AS share_element ON TRUE
+      WHERE 
+        (o.code = :provider)
+        AND cr.attainment_date >= :since
+        AND cr.credittypecode = 4
+        AND cr."isStudyModule" IS NOT TRUE
+      `,
+      {
+        type: sequelize.QueryTypes.SELECT,
+        replacements: { provider, codes, since },
+      }
+    )
+  ).map(formatCredit)
+
+const getTransferredCredits = async (provider, since) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.eq]: [9],
+      },
+      isStudyModule: {
+        [Op.not]: true,
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+    },
+  })
+
+const getThesisCredits = async (provider, since, thesisType, studentnumbers) =>
+  await Credit.findAll({
+    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
+    include: {
+      model: Course,
+      attributes: ['code'],
+      required: true,
+      where: {
+        course_unit_type: {
+          [Op.in]: thesisType,
+        },
+      },
+      include: {
+        model: Organization,
+        required: true,
+        where: {
+          code: provider,
+        },
+      },
+    },
+    where: {
+      credittypecode: {
+        [Op.notIn]: [10, 9, 7],
+      },
+      isStudyModule: {
+        [Op.not]: true,
+      },
+      attainment_date: {
+        [Op.gte]: since,
+      },
+      student_studentnumber: whereStudents(studentnumbers),
+    },
+  })
+
+module.exports = {
+  getCreditsForStudyProgramme,
+  getTransferredCredits,
+  getThesisCredits,
+}

services/backend/src/services/studyprogramme/index.js

@@ -0,0 +1,235 @@
+const _ = require('lodash')
+const Sequelize = require('sequelize')
+const {
+  dbConnections: { sequelize },
+} = require('../../database/connection')
+const { Op } = Sequelize
+const { Credit, Course, Organization, ProgrammeModule, Transfer, Enrollment } = require('../../models')
+const { formatTransfer } = require('./studyprogrammeHelpers')
+const logger = require('../../util/logger')
+
+const whereStudents = studentnumbers => {
+  return studentnumbers ? studentnumbers : { [Op.not]: null }
+}
+
+const sinceDate = since => {
+  return since ? { [Op.gte]: since } : { [Op.not]: null }
+}
+
+const transfersAway = async (studytrack, since) =>
+  (
+    await Transfer.findAll({
+      where: {
+        transferdate: {
+          [Op.gte]: since,
+        },
+        sourcecode: studytrack,
+      },
+    })
+  ).map(formatTransfer)
+
+const transfersTo = async (studytrack, since) =>
+  (
+    await Transfer.findAll({
+      where: {
+        transferdate: {
+          [Op.gte]: since,
+        },
+        targetcode: studytrack,
+      },
+    })
+  ).map(formatTransfer)
+
+const allTransfers = async (studytrack, since) => {
+  const transferredTo = await transfersTo(studytrack, since)
+  const transferredAway = await transfersAway(studytrack, since)
+  return [...transferredTo, ...transferredAway]
+}
+
+const getCourseCodesForStudyProgramme = async provider => {
+  const coursesByProvider = await Course.findAll({
+    attributes: ['id', 'code', 'substitutions'],
+    include: {
+      model: Organization,
+      required: true,
+      where: {
+        code: provider,
+      },
+      through: {
+        attributes: [],
+      },
+    },
+  })
+
+  const coursesWithOpenUniSubstitutions = coursesByProvider.map(({ code, substitutions }) => {
+    if (!substitutions || !substitutions.length) return [code]
+    const alternatives = [`AY-${code}`, `AY${code}`, `A-${code}`]
+    return [code].concat(substitutions.filter(sub => alternatives.includes(sub)))
+  })
+
+  return coursesWithOpenUniSubstitutions.flat()
+}
+
+const getAllProgrammeCourses = async providerCode => {
+  const res = await Course.findAll({
+    attributes: ['id', 'code', 'name', 'substitutions'],
+    include: [
+      {
+        model: Organization,
+        attributes: ['id'],
+        required: true,
+        where: {
+          code: providerCode,
+        },
+        through: {
+          attributes: [],
+        },
+      },
+    ],
+    raw: true,
+  })
+  return res
+}
+
+const getNotCompletedForProgrammeCourses = async (from, to, programmeCourses) => {
+  try {
+    const enrollmentsCourses = await Enrollment.findAll({
+      attributes: ['studentnumber', 'course_code'],
+      where: {
+        course_code: {
+          [Op.in]: programmeCourses,
+        },
+        studentnumber: {
+          [Op.ne]: null,
+        },
+        enrollment_date_time: {
+          [Op.between]: [from, to],
+        },
+        state: 'ENROLLED',
+      },
+    })
+    const getCourseCode = code => {
+      if (code.startsWith('AY')) return code.replace('AY', '')
+      if (code.startsWith('A')) return code.replace('AY', '')
+      return code
+    }
+    const allEnrollments = {}
+    for (const { studentnumber, course_code } of enrollmentsCourses) {
+      const code = getCourseCode(course_code)
+      if (!(code in allEnrollments)) {
+        allEnrollments[code] = []
+      }
+      if (!allEnrollments[code].includes(studentnumber)) allEnrollments[code].push(studentnumber)
+    }
+
+    const credits = await Credit.findAll({
+      attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'isStudyModule'],
+      include: {
+        model: Course,
+        attributes: ['code', 'name'],
+        required: true,
+        where: {
+          code: programmeCourses,
+        },
+      },
+      where: {
+        attainment_date: {
+          [Op.between]: [from, to],
+        },
+      },
+    })
+
+    const creditsObj = credits.map(credit => {
+      return {
+        code: getCourseCode(credit.course_code),
+        studentnumber: credit.student_studentnumber,
+        credittype: credit.credittypecode,
+        courseName: credit.course.name,
+        isStudyModule: credit.isStudyModule,
+      }
+    })
+
+    const passedByCourseCodes = {}
+    const notCompletedByCourseCodes = {}
+    const courses = {}
+    for (const course of creditsObj) {
+      if (!(course.code in courses)) {
+        courses[course.code] = {
+          code: course.code,
+          name: course.courseName,
+          isStudyModule: course.isStudyModule,
+        }
+
+        passedByCourseCodes[course.code] = []
+        notCompletedByCourseCodes[course.code] = []
+      }
+      if ([4, 7, 9].includes(course.credittype)) {
+        passedByCourseCodes[course.code].push(course.studentnumber)
+      }
+      if (course.credittype === 10 && !passedByCourseCodes[course.code].includes(course.studentnumber)) {
+        notCompletedByCourseCodes[course.code].push(course.studentnumber)
+      }
+    }
+    // If student has enrollments, but no attainment for a particular course, they have no credit info.
+    programmeCourses.forEach(courseCode => {
+      if (allEnrollments[courseCode]) {
+        allEnrollments[courseCode].forEach(studentnumber => {
+          if (passedByCourseCodes[courseCode] && !passedByCourseCodes[courseCode].includes(studentnumber)) {
+            notCompletedByCourseCodes[courseCode].push(studentnumber)
+          }
+        })
+      }
+    })
+
+    return Object.keys(courses)
+      .reduce((acc, val) => [...acc, { ...courses[val] }], [])
+      .map(course => ({
+        code: course.code,
+        name: course.name,
+        totalNotCompleted: [...new Set(notCompletedByCourseCodes[course.code])].length,
+        type: 'notCompleted',
+        isStudyModule: course.isStudyModule,
+      }))
+  } catch (e) {
+    logger.error(`getNotCompletedForProgrammeCourses failed ${e}`)
+  }
+}
+
+const getCurrentStudyYearStartDate = _.memoize(
+  async unixMillis =>
+    new Date(
+      (
+        await sequelize.query(
+          `
+      SELECT startdate FROM SEMESTERS s WHERE yearcode = (SELECT yearcode FROM SEMESTERS WHERE startdate < :a ORDER BY startdate DESC LIMIT 1) ORDER BY startdate LIMIT 1;
+      `,
+          {
+            type: sequelize.QueryTypes.SELECT,
+            replacements: { a: new Date(unixMillis) },
+          }
+        )
+      )[0].startdate
+    )
+)
+
+const getProgrammeName = async code => {
+  return ProgrammeModule.findOne({
+    attributes: ['name'],
+    where: {
+      code,
+    },
+  })
+}
+
+module.exports = {
+  transfersAway,
+  transfersTo,
+  allTransfers,
+  getCurrentStudyYearStartDate,
+  getAllProgrammeCourses,
+  getCourseCodesForStudyProgramme,
+  getProgrammeName,
+  getNotCompletedForProgrammeCourses,
+  whereStudents,
+  sinceDate,
+}