[Completed courses] Make visible to all

Also improved the earlier fix regarding mocked user iamgroups, it was cached wrong

.github/actions/run-tests/action.yaml

@@ -15,24 +15,17 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: 'Download images'
-      uses: actions/download-artifact@v3
-      with:
-        path: /tmp/
-    - name: 'Load built images'
-      shell: bash
-      run: |
-        docker load --input /tmp/artifact/backend.tar
-        docker load --input /tmp/artifact/frontend.tar
-    - name: 'Login to toska docker registry'
+    - name: 'Login to toska Docker registry'
       uses: docker/login-action@v2
       with:
-        registry: registry.toska.cs.helsinki.fi
+        registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
+
     - name: 'Run services, pull rest of the needed images'
       shell: bash
-      run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      run: docker-compose --file docker-compose.ci.yml up --detach
+
     - name: 'Run cypress'
       uses: cypress-io/github-action@v4
       with:
@@ -40,12 +33,14 @@ runs:
         spec: ${{ inputs.spec }}
       env:
         CYPRESS_baseUrl: http://localhost:1337
+
     - name: 'Upload video if tests fail'
       uses: actions/upload-artifact@v3
       if: failure()
       with:
         name: Cypress videos
         path: ./cypress/videos/*
+
     - name: 'Print logs on failure'
       shell: bash
       if: failure()

.github/workflows/production.yaml

@@ -1,82 +1,101 @@
-name: 'Release oodikone to production'
+name: 'Release to production'
 
 on:
   release:
     types: [published]
 
 jobs:
-  build_and_release_backend:
-    name: 'Build and release backend'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - name: 'Login to DockerHub'
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: 'Extract release version'
-        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
-        # This steps extracts release version from GITHUB_REF env var
-        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
-        id: extract_version
-        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
-      - name: 'Build and push'
-        uses: docker/build-push-action@v3
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
-          cache-from: |
-            type=registry,ref=toska/oodikone2-backend:staging
-            type=registry,ref=toska/oodikone2-backend:latest
-          cache-to: type=inline
-          context: services/backend
-          push: true
-          tags: toska/oodikone2-backend:latest
-          target: production
-
-  build_and_release_frontend:
+  release_frontend:
     name: 'Build and release frontend'
     runs-on: ubuntu-latest
+
     steps:
       - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - name: 'Login to DockerHub'
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - name: 'Get current time'
         id: get_current_time
         run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-      - name: 'Extract release version'
-        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
-        # This steps extracts release version from GITHUB_REF env var
-        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
+
+      - name: Extract release version
         id: extract_version
         run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
-      - name: 'Build and push'
-        uses: docker/build-push-action@v3
+
+      - name: Build frontend image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
         with:
+          image: oodikone-frontend
+          tags: production ${{ github.sha }}
+          context: services/frontend
+          containerfiles: |
+            services/frontend/Dockerfile
+          build-args: |
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
+            SENTRY_ENVIRONMENT=production
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_backend:
+    name: 'Build and release backend'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Extract release version
+        id: extract_version
+        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+
+      - name: Build backend image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: oodikone-backend
+          tags: production ${{ github.sha }}
+          context: services/backend
+          containerfiles: |
+            services/backend/Dockerfile
           build-args: |
             SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
             SENTRY_ENVIRONMENT=production
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-          cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:staging
-            type=registry,ref=toska/oodikone2-frontend:latest
-          cache-to: type=inline
-          context: services/frontend
-          push: true
-          tags: toska/oodikone2-frontend:latest
-          target: production
-      - name: 'Create a container from built image '
-        run: docker create --name frontend toska/oodikone2-frontend:latest
-      - name: 'Copy assets from the container'
-        run: docker cp frontend:/usr/src/app/build ./build
-      - name: 'Create Sentry release'
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  sentry_release:
+    name: 'Create Sentry release'
+    runs-on: ubuntu-latest
+    needs:
+      - release_frontend
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Extract release version
+        id: extract_version
+        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+
+      - name: Pull image and create a container
+        run: docker create --name frontend quay.io/toska/oodikone-frontend:production
+
+      - name: Copy assets from the container
+        run: docker cp frontend:/opt/app-root/src/build ./build
+
+      - name: Create Sentry release
         uses: getsentry/action-release@v1
         env:
           SENTRY_URL: ${{ secrets.SENTRY_URL }}
@@ -91,13 +110,17 @@ jobs:
           version: ${{ steps.extract_version.outputs.VERSION }}
 
   report_success:
-    runs-on: ubuntu-latest
     name: 'Report deployment success to oodikone notifications'
+    runs-on: ubuntu-latest
     needs:
-      - build_and_release_backend
-      - build_and_release_frontend
+      - release_frontend
+      - release_backend
+      - sentry_release
+
     if: success()
+
     steps:
+      - uses: actions/checkout@v3
       - name: 'Send notification to slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
@@ -107,13 +130,17 @@ jobs:
           deployment-target: production
 
   report_failure:
-    runs-on: ubuntu-latest
     name: 'Report deployment failure to oodikone notifications'
+    runs-on: ubuntu-latest
     needs:
-      - build_and_release_backend
-      - build_and_release_frontend
+      - release_frontend
+      - release_backend
+      - sentry_release
+
     if: failure()
+
     steps:
+      - uses: actions/checkout@v3
       - name: 'Send notification to slack'
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:

.github/workflows/staging.yaml

@@ -0,0 +1,128 @@
+name: 'Test and release to staging'
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - '.github/workflows/test.yaml'
+      - '.github/workflows/staging.yaml'
+      - '.github/actions/**'
+      - 'cypress/**'
+      - 'cypress.config.json'
+      - 'services/**'
+      - '!services/backend/src/rapodiff.js'
+      - 'docker-compose.ci.yml'
+      - 'package*.json'
+      - 'updater/sis-updater-worker/src/db/migrations/**'
+
+jobs:
+  test:
+    uses: ./.github/workflows/test.yaml
+    secrets:
+      username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+      password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+
+  release_frontend:
+    name: 'Build and release frontend'
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Get current time
+        id: get_current_time
+        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
+
+      - name: Build frontend image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: oodikone-frontend
+          tags: staging ${{ github.sha }}
+          context: services/frontend
+          containerfiles: |
+            services/frontend/Dockerfile
+          build-args: |
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+            BASE_PATH=/oodikone
+            GIT_SHA=${{ github.sha }}
+            STAGING=true
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_backend:
+    name: 'Build and release backend'
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build backend image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: oodikone-backend
+          tags: staging ${{ github.sha }}
+          context: services/backend
+          containerfiles: |
+            services/backend/Dockerfile
+          build-args: |
+            BASE_PATH=/oodikone
+            GIT_SHA=${{ github.sha }}
+            STAGING=true
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  report_success:
+    name: 'Report deployment success to oodikone notifications'
+    runs-on: ubuntu-latest
+    needs:
+      - release_frontend
+      - release_backend
+
+    if: success()
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://toska-staging.cs.helsinki.fi/oodikone
+          deployment-target: staging
+
+  report_failure:
+    name: 'Report deployment failure to oodikone notifications'
+    runs-on: ubuntu-latest
+    needs:
+      - release_frontend
+      - release_backend
+
+    if: failure()
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment-failure
+          deployment-target: staging

.github/workflows/test.yaml

@@ -1,25 +1,17 @@
-name: 'Test oodikone and release to staging'
+name: 'Lint and test'
 
 on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/test-oodikone.yaml'
-      - '.github/actions/**'
-      - 'cypress/**'
-      - 'cypress.config.json'
-      - 'services/**'
-      - '!services/backend/src/rapodiff.js'
-      - 'docker-compose.ci.yml'
-      - 'docker-compose.test.yml'
-      - 'nginx.conf'
-      - 'package*.json'
-      - 'updater/sis-updater-worker/src/db/migrations/**'
+  workflow_call:
+    secrets:
+      username:
+        required: true
+        description: 'Toska registry username'
+      password:
+        required: true
+        description: 'Toska registry password'
 
 jobs:
   # === Static tests / linting ===
-
   lint_backend:
     name: 'Lint backend'
     runs-on: ubuntu-latest
@@ -48,7 +40,7 @@ jobs:
           'eslint --ext .jsx,.js services/frontend' \
           'stylelint services/frontend/src/**/*.css'
 
-  # === Build images ===
+  # === Build database images ===
   build_backend:
     name: 'Build backend'
     runs-on: ubuntu-latest
@@ -61,50 +53,15 @@ jobs:
           build-args: |
             SENTRY_RELEASE=${{ github.sha }}
             SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-backend:staging
-            type=registry,ref=toska/oodikone2-backend:latest
-          cache-to: type=inline
           context: services/backend
           push: false
-          tags: toska/oodikone2-backend:staging
-          target: production
+          tags: oodikone-backend:staging
           outputs: type=docker,dest=/tmp/backend.tar
       - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v3
         with:
           path: /tmp/backend.tar
 
-  build_frontend:
-    name: 'Build frontend'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - name: 'Get current time'
-        id: get_current_time
-        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
-      - name: 'Build image and export it'
-        uses: docker/build-push-action@v3
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-          cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:staging
-            type=registry,ref=toska/oodikone2-frontend:latest
-          cache-to: type=inline
-          context: services/frontend
-          push: false
-          tags: toska/oodikone2-frontend:staging
-          target: production
-          outputs: type=docker,dest=/tmp/frontend.tar
-      - name: 'Upload image for other jobs'
-        uses: actions/upload-artifact@v3
-        with:
-          path: /tmp/frontend.tar
-
   build_and_push_anon_sis_db:
     name: 'Build anon sis-db and push to toska hub if new migrations have been added'
     runs-on: ubuntu-latest
@@ -116,13 +73,13 @@ jobs:
           filters: |
             migrations:
               - 'updater/sis-updater-worker/src/db/migrations/**'
-              - '.github/workflows/test-oodikone.yaml'
+              - '.github/workflows/test.yaml'
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v2
         with:
-          registry: registry.toska.cs.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
       - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
         if: steps.check_migrations.outputs.migrations == 'true'
         run: |
@@ -143,7 +100,7 @@ jobs:
           filters: |
             migrations:
               - 'services/backend/src/database/migrations_kone/**'
-              - '.github/workflows/test-oodikone.yaml'
+              - '.github/workflows/test.yaml'
       - name: 'Download images'
         uses: actions/download-artifact@v3
         with:
@@ -153,9 +110,9 @@ jobs:
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v2
         with:
-          registry: registry.toska.cs.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         if: steps.check_migrations.outputs.migrations == 'true'
         run: |
@@ -176,7 +133,7 @@ jobs:
           filters: |
             migrations:
               - 'services/backend/src/database/migrations_user/**'
-              - '.github/workflows/test-oodikone.yaml'
+              - '.github/workflows/test.yaml'
       - name: 'Download images'
         uses: actions/download-artifact@v3
         with:
@@ -186,9 +143,9 @@ jobs:
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v2
         with:
-          registry: registry.toska.cs.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         if: steps.check_migrations.outputs.migrations == 'true'
         run: |
@@ -200,8 +157,6 @@ jobs:
   test_course_statistics:
     name: 'Test course statistics'
     needs:
-      - build_backend
-      - build_frontend
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
@@ -211,15 +166,13 @@ jobs:
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
           spec: cypress/e2e/Course_statistics.js
 
   test_custom_population:
     name: 'Test custom population'
     needs:
-      - build_backend
-      - build_frontend
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
@@ -229,51 +182,45 @@ jobs:
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
           spec: cypress/e2e/Custom_population.js
 
   test_population_statistics:
     name: 'Test population statistics'
+    runs-on: ubuntu-latest
     needs:
-      - build_backend
-      - build_frontend
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
           spec: cypress/e2e/Population_statistics.js
 
   test_filters:
     name: 'Test filters'
+    runs-on: ubuntu-latest
     needs:
-      - build_backend
-      - build_frontend
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
-    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
           spec: cypress/e2e/Filters.js
 
   test_student_statistics:
     name: 'Test student statistics'
     needs:
-      - build_backend
-      - build_frontend
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
@@ -283,15 +230,29 @@ jobs:
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
           spec: cypress/e2e/Student_statistics.js
 
+  test_completed_courses:
+    name: 'Test completed courses'
+    needs:
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: 'Run tests'
+        uses: ./.github/actions/run-tests
+        with:
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+          spec: cypress/e2e/Completed_courses.js
+
   test_studyprogramme_overview:
     name: 'Test studyprogramme overview'
     needs:
-      - build_backend
-      - build_frontend
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
@@ -301,15 +262,13 @@ jobs:
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
           spec: cypress/e2e/Studyprogramme_overview.js
 
   test_teachers_and_users:
     name: 'Test teachers and users'
     needs:
-      - build_backend
-      - build_frontend
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
@@ -319,8 +278,8 @@ jobs:
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
           spec: |
             cypress/e2e/Teachers.js
             cypress/e2e/Users.js
@@ -328,8 +287,6 @@ jobs:
   test_faculty_statistics:
     name: 'Test faculty statistics'
     needs:
-      - build_backend
-      - build_frontend
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
@@ -339,15 +296,13 @@ jobs:
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
           spec: cypress/e2e/Faculty_statistics.js
 
   test_open_uni_search:
     name: 'Test open uni search'
     needs:
-      - build_backend
-      - build_frontend
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
@@ -357,39 +312,30 @@ jobs:
       - name: 'Run tests'
         uses: ./.github/actions/run-tests
         with:
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
           spec: cypress/e2e/Openuni.js
 
   # === Other tests ===
-
   test_service_exit_statuses:
     name: 'Check services run correctly'
     needs:
-      - build_backend
-      - build_frontend
       - build_and_push_anon_sis_db
       - build_and_push_anon_kone_db
       - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: 'Download images'
-        uses: actions/download-artifact@v3
-        with:
-          path: /tmp/
-      - name: 'Load built images'
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v2
         with:
-          registry: registry.toska.cs.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
+          username: ${{ secrets.username }}
+          password: ${{ secrets.password }}
+
       - name: 'Run services, pull rest of the needed images'
-        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+        run: docker-compose --file docker-compose.ci.yml up --detach
+
       - name: 'Check that exit statuses are not errors'
         run: |
           docker ps -a
@@ -397,101 +343,7 @@ jobs:
             echo 'Some process had nonzero exit code';
             exit 1;
           fi
+
       - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
-
-  # === Release ===
-
-  release_images:
-    name: 'Release built images'
-    runs-on: ubuntu-latest
-    needs:
-      - lint_backend
-      - lint_frontend
-      - test_course_statistics
-      - test_custom_population
-      - test_filters
-      - test_population_statistics
-      - test_student_statistics
-      - test_studyprogramme_overview
-      - test_teachers_and_users
-      - test_faculty_statistics
-      - test_open_uni_search
-      - test_service_exit_statuses
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Download images'
-        uses: actions/download-artifact@v3
-        with:
-          path: /tmp/
-      - name: 'Load built images'
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-      - name: 'Login to DockerHub'
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: 'Create a container from frontend'
-        run: docker create --name frontend toska/oodikone2-frontend:staging
-      - name: 'Copy assets from the container'
-        run: docker cp frontend:/usr/src/app/build ./build
-      - name: 'Create Sentry release'
-        uses: getsentry/action-release@v1
-        env:
-          SENTRY_URL: ${{ secrets.SENTRY_URL }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_ORG: toska
-          SENTRY_PROJECT: oodikone
-        with:
-          environment: staging
-          set_commits: 'skip'
-          sourcemaps: './build/static/js'
-          url_prefix: '~/static/js'
-          version: ${{ github.sha }}
-      - name: 'Push to docker hub'
-        run: |
-          docker push toska/oodikone2-backend:staging
-          docker push toska/oodikone2-frontend:staging
-      - name: 'Send deployment success notification to slack'
-        if: success()
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment
-          softa-url: https://oodikone-staging.cs.helsinki.fi/
-          deployment-target: staging
-      - name: 'Send deployment failure notification to slack'
-        if: failure()
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: staging
-
-  report_test_failure:
-    runs-on: ubuntu-latest
-    name: 'Report test failure to oodikone notifications'
-    needs:
-      - lint_backend
-      - lint_frontend
-      - test_course_statistics
-      - test_custom_population
-      - test_filters
-      - test_population_statistics
-      - test_student_statistics
-      - test_studyprogramme_overview
-      - test_teachers_and_users
-      - test_faculty_statistics
-      - test_open_uni_search
-      - test_service_exit_statuses
-    if: failure()
-    steps:
-      - uses: actions/checkout@v3
-      - name: 'Send notification to slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: test-failure

.github/workflows/updater.yaml

@@ -1,17 +1,17 @@
-name: 'Test and release updater'
+name: 'Test and release Updater'
 
 on:
   push:
     branches:
       - master
     paths:
-      - '.github/workflows/test-and-release-updater.yaml'
+      - '.github/workflows/updater.yaml'
       - 'updater/sis-updater-worker/**'
       - 'updater/sis-updater-scheduler/**'
 
 jobs:
   build_worker:
-    name: 'Build sis-updater-worker'
+    name: 'Build Updater worker'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -19,23 +19,17 @@ jobs:
       - name: 'Build the image'
         uses: docker/build-push-action@v3
         with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=production
           context: updater/sis-updater-worker/
-          target: production
           push: false
           outputs: type=docker,dest=/tmp/worker.tar
           tags: toska/sis-updater-worker:latest
-          cache-from: type=registry,ref=toska/sis-updater-worker:latest
-          cache-to: type=inline
       - name: 'Store built image'
         uses: actions/upload-artifact@v3
         with:
           path: /tmp/worker.tar
 
   build_scheduler:
-    name: 'Build sis-updater-scheduler'
+    name: 'Build Updater scheduler'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -43,23 +37,17 @@ jobs:
       - name: 'Build the image'
         uses: docker/build-push-action@v3
         with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=production
           context: updater/sis-updater-scheduler/
-          target: production
           push: false
           outputs: type=docker,dest=/tmp/scheduler.tar
           tags: toska/sis-updater-scheduler:latest
-          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
-          cache-to: type=inline
       - name: 'Store built image'
         uses: actions/upload-artifact@v3
         with:
           path: /tmp/scheduler.tar
 
   test_updater:
-    name: 'Test updater'
+    name: 'Test Updater'
     runs-on: ubuntu-latest
     needs:
       - build_worker
@@ -69,7 +57,7 @@ jobs:
       - name: 'Login to toska docker registry'
         uses: docker/login-action@v2
         with:
-          registry: registry.toska.cs.helsinki.fi
+          registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
       - name: 'Download built images'
@@ -108,55 +96,56 @@ jobs:
           SCHEDULE_IMMEDIATE: purge
           EXIT_AFTER_IMMEDIATES: yes
 
-  release_updater:
-    name: 'Release updater'
+  release_worker:
+    name: 'Build and release Updater worker'
     runs-on: ubuntu-latest
-    needs:
-      - test_updater
+    needs: test_updater
+
     steps:
       - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - name: 'Login to DockerHub'
-        uses: docker/login-action@v2
+
+      - name: Build worker image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: 'Build and push scheduler'
-        uses: docker/build-push-action@v3
+          image: updater-worker
+          tags: staging production ${{ github.sha }}
+          context: updater/sis-updater-worker
+          containerfiles: |
+            updater/sis-updater-worker/Dockerfile
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
         with:
-          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
-          cache-to: type=inline
-          context: updater/sis-updater-scheduler/
-          push: true
-          tags: toska/sis-updater-scheduler:latest
-          target: production
-      - name: 'Build and push worker'
-        uses: docker/build-push-action@v3
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+  release_scheduler:
+    name: 'Build and release Updater scheduler'
+    runs-on: ubuntu-latest
+    needs: test_updater
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build scheduler image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
         with:
-          cache-from: type=registry,ref=toska/sis-updater-worker:latest
-          cache-to: type=inline
-          context: updater/sis-updater-worker/
-          push: true
-          tags: toska/sis-updater-worker:latest
-          target: production
-      - name: 'Create Sentry release'
-        uses: getsentry/action-release@v1
-        env:
-          SENTRY_URL: ${{ secrets.SENTRY_URL }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_ORG: toska
-          SENTRY_PROJECT: updater
+          image: updater-scheduler
+          tags: staging production ${{ github.sha }}
+          context: updater/sis-updater-scheduler
+          containerfiles: |
+            updater/sis-updater-scheduler/Dockerfile
+
+      - name: Push to quay.io
+        uses: redhat-actions/push-to-registry@v2
         with:
-          environment: production
-          set_commits: 'skip'
-          sourcemaps: './build/static/js'
-          url_prefix: '~/static/js'
-          version: ${{ github.sha }}
-      - name: 'Send notification to slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment
-          softa-url: https://oodikone.cs.helsinki.fi
-          deployment-target: production
-          softa-to-deploy: sis-updater
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/toska
+          username: toska+github
+          password: ${{ secrets.QUAY_IO_TOKEN }}

.lintstagedrc.js

@@ -7,7 +7,7 @@ module.exports = {
   '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
   '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
   '*.css': files => `stylelint --fix ${files.join(' ')}`,
-  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0 hadolint ${relativeFilePaths(files)}`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.12.0 hadolint ${relativeFilePaths(files)}`,
   '*.sh': files => `${dockerCmdBase} koalaman/shellcheck:v0.7.2 ${relativeFilePaths(files)} -x`,
   '.github/{workflows,actions}/*': files => `npm run actionlint ${relativeFilePaths(files)}`,
   'docker-compose*': files => {

README.md

@@ -1,6 +1,6 @@
 # Oodikone
 
-An application for analyzing university data, running at [https://oodikone.cs.helsinki.fi/](https://oodikone.cs.helsinki.fi/).
+An application for analyzing university data, running at [https://oodikone.helsinki.fi/](https://oodikone.helsinki.fi/).
 
 ## ✔️ Requirements
 
@@ -8,7 +8,7 @@ To run Oodikone locally, you will need the following:
 
 1. Applications:
    - [Docker](https://docs.docker.com/install/) (version 20.10+),
-   - [Docker Compose](https://docs.docker.com/compose/install/)(version 1.29+) and
+   - [Docker Compose](https://docs.docker.com/compose/install/) (version 1.29+) and
    - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
 2. Access to Toska Docker Hub,
 3. For real data setup, access to Toskas production servers.
@@ -50,7 +50,48 @@ What different CLI options do:
 
 ### Architecture
 
-![Oodikone architecture](./documentation/Oodikone.png)
+```mermaid
+graph TB
+    subgraph Oodikone & Updater
+        subgraph Oodikone
+            client[React app] --> server[NodeJS server]
+            server --> ok_redis(Redis)
+            konedb[(kone-db)]
+        end
+        server -->|Read| sisdb[(sis-db)]
+        subgraph Updater
+            updaterscheduler[Scheduler] --> |Push jobs| nats
+            nats(NATS server)
+            updaterworker --> |Fetch jobs| nats
+            updaterworker[Worker] -->|Write| sisdb
+            updaterscheduler --> redis(Redis)
+            updaterworker --> redis
+        end
+    end
+
+    subgraph Common Toska services
+        jami[JAMI]
+        pate[Pate]
+        importer[Importer]
+    end
+
+    updaterworker -->|Read db| importer
+    updaterscheduler -->|Read db| importer
+    importer -->|Fetch data| sisu[Sisu export APIs]
+    server -->|Get IAM access| jami
+    server -->|Send mail| pate
+
+    subgraph Analytics
+        direction TB
+        sentry[Sentry]
+        graylog[Graylog]
+        grafana[Grafana]
+    end
+```
+
+In development, backend and updater use the same redis, but in production they are separate.
+
+See also [older graph](documentation/Oodikone.png), which presents more information.
 
 ### Documentation
 
@@ -80,13 +121,13 @@ As said, the development environment runs entirely inside docker containers. To
 
 `./run.sh` is simply a wrapper script to run oodikone, updater or both services in either anon or real mode. If you take a look at `package.json`, you can see that most of the predefined scripts above use `./run.sh` under the hood.
 
-It is recommended to spend some time to become familiar with `docker` and `docker-compose` cli commands. You can then use them directly or with `./run.sh` wrapper. Here is some examples for day-to-day development situations:
+It is recommended to spend some time to become familiar with `docker` and `docker compose` cli commands. You can then use them directly or with `./run.sh` wrapper. Here is some examples for day-to-day development situations:
 
 - `./run.sh oodikone anon pull`: Pull all images related to oodikone development
 - `./run.sh updater real up --build --force-recreate --detach`: Start updater detached (=in the background) in real data mode, but build new images before starting
-- `docker-compose ps`: view the containers in the running environment
-- `docker-compose logs frontend`: print logs for just frontend
-- `docker-compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
+- `docker compose ps`: view the containers in the running environment
+- `docker compose logs frontend`: print logs for just frontend
+- `docker compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
 - `docker exec -it backend sh`: open bash terminal inside backend container
 - `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.
 
@@ -102,31 +143,27 @@ Cli script sets up pre-commit hooks that are used to lint and fix files before c
 
 For more information on how files are tested, take a look at "lint-staged" in `package.json`. Some files (e.g. github action files, dockerfiles, shell scripts) are checked with external tools and may require you to install those tools in case you're modifying files in question.
 
-There's no separate "lintfix" / "format" / "lint" command available, since pre-commit hooks will fix formatting and basic linting errors for you. In case pre-commit fails, you should fix the erroring code manually.
-
-However, if you want to run linting / formatting manually from command line, `package.json` defines `npm run eslint`, `npm run prettier` and `npm run stylelint` which you can use as an entrypoint.
+Pre-commit hooks will fix auto-fixable problems. To set up quick formatting: In VSCode you can go to command palette, select "ESLint: Fix all auto-fixable problems" and assign a hotkey for it, like `SHIFT + CTRL + '.'`.
 
 ## 🔨 Testing & CI
 
-There are three types of tests in this project: static tests with eslint/prettier/other tools, jest integration tests for single service and cypress end-to-end tests. Linting is described above and other do work as follows:
+We use cypress for end-to-end testing. No unit tests are currently used.
 
-- Cypress
-  - can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
-  - are defined in cypress -folder and cypress.config.json
-  - whole cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
-  - There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.
-- Jest
-  - are ran with `docker-compose.test.yml`
-  - test command and environment themselves are in Dockerfile for service in question
-  - can be debugged by overriding default command in `docker-compose.test.yml` and running docker container in interactive mode
+Cypress
+
+- Can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
+- Are defined in cypress -folder and cypress.config.json
+- The entire cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
+- Running tests locally sometimes causes problems after running the whole tests suite due to some login-related issue. Also for this reason, it is usually best practice to only run single views locally, and let the CI run the entire test suite.
+- There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.
 
 Continuous integration (CI) works with Github actions and is defined in workflow files in `.github/workflows` folder:
 
-- oodikone setup for cypress and other tests in CI is defined in `docker-compose.ci.yml`. Take a look at this too if debugging github action workflows.
-- tests are run on every push.
-- after successful test run, oodikone is deployed to staging
-- after creating a release, oodikone is deployed to production
-- updater is deployed to production on every push, no tests included
+- Oodikone setup for cypress and other tests in CI is defined in `docker-compose.ci.yml`. Take a look at this too if debugging github action workflows.
+- Tests are run on every push
+- After a successful test run, oodikone is deployed to staging
+- After creating a release, oodikone is deployed to production
+- Updater is deployed to production when pushed if tests pass
 
 ## ❓FAQ
 
@@ -134,21 +171,19 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 
 You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container.
 
-### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data
+### Studyguidance groups don't work on my machine
 
-Make sure you have your VPN on
-
-### I made a change to sis-db models aand created a migration, but tests are failing in CI!
-
-Migrations are handled by sis-importer-worker, since oodikone is using only replica. Since CI is testing only services included in oodikone, your new changes aren't migrated and thus are failing. Open anonyymioodi-folder in this repo to find more about how to to update the anon sis-db.
+Make sure you have your VPN on.
 
 ### Everything is broken, can't get oodikone running, data is not there etc.
 
 Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.
 
+If you don't want to reset all data, you can delete the docker images `docker compose down --rmi` (or just one) and they will be rebuilt when you start them. This is sometimes necessary if some commit has updated images or you switch to an older commit which has changed the images.
+
 ## ✌🏼 Maintainers and contribution
 
-Toska of course.
+[Toska](https://toska.dev) of course.
 
 University of Helsinki
 

anonyymioodi/commit_and_push_to_toska_hub.sh

@@ -43,6 +43,6 @@ fi
 
 infomsg "Committing and pushing to registry"
 database=${args[0]}
-docker commit "$database" "registry.toska.cs.helsinki.fi/$database"
-docker push "registry.toska.cs.helsinki.fi/$database"
+docker commit "$database" "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
+docker push "registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/$database"
 successmsg "Succesfully pushed"

anonyymioodi/docker-compose.yml

@@ -9,7 +9,7 @@ services:
       PGDATA: /data
       POSTGRES_DB: kone-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry.toska.cs.helsinki.fi/kone-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
     # image: postgres:13.4-alpine
     profiles:
       - oodikone
@@ -21,7 +21,7 @@ services:
       PGDATA: /data
       POSTGRES_DB: sis-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    # image: registry.toska.cs.helsinki.fi/sis-db
+    # image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
     image: postgres:13.2
     shm_size: 2gb
 
@@ -32,7 +32,7 @@ services:
       PGDATA: /data
       POSTGRES_DB: sis-importer-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    # image: registry.toska.cs.helsinki.fi/sis-importer-db
+    # image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
     image: postgres:13.1
     profiles:
       - updater
@@ -44,7 +44,7 @@ services:
       PGDATA: /data
       POSTGRES_DB: user-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry.toska.cs.helsinki.fi/user-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
     # image: postgres:13.4-alpine
     profiles:
       - oodikone

cypress/e2e/Completed_courses.js

@@ -0,0 +1,154 @@
+/// <reference types="Cypress" />
+
+const openCompletedCoursesModal = () => {
+  cy.get('button').contains('Search completed courses of students').click()
+}
+
+const openSearch = () => {
+  cy.init('/completedcoursessearch')
+  hasLanded()
+}
+const selectSavedCourselist = name => {
+  cy.get('[data-cy="history-search"]').children().eq(0).type(name).type('{enter}')
+}
+
+const hasLanded = () => {
+  cy.contains('Search completed courses')
+  cy.contains(
+    'Here you can search by a list of student and course numbers to see whether students have completed certain courses yet'
+  )
+}
+
+const deleteAllSearches = () => {
+  cy.contains('Saved courselists')
+    .get('.dropdown')
+    .then(d => {
+      const searchItems = d.find('div[role=option] > span[class=text]')
+      for (let i = 0; i < searchItems.length; i++) {
+        if (searchItems[i].textContent.includes('TEST-')) {
+          cy.get('[data-cy="history-search"]').children().eq(0).type(searchItems[i].textContent).type('{enter}')
+          cy.get('button').contains('Delete').click()
+        }
+      }
+    })
+}
+
+describe('When search modal is opened', () => {
+  after(() => {
+    cy.init('/completedcoursessearch')
+    hasLanded()
+    cy.url().should('include', '/completedcoursessearch')
+    cy.contains('Search completed courses of students')
+    cy.get('[data-cy="open-completed-courses-modal-button"]').click()
+    deleteAllSearches()
+  })
+
+  it('Modal opens correctly', () => {
+    cy.init('/completedcoursessearch')
+    hasLanded()
+    openCompletedCoursesModal()
+    cy.contains('Search completed courses of students')
+    cy.contains('Insert one or more student numbers, separated by a space, a newline, or a comma.')
+    cy.contains('Insert one or more courses, separated by a space, a newline, or a comma.')
+    cy.contains('Insert name for this course list if you wish to save it')
+  })
+
+  describe('When a search is executed with invalid input', () => {
+    it('Returns empty table and notification', () => {
+      cy.init('/completedcoursessearch')
+      hasLanded()
+      openCompletedCoursesModal()
+      cy.get('[data-cy="student-no-input"]').type('1')
+      cy.get('[data-cy="course-list-input"]').type('1')
+      cy.get('[data-cy="completed-courses-search-button"]').click()
+      cy.get('[data-cy="rights-notification"]').should(
+        'contain.text',
+        'The following students information could not be displayed'
+      )
+      cy.get('[data-cy="rights-notification"]').should('contain.text', '1')
+      cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', 'Completed courses search')
+      cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', 'Student Number')
+      cy.get('[data-cy="completed-courses-table-div"]').should('not.contain', '1')
+    })
+  })
+
+  describe('When a search with correct data is executed', () => {
+    before(() => {
+      cy.init('/completedcoursessearch')
+      hasLanded()
+      openCompletedCoursesModal()
+      cy.fixture('completedCoursesData').then(({ studentSet1, coursesSet1 }) => {
+        cy.get('[data-cy="student-no-input"]').type(studentSet1.join('\n'))
+        cy.get('[data-cy="course-list-input"]').type(coursesSet1.join('\n'))
+        cy.get('[data-cy="completed-courses-search-button"]').click()
+      })
+    })
+
+    it('Pushes the query to url', () => {
+      cy.url().should(
+        'include',
+        '/completedcoursessearch?courseList=TKT10001&courseList=TKT10002&courseList=TKT10003&courseList=TKT10004&studentList=010450938&studentList=010589388&studentList=010614509'
+      )
+    })
+    it('Finds correct students and courses', () => {
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(5000)
+      cy.fixture('completedCoursesData').then(({ studentSet1, coursesSet1 }) => {
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', studentSet1[0])
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', studentSet1[1])
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', studentSet1[2])
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', coursesSet1[0])
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', coursesSet1[1])
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', coursesSet1[2])
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', coursesSet1[3])
+      })
+    })
+  })
+})
+
+const courseListNames = ['TEST-courselist1', 'TEST-courselist2']
+
+describe('Courselist saving-related functions work', () => {
+  beforeEach(() => {
+    openSearch()
+    openCompletedCoursesModal()
+    cy.get('[data-cy="course-list-input"]').type('CSM14204\nTKT10004')
+    cy.get('[data-cy="search-name"]').type(courseListNames[0])
+    cy.get('button').contains('Save').click()
+  })
+
+  it('Course list can be saved', () => {
+    selectSavedCourselist(courseListNames[0])
+    openSearch()
+    openCompletedCoursesModal()
+    selectSavedCourselist(courseListNames[0])
+    cy.get('[data-cy="course-list-input"]').contains('TKT10004').contains('CSM14204')
+  })
+
+  it('Course list can be deleted', () => {
+    openSearch()
+    openCompletedCoursesModal()
+    cy.get('[data-cy="search-name"]').type(courseListNames[1])
+    cy.get('button').contains('Save').click()
+    openSearch()
+    openCompletedCoursesModal()
+    selectSavedCourselist(courseListNames[1])
+    cy.get('button').contains('Delete').click()
+    cy.get('[data-cy="history-search"]')
+      .type(courseListNames[1])
+      .type('{enter}')
+      .should('not.contain', courseListNames[1])
+  })
+
+  it('Course list can be updated', () => {
+    openSearch()
+    openCompletedCoursesModal()
+    selectSavedCourselist(courseListNames[0])
+    cy.get('[data-cy="course-list-input"]').type(`,TKT10001`)
+    cy.get('button').contains('Save').click()
+    openSearch()
+    openCompletedCoursesModal()
+    selectSavedCourselist(courseListNames[0])
+    cy.get('[data-cy="course-list-input"]').contains(`TKT10001`)
+  })
+})

cypress/e2e/Course_statistics.js

@@ -1,3 +1,4 @@
+/* eslint-disable cypress/no-unnecessary-waiting */
 /// <reference types="Cypress" />
 
 describe('Course Statistics tests', () => {
@@ -154,7 +155,7 @@ describe('Course Statistics tests', () => {
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Käyttöjärjestelmät')
       cy.contains('TKT20003')
-
+      cy.wait(5000)
       cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
       cy.contains('Population of course Käyttöjärjestelmät 2019-2020 (open and normal')
       cy.contains('TKT20003')
@@ -166,12 +167,17 @@ describe('Course Statistics tests', () => {
 
     it('Population of course shows grades for each student', () => {
       cy.get("input[placeholder='Search by a course code']").type('TKT20001')
-      cy.contains(/^TKT20001, AYTKT20001$/).click()
-      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.wait(2000)
+      cy.contains(/^TKT20001, AYTKT20001$/).click({ force: true })
+      cy.wait(2000)
+      cy.contains('Fetch statistics').should('be.enabled').click({ force: true })
+      cy.wait(2000)
       cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
-      cy.get('tbody >:nth-child(3) > :nth-child(2) .level').click()
+      cy.get('tbody >:nth-child(3) > :nth-child(2) .level').click({ force: true })
+      cy.wait(2000)
       cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019 (open and normal)')
-      cy.contains('Students (204)').click()
+      cy.contains('Students (204)').click({ force: true })
+      cy.wait(2000)
       cy.contains('td', '010262566').siblings().contains('4')
       cy.contains('td', '010674989').siblings().contains('1')
     })
@@ -182,6 +188,7 @@ describe('Course Statistics tests', () => {
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
       cy.get('[data-cy=unify_radio_reqular]').click()
+      cy.wait(5000)
       cy.get('tbody > :nth-child(3) > :nth-child(2) .level').click()
       cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
       cy.contains('Language distribution').click()
@@ -360,7 +367,7 @@ describe('Course Statistics tests', () => {
     })
   })
 
-  it.skip('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
+  it('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
     cy.init('/coursestatistics', 'onlycoursestatistics')
     cy.get('[data-cy=navbar-courseStatistics]').click()
     cy.get('[data-cy=course-code-input]').type('TKT20003')
@@ -373,11 +380,9 @@ describe('Course Statistics tests', () => {
     cy.cs('viewMode-Attempts').click()
 
     const attemptsTableContents = [
-      // [time, passed, failed, passrate]
-      ['Total', 288, 213, 75, '73.96 %'], //Somehow cypress in CI can't find the number even if it is present in the artifact video
-      ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
-      // ['Total', null, 295, 216, 79, '73.22 %'],
-      // ['2020-2021', null, 5, 2, 3, '40.00 %'],
+      // [time, passed, failed, passrate, total enrollments, enrolled, rejected, aborted]
+      ['Total', null, 288, 213, 75, '73.96 %', 0, 0, 0, 0],
+      ['2020-2021', null, '5 or less students', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA'],
       ['2019-2020', null, 164, 121, 43, '73.78 %'],
       ['2018-2019', null, 85, 60, 25, '70.59 %'],
       ['2017-2018', null, 39, 32, 7, '82.05 %'],

cypress/e2e/Faculty_statistics.js

@@ -1,3 +1,4 @@
+/* eslint-disable cypress/no-unnecessary-waiting */
 /// <reference types="Cypress" />
 
 describe('Faculty overview', () => {
@@ -29,6 +30,7 @@ describe('Faculty overview', () => {
   describe('Faculty basic information: basic user', () => {
     beforeEach(() => {
       cy.init('/faculties')
+      cy.wait(3000)
       cy.contains('td', 'H80').click()
     })
     it('Basic information tab show all graphs and tables', () => {
@@ -98,7 +100,6 @@ describe('Faculty overview', () => {
 
       // I tried to fix this without cy.wait() method without effort. Sorting the programmes
       // or something else causes rerendering of the view and yield to failing tests.
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
       cy.wait(10000)
       cy.get('[data-cy="Button-Show-CreditsProducedByTheFaculty-0"]').click()
       cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
@@ -137,8 +138,11 @@ describe('Faculty overview', () => {
   describe('Average graduation times', () => {
     beforeEach(() => {
       cy.init('/faculties')
+      cy.wait(3000)
       cy.contains('td', 'H60').click()
+      cy.wait(3000)
       cy.contains('Graduation times').click()
+      cy.wait(3000)
     })
 
     it('User can view graduation graphs', () => {
@@ -228,7 +232,9 @@ describe('Faculty overview', () => {
     beforeEach(() => {
       cy.init('/faculties')
       cy.contains('td', 'H50').click()
+      cy.wait(1000)
       cy.contains('Programmes and student populations').click()
+      cy.wait(1000)
     })
 
     it('User can view studentdata table without %', () => {
@@ -269,23 +275,22 @@ describe('Faculty overview', () => {
     })
 
     it('Progress tables can be toggled', () => {
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-6"]').should('not.be.visible')
-      cy.get('[data-cy="Button-Show-FacultyBachelorsProgressTable-6"]').click()
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-0"]').should('not.exist')
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-1"]').should('not.exist')
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-2"]').should('not.exist')
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-3"]').should('not.exist')
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-4"]').should('not.exist')
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-6"]').should('be.visible')
-      cy.get('[data-cy="Button-Hide-FacultyBachelorsProgressTable-6"]').click()
-      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-6"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-5"]').should('not.be.visible')
+      cy.get('[data-cy="Button-Show-FacultyBachelorsProgressTable-5"]').click()
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-0"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-1"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-2"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-3"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-5"]').should('be.visible')
+      cy.get('[data-cy="Button-Hide-FacultyBachelorsProgressTable-5"]').click()
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-5"]').should('not.be.visible')
     })
 
-    it('Sudent statstable can be toggled', () => {
+    it('Studentstats table can be toggled', () => {
       cy.get('[data-cy="FacultyStudentStatsTable"]').should('not.contain', 'TKT')
-      cy.get('[data-cy="Button-FacultyStudentStatsTable-3"]').click()
+      cy.get('[data-cy="Button-FacultyStudentStatsTable-6"]').click()
       cy.get('[data-cy="FacultyStudentStatsTable"]').should('contain', 'TKT')
-      cy.get('[data-cy="Button-FacultyStudentStatsTable-3"]').click()
+      cy.get('[data-cy="Button-FacultyStudentStatsTable-6"]').click()
       cy.get('[data-cy="FacultyStudentStatsTable"]').should('not.contain', 'TKT')
     })
 

cypress/e2e/Filters.js

@@ -1,3 +1,4 @@
+/* eslint-disable cypress/no-unnecessary-waiting */
 /// <reference types="Cypress" />
 
 // const baseUrl = Cypress.config().baseUrl
@@ -20,6 +21,7 @@ const clearSingleDropdownSelection = dataCyAttribute => {
 }
 
 const testRangeFilter = (parentEl, min, max, expected) => {
+  cy.clock(MOCKED_DATE, ['Date'])
   cy.cs(parentEl)
     .cs('range-selector-min')
     .find('input')
@@ -49,8 +51,6 @@ const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
     const card = cy.cs(`${filterName}-filter-card`)
 
     card.invoke('attr', 'data-open').then(open => {
-      console.log('open', open)
-
       const getHeader = () => cy.cs(`${filterName}-header`)
 
       if (open === 'false') {
@@ -68,6 +68,25 @@ const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
   }
 }
 
+describe('Population Statistics with CS master', () => {
+  const pathToCSMast2019 =
+    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22MH50_009%22}&tag&year=2019'
+  const defaultAmountOfStudents = 27
+  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
+  before(() => {
+    cy.init(pathToCSMast2019)
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+})
+
 describe('Population Statistics', () => {
   const pathToCSBach2018 =
     '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2018'
@@ -87,18 +106,6 @@ describe('Population Statistics', () => {
     })
   })
 
-  // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
-  // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
-  // So fix this
-  it.skip('Study track filter works', () => {
-    runTestStepWithPreAndPostParts('StudyTrack', () => {
-      const card = cy.cs('StudyTrack-filter-card')
-      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
-      checkFilteringResult(1)
-      programmeDropdown.get('i.delete').click()
-    })
-  })
-
   it('Graduation filter works', () => {
     runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
       const getCard = () => cy.cs('GraduatedFromProgramme-filter-card')
@@ -110,6 +117,7 @@ describe('Population Statistics', () => {
       checkFilteringResult(defaultAmountOfStudents - graduated)
       getCard().cs('option-all').click()
     })
+    cy.cs('reset-all-filters').click()
   })
 
   it('Transfer filter works', () => {
@@ -121,12 +129,13 @@ describe('Population Statistics', () => {
       checkFilteringResult(defaultAmountOfStudents + transferred)
       cy.cs('TransferredToProgramme-filter-card').cs('option-havenot').click()
     })
+    cy.cs('reset-all-filters').click()
   })
 
   it('Enrollment filter works', () => {
     runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
       cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
-      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(0)
+      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(5)
       checkFilteringResult(148)
       clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
     })
@@ -136,6 +145,7 @@ describe('Population Statistics', () => {
     runTestStepWithPreAndPostParts('CreditsEarned', () => {
       testRangeFilter('CreditsEarned-filter-card', 50, 150, 118)
     })
+    cy.cs('reset-all-filters').click()
   })
 
   it('Age filter works', () => {
@@ -146,6 +156,7 @@ describe('Population Statistics', () => {
   })
 
   it('Gender filter works', () => {
+    cy.cs('reset-all-filters').click()
     runTestStepWithPreAndPostParts('Gender', () => {
       cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(42)
@@ -181,21 +192,24 @@ describe('Population Statistics', () => {
     })
   })
 
-  // plz fix
   it.skip('Courses filter works', () => {
-    // courses takes some time to load, wait for it to complete
-    cy.wait(10000)
+    // courses takes long time to load, wait for it to complete
+    cy.wait(25000)
+    cy.contains('Courses of class').click()
     runTestStepWithPreAndPostParts('Courses', () => {
-      cy.cs('courseFilter-course-dropdown').click()
-      const courses = ['TKT20001', 'MAT11002']
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
-      checkFilteringResult(140)
+      const courses = ['TKT20002', 'MAT11002']
+      cy.cs('courseFilter-course-dropdown').click().contains(`${courses[0]} - Ohjelmistotekniikka`).click()
+      checkFilteringResult(118)
       cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
-      checkFilteringResult(131)
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
-      checkFilteringResult(56)
+      checkFilteringResult(117)
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(`${courses[1]} - Lineaarialgebra ja matriisilaskenta I`)
+        .click()
+      checkFilteringResult(50)
       cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
-      checkFilteringResult(2)
+      checkFilteringResult(1)
+      cy.contains('Courses of class').click()
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })
   })
@@ -263,12 +277,14 @@ describe('Course Statistics', () => {
 
   it('Age filter works', () => {
     cy.clock(MOCKED_DATE, ['Date'])
+    cy.cs('reset-all-filters').click()
     runTestStepWithPreAndPostParts('Age', () => {
       testRangeFilter('Age-filter-card', 20, 40, 33)
     })
   })
 
   it('Gender filter works', () => {
+    cy.cs('reset-all-filters').click()
     runTestStepWithPreAndPostParts('Gender', () => {
       cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(27)
@@ -325,6 +341,7 @@ describe('Custom Population Statistics', () => {
     runTestStepWithPreAndPostParts('Age', () => {
       testRangeFilter('Age-filter-card', 20, 30, 1)
     })
+    cy.cs('reset-all-filters').click()
   })
 
   it('Programme defaults to "Active Study Right" mode', () => {
@@ -364,22 +381,16 @@ describe('Custom Population Statistics', () => {
     })
   })
 
-  it.skip('Courses filter works', () => {
+  it('Courses filter works', () => {
     runTestStepWithPreAndPostParts('Courses', () => {
       const courses = ['MAT11001', 'TKT20004']
-      cy.cs('courseFilter-course-dropdown')
-        .click()
-        .contains(courses[0] + ' ')
-        .click()
-      checkFilteringResult(5)
+      cy.cs('courseFilter-course-dropdown').click().contains(`${courses[0]} - Johdatus yliopistomatematiikkaan`).click()
+      checkFilteringResult(4)
       cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
-      checkFilteringResult(5)
-      cy.cs('courseFilter-course-dropdown')
-        .click()
-        .contains(courses[1] + ' ')
-        .click()
+      checkFilteringResult(4)
+      cy.cs('courseFilter-course-dropdown').click().contains(`${courses[1]} - Tietoliikenteen perusteet`).click()
       checkFilteringResult(3)
-      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(3)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
       checkFilteringResult(1)
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })

cypress/e2e/Population_statistics.js

@@ -99,34 +99,25 @@ describe('Population Statistics tests', () => {
       cy.contains('Create new tag')
     })
 
-    // TODO: this fails in CI, but not locally. Investigate and fix
-    it.skip('Advanced settings work', () => {
+    it('Advanced settings work', () => {
       cy.visit(pathToCSBach2017)
       cy.get('[data-cy=advanced-toggle]').click()
       // only spring
       cy.cs('toggle-fall').click()
       cy.contains('Fetch class').click()
-
-      // moar waiting hack
       // eslint-disable-next-line cypress/no-unnecessary-waiting
       cy.wait(10000)
-
-      cy.contains('No statistics found for the given query.')
-
+      cy.contains('Students (0)')
       // only fall
       cy.get('[data-cy=advanced-toggle]').click()
       cy.cs('toggle-fall').click()
       cy.cs('toggle-spring').click()
-
       cy.contains('Fetch class').click()
-
       cy.contains('Credit accumulation (for 170 students)')
-
       // spring + fall
       cy.get('[data-cy=advanced-toggle]').click()
       cy.cs('toggle-spring').click()
       cy.contains('Fetch class').click()
-
       cy.contains('Credit accumulation (for 170 students)')
     })
 
@@ -158,7 +149,6 @@ describe('Population Statistics tests', () => {
 
       cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
         const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)
-
         expect(sum).to.equal(170)
       })
 

cypress/e2e/Student_statistics.js

@@ -78,7 +78,7 @@ describe('Student Statistics tests', () => {
     cy.contains('td a', student.studentnumber).click()
     cy.get('[data-cy=sisulink] > a')
       .should('have.attr', 'href')
-      .and('include', 'https://sis-helsinki.funidata.fi/tutor/role/staff/student/hy-hlo-84830887/basic/basic-info')
+      .and('include', 'https://sisu.helsinki.fi/tutor/role/staff/student/hy-hlo-84830887/basic/basic-info')
   })
 
   it('Searching with bad inputs doesnt yield results', () => {

cypress/e2e/Studyprogramme_overview.js

@@ -1,3 +1,4 @@
+/* eslint-disable cypress/no-unnecessary-waiting */
 /// <reference types="Cypress" />
 
 const deleteTag = name => {
@@ -8,9 +9,11 @@ const deleteTag = name => {
 }
 
 const getEmptyYears = isAcademicYear => {
-  const thisYear = new Date().getFullYear()
+  const today = new Date()
+  const latestYear = isAcademicYear && today.getMonth() < 7 ? today.getFullYear() - 1 : today.getFullYear()
+
   const years = []
-  for (let year = thisYear; year > 2021; year--) {
+  for (let year = latestYear; year > 2021; year--) {
     if (isAcademicYear) {
       years.push(`${year} - ${year + 1}`)
     } else {
@@ -21,6 +24,20 @@ const getEmptyYears = isAcademicYear => {
 }
 
 describe('Studyprogramme overview', () => {
+  before(() => {
+    cy.init('/study-programme', 'admin')
+    cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click({ force: true })
+    cy.get('.attached').contains('Update statistics').click()
+    // eslint-disable-next-line cypress/no-unnecessary-waiting
+    cy.wait(5000)
+    cy.get('button').eq(0).click({ force: true })
+    // eslint-disable-next-line cypress/no-unnecessary-waiting
+    cy.wait(10000)
+    cy.get('button').eq(1).click({ force: true })
+    // eslint-disable-next-line cypress/no-unnecessary-waiting
+    cy.wait(10000)
+  })
+
   /* Basic information overview -tests*/
   describe('Basic information -view works for basic user', () => {
     beforeEach(() => {
@@ -46,8 +63,8 @@ describe('Studyprogramme overview', () => {
         // [Year, Started, Graduated, Transferred Away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0]),
         [2021, 0, 0, 1, 0],
-        [2020, 12, 35, 3, 2],
-        [2019, 87, 17, 0, 1],
+        [2020, 10, 35, 3, 2],
+        [2019, 86, 17, 0, 1],
         [2018, 161, 1, 0, 0],
         [2017, 171, 0, 0, 0],
       ]
@@ -62,8 +79,8 @@ describe('Studyprogramme overview', () => {
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
         [2021, 0, 0, 0, 0, 0],
         [2020, 5860, 5591, 102, 0, 167],
-        [2019, 8696, 7737, 577, 33, 349],
-        [2018, 7818, 6584, 852, 110, 272],
+        [2019, 8696, 7737, 587, 23, 349],
+        [2018, 7818, 6584, 837, 125, 272],
         [2017, 3085, 2492, 153, 52, 388],
       ]
 
@@ -85,17 +102,19 @@ describe('Studyprogramme overview', () => {
       cy.checkTableStats(tableContents, 'GraduatedAndThesisWritersOfTheProgramme')
     })
 
-    it.skip('Special studyrights can be excluded and basic data changes accordingly', () => {
+    it('Special studyrights can be excluded and basic data changes accordingly', () => {
       cy.get('[data-cy=StudentToggle]').click()
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(15000)
       const years = getEmptyYears()
       const studentTableContents = [
         // [Year, Started, Graduated]
         ...years.map(year => [year, 0, 0]),
         [2021, 0, 0],
-        [2020, 22, 0],
-        [2019, 12, 0],
-        [2018, 0, 0],
-        [2017, 0, 0],
+        [2020, 10, 35],
+        [2019, 86, 17],
+        [2018, 161, 1],
+        [2017, 171, 0],
       ]
 
       cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
@@ -104,10 +123,10 @@ describe('Studyprogramme overview', () => {
         // [Year, Total, Major students credits, Transferred credits]
         ...years.map(year => [year, 0, 0, 0]),
         [2021, 0, 0, 0],
-        [2020, 117.5, 0, 7.5],
-        [2019, 67, 0, 0],
-        [2018, 0, 0, 0],
-        [2017, 0, 0, 0],
+        [2020, 5018, 4694, 167],
+        [2019, 6185, 5702, 349],
+        [2018, 4885, 4538, 272],
+        [2017, 1988, 1600, 388],
       ]
 
       cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
@@ -116,9 +135,9 @@ describe('Studyprogramme overview', () => {
         // [Year, Graduated, Wrote thesis]
         ...years.map(year => [year, 0, 0]),
         [2021, 0, 0],
-        [2020, 0, 0],
-        [2019, 0, 0],
-        [2018, 0, 0],
+        [2020, 35, 35],
+        [2019, 17, 25],
+        [2018, 1, 5],
         [2017, 0, 0],
       ]
 
@@ -127,17 +146,18 @@ describe('Studyprogramme overview', () => {
     })
 
     it('Year can be changed to academic year, and data changes accordingly', () => {
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
       cy.get('[data-cy=YearToggle]').click({ force: true })
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(15000)
       const isAcademicYear = true
       const years = getEmptyYears(isAcademicYear)
       const studentTableContents = [
         // [Year, Started, Graduated, Transferred away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0]),
         ['2021 - 2022', 0, 0, 0, 0],
-        ['2020 - 2021', 12, 6, 3, 2],
-        ['2019 - 2020', 86, 41, 1, 0],
-        ['2018 - 2019', 162, 5, 0, 1],
+        ['2020 - 2021', 10, 6, 3, 2],
+        ['2019 - 2020', 86, 38, 1, 0],
+        ['2018 - 2019', 161, 8, 0, 1],
         ['2017 - 2018', 171, 1, 0, 0],
       ]
       // eslint-disable-next-line cypress/no-unnecessary-waiting
@@ -148,10 +168,10 @@ describe('Studyprogramme overview', () => {
         // [Year, Total, Major students credits, Non-major students credits, Non-degree credits, Transferred credits]
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
         ['2021 - 2022', 0, 0, 0, 0, 0],
-        ['2020 - 2021', 566, 486, 20, 0, 60],
-        ['2019 - 2020', 8595, 8271, 106, 0, 218],
-        ['2018 - 2019', 9992, 8855, 781, 53, 303],
-        ['2017 - 2018', 6055, 4792, 777, 142, 344],
+        ['2020 - 2021', 541, 471, 10, 0, 60],
+        ['2019 - 2020', 8575, 8266, 116, 0, 193],
+        ['2018 - 2019', 10013, 8851, 791, 43, 328],
+        ['2017 - 2018', 6079, 4816, 762, 157, 344],
       ]
 
       cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
@@ -169,8 +189,8 @@ describe('Studyprogramme overview', () => {
         .should('contain', 'Major students credits')
         .should('contain', 'Non-major students credits')
         .should('contain', 'Transferred credits')
-        .should('contain', 852)
-        .should('contain', 577)
+        .should('contain', 7737)
+        .should('contain', 6584)
 
       cy.get('[data-cy=Graph-GraduatedAndThesisWritersOfTheProgramme')
         .should('contain', 'Graduated students')
@@ -181,9 +201,9 @@ describe('Studyprogramme overview', () => {
         .should('contain', 25)
         .should('contain', 35)
 
-      cy.get('[data-cy=graduation-times-graph-breakdown]')
+      cy.get('[data-cy=graduation-times-graph-breakdownBachelor]')
       cy.get('[data-cy=GraduationTimeToggle]').click()
-      cy.get('[data-cy=graduation-times-graph]').within(() => {
+      cy.get('[data-cy=graduation-times-graphBachelor]').within(() => {
         cy.contains('Graduation year')
         cy.contains('2019')
         cy.contains('17 graduated').trigger('mouseover')
@@ -236,7 +256,9 @@ describe('Studyprogramme overview', () => {
   describe('Studytrack overview works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
+      cy.wait(10000)
       cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.wait(5000)
       cy.get('.attached').contains('Studytracks and class statistics').click()
     })
 
@@ -249,12 +271,12 @@ describe('Studyprogramme overview', () => {
 
     it('Students of the studyprogramme are shown correctly', () => {
       const tableContents = [
-        // [Year, All, Started studying, Currently enrolled, Absent, Cancelled, Graduated, Men, Women, Finnish]
-        ['2020 - 2021', 12, 10, 0, 0, 0, 0, 10, 2, 12],
-        ['2019 - 2020', 86, 86, 0, 0, 1, 1, 65, 21, 86],
-        ['2018 - 2019', 162, 161, 0, 0, 0, 11, 118, 44, 161],
-        ['2017 - 2018', 171, 171, 0, 0, 4, 42, 135, 36, 166],
-        ['Total', 431, 428, 0, 0, 5, 54, 328, 103, 425],
+        // [Year, All, Started studying, Currently enrolled, Absent, Inactive, Graduated, Men, Women, other/unknown Finnish, other]
+        ['2020 - 2021', 12, 10, 0, 0, 0, 0, 10, 2, 0, 12, 0],
+        ['2019 - 2020', 86, 86, 0, 0, 0, 1, 65, 21, 0, 86, 0],
+        ['2018 - 2019', 162, 161, 0, 0, 0, 11, 118, 44, 0, 161, 1],
+        ['2017 - 2018', 171, 171, 0, 0, 0, 42, 135, 36, 0, 166, 5],
+        ['Total', 431, 428, 0, 0, 0, 54, 328, 103, 0, 425, 6],
       ]
 
       cy.checkTableStats(tableContents, 'StudytrackOverview')
@@ -263,10 +285,10 @@ describe('Studyprogramme overview', () => {
     it('Student progress data is shown correctly', () => {
       const tableContents = [
         ['2020 - 2021', 12, 12, 0, 0, 0, 0, 0, 0, 0],
-        ['2019 - 2020', 86, 11, 15, 35, 20, 5, 0, 0, 0],
-        ['2018 - 2019', 162, 3, 9, 28, 50, 33, 25, 11, 3],
-        ['2017 - 2018', 171, 7, 13, 22, 32, 18, 23, 32, 24],
-        ['Total', 431, 33, 37, 85, 102, 56, 48, 43, 27],
+        ['2019 - 2020', 86, 86, 0, 0, 0, 0, 0, 0, 0],
+        ['2018 - 2019', 162, 162, 0, 0, 0, 0, 0, 0, 0],
+        ['2017 - 2018', 171, 171, 0, 0, 0, 0, 0, 0, 0],
+        ['Total', 431, 431, 0, 0, 0, 0, 0, 0, 0],
       ]
 
       cy.checkTableStats(tableContents, 'StudytrackProgress')
@@ -277,14 +299,12 @@ describe('Studyprogramme overview', () => {
         .should('contain', 'Less than 15 credits')
         .should('contain', '30-59 credits')
         .should('contain', '180 or more credits')
-        .should('contain', '4.1%') // The percentage for less than 15 credits in 2017-2018, to check that the graph renders
-        .should('contain', '5.6%') // The percentage for 15-29 credits in 2018-2019
-        .should('contain', '12.8%') // The percentage for less than 15 credits in 2019-2020
+        .should('contain', '100.0%') // The percentage for less than 15 credits in 2017-2018, to check that the graph renders
 
-      cy.get('[data-cy=graduation-times-graph-breakdown]')
+      cy.get('[data-cy=graduation-times-graph-breakdownBachelor]')
       cy.get('[data-cy=GraduationTimeToggle]').click()
 
-      cy.get('[data-cy=graduation-times-graph]').within(() => {
+      cy.get('[data-cy=graduation-times-graphBachelor]').within(() => {
         cy.contains('Start year')
         cy.contains('2019 - 2020')
         cy.contains('42 graduated').trigger('mouseover')
@@ -298,6 +318,8 @@ describe('Studyprogramme overview', () => {
   describe('Programme courses works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
       cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('.attached').contains('Programme courses').click()
     })
@@ -349,7 +371,7 @@ describe('Studyprogramme overview', () => {
       cy.get('[data-cy=CoursesSortableTable]').within(() => {
         // Test sorting by different columns
         cy.get('th').eq(0).click()
-        cy.get('tr').eq(2).contains('Tietojenkäsittelytieteen kisälliopetus: Java-ohjelmointi')
+        cy.get('tr').eq(2).contains('Tietojenkäsittelytieteen kisälliopetus')
         cy.get('th').eq(1).click()
         cy.get('tr').eq(1).contains('Äidinkielinen viestintä')
         cy.get('th').eq(2).click()
@@ -365,7 +387,7 @@ describe('Studyprogramme overview', () => {
       cy.get('[data-cy=creditsStudentsToggle]').first().click()
 
       cy.get('[data-cy=CoursesSortableTable]').within(() => {
-        cy.get('tr').eq(1).contains('267')
+        cy.get('tr').eq(2).contains('267')
       })
     })
   })
@@ -374,6 +396,8 @@ describe('Studyprogramme overview', () => {
   describe('Basic information -view works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
       cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('.attached').contains('Tags').click()
     })
@@ -449,7 +473,7 @@ describe('Studyprogramme overview', () => {
       cy.contains('Studytracks and class statistics')
 
       cy.contains('Update statistics').should('not.exist')
-      cy.contains('Degree Courses').should('not.exist')
+      cy.contains('Degree courses').should('not.exist')
     })
 
     it('can access basic information', () => {

cypress/fixtures/completedCoursesData.json

@@ -0,0 +1,4 @@
+{
+  "studentSet1": ["010450938", "010589388", "010614509"],
+  "coursesSet1": ["TKT10001", "TKT10002", "TKT10003", "TKT10004"]
+}

docker-compose.ci.yml

@@ -19,12 +19,16 @@ services:
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-    image: toska/oodikone2-backend:staging
+      IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
+      JAMI_URL: 'http://jami:3003/'
+    build:
+      context: services/backend
 
   frontend:
     <<: *defaults
     container_name: frontend
-    image: toska/oodikone2-frontend:staging
+    build:
+      context: services/frontend
 
   kone-db:
     <<: *defaults
@@ -33,20 +37,20 @@ services:
       PGDATA: /data
       POSTGRES_DB: kone-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry.toska.cs.helsinki.fi/kone-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
 
   redis:
     <<: *defaults
-    command: 'redis-server --appendonly yes'
-    container_name: redis
-    image: redis
+    command: >
+      sh -c 'redis-server --appendonly yes && redis-cli flushall'
+    image: redis:6.2.13
     ports:
       - 6379:6379
 
   sis-db:
     <<: *defaults
     container_name: sis-db
-    image: registry.toska.cs.helsinki.fi/sis-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
     environment:
       PGDATA: /data
       POSTGRES_DB: sis-db
@@ -60,18 +64,15 @@ services:
       PGDATA: /data
       POSTGRES_DB: user-db
       POSTGRES_HOST_AUTH_METHOD: trust
-    image: registry.toska.cs.helsinki.fi/user-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
 
   jami:
     <<: *defaults
-    image: toska/jami
+    image: quay.io/toska/jami:production
     environment:
       - PORT=3003
       - NODE_ENV=production
-      - POSTGRES_HOST=jami-db
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DATABASE=postgres
+      - DATABASE_URL=postgres://postgres:postgres@jami-db:5432/postgres
     ports:
       - 3003:3003
     container_name: jami
@@ -101,14 +102,13 @@ services:
   sis-importer-db:
     <<: *defaults
     container_name: sis-importer-db
-    image: registry.toska.cs.helsinki.fi/sis-importer-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
     profiles:
       - updater
 
   sis-updater-worker:
     <<: *defaults
     build:
-      target: development
       context: updater/sis-updater-worker
     depends_on:
       - redis
@@ -116,12 +116,14 @@ services:
       - sis-importer-db
       - sis-updater-nats
     environment:
+      CI: 'true'
       DB_URL: postgres://postgres@sis-db:5432/sis-db
       REDIS_URI: //redis:6379
       SIS_IMPORTER_DATABASE: sis-importer-db
       SIS_IMPORTER_HOST: sis-importer-db
       SIS_IMPORTER_PASSWORD: postgres
       SIS_IMPORTER_USER: postgres
+      SIS_PASSWORD: postgres
       SIS_NATS_TOKEN: dev
       SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
     profiles:
@@ -130,7 +132,6 @@ services:
   sis-updater-scheduler:
     <<: *defaults
     build:
-      target: development
       context: updater/sis-updater-scheduler
     container_name: sis-updater-scheduler
     depends_on:
@@ -138,6 +139,7 @@ services:
       - sis-importer-db
       - sis-updater-nats
     environment:
+      CI: 'true'
       REDIS_URI: //redis:6379
       SECRET_TOKEN: dev
       SIS_IMPORTER_DATABASE: sis-importer-db

docker-compose.yml

@@ -4,7 +4,7 @@ x-defaults: &defaults
   restart: unless-stopped
 
 x-build_defaults: &build_defaults
-  target: development
+  dockerfile: dev.Dockerfile
 
 services:
   adminer:
@@ -39,13 +39,15 @@ services:
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
+      IMPORTER_DB_API_URL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
+      JAMI_URL: 'http://jami:3003/'
     ports:
       - 8080:8080
     profiles:
       - oodikone
     volumes:
-      - backend-nodemod:/usr/src/app/node_modules
-      - ./services/backend/:/usr/src/app
+      - backend-nodemod:/opt/app-root/src/node_modules
+      - ./services/backend/:/opt/app-root/src
 
   frontend:
     <<: *defaults
@@ -58,13 +60,13 @@ services:
     profiles:
       - oodikone
     volumes:
-      - frontend-nodemod:/usr/src/app/node_modules
-      - ./services/frontend/:/usr/src/app
+      - frontend-nodemod:/opt/app-root/src/node_modules
+      - ./services/frontend/:/opt/app-root/src
 
   kone-db:
     <<: *defaults
     container_name: kone-db
-    image: registry.toska.cs.helsinki.fi/kone-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db
     profiles:
       - oodikone
     volumes:
@@ -74,7 +76,7 @@ services:
     <<: *defaults
     command: 'redis-server --appendonly yes'
     container_name: redis
-    image: redis
+    image: redis:6.2.13
     ports:
       - '6379:6379'
     volumes:
@@ -83,7 +85,7 @@ services:
   sis-db:
     <<: *defaults
     container_name: sis-db
-    image: registry.toska.cs.helsinki.fi/sis-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db
     volumes:
       - sis-db-data:/data
     shm_size: 2gb
@@ -91,7 +93,7 @@ services:
   sis-importer-db:
     <<: *defaults
     container_name: sis-importer-db
-    image: registry.toska.cs.helsinki.fi/sis-importer-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db
     profiles:
       - updater
     volumes:
@@ -131,8 +133,8 @@ services:
     profiles:
       - updater
     volumes:
-      - sis-updater-scheduler-nodemod:/usr/src/app/node_modules
-      - ./updater/sis-updater-scheduler/:/usr/src/app
+      - sis-updater-scheduler-nodemod:/opt/app-root/src/node_modules
+      - ./updater/sis-updater-scheduler/:/opt/app-root/src
 
   sis-updater-worker:
     <<: *defaults
@@ -151,18 +153,19 @@ services:
       SIS_IMPORTER_HOST: sis-importer-db
       SIS_IMPORTER_PASSWORD: postgres
       SIS_IMPORTER_USER: postgres
+      SIS_PASSWORD: postgres
       SIS_NATS_TOKEN: dev
       SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
     profiles:
       - updater
     volumes:
-      - sis-updater-worker-nodemod:/usr/src/app/node_modules
-      - ./updater/sis-updater-worker/:/usr/src/app
+      - sis-updater-worker-nodemod:/opt/app-root/src/node_modules
+      - ./updater/sis-updater-worker/:/opt/app-root/src
 
   user-db:
     <<: *defaults
     container_name: user-db
-    image: registry.toska.cs.helsinki.fi/user-db
+    image: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db
     profiles:
       - oodikone
     volumes:

documentation/combined_programme.md

@@ -0,0 +1,96 @@
+# Combined programmes
+
+At the moment only programmes KH90_001 and MH90_001 have a combined view.
+
+The made changes do not have any effects on faculty or trends views. **Needs manual work** is added into those parts that needs visit, if this functionality is added to other programmes.
+
+Basic rule of thumb is that for the master/licentiate programme I used names `combinedProgramme` and in some rare cases `secondProgramme` as whole or part of the variable names.
+
+## Study programme
+
+**Needs manual work** in the function `getCombinedProgrammeId` in `services/frontend/src/common/index.js` if new programmes is added. For showing the both programme data correctly, the filtering mechanism is changed to add both programmes and it is not caring for which programme user has acual rights or IAMrights. However, the links for the class stats level are hidden in studytrack tables, if the user has no rights for the programme.
+
+- Search by class
+  - **Needs manual work** Combine programmes here
+- Overview
+  - **Needs manual work**
+
+**Redis nightly study programme update**
+
+- Updates automatically combined programme stats. **Needs manual work** here `services/backend/src/services/studyprogrammeHelpers.js` in the constant `combinedStudyprogrammes`. This might needs refactoring if bachelor programme can have many separate master programmes. Then remember to change function `refreshNewOverview` in `services/backend/src/events.js` accordingly to handle lists.
+
+**NOTE:** `combinedStudyprogrammes` is also used to fetch course modules (mandatory courses). If the logic of this constant is changed in `studyProgrammeHelpers.js` -> the needed changes must be done also here `services/backend/src/services/programmeModules.js`. While writing this, the cleverer idea of doing this might have been to send `code` in form `KHxx_xxx+MHxx_xxx` and do the splitting into two codes in the backend as done in Study guidance groups logic. Changes were done here 'services/backend/src/routes/population.js', acual place is near `getFilteredAssociations` function.
+
+**Basic information**
+
+All things listed below should work automatically, if new combinations are done.
+
+- Credits produced byt the studyprogramme
+  - credits are computed for both programmes separately
+- Average graduation times
+  - For bachelor + master/licentiate studyright the time is the whole study time starting from bachelor
+- _Students of the programme_ and _Graduated and thesis writers of the programme_
+  - separate fields for started, graduated and thesis writers
+
+**Studytracks and class statistics**
+
+- _Students of the studyprogramme by starting year_
+  - Graduated field separated for both programmes
+  - All = currently enrolled + absent + inactive + graduated licentiate
+- _Progress of students of the studyprogramme by starting year_
+  - computed for whole bachelor + master/licentiate studyright period
+- _Average graduation times_
+  - bachelor + master/licentiate graph: start is bachelor studyright start
+
+**Programme courses**
+
+- Fetch all courses for both programmes and show them in one table.
+
+**Degree courses**
+
+- Possibility to add criteria for whole bachelor-master study time
+- For eläinlääkis this criteria is saved by bachelor programme code. Other cases may need other approaches.
+  In eläinklääkis this done by adding the criteria for 6 years -> the whole study time criteria is saved with the bachelor programme code. That is, same criteria is found, if bachelor or combined programme view is shown. This approach might need extra thinking for different programmes.
+- Show own tables for both programmes
+  - To fetch data **needs manual work** here `services/backend/src/services/studyprogrammeHelpers.js` in the constant `combinedStudyprogrammes`.
+
+**Tags**
+
+- Possibility to add tags for combined programme.
+
+## Class statistics view
+
+**Credit accumulation**
+
+- Show graduations for both programmes
+
+**Credit statistics**
+
+- Bachelor and bachelor+master combo have own graphs.
+
+**Courses of class**
+
+- Show courses for both programmes
+
+**Students**
+
+- **General tab** - Additional fields for graduation and hops
+- **Courses tab** - show all courses for both programmes. The courses are fetched with same logic as for Degree programmes.
+- **Tags tab** - show tabs related to this combination.
+- **Progress tab** - possibility to have progress for whole bachelor-master study time. **See Degree courses approach** to add criteria. **May need future manual work**, depending on the programme id with which the data is saved on the database.
+
+## Filters
+
+- **Hops (study plan)** - possibility filter out credits separate for both programmes or for both at the same time
+
+- **Graduated from programme** - possibility to filter out students based on graduation either in bachelor's or master's.
+
+- **Studyright status** - active or inactive based on the chosen degree
+
+- **Studytrack filter** (needs programme code) is not touched at the moment, thus **may need future manual work**.
+
+## Study guidance groups
+
+- Backend: student data is fetched by bachelor study programme code in `/v3/populationstatisticsbystudentnumbers`. Separation from code 'KHxx_xxx+MHxx_xxx' is done in backend.
+- Frontend: things are done in the same way as with class stats.
+- **Need manual work**: `./oodikone/services/frontend/src/redux/elementdetails.js` add combined programme codes (both bachelor and master) to `combinedProgrammeCodes` and modify `combinedOptions` list accordinly in `useFilterAndFormattedElementDetails()` function.

documentation/numeric_types.md

@@ -19,7 +19,7 @@ These are actually defined in db table studyright_extents. Tells studyright leve
 
 5: Rescinded (Seemingly not in use anymore)
 
-6: Option
+6: Option (Bachelor+master studyright students can add a master programme to their study plan before graduation in bachelor programme: marked as option in Oodikone)
 
 30: Graduated
 

package.json

@@ -16,8 +16,9 @@
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
     "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
+    "flushredis": "docker exec -it redis redis-cli flushall",
     "prepare": "husky install",
-    "rapodiff": "docker exec backend node /usr/src/app/src/rapodiff.js"
+    "rapodiff": "docker exec backend node /opt/app-root/src/src/rapodiff.js"
   },
   "repository": {
     "type": "git",
@@ -29,28 +30,28 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@babel/core": "^7.20.12",
+    "@babel/core": "^7.19.3",
     "@babel/eslint-parser": "^7.19.1",
     "@babel/plugin-syntax-jsx": "^7.16.7",
-    "concurrently": "^7.6.0",
+    "concurrently": "^7.4.0",
     "cypress": "^10.9.0",
-    "eslint": "^8.31.0",
+    "eslint": "^8.25.0",
     "eslint-config-airbnb": "^19.0.4",
-    "eslint-config-prettier": "^8.6.0",
+    "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-babel": "^5.3.1",
     "eslint-plugin-cypress": "^2.12.1",
     "eslint-plugin-import": "^2.25.4",
     "eslint-plugin-jsx-a11y": "^6.5.1",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
     "eslint-plugin-prettier": "^4.2.1",
-    "eslint-plugin-react": "^7.31.11",
+    "eslint-plugin-react": "^7.31.9",
     "eslint-plugin-react-hooks": "^4.3.0",
-    "husky": "^8.0.3",
-    "lint-staged": "^13.1.0",
+    "husky": "^8.0.1",
+    "lint-staged": "^13.0.3",
     "node-actionlint": "^1.2.2",
-    "prettier": "^2.8.2",
-    "stylelint": "^14.16.1",
-    "stylelint-config-prettier": "^9.0.4",
+    "prettier": "^2.5.1",
+    "stylelint": "^14.13.0",
+    "stylelint-config-prettier": "^9.0.3",
     "stylelint-config-standard": "^26.0.0",
     "stylelint-prettier": "^2.0.0"
   },

scripts/functions_for_setup.sh

@@ -21,10 +21,10 @@ USER_DB_NAME="user-db"
 DATABASES=("$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")
 
 ## Urls should be in same order as databases as both are iterated through by indexes.
-KONE_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-kone-db.sqz"
-SIS_DB_REAL_DUMP_URL="svm-96.cs.helsinki.fi:/home/updater_user/backups/latest-sis.sqz"
-SIS_IMPORTER_DB_REAL_DUMP_URL="svm-92.cs.helsinki.fi:/home/importer_user/importer-db/backup/importer-db.sqz"
-USER_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-user-db.sqz"
+KONE_DB_REAL_DUMP_URL="svm-116.cs.helsinki.fi:/home/toska_user/most_recent_backup_store/oodikone-kone-db.sql.gz"
+SIS_DB_REAL_DUMP_URL="svm-116.cs.helsinki.fi:/home/toska_user/most_recent_backup_store/oodikone-sis-db.sql.gz"
+SIS_IMPORTER_DB_REAL_DUMP_URL="svm-116.cs.helsinki.fi:/home/toska_user/most_recent_backup_store/importer.sql.gz"
+USER_DB_REAL_DUMP_URL="svm-116.cs.helsinki.fi:/home/toska_user/most_recent_backup_store/oodikone-user-db.sql.gz"
 REAL_DUMP_URLS=("$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")
 
 # Source utility functions
@@ -56,7 +56,7 @@ retry() {
 download_real_dump() {
   local database=$1
   local pannu_url=$2
-  local dump_destination="$DUMP_DIR/$database.sqz"
+  local dump_destination="$DUMP_DIR/$database.sql.gz"
   scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkki.cs.helsinki.fi" "$username@$pannu_url" "$dump_destination"
 }
 
@@ -76,7 +76,7 @@ reset_databases() {
   docker-compose up -d "${databases[@]}"
 
   for database in "${databases[@]}"; do
-    local database_dump="$database_dump_dir/$database.sqz"
+    local database_dump="$database_dump_dir/$database.sql.gz"
     local database_container="$database"
     local database_name="$database-real"
 
@@ -91,10 +91,8 @@ reset_databases() {
     docker exec -u postgres "$database_container" createdb "$database_name" || warningmsg "This is okay, continuing"
 
     infomsg "Restoring database from dump database"
-    msg "1. Copying dump..."
-    docker cp "$database_dump" "$database_container:/asd.sqz"
-    msg "2. Writing database..."
-    docker exec "$database_container" pg_restore -U postgres --no-owner -F c --dbname="$database_name" -j4 /asd.sqz
+    msg "Writing database..."
+    docker exec -i "$database_container" /bin/bash  -c "gunzip | psql -U postgres -d $database_name" < "$database_dump" 2> /dev/null
     msg ""
   done
 
@@ -103,8 +101,8 @@ reset_databases() {
 
 reset_jami_data() {
   local database="jami-db"
-  local pannu_url="svm-85.cs.helsinki.fi:/home/toska_user/most_recent_backup_store/jami.sql.gz"
-  local dump_destination="$DUMP_DIR/$database.sqz"
+  local pannu_url="svm-116.cs.helsinki.fi:/home/toska_user/most_recent_backup_store/jami.sql.gz"
+  local dump_destination="$DUMP_DIR/$database.sql.gz"
 
   infomsg "Downloading Jami data"
   scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkki.cs.helsinki.fi" "$username@$pannu_url" "$dump_destination"
@@ -118,7 +116,7 @@ reset_jami_data() {
   retry docker-compose -f "$DOCKER_COMPOSE" exec "$database" pg_isready --dbname="postgres"
 
   infomsg "Populating Jami"
-  docker exec -i "$database" /bin/bash -c "gunzip | psql -U postgres" < "$dump_destination"
+  docker exec -i "$database" /bin/bash -c "gunzip | psql -U postgres" < "$dump_destination" 2> /dev/null
   docker-compose stop "$database"
   msg ""
 }
@@ -130,8 +128,8 @@ reset_all_real_data() {
     local url="${REAL_DUMP_URLS[$i]}"
     download_real_dump "$database" "$url"
   done
-  reset_databases "${DATABASES[@]}"
   reset_jami_data
+  reset_databases "${DATABASES[@]}"
 }
 
 reset_single_database() {

services/backend/.dockerignore

@@ -1,3 +1,3 @@
 .dockerignore
-Dockerfile
+*Dockerfile
 node_modules/

services/backend/Dockerfile

@@ -1,23 +1,27 @@
-FROM node:14.17-alpine3.13 as base
+FROM registry.access.redhat.com/ubi8/nodejs-14-minimal
+
+ENV TZ="Europe/Helsinki"
+
+WORKDIR /opt/app-root/src
 
-WORKDIR /usr/src/app
 EXPOSE 8080
 
-FROM base as development
-ENV NODE_ENV=development
-COPY ./package* /usr/src/app/
-RUN npm ci
-COPY . /usr/src/app/
-CMD ["npm", "run", "dev"]
-
-FROM base as production
 ENV NODE_ENV=production
+
+ARG BASE_PATH
+ENV PUBLIC_URL=$BASE_PATH
+
+ARG STAGING
+ENV REACT_APP_STAGING=$STAGING
+
 ARG SENTRY_ENVIRONMENT
 ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
+
 ARG SENTRY_RELEASE
 ENV SENTRY_RELEASE=$SENTRY_RELEASE
-COPY --chown=node:node ./package* /usr/src/app/
-RUN npm ci --production && apk add --no-cache vim curl nano
-COPY --chown=node:node . /usr/src/app/
-USER node
+
+COPY ./package* ./
+RUN npm ci --omit-dev
+COPY . .
+
 CMD ["npm", "start"]

services/backend/config/programmeCodes.js

@@ -125,7 +125,7 @@ const codes = {
   T922106: 'foodhealth',
   T923101: 'papu',
   T923102: 'geodoc',
-  T923103: 'atm',
+  T923103: 'atm-dp',
   T923104: 'chems',
   T923105: 'domast',
   T923106: 'matrena',

services/backend/dev.Dockerfile

@@ -0,0 +1,15 @@
+FROM node:14.17-alpine3.13
+
+ENV TZ="Europe/Helsinki"
+
+WORKDIR /opt/app-root/src
+
+EXPOSE 8080
+
+ENV NODE_ENV=development
+
+COPY ./package* ./
+RUN npm ci
+COPY . .
+
+CMD ["npm", "run", "dev"]

services/backend/package.json

@@ -4,13 +4,13 @@
   "scripts": {
     "start": "node --max_old_space_size=4096 index.js",
     "dev": "nodemon --max_old_space_size=4096 index.js",
-    "rapodiff": "node /usr/src/app/src/rapodiff.js"
+    "rapodiff": "node /opt/app-root/src/src/rapodiff.js"
   },
   "dependencies": {
     "@sentry/node": "^5.30.0",
     "@sentry/tracing": "^6.18.2",
     "async": "^3.2.3",
-    "async-lock": "^1.4.0",
+    "async-lock": "^1.3.1",
     "axios": "^0.21.4",
     "bluebird": "^3.7.2",
     "cors": "^2.8.5",
@@ -20,15 +20,16 @@
     "lodash": "^4.17.21",
     "lru-cache": "^5.1.1",
     "moment": "^2.29.1",
-    "moment-timezone": "^0.5.40",
+    "moment-timezone": "^0.5.37",
     "morgan": "^1.10.0",
     "pg": "^8.8.0",
     "redis": "^3.1.2",
-    "sequelize": "^6.28.0",
+    "sequelize": "^6.24.0",
     "umzug": "^2.3.0",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "uuid": "^3.4.0",
-    "winston": "^3.8.2"
+    "winston": "^3.8.2",
+    "winston-gelf-transporter": "^1.0.2"
   },
   "devDependencies": {
     "nodemon": "^1.19.4"

services/backend/src/conf-backend.js

@@ -1,7 +1,7 @@
 // Node env to use
-const isStaging = process.env.NODE_ENV === 'staging'
 const isDev = process.env.NODE_ENV === 'development'
-const isProduction = process.env.NODE_ENV === 'production'
+const isStaging = process.env.REACT_APP_STAGING === 'true'
+const isProduction = !isStaging && process.env.NODE_ENV === 'production'
 
 // Sentry
 const sentryRelease = process.env.SENTRY_RELEASE || ''
@@ -17,9 +17,10 @@ const requiredGroup = isStaging
 const pateToken = process.env.PATE_API_TOKEN || ''
 
 // Jami
-const jamiUrl = isDev || runningInCI ? 'http://jami:3003/' : 'https://importer.cs.helsinki.fi/api/auth'
+const jamiUrl = process.env.JAMI_URL || ''
 
 // Importer client
+const importerUrl = process.env.IMPORTER_DB_API_URL || ''
 const importerToken = process.env.IMPORTER_DB_API_TOKEN || ''
 
 // Networking: Urls & ports
@@ -27,16 +28,6 @@ const baseUrl = isDev ? '/api' : ''
 const frontUrl = process.env.FRONT_URL
 const backendPort = 8080
 
-// Analytics
-const ANALYTICS_INFLUXDB_URL = process.env.ANALYTICS_INFLUXDB_URL
-const ANALYTICS_INFLUXDB_USER = process.env.ANALYTICS_INFLUXDB_USER
-const ANALYTICS_INFLUXDB_PASSWORD = process.env.ANALYTICS_INFLUXDB_PASSWORD
-const ANALYTICS_INFLUXDB_DB = process.env.ANALYTICS_INFLUXDB_DB
-
-// Matomo
-const MATOMO_SITE_ID = process.env.MATOMO_SITE_ID
-const MATOMO_URL = process.env.MATOMO_URL
-
 // Other stuff
 const DB_URL_KONE = process.env.DB_URL_KONE
 const DB_URL_USER = process.env.DB_URL_USER
@@ -55,11 +46,9 @@ const SECRET_TOKEN = process.env.SECRET_TOKEN
 const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
 const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL
 
-const formatURL = url => {
-  return !!url && !url.startsWith('http') ? `http://${url}` : url
-}
-
-const addSlashToEnd = url => (url.endsWith('/') ? url : url + '/')
+const KONE_PASSWORD = process.env.KONE_PASSWORD || 'postgres'
+const USER_PASSWORD = process.env.USER_PASSWORD || 'postgres'
+const SIS_PASSWORD = process.env.SIS_PASSWORD || 'postgres'
 
 module.exports = {
   DB_URL_USER,
@@ -69,14 +58,11 @@ module.exports = {
   redis,
   SECRET_TOKEN,
   DB_SCHEMA_KONE,
+  KONE_PASSWORD,
+  USER_PASSWORD,
+  SIS_PASSWORD,
   requiredGroup,
   SIS_UPDATER_URL,
-  ANALYTICS_INFLUXDB_URL: formatURL(ANALYTICS_INFLUXDB_URL),
-  ANALYTICS_INFLUXDB_USER,
-  ANALYTICS_INFLUXDB_PASSWORD,
-  ANALYTICS_INFLUXDB_DB,
-  MATOMO_SITE_ID,
-  MATOMO_URL: MATOMO_URL && addSlashToEnd(MATOMO_URL),
   runningInCI,
   sentryRelease,
   sentryEnvironment,
@@ -86,5 +72,6 @@ module.exports = {
   baseUrl,
   frontUrl,
   backendPort,
+  importerUrl,
   importerToken,
 }

services/backend/src/database/connection.js

@@ -20,6 +20,7 @@ class DbConnection extends EventEmitter {
         idle: 300000000,
       },
       logging: false,
+      password: conf.SIS_PASSWORD,
     })
   }
 
@@ -44,12 +45,14 @@ const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
   schema: conf.DB_SCHEMA_KONE,
   searchPath: conf.DB_SCHEMA_KONE,
   logging: false,
+  password: conf.KONE_PASSWORD,
 })
 
 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)
 
 const sequelizeUser = new Sequelize(conf.DB_URL_USER, {
   logging: false,
+  password: conf.USER_PASSWORD,
 })
 
 const initializeDatabaseConnection = async () => {

services/backend/src/database/migrations_kone/20230202_00_create_table_progress_criteria.js

@@ -0,0 +1,33 @@
+const { INTEGER, STRING, ARRAY, DATE } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.createTable('progress_criteria', {
+      code: {
+        type: STRING,
+        primaryKey: true,
+      },
+      courses_year_one: {
+        type: ARRAY(STRING),
+      },
+      courses_year_two: {
+        type: ARRAY(STRING),
+      },
+      courses_year_three: {
+        type: ARRAY(STRING),
+      },
+      credits_year_one: {
+        type: INTEGER,
+      },
+      credits_year_two: {
+        type: INTEGER,
+      },
+      credits_year_three: {
+        type: INTEGER,
+      },
+      created_at: DATE,
+      updated_at: DATE,
+    })
+  },
+  down: () => {},
+}

services/backend/src/database/migrations_kone/20230412_00_add_columns_criteria.js

@@ -0,0 +1,23 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.addColumn('progress_criteria', 'courses_year_four', {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    })
+    await queryInterface.addColumn('progress_criteria', 'courses_year_five', {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    })
+    await queryInterface.addColumn('progress_criteria', 'courses_year_six', {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    })
+    await queryInterface.addColumn('progress_criteria', 'credits_year_four', {
+      type: Sequelize.INTEGER,
+    })
+    await queryInterface.addColumn('progress_criteria', 'credits_year_five', {
+      type: Sequelize.INTEGER,
+    })
+    await queryInterface.addColumn('progress_criteria', 'credits_year_six', {
+      type: Sequelize.INTEGER,
+    })
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_kone/20230714_00_remove_mandatory_courses.js

@@ -0,0 +1,7 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('mandatory_courses')
+    await queryInterface.dropTable('mandatory_course_labels')
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_kone/20230714_01_remove_mandatory_course_sequences.js

@@ -0,0 +1,7 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.sequelize.query('DROP SEQUENCE "mandatory_courses_id_seq"')
+    await queryInterface.sequelize.query('DROP SEQUENCE "mandatory_course_labels_id_seq"')
+  },
+  down: async () => {},
+}

services/backend/src/events.js

@@ -6,10 +6,12 @@ const { getAllProgrammes } = require('./services/studyrights')
 const { updateBasicView, updateStudytrackView } = require('./services/studyprogrammeUpdates')
 const { findAndSaveTeachers } = require('./services/topteachers')
 const { faculties } = require('./services/organisations')
+const { combinedStudyprogrammes } = require('./services/studyprogrammeHelpers')
 const { updateFacultyOverview, updateFacultyProgressOverview } = require('./services/faculty/facultyUpdates')
 const { isProduction } = require('./conf-backend')
 const { getCurrentSemester } = require('./services/semesters')
 const logger = require('./util/logger')
+const { getAssociations } = require('./services/studyrights')
 
 const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
 
@@ -29,9 +31,9 @@ const refreshFaculties = async () => {
       await updateFacultyProgressOverview(faculty.code)
       ready += 1
     } catch (e) {
-      logger.error({ message: `Failed to update stats for faculty ${faculty?.code} with type ALL`, meta: e })
+      logger.error({ message: `Failed to update stats for faculty ${faculty?.code} with type ALL`, meta: `${e}` })
     }
-    logger.info(`${ready}/${facultyList.length} faculties done`)
+    logger.info(`${ready}/${facultyList.length} faculties done; faculty ${faculty?.code}`)
   }
   logger.info('Faculty stats refreshed!')
 }
@@ -48,16 +50,29 @@ const refreshNewOverviews = async () => {
         (code.includes('MH') && !code.startsWith('2_MH') && !code.endsWith('_2')) ||
         /^(T)[0-9]{6}$/.test(code)
     )
+  const associations = await getAssociations()
   let ready = 0
   for (const code of codes) {
+    let combinedProgramme = ''
+    // If combined programme is given, this updates only the bachelor programme
     try {
-      await updateBasicView(code)
-      await updateStudytrackView(code)
+      await updateBasicView(code, combinedProgramme)
+      await updateStudytrackView(code, combinedProgramme, associations)
       ready += 1
     } catch (e) {
-      logger.error({ message: `Failed to update overview stats for programme ${code}`, meta: e })
+      logger.error({ message: `Failed to update overview stats for programme ${code}`, meta: `${e}` })
     }
-    logger.info(`${ready}/${codes.length} programmes done`)
+    combinedProgramme = combinedStudyprogrammes[code] || ''
+    // If bachelor programme is combined with master programme, then update the combination.
+    if (combinedProgramme) {
+      try {
+        await updateBasicView(code, combinedProgramme)
+        await updateStudytrackView(code, combinedProgramme, associations)
+      } catch (e) {
+        logger.error({ message: `Failed to update overview stats for combined programme ${code}`, meta: `${e}` })
+      }
+    }
+    logger.info(`${ready}/${codes.length} programmes done: code ${code}`)
   }
   logger.info('Studyprogramme and studytrack overview stats refreshed!')
 }
@@ -98,7 +113,7 @@ const refreshUberToRedis = async () => {
       await refreshUber(oldAttainmentsQuery)
       logger.info(`Refreshing CDS uber year ${formattedYear} doned`)
     } catch (e) {
-      logger.error({ message: `Error when refreshing CDS uber year ${formattedYear}`, meta: e })
+      logger.error({ message: `Error when refreshing CDS uber year ${formattedYear}`, meta: `${e}` })
     }
   }
   logger.info(`Refreshing CDS Uber data doned`)
@@ -139,4 +154,5 @@ module.exports = {
   refreshStatistics,
   refreshTrends,
   refreshFaculties,
+  refreshNewOverviews,
 }

services/backend/src/middleware/currentUserMiddleware.js

@@ -28,7 +28,7 @@ const currentUserMiddleware = async (req, _res, next) => {
   }
 
   const iamGroups = parseIamGroups(hygroupcn)
-  const { access = {}, specialGroup = {} } = await getOrganizationAccess({ iamGroups })
+  const { access = {}, specialGroup = {} } = await getOrganizationAccess({ id: sisId, iamGroups })
   const iamRights = Object.keys(access)
 
   if (!hasRequiredIamGroup(iamGroups, iamRights)) {

services/backend/src/models/index.js

@@ -67,6 +67,9 @@ Studyright.belongsTo(Organization, { foreignKey: 'facultyCode', targetKey: 'code
 Studyplan.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
 Student.hasMany(Studyplan, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
 
+Studyplan.belongsTo(Studyright, { foreignKey: 'studyrightid', targetKey: 'studyrightid' })
+Studyright.hasMany(Studyplan, { foreignKey: 'studyrightid', sourceKey: 'studyrightid' })
+
 Studyright.belongsTo(Student, { foreignKey: 'studentStudentnumber', targetKey: 'studentnumber' })
 Student.hasMany(Studyright, { foreignKey: 'studentStudentnumber', sourceKey: 'studentnumber' })
 

services/backend/src/models/models_kone.js

@@ -1,3 +1,4 @@
+const { STRING } = require('sequelize')
 const Sequelize = require('sequelize')
 const { sequelizeKone } = require('../database/connection')
 
@@ -32,18 +33,6 @@ const ExcludedCourse = sequelizeKone.define(
   }
 )
 
-const MandatoryCourse = sequelizeKone.define('mandatory_courses', {
-  course_code: {
-    type: Sequelize.STRING,
-  },
-  studyprogramme_id: {
-    type: Sequelize.STRING,
-  },
-  label: {
-    type: Sequelize.BIGINT,
-  },
-})
-
 const TagStudent = sequelizeKone.define(
   'tag_student',
   {
@@ -87,23 +76,6 @@ const Tag = sequelizeKone.define(
   }
 )
 
-const MandatoryCourseLabels = sequelizeKone.define('mandatory_course_labels', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true,
-  },
-  studyprogramme_id: {
-    type: Sequelize.STRING,
-  },
-  label: {
-    type: Sequelize.STRING,
-  },
-  orderNumber: {
-    type: Sequelize.INTEGER,
-  },
-})
-
 const CustomPopulationSearch = sequelizeKone.define('custom_population_searches', {
   id: {
     primaryKey: true,
@@ -121,6 +93,7 @@ const CustomPopulationSearch = sequelizeKone.define('custom_population_searches'
   },
 })
 
+// also used in completed courses search feature
 const OpenUniPopulationSearch = sequelizeKone.define(
   'open_uni_population_searches',
   {
@@ -143,6 +116,53 @@ const OpenUniPopulationSearch = sequelizeKone.define(
   { underscored: true, timestamps: true }
 )
 
+const ProgressCriteria = sequelizeKone.define(
+  'progress_criteria',
+  {
+    code: {
+      primaryKey: true,
+      type: STRING,
+    },
+    coursesYearOne: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+    coursesYearTwo: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+    coursesYearThree: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+    coursesYearFour: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+    coursesYearFive: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+    coursesYearSix: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+    creditsYearOne: {
+      type: Sequelize.INTEGER,
+    },
+    creditsYearTwo: {
+      type: Sequelize.INTEGER,
+    },
+    creditsYearThree: {
+      type: Sequelize.INTEGER,
+    },
+    creditsYearFour: {
+      type: Sequelize.INTEGER,
+    },
+    creditsYearFive: {
+      type: Sequelize.INTEGER,
+    },
+    creditsYearSix: {
+      type: Sequelize.INTEGER,
+    },
+  },
+  { underscored: true, timestamps: true }
+)
+
 const StudyGuidanceGroupTag = sequelizeKone.define(
   'study_guidance_group_tags',
   {
@@ -165,27 +185,17 @@ const StudyGuidanceGroupTag = sequelizeKone.define(
   { underscored: true, timestamps: true }
 )
 
-MandatoryCourse.belongsTo(MandatoryCourseLabels, {
-  foreignKey: 'label',
-  sourceKey: 'id',
-})
-MandatoryCourseLabels.hasMany(MandatoryCourse, {
-  foreignKey: 'label',
-  sourceKey: 'id',
-})
-
 TagStudent.belongsTo(Tag, { foreignKey: 'tag_id', sourceKey: 'tag_id' })
 Tag.hasMany(TagStudent, { foreignKey: 'tag_id', sourceKey: 'tag_id' })
 
 module.exports = {
-  MandatoryCourse,
   TagStudent,
   Tag,
   ThesisTypeEnums,
-  MandatoryCourseLabels,
   CustomPopulationSearch,
   OpenUniPopulationSearch,
   sequelizeKone,
   ExcludedCourse,
   StudyGuidanceGroupTag,
+  ProgressCriteria,
 }

services/backend/src/models/student.js

@@ -1,4 +1,4 @@
-const { Model, STRING, DATE, INTEGER, BOOLEAN } = require('sequelize')
+const { Model, STRING, DATE, INTEGER } = require('sequelize')
 const { dbConnections } = require('../database/connection')
 
 class Student extends Model {}
@@ -27,7 +27,6 @@ Student.init(
     home_country_en: { type: STRING },
     gender_code: { type: STRING },
     sis_person_id: { type: STRING },
-    dissemination_info_allowed: { type: BOOLEAN },
     createdAt: {
       type: DATE,
     },

services/backend/src/models/studyplan.js

@@ -20,6 +20,16 @@ Studyplan.init(
       onDelete: 'cascade',
       allowNull: false,
     },
+    studyrightid: {
+      type: STRING,
+      references: {
+        model: 'studyright',
+        key: 'studyrightid',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+      allowNull: false,
+    },
     programme_code: {
       type: STRING,
       allowNull: false,
@@ -30,6 +40,9 @@ Studyplan.init(
     completed_credits: {
       type: INTEGER,
     },
+    sisu_id: {
+      type: STRING,
+    },
     createdAt: {
       type: DATE,
     },

services/backend/src/models/studyright.js

@@ -1,4 +1,4 @@
-const { Model, STRING, DATE, INTEGER } = require('sequelize')
+const { Model, STRING, DATE, INTEGER, BOOLEAN } = require('sequelize')
 const { dbConnections } = require('../database/connection')
 
 class Studyright extends Model {}
@@ -27,7 +27,9 @@ Studyright.init(
     active: {
       type: INTEGER,
     },
-    // irtisanomisperuste
+    cancelled: {
+      type: BOOLEAN,
+    },
     studentStudentnumber: {
       type: STRING,
       references: {

services/backend/src/routes.js

@@ -3,11 +3,9 @@ const express = require('express')
 const cors = require('cors')
 const { frontUrl } = require('./conf-backend')
 const shibbolethCharsetMiddleware = require('./middleware/shibbolethCharsetMiddleware')
-const matomoInit = require('./routes/matomo-init')
 const currentUserMiddleware = require('./middleware/currentUserMiddleware')
 const accessLogger = require('./middleware/accessLogger')
 const auth = require('./middleware/auth')
-
 const courses = require('./routes/courses')
 const students = require('./routes/students')
 const population = require('./routes/population')
@@ -19,20 +17,19 @@ const teachers = require('./routes/teachers')
 const providers = require('./routes/providers')
 const faculties = require('./routes/faculties')
 const semesters = require('./routes/semesters')
-const mandatoryCourses = require('./routes/mandatorycourses')
-const mandatoryCourseLabels = require('./routes/mandatorycourselabels')
 const feedback = require('./routes/feedback')
 const tags = require('./routes/tags')
 const updater = require('./routes/updater')
-const tsaAnalytics = require('./routes/tsaAnalytics')
 const customPopulationSearch = require('./routes/customPopulationSearch')
+const languageCenterData = require('./routes/languageCenterData')
 const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
 const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
 const studyProgramme = require('./routes/studyProgramme')
 const customOpenUniSearch = require('./routes/customOpenUniSearch')
+const studyProgrammeCriteria = require('./routes/studyProgrammeCriteria')
 const initializeSentry = require('./util/sentry')
-
+const completedCoursesSearch = require('./routes/completedCoursesSearch')
 const errorMiddleware = require('./middleware/errorMiddleware')
 
 module.exports = (app, url) => {
@@ -44,7 +41,6 @@ module.exports = (app, url) => {
   app.use(express.json())
 
   app.use(shibbolethCharsetMiddleware)
-  app.use(url, matomoInit)
   app.use(currentUserMiddleware)
   app.use(accessLogger)
   app.use(url, login)
@@ -58,15 +54,15 @@ module.exports = (app, url) => {
   app.use(url, tags)
   app.use(url, programmeModules)
   app.use(url, studyProgramme)
+  app.use(`${url}/programmecriteria`, studyProgrammeCriteria)
   app.use(`${url}/openunisearch`, auth.roles(['openUniSearch']), customOpenUniSearch)
+  app.use(`${url}/completedcoursessearch`, completedCoursesSearch)
+  app.use(`${url}/languagecenterdata`, languageCenterData)
   app.use(`${url}/faculties`, auth.roles(['facultyStatistics']), faculties)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
   app.use(`${url}/users`, auth.roles(['admin']), users)
   app.use(`${url}/feedback`, feedback)
-  app.use(`${url}/mandatory_courses`, mandatoryCourses)
-  app.use(`${url}/mandatory-course-labels`, mandatoryCourseLabels)
-  app.use(`${url}/tsa`, tsaAnalytics)
   app.use(`${url}/custom-population-search`, customPopulationSearch)
   app.use(`${url}/cool-data-science`, trends)
   app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)

services/backend/src/routes/completedCoursesSearch.js

@@ -0,0 +1,101 @@
+const router = require('express').Router()
+const { getCompletedCourses } = require('../services/completedCoursesSearch')
+const { getImporterClient } = require('../util/importerClient')
+const importerClient = getImporterClient()
+const _ = require('lodash')
+const {
+  getOpenUniSearches,
+  createNewSearch,
+  deleteSearch,
+  updateSearch,
+} = require('../services/openUni/openUniManageSearches')
+const logger = require('../util/logger')
+
+router.get('/', async (req, res) => {
+  const studentNumbers = JSON.parse(req.query?.studentlist) || []
+  const courseCodes = JSON.parse(req.query?.courselist) || []
+  const {
+    user: { isAdmin, studentsUserCanAccess, sisPersonId: teacherId },
+  } = req
+  if (!Array.isArray(studentNumbers)) return res.status(400).json({ error: 'Student numbers must be of type array' })
+  if (!Array.isArray(courseCodes)) return res.status(400).json({ error: 'Courses must be of type array' })
+
+  const answerTimeout = new Promise(resolve => setTimeout(resolve, 6000))
+
+  try {
+    // Teachers also can get rights to students via importer if the students
+    // have enrolled to their courses in last 8 months
+    // (acual logic a bit different, see importer)
+    const teacherRightsToStudents = await Promise.race([
+      importerClient.post(`/teacher-rights/`, { teacherId, studentNumbers }),
+      answerTimeout,
+    ])
+    if (teacherRightsToStudents && Array.isArray(teacherRightsToStudents.data))
+      studentsUserCanAccess.push(...teacherRightsToStudents.data)
+  } catch (e) {
+    logger.error(`Importer teacher-rights request failed with message: ${e?.message}`)
+  }
+
+  const filteredStudentNumbers = isAdmin ? studentNumbers : _.intersection(studentNumbers, studentsUserCanAccess)
+  const completedCourses = await getCompletedCourses(filteredStudentNumbers, courseCodes)
+  const discardedStudentNumbers = _.difference(
+    studentNumbers,
+    completedCourses.students.map(s => s.studentNumber)
+  )
+  return res.json({ discardedStudentNumbers, ...completedCourses })
+})
+
+router.get('/searches', async (req, res) => {
+  const userId = req.user.id
+  const foundSearches = await getOpenUniSearches(userId)
+  return res.json(foundSearches)
+})
+
+router.post('/searches', async (req, res) => {
+  const courseCodes = req.body?.courselist || []
+  const userId = req.user.id
+  const name = req.body?.name
+
+  if (!name) return res.status(400).json({ error: 'Courselist name missing' })
+  if (courseCodes && !Array.isArray(courseCodes))
+    return res.status(400).json({ error: 'Course codes must be type of array' })
+
+  const createdSearch = await createNewSearch(userId, name, courseCodes)
+  if (!createdSearch) return res.status(400).json({ error: 'Failed to create courselist' })
+  return res.status(201).json({
+    id: createdSearch.id,
+    userId: createdSearch.userId,
+    name: createdSearch.name,
+    courseList: createdSearch.courseCodes,
+    updatedAt: createdSearch.updatedAt,
+  })
+})
+
+router.put('/searches/:id', async (req, res) => {
+  const id = req.params?.id
+  const courseCodes = req.body?.courselist || []
+  const userId = req.user.id
+  if (!id || !userId) return res.status(422).end()
+  const updatedSearch = await updateSearch(userId, id, courseCodes)
+
+  if (!updatedSearch) return res.status(404).json({ error: 'Courselist could not be found' })
+  return res.json({
+    id: updatedSearch.id,
+    userId: updatedSearch.userId,
+    name: updatedSearch.name,
+    courseList: updatedSearch.courseCodes,
+    updatedAt: updatedSearch.updatedAt,
+  })
+})
+
+router.delete('/searches/:id', async (req, res) => {
+  const id = req.params?.id
+  const userId = req.user.id
+  if (!id || !userId) return res.status(422).end()
+
+  const deletedSearch = await deleteSearch(userId, id)
+  if (!deletedSearch) return res.status(404).json({ error: 'Courselist could not be found' })
+  return res.json(id)
+})
+
+module.exports = router

services/backend/src/routes/customOpenUniSearch.js

@@ -1,5 +1,5 @@
 const router = require('express').Router()
-var moment = require('moment-timezone')
+const moment = require('moment-timezone')
 const { getCustomOpenUniCourses } = require('../services/openUni/openUniStats')
 const {
   getOpenUniSearches,

services/backend/src/routes/elementdetails.js

@@ -1,5 +1,4 @@
 const router = require('express').Router()
-const MandatoryCourses = require('../services/mandatoryCourses')
 const { getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
 
 router.get('/elementdetails/all', async (req, res) => {
@@ -12,13 +11,4 @@ router.get('/studyprogrammes', async (req, res) => {
   res.json(studyrights)
 })
 
-router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
-  if (req.params.id) {
-    const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
-    res.json(codes)
-  } else {
-    res.status(422).end()
-  }
-})
-
 module.exports = router

services/backend/src/routes/faculties.js

@@ -2,7 +2,6 @@ const router = require('express').Router()
 const { faculties } = require('../services/organisations')
 const { combineFacultyBasics } = require('../services/faculty/facultyBasics')
 const { combineFacultyCredits } = require('../services/faculty/facultyCredits')
-const { findFacultyProgrammeCodes } = require('../services/faculty/faculty')
 const { combineFacultyThesisWriters } = require('../services/faculty/facultyThesisWriters')
 const { countGraduationTimes } = require('../services/faculty/facultyGraduationTimes')
 const { updateFacultyOverview, updateFacultyProgressOverview } = require('../services/faculty/facultyUpdates')
@@ -10,8 +9,7 @@ const { combineFacultyStudentProgress } = require('../services/faculty/facultySt
 const { combineFacultyStudents } = require('../services/faculty/facultyStudents')
 
 const {
-  getFacultyProgrammes,
-  setFacultyProgrammes,
+  getProgrammes,
   getBasicStats,
   setBasicStats,
   getCreditStats,
@@ -30,15 +28,6 @@ const logger = require('../util/logger')
 // Faculty uses a lot of tools designed for Study programme.
 // Some of them have been copied here and slightly edited for faculty purpose.
 
-const getProgrammes = async (code, programmeFilter) => {
-  const programmes = await getFacultyProgrammes(code, programmeFilter)
-  if (programmes) return programmes
-  let updatedProgrammes = await findFacultyProgrammeCodes(code, programmeFilter)
-  if (updatedProgrammes) updatedProgrammes = await setFacultyProgrammes(code, updatedProgrammes, programmeFilter)
-
-  return updatedProgrammes
-}
-
 router.get('/', async (req, res) => {
   const ignore = ['Y', 'H99', 'Y01', 'H92', 'H930']
   const facultyList = (await faculties()).filter(f => !ignore.includes(f.code))
@@ -84,11 +73,10 @@ router.get('/:id/basicstats', async (req, res) => {
 router.get('/:id/creditstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
-  const specialGroups = req.query?.special_groups
   const programmeFilter = req.query?.programme_filter
 
   if (!code) return res.status(422).end()
-  const data = await getCreditStats(code, yearType, programmeFilter, specialGroups)
+  const data = await getCreditStats(code, yearType, programmeFilter)
   if (data) return res.json(data)
 
   const programmes = await getProgrammes(code, programmeFilter)
@@ -97,9 +85,9 @@ router.get('/:id/creditstats', async (req, res) => {
   // list of all programmes is also needed in classification of credits
   const allProgrammes = programmeFilter === 'ALL_PROGRAMMES' ? programmes : await getProgrammes(code, 'ALL_PROGRAMMES')
 
-  let updatedStats = await combineFacultyCredits(code, programmes.data, allProgrammes.data, yearType, specialGroups)
+  let updatedStats = await combineFacultyCredits(code, programmes.data, allProgrammes.data, yearType)
   if (updatedStats) {
-    updatedStats = await setCreditStats(updatedStats, yearType, programmeFilter, specialGroups)
+    updatedStats = await setCreditStats(updatedStats, yearType, programmeFilter)
   }
 
   return res.json(updatedStats)
@@ -141,28 +129,16 @@ router.get('/:id/graduationtimes', async (req, res) => {
 
 router.get('/:id/progressstats', async (req, res) => {
   const code = req.params.id
-  const programmeFilter = req.query.programme_filter
   const specialGroups = req.query?.special_groups
   const graduated = req.query?.graduated
-  const programmes = await getProgrammes(code, programmeFilter)
+  const programmes = await getProgrammes(code, 'NEW_STUDY_PROGRAMMES')
   if (!programmes) return res.status(422).end()
 
-  //all programmes are required for correct sorting of transfers
-  let allProgrammeCodes = []
-  const allProgs = await getProgrammes(code, 'ALL_PROGRAMMES')
-  allProgs?.data.forEach(prog => allProgrammeCodes.push(prog.code))
-
   if (!code) return res.status(422).end()
   const data = await getFacultyProgressStats(code, specialGroups, graduated)
   if (data) return res.json(data)
 
-  let updateStats = await combineFacultyStudentProgress(
-    code,
-    programmes.data,
-    allProgrammeCodes,
-    specialGroups,
-    graduated
-  )
+  let updateStats = await combineFacultyStudentProgress(code, programmes.data, specialGroups, graduated)
   if (updateStats) {
     updateStats = await setFacultyProgressStats(updateStats, specialGroups, graduated)
   }
@@ -171,21 +147,16 @@ router.get('/:id/progressstats', async (req, res) => {
 
 router.get('/:id/studentstats', async (req, res) => {
   const code = req.params.id
-  const programmeFilter = req.query.programme_filter
   const specialGroups = req.query?.special_groups
   const graduated = req.query?.graduated
 
   if (!code) return res.status(422).end()
   const data = await getFacultyStudentStats(code, specialGroups, graduated)
   if (data) return res.json(data)
-  const programmes = await getProgrammes(code, programmeFilter)
-  if (!programmes) return res.status(422).end()
-  //all programmes are required for correct sorting of transfers
-  let allProgrammeCodes = []
-  const allProgs = await getProgrammes(code, 'ALL_PROGRAMMES')
-  allProgs?.data.forEach(prog => allProgrammeCodes.push(prog.code))
+  const newProgrammes = await getProgrammes(code, 'NEW_STUDY_PROGRAMMES')
+  if (!newProgrammes) return res.status(422).end()
 
-  let updateStats = await combineFacultyStudents(code, programmes.data, allProgrammeCodes, specialGroups, graduated)
+  let updateStats = await combineFacultyStudents(code, newProgrammes.data, specialGroups, graduated)
   if (updateStats) {
     updateStats = await setFacultyStudentStats(updateStats, specialGroups, graduated)
   }

services/backend/src/routes/languageCenterData.js

@@ -0,0 +1,13 @@
+const router = require('express').Router()
+const { getLanguageCenterData } = require('../services/languageCenterData')
+
+router.get('/', async (req, res) => {
+  const { iamGroups, isAdmin } = req.user
+  if (!isAdmin && !iamGroups.includes('grp-kielikeskus-esihenkilot')) {
+    return res.status(403).json({ error: 'Request failed because of missing rights' })
+  }
+  const result = await getLanguageCenterData()
+  return res.json(result)
+})
+
+module.exports = router

services/backend/src/routes/mandatorycourselabels.js

@@ -1,71 +0,0 @@
-const router = require('express').Router()
-const MandatoryCourseLabels = require('../services/mandatoryCourseLabels')
-const logger = require('../util/logger')
-
-router.get('/:programme/labels', async (req, res) => {
-  const { programme } = req.params
-  if (programme) {
-    try {
-      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
-      res.status(200).json(labels)
-    } catch (err) {
-      logger.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.delete('/:programme/labels', async (req, res) => {
-  const { programme } = req.params
-  const { label } = req.body
-  if (programme && label) {
-    try {
-      await MandatoryCourseLabels.destroy(programme, label)
-      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
-      res.status(200).json(labels)
-    } catch (err) {
-      logger.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.post('/:programme/move', async (req, res) => {
-  const { programme } = req.params
-  const { label, direction } = req.body
-  if (programme && label) {
-    try {
-      await MandatoryCourseLabels.move(programme, label, direction)
-      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
-      res.status(200).json(labels)
-    } catch (err) {
-      logger.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.post('/:programme/labels', async (req, res) => {
-  const { programme } = req.params
-  const { label } = req.body
-  if (programme && label) {
-    try {
-      await MandatoryCourseLabels.create(programme, label)
-      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
-      res.status(200).json(labels)
-    } catch (err) {
-      logger.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
-  }
-})
-
-module.exports = router

services/backend/src/routes/mandatorycourses.js

@@ -1,36 +0,0 @@
-const router = require('express').Router()
-const MandatoryCourses = require('../services/mandatoryCourses')
-
-router.delete('/:programme', async (req, res) => {
-  const { programme } = req.params
-  const { course } = req.body
-  if (!programme || !course) return res.status(400).end()
-  await MandatoryCourses.remove(programme, course)
-  res.status(200).json(course)
-})
-
-router.post('/:programme', async (req, res) => {
-  const { programme } = req.params
-  const { course } = req.body
-  if (!programme || !course) return res.status(400).end()
-  await MandatoryCourses.create(programme, course)
-  const newCourse = await MandatoryCourses.find(programme, course)
-  if (!newCourse) return res.status(400).end()
-  res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
-})
-
-router.post('/:programme/label/:course', async (req, res) => {
-  const { programme, course } = req.params
-  const { label } = req.body
-  if (!programme || !course) return res.status(400).end()
-  await MandatoryCourses.updateLabel(programme, course, label)
-  const updatedCourse = await MandatoryCourses.find(programme, course)
-  if (!updatedCourse) return res.status(400).end()
-  res.status(200).json({
-    name: updatedCourse.course.name,
-    code: updatedCourse.course_code,
-    label: updatedCourse.mandatory_course_label,
-  })
-})
-
-module.exports = router

services/backend/src/routes/matomo-init.js

@@ -1,36 +0,0 @@
-const router = require('express').Router()
-const config = require('../conf-backend')
-
-// Produces the Matomo tracker initialization script for the frontend.
-// Matomo URL and site id are baked here instead of in the frontend
-// so we can fix them on the fly instead of rebuilding the entire frontend.
-
-const acual = `
-var _paq = window._paq || [];
-/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
-_paq.push(["trackPageView"]);
-_paq.push(["enableLinkTracking"]);
-(function() {
-  var u = "${config.MATOMO_URL}";
-  _paq.push(["setTrackerUrl", u + "matomo.php"]);
-  _paq.push(["setSiteId", "${config.MATOMO_SITE_ID}"]);
-  _paq.push(["enableHeartBeatTimer"]);
-  var d = document,
-    g = d.createElement("script"),
-    s = d.getElementsByTagName("script")[0];
-  g.type = "text/javascript";
-  g.async = true;
-  g.defer = true;
-  g.src = u + "matomo.js";
-  s.parentNode.insertBefore(g, s);
-})();
-`
-
-const script = config.MATOMO_URL ? acual : ''
-
-router.get('/matomo-init', (_, res) => {
-  res.contentType('text/javascript; charset=UTF-8')
-  res.send(script)
-})
-
-module.exports = router

services/backend/src/routes/population.js

@@ -82,60 +82,10 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
       return
     }
 
-    //These cause problems in front (where they were originally calculated)
-    // Do them here for the moment
-    for (const course of result.coursestatistics) {
-      // course.students.notParticipated = {}
-      course.students.didNotPass = {}
-
-      for (const student of Object.keys(course.students.all)) {
-        if (
-          !Object.keys(course.students.passed).includes(student) &&
-          !Object.keys(course.students.retryPassed).includes(student)
-        )
-          course.students.didNotPass[student] = true
-      }
-    }
     res.json(result)
   }
 })
 
-router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => {
-  const { from, to, coursecodes } = req.body
-  if (!from || !to || !coursecodes) {
-    return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
-  }
-  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(coursecodes)
-  if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
-    return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
-  }
-
-  const {
-    user: { isAdmin, studentsUserCanAccess },
-  } = req
-
-  const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
-  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
-
-  const result = await Population.bottlenecksOf(
-    {
-      year: 1900,
-      studyRights: [],
-      semesters: ['FALL', 'SPRING'],
-      months: 10000,
-    },
-    studentnumberlist
-  )
-
-  if (result.error) {
-    Sentry.captureException(new Error(result.error))
-    res.status(400).json(result)
-    return
-  }
-
-  res.json(result)
-})
-
 router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
   const { tag } = req.body
   if (!tag) {
@@ -171,7 +121,7 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
 router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
   const {
     user: { isAdmin, studentsUserCanAccess },
-    body: { studentnumberlist: studentnumbersInReq, studentCountLimit, ...query },
+    body: { studentnumberlist: studentnumbersInReq, ...query },
   } = req
 
   if (!studentnumbersInReq) {
@@ -179,14 +129,12 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
   }
 
   const studentnumberlist = isAdmin ? studentnumbersInReq : _.intersection(studentnumbersInReq, studentsUserCanAccess)
-
   const result = await Population.bottlenecksOf(
     {
       year: query?.year ?? 1900,
       studyRights: query?.studyRights ?? [],
       semesters: query?.semesters ?? ['FALL', 'SPRING'],
       months: query?.months ?? 10000,
-      studentCountLimit,
     },
     studentnumberlist
   )
@@ -214,7 +162,13 @@ router.get('/v3/populationstatistics', async (req, res) => {
   } = req
 
   try {
-    if (!isAdmin && !rights.includes(studyRights.programme) && !iamRights.includes(studyRights.programme)) {
+    if (
+      !isAdmin &&
+      !rights.includes(studyRights.programme) &&
+      !rights.includes(studyRights.combinedProgramme) &&
+      !iamRights.includes(studyRights.programme) &&
+      !iamRights.includes(studyRights.combinedProgramme)
+    ) {
       res.status(403).json([])
       return
     }
@@ -223,7 +177,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
     return
   }
 
-  if (req.query.months == null) {
+  if (req.query.months === null) {
     req.query.months = 12
   }
 
@@ -232,12 +186,15 @@ router.get('/v3/populationstatistics', async (req, res) => {
     const multipopulationstudentPromises = Promise.all(
       req.query.years.map(year => {
         const newMonths = (upperYearBound - Number(year)) * 12
-        const populationStudents = Population.optimizedStatisticsOf({
-          ...req.query,
-          studyRights,
-          year,
-          months: newMonths,
-        })
+        const populationStudents = Population.optimizedStatisticsOf(
+          {
+            ...req.query,
+            studyRights: { programme: studyRights.programme },
+            year,
+            months: newMonths,
+          },
+          null
+        )
         return populationStudents
       })
     )
@@ -253,7 +210,11 @@ router.get('/v3/populationstatistics', async (req, res) => {
 
     res.json(filterPersonalTags(result, user.id))
   } else {
-    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
+    const result = await Population.optimizedStatisticsOf({
+      ...req.query,
+      studyRights: { programme: studyRights.programme },
+    })
+
     if (result.error) {
       Sentry.captureException(new Error(result.error))
       res.status(400).end()
@@ -261,7 +222,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
     }
 
     // Obfuscate if user has only iam rights
-    if (!isAdmin && !rights.includes(studyRights.programme)) {
+    if (!isAdmin && !rights.includes(studyRights.programme) && !rights.includes(studyRights.combinedProgramme)) {
       result.students = result.students.map(student => {
         const { iv, encryptedData } = encrypt(student.studentNumber)
         const obfuscatedBirthDate = new Date(new Date(student.birthdate).setMonth(0, 0)) // only birth year for age distribution
@@ -403,17 +364,20 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
 })
 
 router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
-  const { studentnumberlist } = req.body
+  const { studentnumberlist, tags } = req.body
   const {
     user: { isAdmin, id, studentsUserCanAccess },
   } = req
-
   const filteredStudentNumbers = isAdmin ? studentnumberlist : _.intersection(studentnumberlist, studentsUserCanAccess)
 
+  const studyProgrammeCode =
+    tags?.studyProgramme && tags?.studyProgramme.includes('+')
+      ? tags?.studyProgramme.split('+')[0]
+      : tags?.studyProgramme
   const result = await Population.optimizedStatisticsOf(
     {
-      year: 1900,
-      studyRights: [],
+      year: tags?.year || 1900,
+      studyRights: studyProgrammeCode ? [studyProgrammeCode] : [],
       semesters: ['FALL', 'SPRING'],
       months: 10000,
     },
@@ -423,7 +387,12 @@ router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
     Sentry.captureException(new Error(result.error))
     return
   }
-  res.status(200).json(filterPersonalTags(result, id))
+
+  result.studyProgramme = tags?.studyProgramme
+
+  const discardedStudentNumbers = _.difference(studentnumberlist, filteredStudentNumbers)
+
+  res.status(200).json({ ...filterPersonalTags(result, id), discardedStudentNumbers })
 })
 
 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
@@ -435,7 +404,11 @@ router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
     mapCodesToIds(studyrights.programmes)
     res.json(studyrights)
   } else {
-    const studyrights = await StudyrightService.getFilteredAssociations(rights.concat(iamRights))
+    const allRights = rights.concat(iamRights)
+    // For combined programme
+    // If more programmes are combined, then a function might be a better idea to add moar rights
+    if (allRights.includes('KH90_001') || allRights.includes('MH90_001')) allRights.push('KH90_001', 'MH90_001')
+    const studyrights = await StudyrightService.getFilteredAssociations(allRights)
     mapCodesToIds(studyrights.programmes)
     res.json(studyrights)
   }

services/backend/src/routes/programmeModules.js

@@ -30,8 +30,8 @@ router.delete('/v3/programme_modules', async (req, res) => {
 })
 
 router.post('/v3/programme_modules/:programmecode/', async (req, res) => {
-  const { programmecode, coursecodes } = req.body
-  await addExcludedCourses(programmecode, coursecodes)
+  const { programmecode, excludeFromProgramme, coursecodes } = req.body
+  await addExcludedCourses(excludeFromProgramme, coursecodes)
   const result = await byProgrammeCode(programmecode)
   if (!result) {
     res.status(400).end()

services/backend/src/routes/students.js

@@ -44,9 +44,9 @@ router.get('/students/:id', async (req, res) => {
   if (!isAdmin && !studentsUserCanAccess.includes(studentId)) {
     throw new ApplicationError('Error finding student', 400)
   }
-
   const student = await Student.withId(studentId)
-  res.json(filterStudentTags(student, id))
+  const filteredTags = filterStudentTags(student, id)
+  res.json(filteredTags)
 })
 
 module.exports = router

services/backend/src/routes/studyProgramme.js

@@ -15,20 +15,35 @@ const {
   setStudytrackStats,
 } = require('../services/analyticsService')
 const { updateBasicView, updateStudytrackView } = require('../services/studyprogrammeUpdates')
+const { getProgrammeName } = require('../services/studyprogramme')
 const logger = require('../util/logger')
+const { getAssociations } = require('../services/studyrights')
+const { ElementDetail } = require('../models')
+
+// For grafana statistics (idea stolen from Norppa)
+const logInfoForGrafana = async (code, combinedProgramme) => {
+  const programme = await ElementDetail.findOne({ where: { code: code } })
+  const programmeCode = combinedProgramme ? `${programme.code}-${combinedProgramme}` : programme.code
+  logger.info('Study Programme', {
+    studyprogrammeName: combinedProgramme ? `${programme.name.fi} + maisteri` : programme.name.fi,
+    studyprogrammeCode: programmeCode,
+  })
+}
 
 router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
   const specialGroups = req.query?.special_groups
-
+  const combinedProgramme = req.query?.combined_programme
   if (!code) return res.status(422).end()
 
-  const data = await getBasicStats(code, yearType, specialGroups)
+  logInfoForGrafana(code, combinedProgramme)
+  const data = await getBasicStats(code, combinedProgramme, yearType, specialGroups)
   if (data) return res.json(data)
 
   const updated = await getBasicStatsForStudytrack({
     studyprogramme: req.params.id,
+    combinedProgramme,
     settings: {
       isAcademicYear: yearType === 'ACADEMIC_YEAR',
       includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
@@ -42,13 +57,15 @@ router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
   const specialGroups = req.query?.special_groups
+  const combinedProgramme = req.query?.combined_programme
 
   if (!code) return res.status(422).end()
 
-  const data = await getCreditStats(code, yearType, specialGroups)
+  const data = await getCreditStats(code, combinedProgramme, yearType, specialGroups)
   if (data) return res.json(data)
   const updatedStats = await getCreditStatsForStudytrack({
     studyprogramme: req.params.id,
+    combinedProgramme,
     settings: {
       isAcademicYear: yearType === 'ACADEMIC_YEAR',
       includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
@@ -62,13 +79,15 @@ router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
   const specialGroups = req.query?.special_groups
+  const combinedProgramme = req.query?.combined_programme
 
   if (!code) return res.status(422).end()
 
-  const data = await getGraduationStats(code, yearType, specialGroups)
+  const data = await getGraduationStats(code, combinedProgramme, yearType, specialGroups)
   if (data) return res.json(data)
   const updatedStats = await getGraduationStatsForStudytrack({
     studyprogramme: req.params.id,
+    combinedProgramme,
     settings: {
       isAcademicYear: yearType === 'ACADEMIC_YEAR',
       includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
@@ -81,14 +100,16 @@ router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
 router.get('/v2/studyprogrammes/:id/coursestats', async (req, res) => {
   const code = req.params.id
   const showByYear = req.query.academicyear
+  const combinedProgramme = req.query?.combined_programme
   const date = new Date()
   date.setHours(23, 59, 59, 999)
 
+  logInfoForGrafana(code, combinedProgramme)
   try {
-    const data = await getStudyprogrammeCoursesForStudytrack(date.getTime(), code, showByYear)
+    const data = await getStudyprogrammeCoursesForStudytrack(date.getTime(), code, showByYear, combinedProgramme)
     return res.json(data)
   } catch (e) {
-    logger.error(`Failed to get code ${code} programme courses stats: ${e}`)
+    logger.error({ message: `Failed to get code ${code} programme courses stats`, meta: `${e}` })
   }
 })
 
@@ -96,18 +117,23 @@ router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
   const code = req.params.id
   const graduated = req.query?.graduated
   const specialGroups = req.query?.special_groups
+  const combinedProgramme = req.query?.combined_programme
 
   if (!code) return res.status(422).end()
 
-  const data = await getStudytrackStats(code, graduated, specialGroups)
+  logInfoForGrafana(code, combinedProgramme)
+  const data = await getStudytrackStats(code, combinedProgramme, graduated, specialGroups)
   if (data) return res.json(data)
 
+  const associations = await getAssociations()
   const updated = await getStudytrackStatsForStudyprogramme({
     studyprogramme: code,
+    combinedProgramme,
     settings: {
       graduated: graduated === 'GRADUATED_INCLUDED',
       specialGroups: specialGroups === 'SPECIAL_INCLUDED',
     },
+    associations,
   })
   if (updated) await setStudytrackStats(updated, graduated, specialGroups)
   return res.json(updated)
@@ -115,12 +141,13 @@ router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
 
 router.get('/v2/studyprogrammes/:id/update_basicview', async (req, res) => {
   const code = req.params.id
+  const combinedProgramme = req.query?.combined_programme
   if (code) {
     let result = null
     try {
-      result = await updateBasicView(code)
+      result = await updateBasicView(code, combinedProgramme)
     } catch (e) {
-      logger.error(`Failed to update code ${code} basic stats: ${e}`)
+      logger.error({ message: `Failed to update code ${code} ${combinedProgramme} basic stats`, meta: `${e}` })
     }
     return res.json(result)
   } else {
@@ -130,12 +157,14 @@ router.get('/v2/studyprogrammes/:id/update_basicview', async (req, res) => {
 
 router.get('/v2/studyprogrammes/:id/update_studytrackview', async (req, res) => {
   const code = req.params.id
+  const combinedProgramme = req.query?.combined_programme
   if (code) {
     let result = null
     try {
-      result = await updateStudytrackView(code)
+      const associations = await getAssociations()
+      result = await updateStudytrackView(code, combinedProgramme, associations)
     } catch (e) {
-      logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
+      logger.error({ message: `Failed to update code ${code} ${combinedProgramme} studytrack stats`, meta: `${e}` })
     }
     return res.json(result)
   } else {
@@ -143,4 +172,76 @@ router.get('/v2/studyprogrammes/:id/update_studytrackview', async (req, res) =>
   }
 })
 
+router.get('/v2/studyprogrammes/:id/evaluationstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
+  const graduated = req.query?.graduated
+
+  if (!code) return res.status(422).end()
+
+  // Statistics for Tilannekuvalomake view
+  const combinedProgramme = ''
+  let gradData = await getGraduationStats(code, combinedProgramme, yearType, specialGroups)
+  if (!gradData) {
+    const updatedStats = await getGraduationStatsForStudytrack({
+      studyprogramme: req.params.id,
+      combinedProgramme,
+      settings: {
+        isAcademicYear: yearType === 'ACADEMIC_YEAR',
+        includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
+      },
+    })
+    if (updatedStats) {
+      await setGraduationStats(updatedStats, yearType, specialGroups)
+      gradData = updatedStats
+    }
+  }
+
+  let progressData = await getStudytrackStats(code, combinedProgramme, graduated, specialGroups)
+  if (!progressData) {
+    const associations = await getAssociations()
+    const updated = await getStudytrackStatsForStudyprogramme({
+      studyprogramme: code,
+      combinedProgramme,
+      settings: {
+        graduated: graduated === 'GRADUATED_INCLUDED',
+        specialGroups: specialGroups === 'SPECIAL_INCLUDED',
+      },
+      associations,
+    })
+    if (updated) {
+      await setStudytrackStats(updated, graduated, specialGroups)
+      progressData = updated
+    }
+  }
+
+  const programmeName = await getProgrammeName(code)
+
+  delete gradData.tableStats
+  delete gradData.graphStats
+  delete gradData.titles
+  const data = {
+    id: code,
+    programmeName: programmeName?.name,
+    status: gradData?.status,
+    lastUpdated: gradData.lastUpdated,
+    graduations: gradData,
+    progress: {
+      creditTableStats: {},
+      creditTableTitles: progressData?.creditTableTitles,
+      graphData: {
+        creditGraphStats: {},
+        id: progressData?.id,
+        years: progressData?.years,
+      },
+    },
+  }
+
+  data.progress.creditTableStats[code] = progressData?.creditTableStats[code]
+  data.progress.graphData.creditGraphStats[code] = progressData?.creditGraphStats[code]
+
+  return res.json(data)
+})
+
 module.exports = router

services/backend/src/routes/studyProgrammeCriteria.js

@@ -0,0 +1,30 @@
+const {
+  getCriteria,
+  saveYearlyCourseCriteria,
+  saveYearlyCreditCriteria,
+} = require('../services/studyProgrammeCriteria')
+
+const router = require('express').Router()
+
+router.get('/', async (req, res) => {
+  const studyProgramme = req.query?.programmecode
+  if (studyProgramme !== '' && !studyProgramme) return res.status(422).end()
+  const studyProgrammeCriteria = await getCriteria(studyProgramme)
+  return res.json(studyProgrammeCriteria)
+})
+
+router.post('/courses', async (req, res) => {
+  const { code, courses, year } = req.body
+  if (!code || !courses || !year) return res.status(400).end()
+  const studyProgrammeCriteria = await saveYearlyCourseCriteria(code, courses, year)
+  return res.json(studyProgrammeCriteria)
+})
+
+router.post('/credits', async (req, res) => {
+  const { code, credits } = req.body
+  if (!code || !credits) return res.status(400).end()
+  const studyProgrammeCriteria = await saveYearlyCreditCriteria(code, credits)
+  return res.json(studyProgrammeCriteria)
+})
+
+module.exports = router

services/backend/src/routes/tags.js

@@ -22,8 +22,9 @@ router.get('/tags/:studytrack', async (req, res) => {
   const {
     user: { rights, roles, id },
   } = req
-
-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
+  const programmeCodes = studytrack.includes('KH') && studytrack.includes('MH') ? studytrack.split('-') : [studytrack]
+  if (!rights.includes(programmeCodes[0]) && !rights.includes(programmeCodes[1]) && !roles?.includes('admin'))
+    return res.status(403).end()
 
   const tags = await Tags.findTagsByStudytrack(studytrack)
   res.status(200).json(filterRelevantTags(tags, id))
@@ -37,7 +38,9 @@ router.post('/tags', async (req, res) => {
     user: { rights, roles, id },
   } = req
 
-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
+  const programmeCodes = studytrack.includes('KH') && studytrack.includes('MH') ? studytrack.split('-') : [studytrack]
+  if (!rights.includes(programmeCodes[0]) && !rights.includes(programmeCodes[1]) && !roles?.includes('admin'))
+    return res.status(403).end()
 
   await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
   const tags = await Tags.findTagsByStudytrack(studytrack)
@@ -50,7 +53,15 @@ router.delete('/tags', async (req, res) => {
     user: { rights, roles, id },
   } = req
 
-  if (!rights.includes(tag.studytrack) && !roles?.includes('admin') && !(tag.personal_user_id === id))
+  const programmeCodes =
+    tag.studytrack.includes('KH') && tag.studytrack.includes('MH') ? tag.studytrack.split('-') : [tag.studytrack]
+
+  if (
+    !rights.includes(programmeCodes[0]) &&
+    !rights.includes(programmeCodes[1]) &&
+    !roles?.includes('admin') &&
+    !(tag.personal_user_id === id)
+  )
     return res.status(403).end()
 
   await Tags.deleteTag(tag)
@@ -69,7 +80,9 @@ router.get('/studenttags/:studytrack', async (req, res) => {
     user: { rights, roles, id },
   } = req
 
-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
+  const programmeCodes = studytrack.includes('KH') && studytrack.includes('MH') ? studytrack.split('-') : [studytrack]
+  if (!rights.includes(programmeCodes[0]) && !rights.includes(programmeCodes[1]) && !roles?.includes('admin'))
+    return res.status(403).end()
 
   const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
   res.status(200).json(filterRelevantStudentTags(result, id))
@@ -83,33 +96,38 @@ router.get('/studenttags/:studentnumber', async (req, res) => {
 })
 
 router.post('/studenttags/:studentnumber', async (req, res) => {
-  const { tag, studytrack } = req.body
+  const { tag, studytrack, combinedProgramme } = req.body
   const {
     user: { roles, rights, id },
   } = req
 
-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
+  if (!rights.includes(studytrack) && !rights.includes(combinedProgramme) && !roles?.includes('admin'))
+    return res.status(403).end()
 
   await TagStudent.createStudentTag(tag)
-  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  const studytrackCode = combinedProgramme ? `${studytrack}-${combinedProgramme}` : studytrack
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrackCode)
   res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.post('/studenttags', async (req, res) => {
-  const { tags, studytrack } = req.body
+  const { tags, studytrack, combinedProgramme } = req.body
   const {
     user: { rights, roles, id },
   } = req
 
-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
+  if (!rights.includes(studytrack) && !rights.includes(combinedProgramme) && !roles?.includes('admin'))
+    return res.status(403).end()
 
-  const existingtags = await Tags.findTagsByStudytrack(studytrack)
-  const existingTagids = existingtags.map(t => t.tag_id)
+  const studytrackCode = combinedProgramme ? `${studytrack}-${combinedProgramme}` : studytrack
+  const existingTags = await Tags.findTagsByStudytrack(studytrackCode)
+  const existingTagids = existingTags.map(t => t.tag_id)
   const tagids = [...new Set(tags.map(t => t.tag_id))]
   if (!tagids.find(t => existingTagids.includes(t))) return res.status(400).json({ error: 'The tag does not exist' })
 
   const studentnumbers = tags.map(t => t.studentnumber)
-  const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
+  const studentFromProgrammes = combinedProgramme ? [studytrack, combinedProgramme] : [studytrack]
+  const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, studentFromProgrammes)
   const missingStudents = difference(studentnumbers, students)
   if (missingStudents.length !== 0)
     return res
@@ -117,39 +135,41 @@ router.post('/studenttags', async (req, res) => {
       .json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
 
   await TagStudent.createMultipleStudentTags(tags)
-  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrackCode)
   res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.delete('/studenttags/delete_one', async (req, res) => {
-  const { tag_id, studentnumber, studytrack } = req.body
+  const { tag_id, studentnumber, studytrack, combinedProgramme } = req.body
   const {
     user: { rights, roles, id },
   } = req
 
-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
-
-  const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
+  if (!rights.includes(studytrack) && !rights.includes(combinedProgramme) && !roles?.includes('admin'))
+    return res.status(403).end()
+  const studytrackCode = combinedProgramme ? `${studytrack}-${combinedProgramme}` : studytrack
+  const tags = await Tags.findTagsFromStudytrackById(studytrackCode, [tag_id])
   if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
 
   await TagStudent.deleteStudentTag(studentnumber, tag_id)
-  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrackCode)
   res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.delete('/studenttags/delete_many', async (req, res) => {
-  const { tagId, studentnumbers, studytrack } = req.body
+  const { tagId, studentnumbers, studytrack, combinedProgramme } = req.body
   const {
     user: { rights, roles, id },
   } = req
 
-  if (!rights.includes(studytrack) && !roles?.includes('admin')) return res.status(403).end()
-
-  const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
+  if (!rights.includes(studytrack) && !rights.includes(combinedProgramme) && !roles?.includes('admin'))
+    return res.status(403).end()
+  const studytrackCode = combinedProgramme ? `${studytrack}-${combinedProgramme}` : studytrack
+  const tags = await Tags.findTagsFromStudytrackById(studytrackCode, [tagId])
   if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
 
   await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
-  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrackCode)
   res.status(200).json(filterRelevantStudentTags(result, id))
 })
 

services/backend/src/routes/tsaAnalytics.js

@@ -1,31 +0,0 @@
-const Sentry = require('@sentry/node')
-const router = require('express').Router()
-const tsaService = require('../services/tsaService')
-
-router.post('/event', (req, res) => {
-  const { group, name, label, value } = req.body
-
-  if (!group || !name || !value) {
-    return res.status(400).json({ error: 'group, name and value are required' })
-  }
-
-  const {
-    user: { userId },
-  } = req
-
-  // don't await here because frontend doesn't care if it succeeds
-  tsaService.sendTsaEvent(userId, { group, name, label, value }).catch(e => {
-    Sentry.withScope(scope => {
-      // if error was caused by a 400, dig out the JSON response for Sentry
-      if (e.isAxiosError && e.response && e.response.data) {
-        scope.setExtra({ error: e.response.data })
-      }
-
-      Sentry.captureException(e)
-    })
-  })
-
-  res.status(200).end()
-})
-
-module.exports = router

services/backend/src/routes/updater.js

@@ -8,7 +8,7 @@ const {
   abort,
   updateCoursesByCourseCode,
 } = require('../services/sisUpdaterService')
-const { refreshStatistics, refreshTrends } = require('../events')
+const { refreshStatistics, refreshTrends, refreshFaculties, refreshNewOverviews } = require('../events')
 
 router.get('/update/v2/meta', async (req, res) => {
   const response = await updateSISMetadata(req)
@@ -54,12 +54,22 @@ router.get('/refresh_redis_cache', async (req, res) => {
 
 router.post('/refresh_statistic_v2', async (req, res) => {
   await refreshStatistics()
-  res.status(200).json('Refreshing sis statistics')
+  res.status(200).json('Teacher and study programme statistics refreshed')
+})
+
+router.post('/refresh_study_programmes_v2', async (req, res) => {
+  await refreshNewOverviews()
+  res.status(200).json('New study programme overviews refreshed')
+})
+
+router.post('/refresh_faculties_v2', async (req, res) => {
+  await refreshFaculties()
+  res.status(200).json('Faculties refreshed')
 })
 
 router.post('/refresh_trends', async (req, res) => {
   await refreshTrends()
-  res.status(200).json('Refreshing trends')
+  res.status(200).json('Trends refreshed')
 })
 
 router.get('/abort', async (req, res) => {

services/backend/src/services/analyticsService.js

@@ -10,9 +10,10 @@ const createRedisKeyForGraduationStats = (id, yearType, specialGroups) =>
 const createRedisKeyForStudytrackStats = (id, graduated, specialGroups) =>
   `STUDYTRACK_STATS_${id}_${graduated}_${specialGroups}`
 
-const getBasicStats = async (id, yearType, specialGroups) => {
+const getBasicStats = async (id, combinedProgramme, yearType, specialGroups) => {
   if (!isUpdatedNewProgramme(id)) return null
-  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
+  let searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
+  const redisKey = createRedisKeyForBasicStats(searchkey, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
@@ -32,9 +33,10 @@ const setBasicStats = async (data, yearType, specialGroups) => {
   return dataToRedis
 }
 
-const getCreditStats = async (id, yearType, specialGroups) => {
+const getCreditStats = async (id, combinedProgramme, yearType, specialGroups) => {
   if (!isUpdatedNewProgramme(id)) return null
-  const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
+  let searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
+  const redisKey = createRedisKeyForCreditStats(searchkey, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
@@ -54,9 +56,10 @@ const setCreditStats = async (data, yearType, specialGroups) => {
   return dataToRedis
 }
 
-const getGraduationStats = async (id, yearType, specialGroups) => {
+const getGraduationStats = async (id, combinedProgramme, yearType, specialGroups) => {
   if (!isUpdatedNewProgramme(id)) return null
-  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
+  let searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
+  const redisKey = createRedisKeyForGraduationStats(searchkey, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
@@ -76,9 +79,10 @@ const setGraduationStats = async (data, yearType, specialGroups) => {
   return dataToRedis
 }
 
-const getStudytrackStats = async (id, graduated, specialGroups) => {
+const getStudytrackStats = async (id, combinedProgramme, graduated, specialGroups) => {
   if (!isUpdatedNewProgramme(id)) return null
-  const redisKey = createRedisKeyForStudytrackStats(id, graduated, specialGroups)
+  let searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
+  const redisKey = createRedisKeyForStudytrackStats(searchkey, graduated, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)

services/backend/src/services/completedCoursesSearch.js

@@ -0,0 +1,164 @@
+const { Credit, Enrollment, Student, Course } = require('../models')
+const { Op } = require('sequelize')
+const _ = require('lodash')
+
+const getCompletedCourses = async (studentNumbers, courseCodes) => {
+  let courses = await Course.findAll({
+    attributes: ['code', 'name', 'substitutions'],
+    where: {
+      code: {
+        [Op.in]: courseCodes,
+      },
+    },
+  })
+
+  let fullCourseCodes = [
+    ...courseCodes,
+    ...new Set([...courses.reduce((acc, course) => [...acc, course.code, ...course.substitutions], [])]),
+  ]
+
+  let credits = await Credit.findAll({
+    attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'attainment_date'],
+    where: {
+      course_code: {
+        [Op.in]: fullCourseCodes,
+      },
+      student_studentnumber: {
+        [Op.in]: studentNumbers,
+      },
+    },
+  })
+
+  let enrollments = await Enrollment.findAll({
+    attributes: ['course_code', 'enrollment_date_time', 'studentnumber', 'state'],
+    where: {
+      course_code: {
+        [Op.in]: fullCourseCodes,
+      },
+      studentnumber: {
+        [Op.in]: studentNumbers,
+      },
+      state: {
+        [Op.eq]: 'ENROLLED',
+      },
+    },
+  })
+
+  const studentInfo = await Student.findAll({
+    attributes: ['studentnumber', 'firstnames', 'lastname', 'email', 'sis_person_id'],
+    where: {
+      studentnumber: {
+        [Op.in]: studentNumbers,
+      },
+    },
+  })
+
+  courses = courses.map(course => {
+    const { code, name, substitutions } = course
+    return { code, name, substitutions }
+  })
+
+  credits = credits.map(credit => {
+    const { course_code, student_studentnumber, credittypecode, attainment_date } = credit
+    const originalCode = courseCodes.includes(course_code)
+      ? null
+      : courses.find(course => course.substitutions.includes(course_code))?.code
+    return {
+      courseCode: originalCode ? originalCode : course_code,
+      substitution: originalCode ? course_code : null,
+      studentNumber: student_studentnumber,
+      creditType: credittypecode,
+      date: attainment_date,
+    }
+  })
+
+  enrollments = enrollments.map(enrollment => {
+    const originalCode = courses.find(course => course.substitutions.includes(enrollment.course_code))?.code
+    return {
+      courseCode: originalCode ? originalCode : enrollment.course_code,
+      substitution: originalCode ? enrollment.course_code : null,
+      studentNumber: enrollment.studentnumber,
+      date: enrollment.enrollment_date_time,
+    }
+  })
+
+  const studentCredits = {}
+
+  studentInfo.forEach(s => {
+    if (!studentCredits[s.studentnumber]) {
+      studentCredits[s.studentnumber] = { credits: [], enrollments: [] }
+    }
+    studentCredits[s.studentnumber].firstnames = s.firstnames
+    studentCredits[s.studentnumber].lastname = s.lastname
+    studentCredits[s.studentnumber].email = s.email
+    studentCredits[s.studentnumber].sis_person_id = s.sis_person_id
+  })
+
+  credits.forEach(credit => {
+    if (credit.creditType === 10) {
+      return
+    }
+    const previous = studentCredits[credit.studentNumber].credits?.find(c => credit.courseCode === c.courseCode)
+    if (previous && previous.date > credit.date) {
+      return
+    } else if (previous) {
+      studentCredits[credit.studentNumber].credits = studentCredits[credit.studentNumber].credits.filter(
+        c => credit.courseCode !== c.courseCode
+      )
+    }
+    studentCredits[credit.studentNumber].credits.push({
+      date: credit.date,
+      courseCode: credit.courseCode,
+      creditType: credit.creditType,
+      substitution: credit.substitution,
+    })
+  })
+
+  enrollments.forEach(enrollment => {
+    if (
+      credits.find(
+        credit => credit.courseCode === enrollment.courseCode && credit.studentNumber === enrollment.studentNumber
+      )
+    ) {
+      return
+    }
+    studentCredits[enrollment.studentNumber].enrollments.push({
+      date: enrollment.date,
+      courseCode: enrollment.courseCode,
+      state: enrollment.state,
+    })
+  })
+
+  const students = Object.keys(studentCredits).reduce(
+    (acc, student) => [
+      ...acc,
+      {
+        studentNumber: student,
+        sis_person_id: studentCredits[student].sis_person_id,
+        credits: studentCredits[student].credits,
+        enrollments: studentCredits[student].enrollments,
+        firstnames: studentCredits[student].firstnames,
+        lastname: studentCredits[student].lastname,
+        email: studentCredits[student].email,
+      },
+    ],
+    []
+  )
+
+  students.forEach(student => {
+    courseCodes.forEach(courseCode => {
+      let [hits, rest] = _.partition(student.enrollments, e => e.courseCode === courseCode)
+      if (hits?.length === 0) return
+      let latest = hits[0]
+      for (let hit of hits) {
+        if (hit > latest) {
+          latest = hit
+        }
+      }
+      student.enrollments = [latest, ...rest]
+    })
+  })
+  return { students, courses }
+}
+
+module.exports = { getCompletedCourses }

services/backend/src/services/course_stats_counter.js

@@ -26,7 +26,9 @@ class CourseStatsCounter {
       failedMany: {},
       improvedPassedGrade: {},
       markedToSemester: {},
+      enrolledNoGrade: {},
     }
+
     this.stats = {
       students: 0,
       passed: 0,
@@ -38,10 +40,10 @@ class CourseStatsCounter {
       percentage: undefined,
       passedOfPopulation: undefined,
       triedOfPopulation: undefined,
+      perStudent: undefined,
       passingSemesters: this.initializePassingSemesters(),
     }
     this.enrollments = {
-      enrolledNoGrade: new Set(),
       semesters: this.initializePassingSemesters({}),
     }
     this.grades = {}
@@ -57,7 +59,6 @@ class CourseStatsCounter {
       passingSemesters[fall[i]] = initialValue
       passingSemesters[spring[i]] = initialValue
     }
-
     return passingSemesters
   }
 
@@ -184,11 +185,20 @@ class CourseStatsCounter {
   getFinalStats() {
     const stats = this.stats
     const students = this.students
-    const allStudents = Object.keys(students.all)
-    const totalEnrolledNoGrade = [...(this.enrollments.ENROLLED || [])].filter(
-      student => !allStudents.includes(student)
-    ).length
+    const studs = this.enrollments
+      ? Object.keys(this.enrollments)
+          .filter(key => key !== 'ENROLLED' && key !== 'semesters')
+          .reduce((acc, key) => [...acc, ...[...this.enrollments[key]].map(studentnumber => studentnumber)], [])
+      : []
+    const allStudents = Object.keys(students.all).map(student => student)
+    const fileredEnrolledNoGrade = this.enrollments['ENROLLED']
+      ? [...this.enrollments['ENROLLED']]
+          .filter(student => !studs.includes(student) && !allStudents.includes(student))
+          .reduce((acc, student) => ({ ...acc, [student]: true }), {})
+      : {}
 
+    students.all = { ...students.all, ...fileredEnrolledNoGrade }
+    students.enrolledNoGrade = fileredEnrolledNoGrade
     stats.students = lengthOf(students.all)
     stats.passed = lengthOf(students.passed)
     stats.failed = lengthOf(students.failed)
@@ -200,10 +210,9 @@ class CourseStatsCounter {
     stats.triedOfPopulation = percentageOf(stats.students, this.studentsInTotal)
     stats.perStudent = stats.attempts / (stats.passed + stats.failed)
     stats.passingSemestersCumulative = this.getPassingSemestersCumulative()
-    stats.totalStudents = stats.students + totalEnrolledNoGrade
-    stats.totalEnrolledNoGrade = totalEnrolledNoGrade
+    stats.totalStudents = stats.students
+    stats.totalEnrolledNoGrade = lengthOf(fileredEnrolledNoGrade)
     stats.percentageWithEnrollments = percentageOf(stats.passed, stats.totalStudents)
-
     return {
       stats,
       students,

services/backend/src/services/faculty/faculty.js

@@ -22,7 +22,9 @@ const {
   mapCodesToIds,
   facultyProgrammeStudents,
 } = require('./facultyHelpers')
-
+const {
+  dbConnections: { sequelize },
+} = require('../../database/connection')
 const getTransferredToAndAway = async (programmeCodes, allProgrammeCodes, since) => {
   const awayTransfers = await transferredAway(programmeCodes, allProgrammeCodes, since)
   const toTransfers = await transferredTo(programmeCodes, allProgrammeCodes, since)
@@ -33,163 +35,179 @@ const getTransferredInside = async (programmeCodes, allProgrammeCodes, since) =>
   return await transferredInsideFaculty(programmeCodes, allProgrammeCodes, since)
 }
 
-const getTransfers = async (programmeCode, allProgrammeCodes, since, end) => {
-  return await transferredFaculty(programmeCode, allProgrammeCodes, since, end)
+const getTransfersOut = async (programmeCode, start, end) => {
+  return await transferredFaculty([], [programmeCode], start, end)
 }
 
-const transferredFaculty = async (programme, allProgrammeCodes, since, end) =>
+const getTransfersIn = async (programmeCode, start, end) => {
+  return await transferredFaculty([programmeCode], [], start, end)
+}
+const transferredFaculty = async (programmeCodeIn, programmeCodeOut, start, end) =>
   (
     await Transfer.findAll({
       where: {
         transferdate: {
-          [Op.between]: [since, end],
+          [Op.between]: [start, end],
         },
         [Op.or]: [
           {
-            sourcecode: allProgrammeCodes,
-            targetcode: programme,
+            sourcecode: programmeCodeOut,
           },
           {
-            sourcecode: programme,
-            targetcode: {
-              [Op.notIn]: allProgrammeCodes,
-            },
-          },
-          {
-            sourcecode: {
-              [Op.notIn]: allProgrammeCodes,
-            },
-            targetcode: programme,
+            targetcode: programmeCodeIn,
           },
         ],
       },
     })
   ).map(formatFacultyTransfer)
 
-const startedStudyrights = async (faculty, since, studyRightWhere) =>
+const startedStudyrights = async (faculty, code, since, studyRightWhere) =>
   (
     await Studyright.findAll({
       include: [
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
-            required: true,
-          },
-        },
-      ],
-      where: {
-        faculty_code: faculty,
-        studystartdate: {
-          [Op.gte]: since,
-        },
-        student_studentnumber: { [Op.not]: null },
-        ...studyRightWhere,
-      },
-    })
-  ).map(facultyFormatStudyright)
-
-const graduatedStudyrights = async (faculty, since, studyrightWhere) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
-            required: true,
-          },
-        },
-      ],
-      where: {
-        faculty_code: faculty,
-        enddate: {
-          [Op.gte]: since,
-        },
-        graduated: 1,
-        student_studentnumber: { [Op.not]: null },
-        ...studyrightWhere,
-      },
-    })
-  ).map(facultyFormatStudyright)
-
-const studyrightsByRightStartYear = async (faculty, since, graduated = 1) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
-            required: true,
-          },
-        },
-      ],
-      where: {
-        faculty_code: faculty,
-        startdate: {
-          [Op.gte]: since,
-        },
-        graduated: graduated,
-        student_studentnumber: { [Op.not]: null },
-      },
-    })
-  ).map(facultyFormatStudyright)
-
-const getStudyRightsByExtent = async (faculty, elementStart, studyrightStart, code, extents, graduated) =>
-  (
-    await Studyright.findAll({
-      include: [
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
         {
           model: StudyrightElement,
           required: true,
           where: {
             code: code,
-            ...elementStart,
           },
           include: {
             model: ElementDetail,
             required: true,
-            where: {
-              type: 20,
-            },
           },
         },
       ],
       where: {
-        faculty_code: faculty,
+        facultyCode: faculty,
+        startdate: {
+          [Op.gte]: since,
+        },
+        ...studyRightWhere,
+      },
+    })
+  ).map(facultyFormatStudyright)
+
+const graduatedStudyrights = async (faculty, code, since, studyrightWhere) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          where: {
+            code: code,
+          },
+          include: {
+            model: ElementDetail,
+            required: true,
+          },
+        },
+      ],
+      where: {
+        facultyCode: faculty,
+        enddate: {
+          [Op.gte]: since,
+        },
+        graduated: 1,
+        ...studyrightWhere,
+      },
+    })
+  ).map(facultyFormatStudyright)
+
+const studyrightsByRightStartYear = async (faculty, code, since, graduated = 1) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          attributes: ['code', 'startdate'],
+          where: {
+            code: code,
+          },
+          required: true,
+          include: {
+            model: ElementDetail,
+            required: true,
+          },
+        },
+      ],
+      where: {
+        facultyCode: faculty,
+        startdate: {
+          [Op.gte]: since,
+        },
+        graduated: graduated,
+      },
+    })
+  ).map(facultyFormatStudyright)
+
+const getStudyRightsByExtent = async (faculty, startDate, endDate, code, extents, graduated) =>
+  (
+    await Studyright.findAll({
+      include: {
+        model: StudyrightElement,
+        attributes: [],
+        required: true,
+        where: {
+          code: code,
+        },
+        include: {
+          model: ElementDetail,
+          attributes: [],
+        },
+      },
+      group: [sequelize.col('studyright.studyrightid')],
+      where: {
+        facultyCode: faculty,
         extentcode: {
           [Op.in]: extents,
         },
+        prioritycode: {
+          [Op.not]: 6,
+        },
         graduated: {
           [Op.in]: graduated,
         },
-        studystartdate: { [Op.not]: null },
-        ...studyrightStart,
-        student_studentnumber: { [Op.not]: null },
+        [Op.and]: [
+          sequelize.where(
+            sequelize.fn(
+              'GREATEST',
+              sequelize.col('studyright_elements.startdate'),
+              sequelize.col('studyright.startdate')
+            ),
+            {
+              [Op.between]: [startDate, endDate],
+            }
+          ),
+        ],
+      },
+    })
+  ).map(facultyFormatStudyright)
+
+const getStudyRightsByBachelorStart = async (faculty, startDate, endDate, code, extents, graduated) =>
+  (
+    await Studyright.findAll({
+      include: {
+        model: StudyrightElement,
+        required: true,
+        where: {
+          code: code,
+        },
+      },
+      where: {
+        facultyCode: faculty,
+        extentcode: {
+          [Op.in]: extents,
+        },
+        prioritycode: {
+          [Op.not]: 6,
+        },
+        graduated: {
+          [Op.in]: graduated,
+        },
+        startdate: {
+          [Op.between]: [startDate, endDate],
+        },
       },
     })
   ).map(facultyFormatStudyright)
@@ -396,6 +414,8 @@ module.exports = {
   getTransferredToAndAway,
   getTransferredInside,
   getStudyRightsByExtent,
+  getStudyRightsByBachelorStart,
   getStudentsByStudentnumbers,
-  getTransfers,
+  getTransfersOut,
+  getTransfersIn,
 }

services/backend/src/services/faculty/facultyBasics.js

@@ -1,4 +1,4 @@
-const { indexOf } = require('lodash')
+const { indexOf, isArray } = require('lodash')
 const moment = require('moment')
 const {
   startedStudyrights,
@@ -9,6 +9,7 @@ const {
   getTransferredToAndAway,
   getTransferredInside,
   studyrightsByRightStartYear,
+  getTransfersIn,
 } = require('./faculty')
 const { getStatsBasis, getYearsArray, defineYear } = require('../studyprogrammeHelpers')
 const { findRightProgramme, isNewProgramme, checkTransfers, getExtentFilter } = require('./facultyHelpers')
@@ -16,17 +17,12 @@ const { codes } = require('../../../config/programmeCodes')
 
 const filterDuplicateStudyrights = studyrights => {
   // bachelor+master students have two studyrights (separated by two last digits in studyrightid)
-  // choose only the earlier started one, so we don't count start of masters as starting in faculty
+  // choose only bachelor one, so we don't count start of masters as starting in faculty
   let rightsToCount = {}
 
   studyrights.forEach(right => {
     const id = right.studyrightid.slice(0, -2)
-    const start = new Date(right.studystartdate)
-    if (id in rightsToCount) {
-      if (new Date(rightsToCount[id].studystartdate) > start) {
-        rightsToCount[id] = right
-      }
-    } else {
+    if (right.studyrightid.slice(-2) === '-1') {
       rightsToCount[id] = right
     }
   })
@@ -60,6 +56,7 @@ const addProgramme = async (
 
 const getFacultyStarters = async (
   faculty,
+  programmes,
   transfersToOrAwayStudyrights,
   insideTransfersStudyrights,
   since,
@@ -75,43 +72,50 @@ const getFacultyStarters = async (
 ) => {
   const startedGraphStats = [...graphStats]
   const startedTableStats = { ...tableStats }
-  const studyrightWhere = getExtentFilter(includeAllSpecials)
-  const studyrights = await startedStudyrights(faculty, since, studyrightWhere)
-  let filteredStudyrights = filterDuplicateStudyrights(studyrights)
-  const keys = Object.keys(codes)
   const programmeTableStats = {}
-  if (!includeAllSpecials) {
-    filteredStudyrights = filteredStudyrights.filter(
-      s => !checkTransfers(s, insideTransfersStudyrights, transfersToOrAwayStudyrights)
-    )
+  const studyrightWhere = getExtentFilter(includeAllSpecials)
+  const start = new Date('2017-01-01').toUTCString()
+  const end = new Date()
+  for (const code of programmes) {
+    const studyrights = await startedStudyrights(faculty, code, since, studyrightWhere)
+    let filteredStudyrights = filterDuplicateStudyrights(studyrights)
+    const insideTransfers = await getTransfersIn(code, start, end)
+    filteredStudyrights = filteredStudyrights.filter(s => !checkTransfers(s, insideTransfers, insideTransfers))
+    const keys = Object.keys(codes)
+
+    if (!includeAllSpecials) {
+      // We do not include inside transferred studyrights into faculty starters, but function below expect two arguments
+      filteredStudyrights = filteredStudyrights.filter(
+        s => !checkTransfers(s, insideTransfersStudyrights, transfersToOrAwayStudyrights)
+      )
+    }
+
+    filteredStudyrights.forEach(({ startdate, studyrightElements }) => {
+      const { programme, programmeName } = findRightProgramme(studyrightElements, code)
+      let programmeId = programme
+
+      if (keys.includes(programme)) {
+        programmeId = codes[programme].toUpperCase()
+      }
+
+      if (programmeFilter === 'ALL_PROGRAMMES' || isNewProgramme(programme)) {
+        const startYear = defineYear(startdate, isAcademicYear)
+
+        startedGraphStats[indexOf(yearsArray, startYear)] += 1
+        startedTableStats[startYear] += 1
+
+        if (!(programmeId in programmeTableStats)) {
+          programmeTableStats[programmeId] = { ...tableStats }
+        }
+        programmeTableStats[programmeId][startYear] += 1
+
+        if (!(programmeId in allBasics.programmeNames)) {
+          allBasics.programmeNames[programmeId] = { ...programmeName, code: programme }
+        }
+      }
+    })
   }
-
-  filteredStudyrights.forEach(({ studystartdate, studyrightElements }) => {
-    const { programme, programmeName } = findRightProgramme(studyrightElements, 'started')
-    let programmeId = programme
-
-    if (keys.includes(programme)) {
-      programmeId = codes[programme].toUpperCase()
-    }
-
-    if (programmeFilter === 'ALL_PROGRAMMES' || isNewProgramme(programme)) {
-      const startYear = defineYear(studystartdate, isAcademicYear)
-
-      startedGraphStats[indexOf(yearsArray, startYear)] += 1
-      startedTableStats[startYear] += 1
-
-      if (!(programmeId in programmeTableStats)) {
-        programmeTableStats[programmeId] = { ...tableStats }
-      }
-      programmeTableStats[programmeId][startYear] += 1
-
-      if (!(programmeId in allBasics.programmeNames)) {
-        allBasics.programmeNames[programmeId] = { ...programmeName, code: programme }
-      }
-    }
-  })
-
-  allBasics.studentInfo.graphStats.push({ name: 'Started studying', data: startedGraphStats })
+  allBasics.studentInfo.graphStats.push({ name: 'Started studying (new in faculty)', data: startedGraphStats })
   programmeData['started'] = programmeTableStats
 
   Object.keys(startedTableStats).forEach(year => {
@@ -121,6 +125,7 @@ const getFacultyStarters = async (
 
 const getFacultyGraduates = async (
   faculty,
+  programmes,
   transfersToOrAwayStudyrights,
   insideTransfersStudyrights,
   since,
@@ -137,65 +142,62 @@ const getFacultyGraduates = async (
 ) => {
   const graduatedGraphStats = [[...graphStats], [...graphStats], [...graphStats], [...graphStats], [...graphStats]]
   const graduatedTableStats = {}
+  const programmeTableStats = {}
+  Object.keys(tableStats).forEach(year => (graduatedTableStats[year] = [0, 0, 0, 0, 0]))
   const keys = Object.keys(codes)
   let studyrightWhere = getExtentFilter(includeAllSpecials)
-  let graduatedRights = await graduatedStudyrights(faculty, since, studyrightWhere)
-  const programmeTableStats = {}
+  for (const code of programmes) {
+    let graduatedRights = await graduatedStudyrights(faculty, code, since, studyrightWhere)
+    if (!includeAllSpecials) {
+      graduatedRights = graduatedRights.filter(
+        s => !checkTransfers(s, insideTransfersStudyrights, transfersToOrAwayStudyrights)
+      )
+    }
+    graduatedRights.forEach(({ enddate, extentcode, studyrightElements }) => {
+      const { programme, programmeName } = findRightProgramme(studyrightElements, code)
+      let programmeId = programme
 
-  Object.keys(tableStats).forEach(year => (graduatedTableStats[year] = [0, 0, 0, 0, 0]))
-  if (!includeAllSpecials) {
-    graduatedRights = graduatedRights.filter(
-      s => !checkTransfers(s, insideTransfersStudyrights, transfersToOrAwayStudyrights)
-    )
+      if (keys.includes(programme)) {
+        programmeId = codes[programme].toUpperCase()
+      }
+
+      if (programmeFilter === 'ALL_PROGRAMMES' || isNewProgramme(programme)) {
+        const endYear = defineYear(enddate, isAcademicYear)
+        graduatedGraphStats[0][indexOf(yearsArray, endYear)] += 1
+        graduatedTableStats[endYear][0] += 1
+
+        if (!(programmeId in programmeTableStats)) {
+          programmeTableStats[programmeId] = {}
+          Object.keys(tableStats).forEach(year => (programmeTableStats[programmeId][year] = [0, 0, 0, 0, 0]))
+        }
+        programmeTableStats[programmeId][endYear][0] += 1
+
+        if (extentcode === 1) {
+          graduatedGraphStats[1][indexOf(yearsArray, endYear)] += 1
+          graduatedTableStats[endYear][1] += 1
+          programmeTableStats[programmeId][endYear][1] += 1
+        } else if (extentcode === 2) {
+          graduatedGraphStats[2][indexOf(yearsArray, endYear)] += 1
+          graduatedTableStats[endYear][2] += 1
+          programmeTableStats[programmeId][endYear][2] += 1
+        } else if (extentcode === 4) {
+          graduatedGraphStats[3][indexOf(yearsArray, endYear)] += 1
+          graduatedTableStats[endYear][3] += 1
+          programmeTableStats[programmeId][endYear][3] += 1
+        } else {
+          graduatedGraphStats[4][indexOf(yearsArray, endYear)] += 1
+          graduatedTableStats[endYear][4] += 1
+          programmeTableStats[programmeId][endYear][4] += 1
+        }
+
+        if (!(programmeId in allBasics.programmeNames)) {
+          allBasics.programmeNames[programmeId] = { code: programme, ...programmeName }
+        }
+      }
+    })
   }
-  graduatedRights.forEach(({ enddate, extentcode, studyrightElements }) => {
-    const { programme, programmeName } = findRightProgramme(studyrightElements, 'graduated')
-    let programmeId = programme
-
-    if (keys.includes(programme)) {
-      programmeId = codes[programme].toUpperCase()
-      if (programme === 'T923103') {
-        // Two ATM programmes
-        programmeId += '-DP'
-      }
-    }
-    if (programmeFilter === 'ALL_PROGRAMMES' || isNewProgramme(programme)) {
-      const endYear = defineYear(enddate, isAcademicYear)
-      graduatedGraphStats[0][indexOf(yearsArray, endYear)] += 1
-      graduatedTableStats[endYear][0] += 1
-
-      if (!(programmeId in programmeTableStats)) {
-        programmeTableStats[programmeId] = {}
-        Object.keys(tableStats).forEach(year => (programmeTableStats[programmeId][year] = [0, 0, 0, 0, 0]))
-      }
-      programmeTableStats[programmeId][endYear][0] += 1
-
-      if (extentcode === 1) {
-        graduatedGraphStats[1][indexOf(yearsArray, endYear)] += 1
-        graduatedTableStats[endYear][1] += 1
-        programmeTableStats[programmeId][endYear][1] += 1
-      } else if (extentcode === 2) {
-        graduatedGraphStats[2][indexOf(yearsArray, endYear)] += 1
-        graduatedTableStats[endYear][2] += 1
-        programmeTableStats[programmeId][endYear][2] += 1
-      } else if (extentcode === 4) {
-        graduatedGraphStats[3][indexOf(yearsArray, endYear)] += 1
-        graduatedTableStats[endYear][3] += 1
-        programmeTableStats[programmeId][endYear][3] += 1
-      } else {
-        graduatedGraphStats[4][indexOf(yearsArray, endYear)] += 1
-        graduatedTableStats[endYear][4] += 1
-        programmeTableStats[programmeId][endYear][4] += 1
-      }
-
-      if (!(programmeId in allBasics.programmeNames)) {
-        allBasics.programmeNames[programmeId] = { code: programme, ...programmeName }
-      }
-    }
-  })
 
   allBasics.studentInfo.graphStats.push({ name: 'Graduated', data: graduatedGraphStats[0] })
-
   allBasics.graduationInfo.graphStats.push({ name: 'Bachelors', data: graduatedGraphStats[1] })
   allBasics.graduationInfo.graphStats.push({ name: 'Masters', data: graduatedGraphStats[2] })
   allBasics.graduationInfo.graphStats.push({ name: 'Doctors', data: graduatedGraphStats[3] })
@@ -208,15 +210,15 @@ const getFacultyGraduates = async (
     countsGraduations[year] = graduatedTableStats[year]
   })
 }
-// If includeAllSpecial false, insidetransfers should contain only students started after 1.8.2017.
+// If includeAllSpecial false, inside transfers should contain only students started after 1.8.2017.
 const getInsideTransfers = async (programmeCodes, allProgrammeCodes, since, includeAllSpecials, faculty) => {
   if (includeAllSpecials) {
     return await transferredInsideFaculty(programmeCodes, allProgrammeCodes, since)
   }
   const insiders = await transferredInsideFaculty(programmeCodes, allProgrammeCodes, since)
-  const studyrights = (await studyrightsByRightStartYear(faculty, moment('2017-08-01', 'YYYY-MM-DD'))).map(
-    sr => sr.studyrightid
-  )
+  const studyrights = (
+    await studyrightsByRightStartYear(faculty, new Date(moment('2017-08-01', 'YYYY-MM-DD')).toUTCString())
+  ).map(sr => sr.studyrightid)
   const filteredTransfers = insiders.filter(tr => studyrights.includes(tr.studyrightid))
   return filteredTransfers
 }
@@ -257,7 +259,7 @@ const getFacultyTransfers = async (
   for (const { transferdate, targetcode } of insideTransfers) {
     const transferYear = defineYear(transferdate, isAcademicYear)
     transferGraphStats[0][indexOf(yearsArray, transferYear)] += 1
-    if (isAcademicYear || !(transferYear > moment().year())) {
+    if (isArray(transferTableStats[transferYear])) {
       transferTableStats[transferYear][0] += 1
       await addProgramme(
         programmeTableStats,
@@ -277,7 +279,7 @@ const getFacultyTransfers = async (
 
     for (const { transferdate, sourcecode } of awayTransfers) {
       const transferYear = defineYear(transferdate, isAcademicYear)
-      if (isAcademicYear || !(transferYear > moment().year())) {
+      if (isArray(transferTableStats[transferYear])) {
         transferGraphStats[1][indexOf(yearsArray, transferYear)] += 1
         transferTableStats[transferYear][1] += 1
         await addProgramme(
@@ -295,7 +297,7 @@ const getFacultyTransfers = async (
 
     for (const { transferdate, targetcode } of toTransfers) {
       const transferYear = defineYear(transferdate, isAcademicYear)
-      if (isAcademicYear || !(transferYear > moment().year())) {
+      if (isArray(transferTableStats[transferYear])) {
         transferGraphStats[2][indexOf(yearsArray, transferYear)] += 1
         transferTableStats[transferYear][2] += 1
         await addProgramme(
@@ -361,6 +363,7 @@ const combineFacultyBasics = async (faculty, programmes, yearType, allProgrammeC
   // Started studying in faculty
   await getFacultyStarters(
     faculty,
+    wantedProgrammeCodes,
     transfersToAwayStudyrights,
     transferInsideStudyrights,
     since,
@@ -378,6 +381,7 @@ const combineFacultyBasics = async (faculty, programmes, yearType, allProgrammeC
   // Graduated in faculty
   await getFacultyGraduates(
     faculty,
+    wantedProgrammeCodes,
     transfersToAwayStudyrights,
     transferInsideStudyrights,
     since,

services/backend/src/services/faculty/facultyCredits.js

@@ -1,4 +1,4 @@
-const { indexOf } = require('lodash')
+const { indexOf, uniqBy } = require('lodash')
 const { mapToProviders } = require('../../util/utils')
 const {
   getStatsBasis,
@@ -7,27 +7,25 @@ const {
   isNonMajorCredit,
   getYearsArray,
   getStartDate,
-  isSpecialGroupCredit,
 } = require('../studyprogrammeHelpers')
-const {
-  getStudyRights,
-  getCreditsForStudyProgramme,
-  allTransfers,
-  getCourseCodesForStudyProgramme,
-} = require('../studyprogramme')
+const { getStudyRights, getCreditsForStudyProgramme, getCourseCodesForStudyProgramme } = require('../studyprogramme')
 
-const isFacultyNonMajorCredit = (studyrights, attainment_date, facultyProgrammes) => {
+const isFacultyNonMajorCredit = (studyrights, attainmentDate, facultyProgrammes, facultyCode) => {
   let right = ''
   studyrights.forEach(studyright => {
-    if (studyright) {
-      if (!studyright.graduated && new Date(attainment_date) >= new Date(studyright.studystartdate)) {
+    if (studyright && studyright.facultyCode === facultyCode) {
+      const startDate =
+        studyright.studyrightid.slice(-2) === '-2' && studyright.extentcode === 2
+          ? studyright.studystartdate
+          : studyright.startdate
+      if (!studyright.graduated && new Date(attainmentDate) >= new Date(startDate)) {
         if (!right) right = studyright.code
         else if (right && !facultyProgrammes.includes(right) && facultyProgrammes.includes(studyright.code))
           right = studyright.code
       } else {
         if (
-          new Date(attainment_date) >= new Date(studyright.studystartdate) &&
-          new Date(attainment_date) <= new Date(studyright.enddate)
+          new Date(attainmentDate) >= new Date(startDate) &&
+          new Date(attainmentDate) <= new Date(studyright.enddate)
         ) {
           if (!right) right = studyright.code
           else if (right && !facultyProgrammes.includes(right) && facultyProgrammes.includes(studyright.code))
@@ -38,58 +36,58 @@ const isFacultyNonMajorCredit = (studyrights, attainment_date, facultyProgrammes
   })
   return facultyProgrammes.includes(right)
 }
+const isFacultyOtherCredit = (studyrights, attainmentDate) => {
+  return studyrights.some(studyright => {
+    return (
+      studyright &&
+      [6, 7, 23, 34, 99].includes(studyright.extentcode) &&
+      new Date(attainmentDate) >= new Date(studyright.startdate)
+    )
+  })
+}
 
 // Fetches all the credits for the studyprogramme and divides them into major-students, non-major faculty students,
-// non-major other faculty students adn non-degree student credits
+// non-major other faculty students, other (exhange students) and non-degree student credits.
 // Division is done on the basis that whether the student had a primary studyright to the programme on the attainment_date
 const getFacultyRegularCreditStats = async ({
   studyprogramme,
   since,
   years,
   isAcademicYear,
-  includeAllSpecials,
   facultyProgrammes,
+  facultyCode,
 }) => {
   const providercode = mapToProviders([studyprogramme])[0]
-  const courses = await getCourseCodesForStudyProgramme(providercode)
-  const credits = await getCreditsForStudyProgramme(courses, since)
-  const students = [...new Set(credits.map(({ student_studentnumber }) => student_studentnumber))]
+  const providedByProgramme = await getCourseCodesForStudyProgramme(providercode)
+  const allCredits = await getCreditsForStudyProgramme(providedByProgramme, since)
+  const credits = uniqBy(allCredits, 'id')
+  const students = [...new Set(credits.map(({ studentNumber }) => studentNumber))]
 
-  let studyrights = await getStudyRights(students)
-  const transfers = (await allTransfers(studyprogramme, since)).map(t => t.studyrightid).filter(s => s !== 'undefined')
-  if (!includeAllSpecials) {
-    studyrights = studyrights.filter(s => !transfers.includes(s.studyrightid))
-  }
+  let studyRights = await getStudyRights(students)
+  let majors = getStatsBasis(years) // Has studyright to programme
+  let facultyNonMajors = getStatsBasis(years) // Has studyright in the same faculty but not in the programme from which credits are
+  let facultyOthers = getStatsBasis(years) // Exchange students, specialization education in Medicine, Dentistry and vetenary medicine, continuous education
+  let otherNonMajors = getStatsBasis(years) // Has studyright in different faculty
+  let nonDegree = getStatsBasis(years) // open uni, non-degree study rights, contract studies, (other) specialisation education, credits outside of the studyright
 
-  let majors = getStatsBasis(years)
-  let allNonMajors = getStatsBasis(years)
-  let facultyNonMajors = getStatsBasis(years)
-  let otherNonMajors = getStatsBasis(years)
-  let nonDegree = getStatsBasis(years)
+  credits.forEach(({ studentNumber, attainmentDate, credits }) => {
+    const studentStudyrights = studyRights.filter(studyright => studyright.studentNumber === studentNumber)
+    const attainmentYear = defineYear(attainmentDate, isAcademicYear)
 
-  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
-    const studentStudyrights = studyrights.filter(studyright => studyright.studentnumber === student_studentnumber)
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-
-    if (!includeAllSpecials && isSpecialGroupCredit(studentStudyrights, attainment_date, transfers)) {
-      return
-    }
-
-    if (isMajorStudentCredit(studentStudyrights, attainment_date, studyprogramme)) {
+    if (isMajorStudentCredit(studentStudyrights, attainmentDate, studyprogramme)) {
       majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
       majors.tableStats[attainmentYear] += credits || 0
-    } else if (isNonMajorCredit(studentStudyrights, attainment_date)) {
-      // for comparison
-      allNonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
-      allNonMajors.tableStats[attainmentYear] += credits || 0
-
-      if (isFacultyNonMajorCredit(studentStudyrights, attainment_date, facultyProgrammes)) {
+    } else if (isNonMajorCredit(studentStudyrights, attainmentDate)) {
+      if (isFacultyNonMajorCredit(studentStudyrights, attainmentDate, facultyProgrammes, facultyCode)) {
         facultyNonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
         facultyNonMajors.tableStats[attainmentYear] += credits || 0
       } else {
         otherNonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
         otherNonMajors.tableStats[attainmentYear] += credits || 0
       }
+    } else if (isFacultyOtherCredit(studentStudyrights, attainmentDate)) {
+      facultyOthers.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      facultyOthers.tableStats[attainmentYear] += credits || 0
     } else {
       nonDegree.graphStats[indexOf(years, attainmentYear)] += credits || 0
       nonDegree.tableStats[attainmentYear] += credits || 0
@@ -110,66 +108,67 @@ const getFacultyRegularCreditStats = async ({
     }
   }
 
-  return { majors, facultyNonMajors, otherNonMajors, nonDegree }
+  return { majors, facultyNonMajors, otherNonMajors, facultyOthers, nonDegree }
 }
 
 // Fetches all credits for the studytrack and combines them into statistics
 // These are for the faculty view which differs from study programme overview
 // (no transfers, non majors split to in- and outside-faculty)
-const getFacultyCreditStatsForStudytrack = async ({ studyprogramme, facultyProgrammes, settings }) => {
-  const { isAcademicYear, includeAllSpecials } = settings
+const getFacultyCreditStatsForStudytrack = async ({
+  studyprogramme,
+  facultyProgrammes,
+  isAcademicYear,
+  facultyCode,
+}) => {
   const since = getStartDate(studyprogramme, isAcademicYear)
   const years = getYearsArray(since.getFullYear(), isAcademicYear)
 
-  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials, facultyProgrammes }
-  const { majors, facultyNonMajors, otherNonMajors, nonDegree } = await getFacultyRegularCreditStats(queryParameters)
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, facultyProgrammes, facultyCode }
+  const { majors, facultyNonMajors, otherNonMajors, facultyOthers, nonDegree } = await getFacultyRegularCreditStats(
+    queryParameters
+  )
 
   const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
   const dataFound = [majors, facultyNonMajors, otherNonMajors].some(d => d.graphStats.length) // , transferred
 
   if (!dataFound) return null
 
-  const tableStats = reversedYears.map(year =>
-    includeAllSpecials
-      ? [
-          year,
-          majors.tableStats[year] +
-            facultyNonMajors.tableStats[year] +
-            otherNonMajors.tableStats[year] +
-            nonDegree.tableStats[year],
-          majors.tableStats[year],
-          facultyNonMajors.tableStats[year],
-          otherNonMajors.tableStats[year],
-          nonDegree.tableStats[year],
-        ]
-      : [year, majors.tableStats[year], majors.tableStats[year]]
-  )
+  const tableStats = reversedYears.map(year => [
+    year,
+    majors.tableStats[year] +
+      facultyNonMajors.tableStats[year] +
+      otherNonMajors.tableStats[year] +
+      nonDegree.tableStats[year] +
+      facultyOthers.tableStats[year],
+    majors.tableStats[year],
+    facultyNonMajors.tableStats[year],
+    otherNonMajors.tableStats[year],
+    nonDegree.tableStats[year],
+    facultyOthers.tableStats[year],
+  ])
 
-  const graphStats = includeAllSpecials
-    ? [
-        {
-          name: 'Major students credits',
-          data: majors.graphStats,
-        },
-        {
-          name: 'Non-major faculty students credits',
-          data: facultyNonMajors.graphStats,
-        },
-        {
-          name: 'Non-major other faculty students credits 2',
-          data: otherNonMajors.graphStats,
-        },
-        {
-          name: 'Non-degree credits',
-          data: nonDegree.graphStats,
-        },
-      ]
-    : [
-        {
-          name: 'Major students credits',
-          data: majors.graphStats,
-        },
-      ]
+  const graphStats = [
+    {
+      name: 'Major students credits',
+      data: majors.graphStats,
+    },
+    {
+      name: 'Non-major faculty students credits',
+      data: facultyNonMajors.graphStats,
+    },
+    {
+      name: 'Non-major other faculty students credits 2',
+      data: otherNonMajors.graphStats,
+    },
+    {
+      name: 'Non-degree credits',
+      data: nonDegree.graphStats,
+    },
+    {
+      other: 'Other credits',
+      data: facultyOthers.graphStats,
+    },
+  ]
 
   return {
     id: studyprogramme,
@@ -179,33 +178,29 @@ const getFacultyCreditStatsForStudytrack = async ({ studyprogramme, facultyProgr
   }
 }
 
-const getProgrammeCredits = async (code, yearType, specialGroups, facultyProgrammes) => {
+const getProgrammeCredits = async (code, yearType, facultyProgrammes, faculty) => {
   const updatedStats = await getFacultyCreditStatsForStudytrack({
+    facultyCode: faculty,
     studyprogramme: code,
     facultyProgrammes,
-    settings: {
-      isAcademicYear: yearType === 'ACADEMIC_YEAR',
-      includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
-    },
+    isAcademicYear: yearType === 'ACADEMIC_YEAR',
   })
 
   return updatedStats
 }
 
-const combineFacultyCredits = async (faculty, programmes, allProgrammes, yearType, specialGroups) => {
+const combineFacultyCredits = async (faculty, programmes, allProgrammes, yearType) => {
   let counts = {}
   let years = []
-  const titles =
-    specialGroups === 'SPECIAL_INCLUDED'
-      ? [
-          '',
-          'Total',
-          'Major students credits',
-          'Non-major faculty students credits',
-          'Non-major other faculty students credits',
-          'Non-degree student credits',
-        ]
-      : ['', 'Total', 'Major students credits']
+  const titles = [
+    '',
+    'Total',
+    'Major students credits',
+    'Non-major faculty students credits',
+    'Non-major other faculty students credits',
+    'Non-degree student credits',
+    'Other non-degree credits',
+  ]
 
   let allCredits = {
     id: faculty,
@@ -220,7 +215,7 @@ const combineFacultyCredits = async (faculty, programmes, allProgrammes, yearTyp
   const progCodes = allProgrammes.map(p => p.code)
   for (const prog of programmes) {
     if (prog.code === 'MH70_008_2') continue
-    const data = await getProgrammeCredits(prog.code, yearType, specialGroups, progCodes)
+    const data = await getProgrammeCredits(prog.code, yearType, progCodes, faculty)
     if (data) {
       allCredits.programmeNames[prog.progId] = { code: prog.code, ...prog.name }
       allCredits.programmeTableStats[prog.progId] = data.tableStats
@@ -251,25 +246,24 @@ const combineFacultyCredits = async (faculty, programmes, allProgrammes, yearTyp
   let majors = []
   let facultyNonMajor = []
   let otherNonMajor = []
+  let facultyOthers = []
   let nonDegree = []
 
   years.forEach(year => {
     majors.push(counts[year][1])
     facultyNonMajor.push(counts[year][2])
     otherNonMajor.push(counts[year][3])
-    nonDegree.push(counts[year][4])
+    facultyOthers.push(counts[year][4])
+    nonDegree.push(counts[year][5])
   })
 
-  allCredits.graphStats =
-    specialGroups === 'SPECIAL_INCLUDED'
-      ? [
-          { name: 'Major students credits', data: majors },
-          { name: 'Non-major faculty students credits', data: facultyNonMajor },
-          { name: 'Non-major other faculty students credits', data: otherNonMajor },
-          { name: 'Non-degree credits', data: nonDegree },
-        ]
-      : [{ name: 'Major students credits', data: majors }]
-
+  allCredits.graphStats = [
+    { name: 'Major students credits', data: majors },
+    { name: 'Non-major faculty students credits', data: facultyNonMajor },
+    { name: 'Non-major other faculty students credits', data: otherNonMajor },
+    { name: 'Non-degree credits', data: nonDegree },
+    { name: 'Other non-degree credits', data: facultyOthers },
+  ]
   return allCredits
 }
 

services/backend/src/services/faculty/facultyGraduationTimes.js

@@ -4,6 +4,7 @@ const { findRightProgramme, isNewProgramme } = require('./facultyHelpers')
 const { getYearsArray, getYearsObject, getMedian, defineYear } = require('../studyprogrammeHelpers')
 const { codes } = require('../../../config/programmeCodes')
 const { bachelorStudyright, countTimeCategories, getStatutoryAbsences } = require('../graduationHelpers')
+const { getProgrammes } = require('./facultyService')
 
 const sortProgrammes = data => {
   const check = name => {
@@ -112,71 +113,86 @@ const addGraduation = async (
   ]
 }
 
-const getClassSizes = async (faculty, since, classSizes, programmeFilter, years) => {
-  const studyrights = await studyrightsByRightStartYear(faculty, since, [0, 1])
+const getClassSizes = async (faculty, programmeCodes, since, classSizes, programmeFilter, years) => {
+  for (const code of programmeCodes) {
+    const studyrights = await studyrightsByRightStartYear(faculty, code, since, [0, 1])
 
-  // get all received studyrights for each year
-  // a transferred student is counted into new programmes class size i.e.
-  // students who received studyright on year X and graduated/will be graduating from programme Y
-  // if we only counted those who started fresh in the new programme we could get a bigger number of graduates
-  // than the class size, as the graduatation time count only looks at a degree as whole studyright
-  for (const right of studyrights) {
-    const { startdate, studyrightid, extentcode, studyrightElements } = right
-    const { programme } = findRightProgramme(studyrightElements, 'graduated')
-    if (programmeFilter === 'NEW_STUDY_PROGRAMMES' && !isNewProgramme(programme)) continue
+    // get all received studyrights for each year
+    // a transferred student is counted into new programmes class size i.e.
+    // students who received studyright on year X and graduated/will be graduating from programme Y
+    // if we only counted those who started fresh in the new programme we could get a bigger number of graduates
+    // than the class size, as the graduatation time count only looks at a degree as whole studyright
+    for (const right of studyrights) {
+      const { startdate, studyrightid, extentcode, studyrightElements } = right
+      if (programmeFilter === 'NEW_STUDY_PROGRAMMES' && !isNewProgramme(code)) continue
 
-    let actualStartdate = startdate
-    let level = null
+      let actualStartdate = startdate
+      let level = null
 
-    if (programme === 'MH30_001' || programme === 'MH30_003') {
-      level = 'bcMsCombo'
-    } else if (extentcode === 1) {
-      level = 'bachelor'
-    } else if (extentcode === 2) {
-      if (studyrightid.slice(-2) === '-2') {
+      if (code === 'MH30_001' || code === 'MH30_003') {
         level = 'bcMsCombo'
-        actualStartdate = await findBachelorStartdate(studyrightid.replace(/-2$/, '-1'))
+      } else if (extentcode === 1) {
+        level = 'bachelor'
+      } else if (extentcode === 2) {
+        if (studyrightid.slice(-2) === '-2') {
+          level = 'bcMsCombo'
+          actualStartdate = await findBachelorStartdate(studyrightid.replace(/-2$/, '-1'))
+        } else {
+          if (code.includes('KH')) {
+            level = 'bachelor' // Some rare faculties have one/two outliers with extent 2 for bachelor
+          } else {
+            level = 'master'
+          }
+        }
+      } else if (extentcode === 4) {
+        level = 'doctor'
+      } else if (extentcode === 3) {
+        level = 'licentiate'
       } else {
-        level = 'master'
+        continue
       }
-    } else if (extentcode === 4) {
-      level = 'doctor'
-    } else if (extentcode === 3) {
-      level = 'licentiate'
-    } else {
-      continue
-    }
+      const startYear = defineYear(actualStartdate, false)
 
-    const startYear = defineYear(actualStartdate, false)
+      // On faculty level, count started bachelor rights to BcMsCombo as well  --> useful class size even if master studies have yet not started
+      // On programme level BcMsCombo sizes are iffy as many students haven't started masters yet --> no Ms programme element to count
+      if (level !== 'bcMsCombo') {
+        classSizes[level][startYear] += 1
+        if (level === 'bachelor' && (await hasMS(code, studyrightElements, studyrightid))) {
+          classSizes['bcMsCombo'][startYear] += 1
+        }
+      }
 
-    // On faculty level, count started bachelor rights to BcMsCombo as well  --> useful class size even if master studies have yet not started
-    // On programme level BcMsCombo sizes are iffy as many students haven't started masters yet --> no Ms programme element to count
-    if (level !== 'bcMsCombo') {
-      classSizes[level][startYear] += 1
-      if (level === 'bachelor' && (await hasMS(programme, studyrightElements, studyrightid))) {
-        classSizes['bcMsCombo'][startYear] += 1
+      if (!(code in classSizes.programmes)) {
+        if (extentcode === 2) {
+          classSizes.programmes[code] = {}
+          classSizes.programmes[code]['bcMsCombo'] = getYearsObject({ years, emptyArrays: false })
+          classSizes.programmes[code]['master'] = getYearsObject({ years, emptyArrays: false })
+        } else {
+          classSizes.programmes[code] = getYearsObject({ years, emptyArrays: false })
+        }
       }
-    }
 
-    if (!(programme in classSizes.programmes)) {
       if (extentcode === 2) {
-        classSizes.programmes[programme] = {}
-        classSizes.programmes[programme]['bcMsCombo'] = getYearsObject({ years, emptyArrays: false })
-        classSizes.programmes[programme]['master'] = getYearsObject({ years, emptyArrays: false })
+        classSizes.programmes[code][level][startYear] += 1
       } else {
-        classSizes.programmes[programme] = getYearsObject({ years, emptyArrays: false })
+        classSizes.programmes[code][startYear] += 1
       }
     }
-
-    if (extentcode === 2) {
-      classSizes.programmes[programme][level][startYear] += 1
-    } else {
-      classSizes.programmes[programme][startYear] += 1
-    }
   }
 }
 
-const count = async (faculty, since, years, yearsList, levels, programmeFilter, programmeNames, goals, mode) => {
+const count = async (
+  faculty,
+  programmeCodes,
+  since,
+  years,
+  yearsList,
+  levels,
+  programmeFilter,
+  programmeNames,
+  goals,
+  mode
+) => {
   let graduationAmounts = {}
   let graduationTimes = {}
   let programmes = getProgrammeObjectBasis(yearsList, levels)
@@ -192,44 +208,43 @@ const count = async (faculty, since, years, yearsList, levels, programmeFilter,
     graduationTimes[level] = getYearsObject({ years: yearsList, emptyArrays: true })
     data.medians[level] = []
   })
-
-  let studyrights = null
-  if (mode === 'gradYear') {
-    studyrights = await graduatedStudyrights(faculty, since)
-  } else {
-    studyrights = await studyrightsByRightStartYear(faculty, since)
-  }
-
-  for (const right of studyrights) {
-    const { enddate, startdate, studyrightid, extentcode, studyrightElements, studentnumber } = right
-    const { programme, programmeName } = findRightProgramme(studyrightElements, 'graduated')
-    if (programmeFilter === 'NEW_STUDY_PROGRAMMES' && !isNewProgramme(programme)) continue
-
-    if (!(programme in programmeNames)) {
-      programmeNames[programme] = programmeName
-    }
-
-    let year = null
+  for (const code of programmeCodes) {
+    let studyrights = null
     if (mode === 'gradYear') {
-      year = defineYear(enddate, false)
+      studyrights = await graduatedStudyrights(faculty, code, since)
     } else {
-      year = defineYear(startdate, false)
+      studyrights = await studyrightsByRightStartYear(faculty, code, since)
     }
+    for (const right of studyrights) {
+      const { enddate, startdate, studyrightid, extentcode, studyrightElements, studentnumber } = right
+      const { programme, programmeName } = findRightProgramme(studyrightElements, code)
+      if (programmeFilter === 'NEW_STUDY_PROGRAMMES' && !isNewProgramme(programme)) continue
 
-    await addGraduation(
-      extentcode,
-      startdate,
-      studyrightid,
-      enddate,
-      studentnumber,
-      graduationAmounts,
-      graduationTimes,
-      year,
-      programmes,
-      programme
-    )
+      if (!(programme in programmeNames)) {
+        programmeNames[programme] = programmeName
+      }
+
+      let year = null
+      if (mode === 'gradYear') {
+        year = defineYear(enddate, false)
+      } else {
+        year = defineYear(startdate, false)
+      }
+
+      await addGraduation(
+        extentcode,
+        startdate,
+        studyrightid,
+        enddate,
+        studentnumber,
+        graduationAmounts,
+        graduationTimes,
+        year,
+        programmes,
+        programme
+      )
+    }
   }
-
   years.forEach(year => {
     levels.forEach(level => {
       const median = getMedian(graduationTimes[level][year])
@@ -279,7 +294,8 @@ const count = async (faculty, since, years, yearsList, levels, programmeFilter,
 }
 
 const countGraduationTimes = async (faculty, programmeFilter) => {
-  const isAcademicYear = false // = yearType === 'ACADEMIC_YEAR'
+  const programmes = (await getProgrammes(faculty, programmeFilter)).data
+  const isAcademicYear = false
   const since = isAcademicYear ? new Date('2017-08-01') : new Date('2017-01-01')
   const yearsList = getYearsArray(since.getFullYear(), isAcademicYear)
   const levels = ['bachelor', 'bcMsCombo', 'master', 'doctor', 'licentiate']
@@ -310,9 +326,10 @@ const countGraduationTimes = async (faculty, programmeFilter) => {
   // We count studyrights (vs. studyright_elements)
   // This way we get the whole time for a degree, even if the student was transferred to a new programme
   // E.g. started 8/2016 in old Bc, transferred to new 10/2020, graduated from new 1/2021 --> total 53 months (not 3)
-
+  const programmeCodes = programmes.map(prog => prog.code)
   const byGradYear = await count(
     faculty,
+    programmeCodes,
     since,
     years,
     yearsList,
@@ -322,8 +339,10 @@ const countGraduationTimes = async (faculty, programmeFilter) => {
     goals,
     'gradYear'
   )
+
   const byStartYear = await count(
     faculty,
+    programmeCodes,
     since,
     years,
     yearsList,
@@ -336,7 +355,7 @@ const countGraduationTimes = async (faculty, programmeFilter) => {
 
   const classSizes = getProgrammeObjectBasis(yearsList, levels, false)
   classSizes['programmes'] = {}
-  await getClassSizes(faculty, since, classSizes, programmeFilter, years)
+  await getClassSizes(faculty, programmeCodes, since, classSizes, programmeFilter, years)
 
   return { id: faculty, goals, byGradYear, byStartYear, programmeNames, classSizes }
 }

services/backend/src/services/faculty/facultyHelpers.js

@@ -2,26 +2,19 @@ const Sequelize = require('sequelize')
 const { Op } = Sequelize
 const { codes } = require('../../../config/programmeCodes')
 
-const findRightProgramme = (studyrightElements, mode) => {
+const findRightProgramme = (studyrightElements, code) => {
   let programme = ''
   let programmeName = ''
   let studyRightElement = null
 
   if (studyrightElements) {
-    if (mode === 'started') {
-      studyRightElement = studyrightElements
-        .filter(sre => sre.element_detail.type === 20)
-        .sort((a, b) => new Date(a.startdate) - new Date(b.startdate))[0]
-      // this way round counts to old programmes, other way around to new programmes
-    } else {
-      // graduated
-      studyRightElement = studyrightElements
-        .filter(sre => sre.element_detail.type === 20)
-        .sort((a, b) => new Date(b.startdate) - new Date(a.startdate))[0]
-    }
-    if (studyRightElement) {
-      programme = studyRightElement.code
-      programmeName = studyRightElement.element_detail.name
+    studyRightElement = studyrightElements
+      .filter(sre => sre.element_detail.type === 20)
+      .filter(sre => sre.code === code)
+
+    if (studyRightElement.length > 0) {
+      programme = studyRightElement[0].code
+      programmeName = studyRightElement[0].element_detail.name
     }
   }
   return { programme, programmeName }
@@ -37,9 +30,10 @@ const facultyFormatStudyright = studyright => {
     active,
     prioritycode,
     extentcode,
-    student,
+    studentStudentnumber,
     studyright_elements,
     startdate,
+    facultyCode,
   } = studyright
 
   return {
@@ -52,16 +46,17 @@ const facultyFormatStudyright = studyright => {
     active,
     prioritycode,
     extentcode,
-    studentnumber: student.studentnumber,
+    facultyCode,
+    studentnumber: studentStudentnumber,
     studyrightElements: studyright_elements,
   }
 }
 const facultyProgrammeStudents = student => {
   const { studentnumber, home_country_en, gender_code, semester_enrollments } = student
   return {
-    studentnumber,
-    home_country_en,
-    gender_code,
+    stundetNumber: studentnumber,
+    homeCountryEn: home_country_en,
+    genderCode: gender_code,
     semesters: semester_enrollments.map(s => s.dataValues),
   }
 }
@@ -119,7 +114,7 @@ const checkTransfers = (s, insideTransfersStudyrights, transfersToOrAwayStudyrig
 const getExtentFilter = includeAllSpecials => {
   const filteredExtents = [16] // always filter out secondary subject students
   if (!includeAllSpecials) {
-    filteredExtents.push(7, 34, 22, 99, 14, 13, 9)
+    filteredExtents.push(6, 7, 9, 13, 14, 18, 22, 23, 34, 99)
   }
   let studyrightWhere = {
     extentcode: {
@@ -135,7 +130,7 @@ const mapCodesToIds = data => {
   for (const prog of data) {
     if (keys.includes(prog.code)) {
       prog.progId = codes[prog.code].toUpperCase()
-      if (prog.code === 'T923103') {
+      if (prog.code === 'T923103' && !prog.progId.includes('-DP')) {
         prog.progId = prog.progId + '-DP'
       }
     } else {

services/backend/src/services/faculty/facultyService.js

@@ -1,11 +1,12 @@
 const { redisClient } = require('../redis')
 const moment = require('moment')
+const { findFacultyProgrammeCodes } = require('./faculty')
 
 const createRedisKeyForFacultyProgrammes = (id, programmeFilter) => `FACULTY_PROGRAMMES_${id}_${programmeFilter}`
 const createRedisKeyForBasicStats = (id, yearType, programmeFilter, specialGroups) =>
   `FACULTY_BASIC_STATS_${id}_${yearType}_${programmeFilter}_${specialGroups}`
-const createRedisKeyForCreditStats = (id, yearType, programmeFilter, specialGroups) =>
-  `FACULTY_CREDIT_STATS_${id}_${yearType}_${programmeFilter}_${specialGroups}`
+const createRedisKeyForCreditStats = (id, yearType, programmeFilter) =>
+  `FACULTY_CREDIT_STATS_${id}_${yearType}_${programmeFilter}`
 const createRedisKeyForThesiswriters = (id, yearType, programmeFilter, specialGroups) =>
   `FACULTY_THESIS_WRITERS_STATS_${id}_${yearType}_${programmeFilter}_${specialGroups}`
 const createRedisKeyForGraduationTimeStats = (id, programmeFilter) =>
@@ -27,7 +28,16 @@ const setFacultyProgrammes = async (id, data, programmeFilter) => {
   return dataToRedis
 }
 
-const getFacultyProgrammes = async (id, programmeFilter) => {
+const getProgrammes = async (code, programmeFilter) => {
+  const programmes = await getFacultyProgrammesFromRedis(code, programmeFilter)
+  if (programmes) return programmes
+  let updatedProgrammes = await findFacultyProgrammeCodes(code, programmeFilter)
+  if (updatedProgrammes) updatedProgrammes = await setFacultyProgrammes(code, updatedProgrammes, programmeFilter)
+
+  return updatedProgrammes
+}
+
+const getFacultyProgrammesFromRedis = async (id, programmeFilter) => {
   const redisKey = createRedisKeyForFacultyProgrammes(id, programmeFilter)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
@@ -54,9 +64,9 @@ const getBasicStats = async (id, yearType, programmeFilter, specialGroups) => {
   return JSON.parse(dataFromRedis)
 }
 
-const setCreditStats = async (data, yearType, programmeFilter, specialGroups) => {
+const setCreditStats = async (data, yearType, programmeFilter) => {
   const { id } = data
-  const redisKey = createRedisKeyForCreditStats(id, yearType, programmeFilter, specialGroups)
+  const redisKey = createRedisKeyForCreditStats(id, yearType, programmeFilter)
   const dataToRedis = {
     ...data,
     status: 'DONE',
@@ -67,8 +77,8 @@ const setCreditStats = async (data, yearType, programmeFilter, specialGroups) =>
   return dataToRedis
 }
 
-const getCreditStats = async (id, yearType, programmeFilter, specialGroups) => {
-  const redisKey = createRedisKeyForCreditStats(id, yearType, programmeFilter, specialGroups)
+const getCreditStats = async (id, yearType, programmeFilter) => {
+  const redisKey = createRedisKeyForCreditStats(id, yearType, programmeFilter)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
@@ -155,7 +165,7 @@ const setFacultyStudentStats = async (data, specialGroups, graduated) => {
 
 module.exports = {
   setFacultyProgrammes,
-  getFacultyProgrammes,
+  getProgrammes,
   setBasicStats,
   getBasicStats,
   setCreditStats,

services/backend/src/services/faculty/facultyStudentProgress.js

@@ -1,20 +1,18 @@
 const { indexOf } = require('lodash')
-const { Op } = require('sequelize')
 const moment = require('moment')
 const { getAcademicYearDates } = require('../../util/semester')
 const {
   getCreditThresholds,
   getBachelorCreditGraphStats,
   getMasterCreditGraphStats,
+  getBachelorMasterCreditGraphStats,
   getDoctoralCreditGraphStats,
   getYearsArray,
   tableTitles,
-  getOnlyMasterCreditGraphStats,
-  getBcMsThresholds,
   getVetenaryCreditGraphStats,
 } = require('../studyprogrammeHelpers')
 const { studytrackStudents } = require('../studyprogramme')
-const { getStudyRightsByExtent, getTransferredInside, getTransferredToAndAway } = require('./faculty')
+const { getStudyRightsByExtent, getStudyRightsByBachelorStart, getTransfersIn, getTransfersOut } = require('./faculty')
 const { checkTransfers } = require('./facultyHelpers')
 
 const getStudentData = (startDate, students, thresholdKeys, thresholdAmounts, limits = [], limitKeys = [], prog) => {
@@ -80,20 +78,20 @@ const createYearlyTitles = (start, limitList) => {
   ]
 }
 
-const filterOutTransfers = async (studyrights, programmes, allProgrammeCodes, since) => {
-  const transferredInAndOut = await getTransferredToAndAway(programmes, allProgrammeCodes, since)
-  const insideTransfers = await getTransferredInside(programmes, allProgrammeCodes, since)
-  const filteredStudyrights = studyrights.filter(sr => !checkTransfers(sr, insideTransfers, transferredInAndOut))
+const filterOutTransfers = async (studyrights, programmeCode, startDate, endDate) => {
+  const transferredIn = await getTransfersIn(programmeCode, startDate, endDate)
+  const transferredOut = await getTransfersOut(programmeCode, startDate, endDate)
+  const filteredStudyrights = studyrights.filter(sr => !checkTransfers(sr, transferredIn, transferredOut))
   return filteredStudyrights
 }
 
-const combineFacultyStudentProgress = async (faculty, programmes, allProgrammeCodes, specialGroups, graduated) => {
+const combineFacultyStudentProgress = async (faculty, programmes, specialGroups, graduated) => {
   const since = new Date('2017-08-01')
   const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
   const includeGraduated = graduated === 'GRADUATED_INCLUDED'
-  const graduationStatus = [0]
+  let graduationStatus = [0]
   if (includeGraduated) {
-    graduationStatus.push(1)
+    graduationStatus = [0, 1]
   }
   const yearsArray = getYearsArray(since.getFullYear(), true, true)
   const titlesVet = [
@@ -115,12 +113,14 @@ const combineFacultyStudentProgress = async (faculty, programmes, allProgrammeCo
     mastersTableStats: new Array(yearsArray.length),
     doctoralTableStats: new Array(yearsArray.length),
     bachelorsGraphStats: getBachelorCreditGraphStats(yearsArray),
-    bcMsGraphStats: faculty === 'H90' ? getVetenaryCreditGraphStats(yearsArray) : getMasterCreditGraphStats(yearsArray),
-    mastersGraphStats: getOnlyMasterCreditGraphStats(yearsArray),
+    bcMsGraphStats:
+      faculty === 'H90' ? getVetenaryCreditGraphStats(yearsArray) : getBachelorMasterCreditGraphStats(yearsArray),
+    mastersGraphStats:
+      faculty === 'H90' ? getBachelorCreditGraphStats(yearsArray) : getMasterCreditGraphStats(yearsArray),
     doctoralGraphStats: getDoctoralCreditGraphStats(yearsArray),
     bachelorTitles: tableTitles.creditProgress.bachelor,
-    bcMsTitles: faculty === 'H90' ? titlesVet : tableTitles.creditProgress.master,
-    mastersTitles: tableTitles.creditProgress.masterOnly,
+    bcMsTitles: faculty === 'H90' ? titlesVet : tableTitles.creditProgress.bachelorMaster,
+    mastersTitles: faculty === 'H90' ? tableTitles.creditProgress.bachelor : tableTitles.creditProgress.master,
     doctoralTitles: tableTitles.creditProgress.doctoral,
     yearlyBachelorTitles: [],
     yearlyBcMsTitles: [],
@@ -139,6 +139,7 @@ const combineFacultyStudentProgress = async (faculty, programmes, allProgrammeCo
     progressStats.mastersTableStats[indexOf(reversedYears, year)] = [year, 0, 0, 0, 0, 0, 0, 0]
     progressStats.doctoralTableStats[indexOf(reversedYears, year)] = [year, 0, 0, 0, 0, 0, 0]
   })
+
   const limitKeys = ['zero', 't5', 't4', 't3', 't2', 't1']
   let bachelorlimits = []
   let masterlimits = []
@@ -172,29 +173,26 @@ const combineFacultyStudentProgress = async (faculty, programmes, allProgrammeCo
     progressStats.yearlyBachelorTitles = [...progressStats.yearlyBachelorTitles, createYearlyTitles(0, bachelorlimits)]
     progressStats.yearlyMasterTitles = [...progressStats.yearlyMasterTitles, createYearlyTitles(0, masterlimits)]
     progressStats.yearlyBcMsTitles = [...progressStats.yearlyBcMsTitles, createYearlyTitles(0, bcmslimits)]
-    const programmeCodes = programmes.map(prog => prog.code)
 
     for (const { progId, code, name } of programmes) {
       let extentcodes = [1, 2, 3, 4]
       if (includeAllSpecials) {
-        extentcodes = [...extentcodes, ...[7, 9, 34, 22, 99, 14, 13]]
+        extentcodes = [...extentcodes, ...[6, 7, 9, 13, 14, 22, 23, 34, 99]]
       }
-      const startDateWhere = {
-        startdate: {
-          [Op.and]: {
-            [Op.gte]: startDate,
-            [Op.lte]: endDate,
-          },
-        },
-      }
-      let studyrights = await getStudyRightsByExtent(faculty, {}, startDateWhere, code, extentcodes, graduationStatus)
-      let { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(code)
+      // For bachelor-master we want to have the start of the bachelor programme of the student.
+      let studyrights = code.includes('MH')
+        ? await getStudyRightsByBachelorStart(faculty, startDate, endDate, code, extentcodes, graduationStatus)
+        : await getStudyRightsByExtent(faculty, startDate, endDate, code, extentcodes, graduationStatus)
+
+      // Get credit threshold values: 'combined study programme' is false and 'only master studyright' is true
+      let { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(code, false, true)
+
       if (!creditThresholdKeys) continue
       if (!(progId in progressStats.programmeNames)) {
         progressStats.programmeNames[progId] = { code: code, ...name }
       }
       if (!includeAllSpecials) {
-        studyrights = await filterOutTransfers(studyrights, programmeCodes, allProgrammeCodes, since)
+        studyrights = await filterOutTransfers(studyrights, code, startDate, endDate)
       }
       if (code.includes('KH')) {
         const allBachelors = studyrights.filter(sr => sr.extentcode === 1).map(sr => sr.studentnumber)
@@ -237,7 +235,11 @@ const combineFacultyStudentProgress = async (faculty, programmes, allProgrammeCo
         progressStats.mastersTableStats[indexOf(reversedYears, 'Total')][1] += msStudents.length || 0
         progressStats.bcMsTableStats[indexOf(reversedYears, year)][1] += bcMsStudents.length || 0
         progressStats.bcMsTableStats[indexOf(reversedYears, 'Total')][1] += bcMsStudents.length || 0
-        const { creditThresholdKeysBcMs, creditThresholdAmountsBcMs } = getBcMsThresholds()
+
+        // Get credit threshold values both 'combined study programme' and 'only master studyright' are false
+        const { creditThresholdKeys: creditThresholdKeysBcMs, creditThresholdAmounts: creditThresholdAmountsBcMs } =
+          getCreditThresholds(code, false, false)
+
         const { data: bcMsStudentdata, progData: bcMsProgdata } = getStudentData(
           startDate,
           bcMsStudents,
@@ -247,7 +249,6 @@ const combineFacultyStudentProgress = async (faculty, programmes, allProgrammeCo
           limitKeys,
           'MH'
         )
-
         const { data: msStudentdata, progData: msProgdata } = getStudentData(
           startDate,
           msStudents,
@@ -261,7 +262,6 @@ const combineFacultyStudentProgress = async (faculty, programmes, allProgrammeCo
           masterProgress[progId] = new Array(reversedYears.length - 1)
           bcmsProgress[progId] = new Array(reversedYears.length - 1)
         }
-
         masterProgress[progId][indexOf(reversedYears, year)] = limitKeys.map(key => msProgdata[key])
         bcmsProgress[progId][indexOf(reversedYears, year)] = limitKeys.map(key => bcMsProgdata[key])
         for (const key of Object.keys(msStudentdata)) {

services/backend/src/services/faculty/facultyStudents.js

@@ -1,11 +1,9 @@
-const moment = require('moment')
-const { Op } = require('sequelize')
 const { getStartDate, getYearsArray, getPercentage, getYearsObject, tableTitles } = require('../studyprogrammeHelpers')
 const { getAcademicYearDates } = require('../../util/semester')
-const { getStudyRightsByExtent, getStudentsByStudentnumbers, getTransfers } = require('./faculty')
-const { getCurrentSemester } = require('../semesters')
+const { getStudyRightsByExtent, getStudentsByStudentnumbers, getTransfersIn } = require('./faculty')
 const { checkTransfers } = require('./facultyHelpers')
-const { inactiveStudyrights, absentStudents } = require('../studyprogramme')
+const { inactiveStudyrights, absentStudents, enrolledStudents } = require('../studyprogramme')
+const { graduatedStudyRights } = require('../studyprogramme')
 const emptyTotals = () => {
   return {
     total: 0,
@@ -55,22 +53,20 @@ const addTotalsProgramme = (programmeTableStats, progId, year, totals) => {
 
 const getStudentData = (students, facultyExtra, year, code) => {
   let data = { female: 0, male: 0, finnish: 0, otherCountries: 0, otherUnknown: 0 }
-  students.forEach(({ gender_code, home_country_en }) => {
-    data.male += gender_code === '1' ? 1 : 0
-    data.female += gender_code === '2' ? 1 : 0
-    data.otherUnknown += ['0', '3'].includes(gender_code) ? 1 : 0
-    facultyExtra[year][code].otherG += gender_code === '3' ? 1 : 0
-    facultyExtra[year][code].unknownG += gender_code === '0' ? 1 : 0
-    facultyExtra['Total'][code].otherG += gender_code === '3' ? 1 : 0
-    facultyExtra['Total'][code].unknownG += gender_code === '0' ? 1 : 0
-    data.finnish += home_country_en === 'Finland' ? 1 : 0
-    data.otherCountries += home_country_en !== 'Finland' ? 1 : 0
-    if (!Object.keys(facultyExtra[year][code].countries).includes(home_country_en))
-      facultyExtra[year][code].countries[home_country_en] = 0
-    if (!Object.keys(facultyExtra['Total'][code].countries).includes(home_country_en))
-      facultyExtra['Total'][code].countries[home_country_en] = 0
-    facultyExtra[year][code].countries[home_country_en] += 1
-    facultyExtra['Total'][code].countries[home_country_en] += 1
+  students.forEach(({ genderCode, homeCountryEn }) => {
+    data.male += genderCode === '1' ? 1 : 0
+    data.female += genderCode === '2' ? 1 : 0
+    data.otherUnknown += ['0', '3'].includes(genderCode) ? 1 : 0
+    data.finnish += homeCountryEn === 'Finland' ? 1 : 0
+    data.otherCountries += homeCountryEn !== 'Finland' ? 1 : 0
+    if (!Object.keys(facultyExtra[year][code]).includes(homeCountryEn) && homeCountryEn !== 'Finland')
+      facultyExtra[year][code][homeCountryEn] = 0
+    if (!Object.keys(facultyExtra['Total'][code]).includes(homeCountryEn) && homeCountryEn !== 'Finland')
+      facultyExtra['Total'][code][homeCountryEn] = 0
+    if (homeCountryEn !== 'Finland') {
+      facultyExtra[year][code][homeCountryEn] += 1
+      facultyExtra['Total'][code][homeCountryEn] += 1
+    }
   })
   return data
 }
@@ -79,7 +75,6 @@ const getStudentData = (students, facultyExtra, year, code) => {
 const getFacultyDataForYear = async ({
   faculty,
   programmes,
-  allProgrammeCodes,
   since,
   settings,
   year,
@@ -96,68 +91,36 @@ const getFacultyDataForYear = async ({
   let extents = [1, 2, 3, 4]
   let graduated = [0]
   if (includeAllSpecials) {
-    extents = [...extents, ...[7, 9, 34, 22, 99, 14, 13]]
+    extents = [...extents, ...[6, 7, 9, 13, 14, 18, 22, 23, 34, 99]] // 16 is always excluded
   }
   if (includeGraduated) {
     graduated = [...graduated, 1]
   }
   const totals = emptyTotals()
-
   for (const { progId, code, name } of programmes) {
-    let elementStart = {}
-    let studyrightStart = {}
-    const start = moment(`${year.slice(0, 4)}-08-01`, 'YYYY-MM-DD')
-    const end = moment(`${year.slice(-4)}-07-31`, 'YYYY-MM-DD').endOf('day')
-    const allTransfers = await getTransfers(code, allProgrammeCodes, start, end)
-    if (!Object.keys(facultyExtra[year]).includes(code))
-      facultyExtra[year][code] = { unknownG: 0, otherG: 0, countries: {} }
-    if (!Object.keys(facultyExtra['Total']).includes(code))
-      facultyExtra['Total'][code] = { unknownG: 0, otherG: 0, countries: {} }
-    const startDateWhere = {
-      startdate: {
-        [Op.and]: {
-          [Op.gte]: startDate,
-          [Op.lte]: endDate,
-        },
-      },
-    }
-    if (code.includes('MH')) {
-      elementStart = startDateWhere
-    } else {
-      studyrightStart = startDateWhere
-    }
-    const allStudyrights = await getStudyRightsByExtent(
-      faculty,
-      elementStart,
-      studyrightStart,
-      code,
-      extents,
-      graduated
-    )
-    let studyrights = allStudyrights
+    if (!Object.keys(facultyExtra[year]).includes(code)) facultyExtra[year][code] = {}
+    if (!Object.keys(facultyExtra['Total']).includes(code)) facultyExtra['Total'][code] = {}
+
+    const toTransfers = await getTransfersIn(code, startDate, endDate)
+    const allStudyrights = await getStudyRightsByExtent(faculty, startDate, endDate, code, extents, graduated)
     let allStudents = []
-
-    studyrights = allStudyrights.filter(s => !checkTransfers(s, allTransfers, allTransfers))
     if (includeAllSpecials) {
-      allStudents = [...new Set([...allTransfers.map(sr => sr.studentnumber).filter(smbr => smbr !== null)])]
+      allStudents = [...new Set([...allStudyrights.map(sr => sr.studentnumber)])]
     }
-    // Get all the studyrights and students for the calculations
-    const currentSemester = await getCurrentSemester()
-    const studentNbrs = studyrights.map(sr => sr.studentnumber)
-    allStudents = [...studentNbrs, ...allStudents]
+    let studyrights = allStudyrights.filter(s => !checkTransfers(s, toTransfers, toTransfers))
 
+    // Get all the studyrights and students for the calculations
+    const studentNbrs = studyrights.map(sr => sr.studentnumber)
+    allStudents = [...new Set([...studentNbrs, ...allStudents])]
     const students = await getStudentsByStudentnumbers(allStudents)
     const studentData = getStudentData(students, facultyExtra, year, code)
-    const started = await getStudentsByStudentnumbers(studentNbrs)
-    const inactive = await inactiveStudyrights(code, studentNbrs)
-    const graduatedStudents = studyrights.filter(sr => sr.graduated === 1)
-    const absent = await absentStudents(code, studentNbrs)
-    const enrolled = students.filter(
-      s =>
-        s.semesters.filter(semester => semester.semestercode === currentSemester.semestercode)[0]?.enrollmenttype === 1
-    )
+    const started = [...new Set(studentNbrs)]
+    const inactive = await inactiveStudyrights(code, allStudents)
+    const graduatedStudents = await graduatedStudyRights(code, startDate, allStudents)
+    const absent = await absentStudents(code, allStudents)
+    const enrolled = await enrolledStudents(code, allStudents)
     // Count stats for the programme
-    if (students.length === 0) continue
+    if (allStudents.length === 0) continue
     if (!(progId in programmeTableStats)) {
       programmeTableStats[progId] = years.reduce((resultObj, yearKey) => {
         return { ...resultObj, [yearKey]: [] }
@@ -224,7 +187,7 @@ const getFacultyDataForYear = async ({
 }
 
 // Combines all the data for faculty students table
-const combineFacultyStudents = async (code, programmes, allProgrammeCodes, specialGroups, graduated) => {
+const combineFacultyStudents = async (code, programmes, specialGroups, graduated) => {
   // Only academic years are considered
   const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
   const includeGraduated = graduated === 'GRADUATED_INCLUDED'
@@ -242,7 +205,6 @@ const combineFacultyStudents = async (code, programmes, allProgrammeCodes, speci
     const queryParams = {
       faculty: code,
       programmes,
-      allProgrammeCodes,
       since,
       settings,
       year,
@@ -254,9 +216,7 @@ const combineFacultyStudents = async (code, programmes, allProgrammeCodes, speci
       programmeNames,
     }
     const totals = await getFacultyDataForYear(queryParams)
-    if (totals.total !== 0) {
-      addTotals(facultyTableStats, year, totals)
-    }
+    addTotals(facultyTableStats, year, totals)
   }
 
   addTotals(facultyTableStats, 'Total', allTotals)
@@ -270,7 +230,7 @@ const combineFacultyStudents = async (code, programmes, allProgrammeCodes, speci
     facultyTableStats: facultyTableStats,
     facultyTableStatsExtra: facultyExtra,
     programmeStats: programmeTableStats,
-    titles: tableTitles['programmes'],
+    titles: [...tableTitles.studytracksStart, ...tableTitles.studytracksBasic, ...tableTitles.studytracksEnd],
     programmeNames: programmeNames,
   }
   return studentsData

services/backend/src/services/faculty/facultyThesisWriters.js

@@ -26,6 +26,7 @@ const getFacultyThesisWriters = async ({ since, years, isAcademicYear, facultyPr
       startDate: alltimeStartDate,
       endDate: alltimeEndDate,
       includeAllSpecials,
+      includeTransferredTo: includeAllSpecials,
     })
 
     const thesisCourseCodes = await thesisWriters(provider, since, thesisTypes, students)

services/backend/src/services/faculty/facultyUpdates.js

@@ -83,7 +83,7 @@ const updateFacultyOverview = async (faculty, statsType) => {
     newProgrammes = await setFacultyProgrammes(faculty, onlyNew, 'NEW_STUDY_PROGRAMMES')
     allProgrammes?.data.forEach(prog => allProgrammeCodes.push(prog.code))
   } catch (e) {
-    logger.error(e)
+    logger.error(`Faculty updates: programme stats failed with error: ${e}`)
   }
 
   for (const option of options) {
@@ -100,13 +100,12 @@ const updateFacultyOverview = async (faculty, statsType) => {
         )
         await setBasicStats(updatedStudentInfo, option.yearType, option.programmeFilter, option.specialGroups)
       }
-      if (statsType === 'ALL' || statsType === 'CREDITS') {
+      if ((statsType === 'ALL' || statsType === 'CREDITS') && specialGroups !== 'SPECIAL_EXCLUDED') {
         const updatedCredits = await combineFacultyCredits(
           faculty,
           programmeFilter === 'ALL_PROGRAMMES' ? allProgrammes.data : newProgrammes.data,
           allProgrammes.data,
-          yearType,
-          specialGroups
+          yearType
         )
         await setCreditStats(updatedCredits, yearType, programmeFilter, specialGroups)
       }
@@ -120,7 +119,7 @@ const updateFacultyOverview = async (faculty, statsType) => {
         await setThesisWritersStats(updateThesisWriters, yearType, programmeFilter, specialGroups)
       }
     } catch (e) {
-      logger.error(e)
+      logger.error(`Faculty updates: basic/thesis-writers/credits stats failed with error ${e}`)
     }
   }
 
@@ -130,7 +129,7 @@ const updateFacultyOverview = async (faculty, statsType) => {
     const updatedTimesNew = await countGraduationTimes(faculty, 'NEW_STUDY_PROGRAMMES')
     await setGraduationStats(updatedTimesNew, 'NEW_STUDY_PROGRAMMES')
   } catch (e) {
-    logger.error(e)
+    logger.error(`Faculty updates: graduation stats failed with error: ${e}`)
   }
   return 'OK'
 }
@@ -155,16 +154,11 @@ const updateFacultyProgressOverview = async faculty => {
   const options = [specialGraduated, specialNotGraduated, notSpecialGraduated, notSpecialNotGraduated]
 
   let newProgrammes = []
-  let allProgrammes = []
-  let allProgrammeCodes = []
   try {
-    const all = await findFacultyProgrammeCodes(faculty, 'ALL_PROGRAMMES')
     const onlyNew = await findFacultyProgrammeCodes(faculty, 'NEW_STUDY_PROGRAMMES')
-    allProgrammes = await setFacultyProgrammes(faculty, all, 'ALL_PROGRAMMES')
     newProgrammes = await setFacultyProgrammes(faculty, onlyNew, 'NEW_STUDY_PROGRAMMES')
-    allProgrammes?.data.forEach(prog => allProgrammeCodes.push(prog.code))
   } catch (e) {
-    logger.error(e)
+    logger.error(`Faculty stats: all programme stats failed with error ${e}`)
   }
 
   for (const option of options) {
@@ -173,7 +167,6 @@ const updateFacultyProgressOverview = async faculty => {
       const updateFacultyStudentStats = await combineFacultyStudents(
         faculty,
         newProgrammes.data,
-        allProgrammeCodes,
         specialGroups,
         graduated
       )
@@ -181,13 +174,12 @@ const updateFacultyProgressOverview = async faculty => {
       const updateFacultyProgressStats = await combineFacultyStudentProgress(
         faculty,
         newProgrammes.data,
-        allProgrammeCodes,
         specialGroups,
         graduated
       )
       await setFacultyProgressStats(updateFacultyProgressStats, specialGroups, graduated)
     } catch (e) {
-      logger.error(e)
+      logger.error(`Faculty stats: progress stats failed with error ${e}`)
     }
   }
   return 'OK'

services/backend/src/services/languageCenterData.js

@@ -0,0 +1,98 @@
+const { Course, Credit, Enrollment, StudyrightElement } = require('../models')
+const { Op } = require('sequelize')
+const { redisClient } = require('./redis')
+const REDIS_KEY = 'LANGUAGE_CENTER_DATA'
+
+const getLanguageCenterData = async () => {
+  const dataOnRedis = await redisClient.getAsync(REDIS_KEY)
+  if (dataOnRedis) return JSON.parse(dataOnRedis)
+  const freshData = await computeLanguageCenterData()
+  redisClient.setAsync(REDIS_KEY, JSON.stringify(freshData))
+  return freshData
+}
+
+const computeLanguageCenterData = async () => {
+  const courses = await Course.findAll({
+    attributes: ['code', 'name'],
+    where: {
+      [Op.or]: [{ code: { [Op.like]: 'KK%' } }, { code: { [Op.like]: 'AYKK%' } }],
+    },
+  })
+
+  const credits = await Credit.findAll({
+    attributes: ['course_code', 'credittypecode', 'student_studentnumber', 'semestercode', 'createdate'],
+    where: {
+      [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
+    },
+  })
+
+  const enrollments = await Enrollment.findAll({
+    attributes: ['studentnumber', 'semestercode', 'course_code', 'enrollment_date_time'],
+    where: {
+      [Op.or]: [{ course_code: { [Op.like]: 'KK%' } }, { course_code: { [Op.like]: 'AYKK%' } }],
+    },
+  })
+
+  const studentList = new Set()
+  const attemptsByStudents = {}
+
+  credits.forEach(c => {
+    const sn = c.student_studentnumber
+
+    // ignoring 7 (improved), 9 (transferred) and 10 (failed)
+    if (c.credittypecode !== 4) return
+    studentList.add(sn)
+    if (!attemptsByStudents[sn]) {
+      attemptsByStudents[sn] = []
+    }
+    attemptsByStudents[sn].push({ studentNumber: sn, courseCode: c.course_code, completed: true, date: c.createdate })
+  })
+
+  enrollments.forEach(e => {
+    const sn = e.studentnumber
+    if (attemptsByStudents[sn] && attemptsByStudents[sn].find(att => att.courseCode === e.course_code)) return
+    studentList.add(sn)
+    if (!attemptsByStudents[sn]) {
+      attemptsByStudents[sn] = []
+    }
+    attemptsByStudents[sn].push({
+      studentNumber: sn,
+      courseCode: e.course_code,
+      completed: false,
+      date: e.enrollment_date_time,
+    })
+  })
+
+  const studyrightElements = await StudyrightElement.findAll({
+    attributes: ['studentnumber', 'startdate', 'enddate', 'code'],
+    where: {
+      studentnumber: { [Op.in]: Array.from(studentList.values()) },
+    },
+  })
+  const studyrightMap = studyrightElements.reduce((obj, cur) => {
+    if (!obj[cur.studentnumber]) {
+      obj[cur.studentnumber] = [{ startdate: cur.startdate, enddate: cur.enddate, code: cur.code }]
+    } else {
+      obj[cur.studentnumber].push({ startdate: cur.startdate, enddate: cur.enddate, code: cur.code })
+    }
+    return obj
+  }, {})
+
+  const getFaculty = (studentNumber, date) =>
+    studyrightMap[studentNumber]?.find(sr => sr.startdate <= date && sr.enddate >= date)?.code
+
+  const attemptsArray = []
+  studentList.forEach(sn => attemptsArray.push(...attemptsByStudents[sn]))
+
+  attemptsArray.forEach(item => {
+    const faculty = getFaculty(item.studentNumber, item.date)
+    if (faculty?.startsWith('MH') || faculty?.startsWith('KH')) {
+      item.faculty = faculty.substring(1, 4)
+    } else {
+      item.faculty = faculty
+    }
+  })
+  return { attempts: attemptsArray, courses }
+}
+
+module.exports = { getLanguageCenterData }

services/backend/src/services/mailservice.js

@@ -3,7 +3,7 @@ const { isProduction, pateToken } = require('../conf-backend')
 const { ApplicationError } = require('../util/customErrors')
 
 const pateClient = axios.create({
-  baseURL: 'https://importer.cs.helsinki.fi/api/pate/',
+  baseURL: 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/pate/',
   params: {
     token: pateToken,
   },

services/backend/src/services/mandatoryCourseLabels.js

@@ -1,64 +0,0 @@
-const Sequelize = require('sequelize')
-const _ = require('lodash')
-const { MandatoryCourseLabels, sequelizeKone } = require('../models/models_kone')
-const { Op } = Sequelize
-
-const create = async (studyprogramme_id, { label }) => {
-  const labels = await labelsByStudyprogramme(studyprogramme_id)
-  await MandatoryCourseLabels.create({ studyprogramme_id, label, orderNumber: labels.length + 1 })
-}
-
-const destroy = (studyprogramme_id, { id }) => {
-  return MandatoryCourseLabels.destroy({ where: { studyprogramme_id, id } })
-}
-
-const move = async (studyprogramme_id, { id }, direction) => {
-  const labels = await labelsByStudyprogramme(studyprogramme_id).map(e => e.get())
-  const orderedlabels = _.sortBy(labels, ['orderNumber'])
-  const indexToMove = orderedlabels.findIndex(e => e.id === id)
-  const indexToSwap = direction === 'up' ? indexToMove - 1 : indexToMove + 1
-  if (indexToSwap < 0 || indexToSwap >= orderedlabels.length) return null
-  ;[orderedlabels[indexToMove], orderedlabels[indexToSwap]] = [orderedlabels[indexToSwap], orderedlabels[indexToMove]]
-  const newOrder = orderedlabels.map((e, index) => ({ ...e, orderNumber: index }))
-  await sequelizeKone.transaction(async transaction => {
-    for (let e of newOrder) {
-      await MandatoryCourseLabels.update(
-        { orderNumber: e.orderNumber },
-        { where: { studyprogramme_id, id: e.id }, transaction }
-      )
-    }
-  })
-}
-
-const find = (studyprogramme_id, { id }) => {
-  return MandatoryCourseLabels.findOne({
-    attributes: ['id', 'label', 'orderNumber'],
-    where: {
-      studyprogramme_id: {
-        [Op.eq]: studyprogramme_id,
-      },
-      id: {
-        [Op.eq]: id,
-      },
-    },
-  })
-}
-
-const labelsByStudyprogramme = studyProgrammeId => {
-  return MandatoryCourseLabels.findAll({
-    attributes: ['id', 'label', 'orderNumber'],
-    where: {
-      studyprogramme_id: {
-        [Op.eq]: studyProgrammeId,
-      },
-    },
-  })
-}
-
-module.exports = {
-  labelsByStudyprogramme,
-  create,
-  destroy,
-  move,
-  find,
-}

services/backend/src/services/mandatoryCourses.js

@@ -1,108 +0,0 @@
-const Sequelize = require('sequelize')
-const { Course } = require('../models')
-const { MandatoryCourse, MandatoryCourseLabels } = require('../models/models_kone')
-const { Op } = Sequelize
-
-const byStudyprogramme = async studyProgrammeId => {
-  const mandatoryCourses = await MandatoryCourse.findAll({
-    attributes: ['course_code'],
-    include: [
-      {
-        model: MandatoryCourseLabels,
-        attributes: ['id', 'label', 'orderNumber'],
-      },
-    ],
-    where: {
-      studyprogramme_id: {
-        [Op.eq]: studyProgrammeId,
-      },
-    },
-  })
-
-  const courses = await Course.findAll({
-    model: Course,
-    where: {
-      code: {
-        [Op.in]: mandatoryCourses.map(c => c.course_code),
-      },
-    },
-  })
-  const courseCodeToCourse = courses.reduce((acc, c) => {
-    acc[c.code] = c
-    return acc
-  }, {})
-
-  return mandatoryCourses.map(mc => ({
-    name: courseCodeToCourse[mc.course_code].name,
-    code: mc.course_code,
-    label: mc.mandatory_course_label,
-  }))
-}
-
-const create = (studyProgrammeId, code) => {
-  return MandatoryCourse.create({
-    studyprogramme_id: studyProgrammeId,
-    course_code: code,
-  })
-}
-
-const find = async (studyProgrammeId, code) => {
-  const mandatoryCourse = await MandatoryCourse.findOne({
-    attributes: ['course_code', 'label'],
-    include: [
-      {
-        model: MandatoryCourseLabels,
-        attributes: ['id', 'label', 'orderNumber'],
-      },
-    ],
-    where: {
-      studyprogramme_id: {
-        [Op.eq]: studyProgrammeId,
-      },
-      course_code: {
-        [Op.eq]: code,
-      },
-    },
-  })
-  if (!mandatoryCourse) return null
-
-  const course = await Course.findOne({
-    where: {
-      code: mandatoryCourse.course_code,
-    },
-  })
-  if (!course) return null
-
-  return { ...mandatoryCourse.get(), course: course.get() }
-}
-
-const remove = (studyProgrammeId, code) => {
-  return MandatoryCourse.destroy({
-    where: {
-      studyprogramme_id: studyProgrammeId,
-      course_code: code,
-    },
-  })
-}
-
-const updateLabel = (studyProgrammeId, code, { id }) => {
-  return MandatoryCourse.update(
-    {
-      label: id,
-    },
-    {
-      where: {
-        course_code: code,
-        studyprogramme_id: studyProgrammeId,
-      },
-    }
-  )
-}
-
-module.exports = {
-  create,
-  find,
-  remove,
-  byStudyprogramme,
-  updateLabel,
-}

services/backend/src/services/openUni/openUniHelpers.js

@@ -9,12 +9,11 @@ const mapOpenCredits = credit => {
 }
 
 const mapStundentInfo = student => {
-  const { email, secondary_email, studentnumber, dissemination_info_allowed } = student
+  const { email, secondary_email, studentnumber } = student
   return {
     studentnumber,
     email,
     secondary_email,
-    dissemination_info_allowed,
   }
 }
 const mapOpenEnrollments = enrollment => {

services/backend/src/services/openUni/openUniSearches.js

@@ -9,7 +9,6 @@ const {
 } = require('./openUniHelpers')
 const { Op } = require('sequelize')
 
-// 1. iteration: time is hardcoded. Check hyväksytty grades
 const getCredits = async (courseCodes, startdate) =>
   (
     await Credit.findAll({

services/backend/src/services/openUni/openUniStats.js

@@ -51,15 +51,17 @@ const getCustomOpenUniCourses = async (courseCodes, startdate, enddate) => {
     }
   }
 
-  for (const { studentnumber, email, secondary_email, dissemination_info_allowed } of studentInfo) {
+  for (const { studentnumber, email, secondary_email } of studentInfo) {
     // Check if the student has existing studyright: if yes, then stop here
     if (!uniqueStudentsWithCurrentStudyRight.includes(studentnumber)) {
       if (!(studentnumber in studentStats)) {
         studentStats[studentnumber] = {
-          courseInfo: courseCodes.reduce((acc, code) => ({ ...acc, [code]: getEmptyCourseInfo() }), {}),
+          courseInfo: courseCodes.reduce(
+            (acc, code) => ({ ...acc, [code.replace('AY', '')]: getEmptyCourseInfo() }),
+            {}
+          ),
           email: email,
           secondaryEmail: secondary_email,
-          disseminationInfoAllowed: dissemination_info_allowed,
           totals: { passed: 0, failed: 0, unfinished: 0 },
         }
       }

services/backend/src/services/organisations.js

@@ -10,6 +10,7 @@ const facultiesAndProgrammesInUserService = [
   ['H10', 'KH10_001'],
   ['H20', 'KH20_001'],
   ['H20', 'MH20_001'],
+  ['H20', 'MH20_003'],
   ['H20', 'MH20_002'],
   ['H30', 'KH30_001'],
   ['H30', 'KH30_002'],
@@ -48,6 +49,7 @@ const facultiesAndProgrammesInUserService = [
   ['H50', 'KH50_005'],
   ['H50', 'KH50_006'],
   ['H50', 'KH50_007'],
+  ['H50', 'KH50_008'],
   ['H50', 'MH50_001'],
   ['H50', 'MH50_002'],
   ['H50', 'MH50_003'],
@@ -61,8 +63,10 @@ const facultiesAndProgrammesInUserService = [
   ['H50', 'MH50_011'],
   ['H50', 'MH50_012'],
   ['H50', 'MH50_013'],
+  ['H50', 'MH50_014'],
   ['H55', 'KH55_001'],
   ['H55', 'MH55_001'],
+  ['H55', 'MH55_002'],
   ['H57', 'KH57_001'],
   ['H57', 'KH57_002'],
   ['H57', 'KH57_003'],
@@ -76,6 +80,7 @@ const facultiesAndProgrammesInUserService = [
   ['H57', 'MH80_007'],
   ['H60', 'KH60_001'],
   ['H60', 'MH60_001'],
+  ['H60', 'MH60_002'],
   ['H70', 'KH40_001'],
   ['H70', 'KH50_001'],
   ['H70', 'KH70_001'],
@@ -94,6 +99,7 @@ const facultiesAndProgrammesInUserService = [
   ['H70', 'MH70_008'],
   ['H70', 'MH70_009'],
   ['H70', 'MH70_010'],
+  ['H70', 'MH70_011'],
   ['H74', 'KH74_001'],
   ['H80', 'KH57_002'],
   ['H80', 'KH80_001'],
@@ -110,7 +116,42 @@ const facultiesAndProgrammesInUserService = [
   ['H80', 'MH80_005'],
   ['H80', 'MH80_006'],
   ['H80', 'MH80_007'],
+  ['H80', 'MH80_008'],
   ['H90', 'KH90_001'],
+  ['H90', 'MH90_001'],
+  ['H92', 'T920101'],
+  ['H92', 'T920102'],
+  ['H92', 'T920103'],
+  ['H92', 'T920104'],
+  ['H92', 'T920105'],
+  ['H92', 'T920106'],
+  ['H92', 'T920107'],
+  ['H92', 'T920108'],
+  ['H92', 'T920109'],
+  ['H92', 'T920110'],
+  ['H92', 'T920111'],
+  ['H92', 'T921101'],
+  ['H92', 'T921102'],
+  ['H92', 'T921103'],
+  ['H92', 'T921104'],
+  ['H92', 'T921105'],
+  ['H92', 'T921106'],
+  ['H92', 'T921107'],
+  ['H92', 'T921108'],
+  ['H92', 'T921109'],
+  ['H92', 'T922101'],
+  ['H92', 'T922102'],
+  ['H92', 'T922103'],
+  ['H92', 'T922104'],
+  ['H92', 'T922105'],
+  ['H92', 'T922106'],
+  ['H92', 'T923101'],
+  ['H92', 'T923102'],
+  ['H92', 'T923103'],
+  ['H92', 'T923104'],
+  ['H92', 'T923105'],
+  ['H92', 'T923106'],
+  ['H92', 'T923107'],
   ['H30', '447000'],
   ['H30', '447001'],
   ['H30', '447002'],
@@ -143,7 +184,6 @@ const facultiesAndProgrammesInUserService = [
   ['H60', '0005'],
   ['H60', '7900'],
   ['H60', '0152 '],
-  ['H90', 'MH90_001'],
 ]
 
 const facultiesAndProgrammesForTrends = facultiesAndProgrammesInUserService.map(f => ({

services/backend/src/services/populations.js

@@ -25,6 +25,8 @@ const { CourseStatsCounter } = require('./course_stats_counter')
 const { getPassingSemester, semesterEnd, semesterStart } = require('../util/semester')
 const { getAllProgrammes } = require('./studyrights')
 const { encrypt } = require('../services/encrypt')
+const { getCriteria } = require('./studyProgrammeCriteria')
+const { getCurrentSemester } = require('./semesters')
 
 const enrolmentDates = () => {
   const query = 'SELECT DISTINCT s.dateOfUniversityEnrollment as date FROM Student s'
@@ -38,6 +40,61 @@ const universityEnrolmentDates = async () => {
     .filter(d => d)
     .sort()
 }
+// Progress tab related helper functions.
+const createEmptyCriteriaYear = (criteria, year) => {
+  return {
+    credits: false,
+    totalSatisfied: 0,
+    coursesSatisfied:
+      criteria?.courses && criteria?.courses[year]
+        ? criteria.courses[year].reduce((acc, course) => {
+            acc[course] = null
+            return acc
+          }, {})
+        : {},
+  }
+}
+
+const getCreditAmount = (course, hops, courseCode, startDate, addition) => {
+  return moment(course.date).isBetween(moment(startDate), moment(startDate).add(addition, 'year')) &&
+    hops.length > 0 &&
+    (hops[0].included_courses.includes(courseCode) || hops[0].included_courses.includes(course.course_code))
+    ? course.credits
+    : 0
+}
+
+const updateCourseByYear = (criteria, criteriaYear, course, criteriaChecked, yearToAdd, correctCode) => {
+  if (
+    criteria?.courses &&
+    criteria?.courses[criteriaYear] &&
+    (criteria.courses[criteriaYear].includes(course.course_code) ||
+      criteria.courses[criteriaYear].some(
+        c => criteria.allCourses[c] && criteria.allCourses[c].includes(course.course_code)
+      ))
+  ) {
+    if (
+      !criteriaChecked[yearToAdd].coursesSatisfied[correctCode] ||
+      new Date(criteriaChecked[yearToAdd].coursesSatisfied[correctCode]) > new Date(course.date)
+    ) {
+      criteriaChecked[yearToAdd].coursesSatisfied[correctCode] = course.date
+    }
+  }
+}
+
+const updateCreditcriteriaInfo = (criteria, criteriaYear, criteriaChecked, yearToAdd, academicYears, academicYear) => {
+  if (criteria.courses) {
+    if (criteriaChecked?.[yearToAdd]) {
+      criteriaChecked[yearToAdd].totalSatisfied +=
+        Object.keys(criteriaChecked?.[yearToAdd].coursesSatisfied).filter(
+          course => criteriaChecked?.[yearToAdd].coursesSatisfied[course] !== null
+        ).length || 0
+    }
+    if (academicYears[academicYear] >= criteria?.credits[criteriaYear] && criteria?.credits[criteriaYear] > 0) {
+      criteriaChecked[yearToAdd].credits = true
+      criteriaChecked[yearToAdd].totalSatisfied += 1
+    }
+  }
+}
 
 const formatStudentForPopulationStatistics = (
   {
@@ -48,6 +105,7 @@ const formatStudentForPopulationStatistics = (
     creditcount,
     abbreviatedname,
     email,
+    secondary_email,
     phone_number,
     studyrights,
     studyplans,
@@ -56,9 +114,6 @@ const formatStudentForPopulationStatistics = (
     updatedAt,
     createdAt,
     gender_code,
-    gender_fi,
-    gender_sv,
-    gender_en,
     tags,
     option,
     birthdate,
@@ -68,7 +123,10 @@ const formatStudentForPopulationStatistics = (
   credits,
   startDate,
   startDateMoment,
-  endDateMoment
+  endDateMoment,
+  criteria,
+  code,
+  currentSemester
 ) => {
   const toCourse = ({ grade, attainment_date, credits, course_code, credittypecode, isStudyModule, language }) => {
     const attainment_date_normailized =
@@ -86,10 +144,58 @@ const formatStudentForPopulationStatistics = (
       language,
     }
   }
+  const criteriaCoursesBySubstitutions = criteria?.allCourses
+    ? Object.keys(criteria.allCourses).reduce((acc, code) => {
+        acc[code] = code
+        criteria.allCourses[code].map(subst => (acc[subst] = code))
+        return acc
+      }, {})
+    : {}
+
+  const toProgressCriteria = () => {
+    const criteriaChecked = {
+      year1: createEmptyCriteriaYear(criteria, 'yearOne'),
+      year2: createEmptyCriteriaYear(criteria, 'yearTwo'),
+      year3: createEmptyCriteriaYear(criteria, 'yearThree'),
+      year4: createEmptyCriteriaYear(criteria, 'yearFour'),
+      year5: createEmptyCriteriaYear(criteria, 'yearFive'),
+      year6: createEmptyCriteriaYear(criteria, 'yearSix'),
+    }
+    const correctStudyplan = studyplans ? studyplans.filter(plan => plan.programme_code === code) : []
+    const academicYears = { first: 0, second: 0, third: 0, fourth: 0, fifth: 0, sixth: 0 }
+    if (criteria.courses || criteria.credits) {
+      const courses = credits[studentnumber] ? credits[studentnumber].map(toCourse) : []
+      courses.forEach(course => {
+        if (course.passed) {
+          const correctCode = criteriaCoursesBySubstitutions[course.course_code]
+          updateCourseByYear(criteria, 'yearOne', course, criteriaChecked, 'year1', correctCode)
+          updateCourseByYear(criteria, 'yearTwo', course, criteriaChecked, 'year2', correctCode)
+          updateCourseByYear(criteria, 'yearThree', course, criteriaChecked, 'year3', correctCode)
+          updateCourseByYear(criteria, 'yearFour', course, criteriaChecked, 'year4', correctCode)
+          updateCourseByYear(criteria, 'yearFive', course, criteriaChecked, 'year5', correctCode)
+          updateCourseByYear(criteria, 'yearSix', course, criteriaChecked, 'year6', correctCode)
+          academicYears.first += getCreditAmount(course, correctStudyplan, correctCode, startDate, 1)
+          academicYears.second += getCreditAmount(course, correctStudyplan, correctCode, startDate, 2)
+          academicYears.third += getCreditAmount(course, correctStudyplan, correctCode, startDate, 3)
+          academicYears.fourth += getCreditAmount(course, correctStudyplan, correctCode, startDate, 4)
+          academicYears.fifth += getCreditAmount(course, correctStudyplan, correctCode, startDate, 5)
+          academicYears.sixth += getCreditAmount(course, correctStudyplan, correctCode, startDate, 6)
+        }
+      })
+    }
+
+    updateCreditcriteriaInfo(criteria, 'yearOne', criteriaChecked, 'year1', academicYears, 'first')
+    updateCreditcriteriaInfo(criteria, 'yearTwo', criteriaChecked, 'year2', academicYears, 'second')
+    updateCreditcriteriaInfo(criteria, 'yearThree', criteriaChecked, 'year3', academicYears, 'third')
+    updateCreditcriteriaInfo(criteria, 'yearFour', criteriaChecked, 'year4', academicYears, 'fourth')
+    updateCreditcriteriaInfo(criteria, 'yearFive', criteriaChecked, 'year5', academicYears, 'fifth')
+    updateCreditcriteriaInfo(criteria, 'yearSix', criteriaChecked, 'year6', academicYears, 'sixth')
+    return criteriaChecked
+  }
 
   studyrights = studyrights || []
-
   const started = dateofuniversityenrollment
+
   return {
     firstnames,
     lastname,
@@ -102,10 +208,14 @@ const formatStudentForPopulationStatistics = (
     name: abbreviatedname,
     transfers: transfers || [],
     gender_code,
-    gender: { gender_en, gender_fi, gender_sv },
     email,
+    secondaryEmail: secondary_email,
     phoneNumber: phone_number,
-    semesterenrollments: semester_enrollments || [],
+    semesterenrollments: semester_enrollments
+      ? semester_enrollments
+          .sort((a, b) => a.semestercode - b.semestercode)
+          .filter(enrollment => enrollment.semestercode <= currentSemester + (currentSemester % 2))
+      : [],
     updatedAt: updatedAt || createdAt,
     tags: tags || [],
     studyrightStart: startDate,
@@ -114,6 +224,7 @@ const formatStudentForPopulationStatistics = (
     birthdate,
     studyplans,
     sis_person_id,
+    criteriaProgress: toProgressCriteria(),
   }
 }
 
@@ -233,14 +344,12 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
         'creditcount',
         'abbreviatedname',
         'email',
+        'secondary_email',
         'phone_number',
         'updatedAt',
         'gender_code',
         'birthdate',
         'sis_person_id',
-        /* 'gender_fi',
-        'gender_sv',
-        'gender_en' */
       ],
       include: [
         {
@@ -261,6 +370,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
             'faculty_code',
             'studystartdate',
             'admission_type',
+            'cancelled',
           ],
           separate: true,
           include: [
@@ -291,7 +401,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
         },
         {
           model: Studyplan,
-          attributes: ['included_courses', 'programme_code', 'completed_credits'],
+          attributes: ['included_courses', 'programme_code', 'completed_credits', 'studyrightid'],
           separate: true,
         },
       ],
@@ -329,6 +439,9 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
             student_studentnumber: {
               [Op.in]: studentnumbers,
             },
+            prioritycode: {
+              [Op.not]: 6,
+            },
           },
         },
       ],
@@ -400,7 +513,7 @@ const getEarliestYear = async (studentnumberlist, studyRights) => {
   return Math.min(...startyears)
 }
 
-const studentnumbersWithAllStudyrightElements = async (
+const studentnumbersWithAllStudyrightElements = async ({
   studyRights,
   startDate,
   endDate,
@@ -409,24 +522,25 @@ const studentnumbersWithAllStudyrightElements = async (
   transferredOutStudents,
   tag,
   transferredToStudents,
-  graduatedStudents
-) => {
+  graduatedStudents,
+}) => {
   // eslint-disable-line
 
-  // db startdate is formatted to utc so need to change it when querying
-  const formattedStartDate = new Date(moment.tz(startDate, 'Europe/Helsinki').format()).toUTCString()
-
   const filteredExtents = [16] // always filter out secondary subject students
-  let studyrightWhere = {
-    extentcode: {
-      [Op.notIn]: filteredExtents,
-    },
-  }
   if (!exchangeStudents) {
     filteredExtents.push(7, 34)
   }
   if (!nondegreeStudents) {
-    filteredExtents.push(22, 99, 14, 13)
+    filteredExtents.push(6, 9, 13, 14, 18, 22, 23, 99)
+  }
+
+  let studyrightWhere = {
+    extentcode: {
+      [Op.notIn]: filteredExtents,
+    },
+    prioritycode: {
+      [Op.not]: 6,
+    },
   }
 
   let studentWhere = {}
@@ -472,10 +586,10 @@ const studentnumbersWithAllStudyrightElements = async (
           sequelize.fn(
             'GREATEST',
             sequelize.col('studyright_elements.startdate'),
-            sequelize.col('studyright.studystartdate')
+            sequelize.col('studyright.startdate')
           ),
           {
-            [Op.between]: [formattedStartDate, endDate],
+            [Op.between]: [startDate, endDate],
           }
         ),
       ],
@@ -487,12 +601,11 @@ const studentnumbersWithAllStudyrightElements = async (
   })
 
   let studentnumbers = [...new Set(students.map(s => s.student_studentnumber))]
-
   // bit hacky solution, but this is used to filter out studentnumbers who have since changed studytracks
   const rights = await Studyright.findAll({
     attributes: ['studyrightid'],
     where: {
-      student_studentnumber: {
+      studentStudentnumber: {
         [Op.in]: studentnumbers,
       },
     },
@@ -569,7 +682,7 @@ const studentnumbersWithAllStudyrightElements = async (
   }
 
   // fetch students that have transferred to the programme and filter out these studentnumbers
-  if (transferredToStudents) {
+  if (!transferredToStudents) {
     const transfersTo = (
       await Transfer.findAll({
         attributes: ['studentnumber'],
@@ -591,7 +704,7 @@ const studentnumbersWithAllStudyrightElements = async (
   }
 
   // fetch students that have graduated from the programme and filter out these studentnumbers
-  if (graduatedStudents) {
+  if (!graduatedStudents) {
     const graduated = (
       await Student.findAll({
         attributes: ['studentnumber'],
@@ -749,10 +862,14 @@ const formatStudentsForApi = async (
   startDate,
   endDate,
   { studyRights },
-  optionData
+  optionData,
+  criteria,
+  code
 ) => {
   const startDateMoment = moment(startDate)
   const endDateMoment = moment(endDate)
+  const currentSemester = (await getCurrentSemester()).semestercode
+
   elementdetails = elementdetails.reduce(
     (acc, e) => {
       acc.data[e.code] = e
@@ -783,8 +900,19 @@ const formatStudentsForApi = async (
       })
       if (student.studentnumber in optionData) student.option = optionData[student.studentnumber]
       else student.option = null
+
       stats.students.push(
-        formatStudentForPopulationStatistics(student, enrollments, credits, startDate, startDateMoment, endDateMoment)
+        formatStudentForPopulationStatistics(
+          student,
+          enrollments,
+          credits,
+          startDate,
+          startDateMoment,
+          endDateMoment,
+          criteria,
+          code,
+          currentSemester
+        )
       )
       return stats
     },
@@ -825,6 +953,7 @@ const formatStudentsForApi = async (
     courses,
     elementdetails,
   }
+
   return returnvalue
 }
 
@@ -860,24 +989,29 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
 
   const studentnumbers = studentnumberlist
     ? studentnumberlist
-    : await studentnumbersWithAllStudyrightElements(
+    : await studentnumbersWithAllStudyrightElements({
         studyRights,
-        formattedStartDate,
+        startDate: formattedStartDate,
         endDate,
         exchangeStudents,
         nondegreeStudents,
-        transferredStudents
-      )
-  // wtf
-  // plz
+        transferredOutStudents: transferredStudents,
+        tag: null,
+        transferredToStudents: true,
+        graduatedStudents: true,
+      })
+
   const code = studyRights[0] || ''
   let optionData = {}
+  let criteria = {}
   if (code.includes('MH')) {
     optionData = await getOptionsForStudents(studentnumbers, code, 'MSC')
   } else if (code.includes('KH')) {
     optionData = await getOptionsForStudents(studentnumbers, code, 'BSC')
   }
-
+  if (code.includes('KH') || ['MH30_001', 'MH30_003'].includes(code)) {
+    criteria = await getCriteria(code)
+  }
   const { students, enrollments, credits, extents, semesters, elementdetails, courses } =
     await getStudentsIncludeCoursesBetween(
       studentnumbers,
@@ -892,7 +1026,9 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
     startDate,
     endDate,
     formattedQueryParams,
-    optionData
+    optionData,
+    criteria,
+    code
   )
 
   return formattedStudents
@@ -902,10 +1038,50 @@ const getSubstitutions = async codes => {
   const courses = await Course.findAll({ where: { code: codes }, attributes: ['code', 'substitutions'], raw: true })
   return [...new Set(courses.map(({ code, substitutions }) => [code, ...substitutions]).flat())]
 }
-
-const findCourses = async (studentnumbers, beforeDate, courses = [], studentCountLimit = 0) => {
+// This duplicate code is added here to ensure that we get the enrollments in cases no credits found for the selected students.
+const findCourseEnrollments = async (studentnumbers, beforeDate, courses = []) => {
+  const courseCodes = courses.length > 0 ? await getSubstitutions(courses) : ['DUMMY']
+  const res = await sequelize.query(
+    `
+      SELECT DISTINCT ON (course.id)
+        course.code,
+        course.name,
+        course.coursetypecode,
+        course.substitutions,
+        course.main_course_code,
+        course_types.name AS course_type,
+        enrollment.data AS enrollments
+      FROM course
+      INNER JOIN course_types ON course_types.coursetypecode = course.coursetypecode
+      LEFT JOIN (
+        SELECT
+          course_id,
+          ARRAY_AGG(JSON_BUILD_OBJECT(
+            'studentnumber', studentnumber,
+            'state', state,
+            'enrollment_date_time', enrollment_date_time
+          )) AS data
+        FROM enrollment
+        WHERE enrollment.studentnumber IN (:studentnumbers) AND enrollment.enrollment_date_time < :beforeDate
+        GROUP BY enrollment.course_id
+      ) enrollment ON enrollment.course_id = course.id 
+      WHERE :skipCourseCodeFilter OR course.code IN (:courseCodes)
+      -- GROUP BY 1, 2, 3, 4, 5, 6
+    `,
+    {
+      replacements: {
+        studentnumbers: studentnumbers.length > 0 ? studentnumbers : ['DUMMY'],
+        beforeDate,
+        courseCodes,
+        skipCourseCodeFilter: courses.length === 0,
+      },
+      type: sequelize.QueryTypes.SELECT,
+    }
+  )
+  return res
+}
+const findCourses = async (studentnumbers, beforeDate, courses = []) => {
   const courseCodes = courses.length > 0 ? await getSubstitutions(courses) : ['DUMMY']
-
   const res = await sequelize.query(
     `
       SELECT DISTINCT ON (course.id)
@@ -931,7 +1107,6 @@ const findCourses = async (studentnumbers, beforeDate, courses = [], studentCoun
         FROM credit
         WHERE student_studentnumber IN (:studentnumbers) AND attainment_date < :beforeDate
         GROUP BY credit.course_code
-        HAVING COUNT(DISTINCT credit.student_studentnumber) >= :studentCountLimit
       ) credit ON credit.course_code = course.code
       INNER JOIN course_types ON course_types.coursetypecode = course.coursetypecode
       LEFT JOIN (
@@ -953,7 +1128,6 @@ const findCourses = async (studentnumbers, beforeDate, courses = [], studentCoun
       replacements: {
         studentnumbers: studentnumbers.length > 0 ? studentnumbers : ['DUMMY'],
         beforeDate,
-        studentCountLimit,
         courseCodes,
         skipCourseCodeFilter: courses.length === 0,
       },
@@ -992,15 +1166,17 @@ const bottlenecksOf = async (query, studentnumberlist, encryptdata = false) => {
       return { error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' }
     }
     if (query.selectedStudents) {
-      const allStudents = await studentnumbersWithAllStudyrightElements(
+      const allStudents = await studentnumbersWithAllStudyrightElements({
         studyRights,
         startDate,
         endDate,
         exchangeStudents,
         nondegreeStudents,
-        transferredStudents,
-        query.tag
-      )
+        transferredOutStudents: transferredStudents,
+        tag: query.tag,
+        transferredToStudents: true,
+        graduatedStudents: true,
+      })
       const disallowedRequest = checkThatSelectedStudentsAreUnderRequestedStudyright(
         query.selectedStudents,
         allStudents
@@ -1016,38 +1192,45 @@ const bottlenecksOf = async (query, studentnumberlist, encryptdata = false) => {
         params
       const studentnumbers =
         selectedStudents ||
-        (await studentnumbersWithAllStudyrightElements(
+        (await studentnumbersWithAllStudyrightElements({
           studyRights,
           startDate,
           endDate,
           exchangeStudents,
           nondegreeStudents,
-          transferredStudents,
-          tag
-        ))
+          transferredOutStudents: transferredStudents,
+          tag,
+          transferredToStudents: true,
+          graduatedStudents: true,
+        }))
 
       const allstudents = studentnumbers.reduce((numbers, num) => {
         numbers[num] = true
         return numbers
       }, {})
+      const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months), courseCodes)
+      const foundCourseCodes = Object.keys(keyBy(courses, 'code'))
+      const filteredCourseCodes = courseCodes?.filter(code => !foundCourseCodes.includes(code))
 
-      const courses = await findCourses(
+      const courseEnrollements = await findCourseEnrollments(
         studentnumbers,
         dateMonthsFromNow(startDate, months),
-        courseCodes,
-        query.studentCountLimit
+        filteredCourseCodes
       )
-      return [allstudents, courses]
+      return [allstudents, courses, courseEnrollements]
     } else {
       const { months, startDate } = params
       const beforeDate = months && startDate ? dateMonthsFromNow(startDate, months) : new Date()
-
       const allstudents = studentnumberlist.reduce((numbers, num) => {
         numbers[num] = true
         return numbers
       }, {})
-      const courses = await findCourses(studentnumberlist, beforeDate, courseCodes, query.studentCountLimit)
-      return [allstudents, courses]
+      const courses = await findCourses(studentnumberlist, beforeDate, courseCodes)
+      const foundCourseCodes = Object.keys(keyBy(courses, 'code'))
+      const filteredCourseCodes = courseCodes?.filter(code => !foundCourseCodes.includes(code))
+
+      const courseEnrollements = await findCourseEnrollments(studentnumberlist, beforeDate, filteredCourseCodes)
+      return [allstudents, courses, courseEnrollements]
     }
   }
 
@@ -1067,24 +1250,52 @@ const bottlenecksOf = async (query, studentnumberlist, encryptdata = false) => {
   }
 
   const params = parseQueryParams(query)
-  const [[allstudents, courses], error] = await Promise.all([
+  const allStudentsByYears = query?.selectedStudentsByYear
+    ? Object.keys(query.selectedStudentsByYear).reduce(
+        (res, year) => [...res, ...query.selectedStudentsByYear[year]],
+        []
+      )
+    : []
+  // To fix failed and enrolled, no grade filter options some not so clean and nice solutions were added
+  // Get the data with actual 1. courses and filtered students. 2. all students by year, if provided.
+  const [[allstudents, courses, courseEnrollements], [, allCourses], error] = await Promise.all([
     getStudentsAndCourses(query.selectedStudents, studentnumberlist, query.courses),
+    getStudentsAndCourses(allStudentsByYears, null, query.courses),
     isValidRequest(query, params),
   ])
-  if (error) return error
 
+  if (error) return error
+  // Get the substitution codes for the fetch data by selscted students
+  const substitutionCodes = Object.entries(courses).reduce(
+    (res, [, obj]) => [...res, ...(obj?.substitutions || [])],
+    []
+  )
+  const codes = Object.keys(keyBy(courses, 'code')).map(code => code)
+  // Filter substitution courses for fetched courses -> by this we avoid the situation in which there is only
+  // courses with old course codes. Frontend NEEDS in most cases the current course.
+  const substitutionCourses = allCourses.filter(
+    obj => substitutionCodes.includes(obj.code) && !codes.includes(obj.code)
+  )
   const bottlenecks = {
     disciplines: {},
     coursetypes: {},
   }
 
   const stats = {}
-
   const startYear = parseInt(query.year, 10)
   const allstudentslength = Object.keys(allstudents).length
-  const coursesByCode = keyBy(courses, 'code')
+  let coursesToLoop = courses.concat(substitutionCourses)
+  const codesList = coursesToLoop.map(course => course.code)
+  // This fix problem when Enrolled no grade is chosen. The sql query for fetching
+  // credits do not fetch enrollments if no credits found for selected students.
+  // This and other sql query ensures that enrollments are added.
+  const coursesOnlyWithEnrollments = courseEnrollements.filter(
+    course => !codesList.includes(course.code) && course.enrollments
+  )
+  coursesToLoop = coursesToLoop.concat(coursesOnlyWithEnrollments)
 
-  for (const course of courses) {
+  const coursesByCode = keyBy(coursesToLoop, 'code')
+  for (const course of coursesToLoop) {
     let { course_type } = course
     let maincourse = course
 
@@ -1099,30 +1310,33 @@ const bottlenecksOf = async (query, studentnumberlist, encryptdata = false) => {
     if (!stats[maincourse.code]) {
       stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudentslength)
     }
-
     const coursestats = stats[maincourse.code]
-
     coursestats.addCourseType(course_type.coursetypecode, course_type.name)
     coursestats.addCourseSubstitutions(course.substitutions)
     bottlenecks.coursetypes[course_type.coursetypecode] = course_type.name
-
-    course.credits.forEach(credit => {
-      const { studentnumber, passingGrade, improvedGrade, failingGrade, grade, date } = parseCreditInfo(credit)
-      const semester = getPassingSemester(startYear, date)
-      coursestats.markCredit(studentnumber, grade, passingGrade, failingGrade, improvedGrade, semester)
-    })
-
     if (course.enrollments) {
       course.enrollments.forEach(({ studentnumber, state, enrollment_date_time }) => {
-        const semester = getPassingSemester(startYear, enrollment_date_time)
-        coursestats.markEnrollment(studentnumber, state, semester)
+        if ((query?.selectedStudents && query?.selectedStudents.includes(studentnumber)) || !query?.selectedStudents) {
+          const semester = getPassingSemester(startYear, enrollment_date_time)
+          coursestats.markEnrollment(studentnumber, state, semester, enrollment_date_time)
+        }
+      })
+    }
+    if (course.credits) {
+      course.credits.forEach(credit => {
+        const { studentnumber, passingGrade, improvedGrade, failingGrade, grade, date } = parseCreditInfo(credit)
+        if ((query?.selectedStudents && query?.selectedStudents.includes(studentnumber)) || !query?.selectedStudents) {
+          const semester = getPassingSemester(startYear, date)
+          coursestats.markCredit(studentnumber, grade, passingGrade, failingGrade, improvedGrade, semester)
+        }
       })
     }
 
     stats[maincourse.code] = coursestats
   }
 
-  bottlenecks.coursestatistics = Object.values(stats).map(coursestatistics => coursestatistics.getFinalStats())
+  const allStats = Object.values(stats).map(coursestatistics => coursestatistics.getFinalStats())
+  bottlenecks.coursestatistics = allStats.filter(course => course.stats.students > 0)
   bottlenecks.allStudents = allstudentslength
 
   if (encryptdata) encryptStudentnumbers(bottlenecks)

services/backend/src/services/programmeModules.js

@@ -2,6 +2,7 @@ const Sequelize = require('sequelize')
 const { ExcludedCourse } = require('../models/models_kone')
 const { Op } = Sequelize
 const logger = require('../util/logger')
+const { combinedStudyprogrammes } = require('./studyprogrammeHelpers.js')
 
 const { dbConnections } = require('../database/connection')
 
@@ -27,6 +28,20 @@ const recursivelyGetModuleAndChildren = async (code, type) => {
   }
 }
 
+const labelProgammes = (filteredProgrammes, excludedProgrammes) => {
+  return filteredProgrammes.map(module => {
+    const label = {
+      id: module.label_name.fi,
+      label: `${module.label_code}\n${module.label_name.fi}`,
+      orderNumber: module.module_order,
+    }
+    // check if course is excluded, and hide if it is
+    const foundCourse = excludedProgrammes.find(course => course.course_code === module.code)
+    const visible = { visibility: !foundCourse, id: foundCourse ? foundCourse.id : null }
+
+    return { ...module, label, visible }
+  })
+}
 const byProgrammeCode = async code => {
   const result = await recursivelyGetModuleAndChildren(code, 'course')
   if (!result.length) return []
@@ -45,27 +60,52 @@ const byProgrammeCode = async code => {
   )
 
   // this labels the modules to match the old system in frontend
-  const labeled = filtered.map(module => {
-    const label = {
-      id: module.label_name.fi,
-      label: `${module.label_code}\n${module.label_name.fi}`,
-      orderNumber: module.module_order,
+  const labeled = labelProgammes(filtered, excluded)
+
+  // Some studyprogrammes (e.g. eläinlääkis) are combined as bachelor-master
+  const isCombined = Object.keys(combinedStudyprogrammes).includes(code)
+  let secondProgrammelabeled = []
+  if (isCombined) {
+    const secondProgCode = combinedStudyprogrammes[code]
+    const secondProgrammeResult = await recursivelyGetModuleAndChildren(secondProgCode, 'course')
+    // Now do the same things for the second set of results as for the first
+    if (secondProgrammeResult.length) {
+      const excludedSecondProgrammes = await ExcludedCourse.findAll({
+        where: {
+          programme_code: {
+            [Op.eq]: secondProgCode,
+          },
+        },
+      })
+
+      // filter out possible duplicates
+      const secondFiltered = secondProgrammeResult.filter(
+        (course, index, array) =>
+          array.findIndex(c => c.id == course.id && c.code == course.code && c.label_code == course.label_code) == index
+      )
+
+      // this labels the modules to match the old system in frontend
+      secondProgrammelabeled = labelProgammes(secondFiltered, excludedSecondProgrammes)
     }
-    // check if course is excluded, and hide if it is
-    const foundCourse = excluded.find(course => course.course_code === module.code)
-    const visible = { visibility: !foundCourse, id: foundCourse ? foundCourse.id : null }
-
-    return { ...module, label, visible }
-  })
-
-  return labeled
+  }
+  // The second results are for combined programme views
+  return { defaultProgrammeCourses: labeled, secondProgrammeCourses: secondProgrammelabeled }
 }
 
 const addExcludedCourses = async (programmecode, coursecodes) => {
   return ExcludedCourse.bulkCreate(coursecodes.map(c => ({ programme_code: programmecode, course_code: c })))
 }
+
 const modulesByProgrammeCode = async code => {
-  return await recursivelyGetModuleAndChildren(code, 'module')
+  const programmeResults = await recursivelyGetModuleAndChildren(code, 'module')
+
+  let secondProgrammeResults = []
+  if (Object.keys(combinedStudyprogrammes).includes(code)) {
+    const masterProgCode = combinedStudyprogrammes[code]
+    secondProgrammeResults = await recursivelyGetModuleAndChildren(masterProgCode, 'module')
+  }
+  // The second results are for combined programme views
+  return { defaultProgrammeResults: programmeResults, secondProgrammeResults: secondProgrammeResults }
 }
 
 const removeExcludedCourses = async ids => {

services/backend/src/services/redis.js

@@ -1,11 +1,14 @@
 const redis = require('redis')
 const bluebird = require('bluebird')
 const conf = require('../conf-backend.js')
+const logger = require('../util/logger')
 
 bluebird.promisifyAll(redis.RedisClient.prototype)
 
 const redisClient = redis.createClient(6379, conf.redis)
 
+redisClient.on('error', err => logger.error('Redis Client Error', err))
+
 module.exports = {
   redisClient,
 }

services/backend/src/services/semesters.js

@@ -43,16 +43,7 @@ const getSemestersAndYears = async () => {
   return result
 }
 
-const getMaxYearcode = async () => {
-  const aa = await Semester.findAll({
-    attributes: [[sequelize.fn('max', sequelize.col('yearcode')), 'maxYearCode']],
-    raw: true,
-  })
-  return aa[0].maxYearCode
-}
-
 module.exports = {
   getSemestersAndYears,
-  getMaxYearcode,
   getCurrentSemester,
 }

services/backend/src/services/students.js

@@ -13,6 +13,7 @@ const {
   SemesterEnrollment,
   Semester,
   Studyplan,
+  Transfer,
 } = require('../models')
 const { TagStudent, Tag } = require('../models/models_kone')
 const Op = Sequelize.Op
@@ -60,7 +61,10 @@ const byId = async id => {
         },
         {
           model: Studyplan,
-          attributes: ['included_courses', 'programme_code', 'completed_credits'],
+          attributes: ['included_courses', 'programme_code', 'completed_credits', 'studyrightid'],
+        },
+        {
+          model: Transfer,
         },
       ],
     }),
@@ -82,9 +86,7 @@ const byId = async id => {
       },
     },
   })
-
   const semesters = await Semester.findAll()
-
   const mappedEnrollments = student.semester_enrollments.map(enrollment => {
     const semester = semesters.find(sem => sem.semestercode === enrollment.semestercode)
     return {
@@ -94,9 +96,11 @@ const byId = async id => {
       startYear: semester.startYear,
     }
   })
-
   student.semester_enrollments = mappedEnrollments
 
+  const filteredStudyrights = student.studyrights.filter(sr => sr.prioritycode !== 6)
+  student.studyrights = filteredStudyrights
+
   student.tags = tags.map(t => ({
     ...t.get(),
     programme: tagprogrammes.find(p => p.code === t.tag.studytrack),
@@ -267,13 +271,6 @@ const formatStudent = async ({
     .map(toCourse)
     .filter(c => c.course.name !== 'missing')
 
-  const hopsCourses = studyplans.map(sp => sp.included_courses).flat()
-
-  const allCourses = await sequelize.query(
-    'SELECT DISTINCT ON(course_code) course_code, credits FROM credit WHERE course_code IN (:hopsCourses)',
-    { replacements: { hopsCourses: hopsCourses.concat('id-to-fill-array') } }
-  )
-
   return {
     firstnames,
     lastname,
@@ -289,7 +286,6 @@ const formatStudent = async ({
     semesterenrollments,
     updatedAt: updatedAt || createdAt,
     studyplans,
-    allCourses: allCourses[0],
     tags,
     sis_person_id: sis_person_id,
   }
@@ -298,9 +294,10 @@ const formatStudent = async ({
 const withId = async id => {
   try {
     const result = await byId(id)
-    return await formatStudent(result)
+    const formattedStudent = await formatStudent(result)
+    return formattedStudent
   } catch (e) {
-    logger.error({ message: 'error when fetching single student', meta: e })
+    logger.error(`Error when fetching single student ${e}`)
     return {
       error: e,
     }
@@ -347,6 +344,25 @@ const studentnumberLike = terms => {
 const bySearchTerm = async searchterm => {
   const terms = splitByEmptySpace(searchterm)
   const matches = await Student.findAll({
+    include: {
+      model: Studyright,
+      include: {
+        model: StudyrightElement,
+        include: {
+          model: ElementDetail,
+          where: {
+            type: {
+              [Op.in]: [10, 20, 30],
+            },
+          },
+        },
+      },
+      where: {
+        prioritycode: {
+          [Op.not]: 6,
+        },
+      },
+    },
     where: {
       [Op.or]: [nameLike(terms), studentnumberLike(terms)],
     },
@@ -357,6 +373,20 @@ const bySearchTerm = async searchterm => {
 const bySearchTermAndStudentNumbers = async (searchterm, studentnumbers) => {
   const terms = splitByEmptySpace(searchterm)
   const matches = await Student.findAll({
+    include: {
+      model: Studyright,
+      include: {
+        model: StudyrightElement,
+        include: {
+          model: ElementDetail,
+          where: {
+            type: {
+              [Op.in]: [10, 20, 30],
+            },
+          },
+        },
+      },
+    },
     where: {
       [Op.and]: {
         [Op.or]: [nameLike(terms), studentnumberLike(terms)],

services/backend/src/services/studyProgrammeCriteria.js

@@ -0,0 +1,148 @@
+const { ProgressCriteria } = require('../models/models_kone')
+const { Course } = require('../models')
+const logger = require('../util/logger')
+
+const getCriteriaByStudyProgramme = async code => {
+  if (code === '') return null
+  return await ProgressCriteria.findOne({
+    where: {
+      code: code,
+    },
+  })
+}
+
+const getSubstitutions = async codes => {
+  if (codes === []) return []
+  const courses = await Course.findAll({ where: { code: codes }, attributes: ['code', 'substitutions'], raw: true })
+  return courses.reduce((acc, { code, substitutions }) => ({ ...acc, [code]: substitutions }), {})
+}
+
+const formattedData = async data => {
+  const yearOne = data.coursesYearOne ? data.coursesYearOne : []
+  const yearTwo = data.coursesYearTwo ? data.coursesYearTwo : []
+  const yearThree = data.coursesYearThree ? data.coursesYearThree : []
+  const yearFour = data.coursesYearFour ? data.coursesYearFour : []
+  const yearFive = data.coursesYearFive ? data.coursesYearFive : []
+  const yearSix = data.coursesYearSix ? data.coursesYearSix : []
+  const courseCodes = [...yearOne, ...yearTwo, ...yearThree, ...yearFour, ...yearFive, ...yearSix]
+  const allCourses = await getSubstitutions(courseCodes)
+
+  const criteria = {
+    courses: { yearOne, yearTwo, yearThree, yearFour, yearFive, yearSix },
+    allCourses,
+    credits: {
+      yearOne: data.creditsYearOne ? data.creditsYearOne : 0,
+      yearTwo: data.creditsYearTwo ? data.creditsYearTwo : 0,
+      yearThree: data.creditsYearThree ? data.creditsYearThree : 0,
+      yearFour: data.creditsYearFour ? data.creditsYearFour : 0,
+      yearFive: data.creditsYearFive ? data.creditsYearFive : 0,
+      yearSix: data.creditsYearSix ? data.creditsYearSix : 0,
+    },
+  }
+  return criteria
+}
+
+const createCriteria = async (studyProgramme, courseObj, creditsObj) => {
+  const newProgrammeCriteria = {
+    code: studyProgramme,
+    coursesYearOne: courseObj.year1,
+    coursesYearTwo: courseObj.year2,
+    coursesYearThree: courseObj.year3,
+    coursesYearFour: courseObj.year4,
+    coursesYearFive: courseObj.year5,
+    coursesYearSix: courseObj.year6,
+    creditsYearOne: creditsObj.year1,
+    creditsYearTwo: creditsObj.year2,
+    creditsYearThree: creditsObj.year3,
+    creditsYearFour: creditsObj.year4,
+    creditsYearFive: creditsObj.year5,
+    creditsYearSix: creditsObj.year6,
+  }
+  try {
+    const createdData = await ProgressCriteria.create(newProgrammeCriteria)
+    return await formattedData(createdData)
+  } catch (error) {
+    logger.error(`Create criteria failed: ${error}`)
+  }
+}
+
+const saveYearlyCreditCriteria = async (studyProgramme, credits) => {
+  const studyProgrammeToUpdate = await getCriteriaByStudyProgramme(studyProgramme)
+  if (!studyProgrammeToUpdate) {
+    const courseObj = { year1: [], year2: [], year3: [], year4: [], year5: [], year6: [] }
+    const creditsObj = {
+      year1: parseInt(credits.year1),
+      year2: parseInt(credits.year2),
+      year3: parseInt(credits.year3),
+      year4: parseInt(credits.year4),
+      year5: parseInt(credits.year5),
+      year6: parseInt(credits.year6),
+    }
+    return await createCriteria(studyProgramme, courseObj, creditsObj)
+  }
+  const yearlyCredits = {
+    creditsYearOne: parseInt(credits.year1),
+    creditsYearTwo: parseInt(credits.year2),
+    creditsYearThree: parseInt(credits.year3),
+    creditsYearFour: parseInt(credits.year4),
+    creditsYearFive: parseInt(credits.year5),
+    creditsYearSix: parseInt(credits.year6),
+  }
+  try {
+    const updatedData = await studyProgrammeToUpdate.update({ ...yearlyCredits })
+    return await formattedData(updatedData)
+  } catch (error) {
+    logger.error(`Update yearly credit criteria failed: ${error}`)
+  }
+}
+
+const saveYearlyCourseCriteria = async (studyProgramme, courses, year) => {
+  const studyProgrammeToUpdate = await getCriteriaByStudyProgramme(studyProgramme)
+  if (!studyProgrammeToUpdate) {
+    const courseObj = { year1: [], year2: [], year3: [], year4: [], year5: [], year6: [] }
+    const creditObj = { year1: 0, year2: 0, year3: 0, year4: 0, year5: 0, year6: 0 }
+    if (year === 1) {
+      courseObj.year1 = courses
+    } else if (year === 2) {
+      courseObj.year2 = courses
+    } else if (year === 3) {
+      courseObj.year3 = courses
+    } else if (year === 4) {
+      courseObj.year4 = courses
+    } else if (year === 5) {
+      courseObj.year5 = courses
+    } else {
+      courseObj.year6 = courses
+    }
+    return await createCriteria(studyProgramme, courseObj, creditObj)
+  }
+
+  try {
+    let updatedData = {}
+    if (year === 1) updatedData = await studyProgrammeToUpdate.update({ coursesYearOne: courses })
+    if (year === 2) updatedData = await studyProgrammeToUpdate.update({ coursesYearTwo: courses })
+    if (year === 3) updatedData = await studyProgrammeToUpdate.update({ coursesYearThree: courses })
+    if (year === 4) updatedData = await studyProgrammeToUpdate.update({ coursesYearFour: courses })
+    if (year === 5) updatedData = await studyProgrammeToUpdate.update({ coursesYearFive: courses })
+    if (year === 6) updatedData = await studyProgrammeToUpdate.update({ coursesYearSix: courses })
+    return await formattedData(updatedData)
+  } catch (error) {
+    logger.error(`Update yearly credit criteria failed: ${error}`)
+  }
+}
+
+const getCriteria = async studyProgramme => {
+  const studyProgrammeCriteria = await getCriteriaByStudyProgramme(studyProgramme)
+  const criteria = {
+    courses: { yearOne: [], yearTwo: [], yearThree: [], yearFour: [], yearFive: [], yearSix: [] },
+    allCourses: [],
+    credits: { yearOne: 0, yearTwo: 0, yearThree: 0, yearFour: 0, yearFive: 0, yearSix: 0 },
+  }
+  return studyProgrammeCriteria ? await formattedData(studyProgrammeCriteria) : criteria
+}
+
+module.exports = {
+  getCriteria,
+  saveYearlyCourseCriteria,
+  saveYearlyCreditCriteria,
+}

services/backend/src/services/studyprogramme.js

@@ -3,7 +3,6 @@ const Sequelize = require('sequelize')
 const {
   dbConnections: { sequelize },
 } = require('../database/connection')
-const moment = require('moment')
 const { Op } = Sequelize
 const {
   Credit,
@@ -12,13 +11,16 @@ const {
   Studyright,
   StudyrightElement,
   ElementDetail,
+  ProgrammeModule,
   Transfer,
   Student,
   SemesterEnrollment,
   Semester,
+  Enrollment,
 } = require('../models')
-const { formatStudyright, formatStudent, formatTransfer } = require('./studyprogrammeHelpers')
+const { formatStudyright, formatStudent, formatTransfer, formatCredit } = require('./studyprogrammeHelpers')
 const { getCurrentSemester } = require('./semesters')
+const logger = require('../util/logger')
 
 const whereStudents = studentnumbers => {
   return studentnumbers ? studentnumbers : { [Op.not]: null }
@@ -36,7 +38,7 @@ const studytrackStudents = async studentnumbers =>
         separate: true,
         attributes: ['credits', 'attainment_date'],
         where: {
-          [Op.or]: [{ isStudyModule: null }, { isStudyModule: false }],
+          isStudyModule: false,
           credittypecode: {
             [Op.in]: [4, 9],
           },
@@ -50,7 +52,7 @@ const studytrackStudents = async studentnumbers =>
     })
   ).map(formatStudent)
 
-const enrolledStudents = async (studytrack, since, studentnumbers) => {
+const enrolledStudents = async (studytrack, studentnumbers) => {
   const currentSemester = await getCurrentSemester()
 
   const students = await Student.findAll({
@@ -68,14 +70,17 @@ const enrolledStudents = async (studytrack, since, studentnumbers) => {
           },
         ],
         attributes: ['studyrightid'],
+        where: {
+          graduated: 0,
+          active: 1,
+        },
       },
       {
         model: SemesterEnrollment,
-        attributes: ['semestercode'],
+        attributes: ['semestercode', 'enrollmenttype'],
         include: [
           {
             model: Semester,
-            required: true,
             where: {
               semestercode: currentSemester.semestercode,
             },
@@ -113,6 +118,10 @@ const absentStudents = async (studytrack, studentnumbers) => {
           },
         ],
         attributes: ['studyrightid'],
+        where: {
+          graduated: 0,
+          active: 1,
+        },
       },
       {
         model: SemesterEnrollment,
@@ -190,6 +199,38 @@ const startedStudyrights = async (studytrack, since, studentnumbers) =>
     })
   ).map(formatStudyright)
 
+const graduatedStudyRightsByStartDate = async (studytrack, since, academicEnddate) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: {
+            model: ElementDetail,
+            where: {
+              code: studytrack,
+            },
+          },
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        graduated: 1,
+        startdate: {
+          [Op.between]: [since, academicEnddate],
+        },
+        student_studentnumber: {
+          [Op.not]: null,
+        },
+      },
+    })
+  ).map(formatStudyright)
+
 const graduatedStudyRights = async (studytrack, since, studentnumbers) =>
   (
     await Studyright.findAll({
@@ -220,37 +261,46 @@ const graduatedStudyRights = async (studytrack, since, studentnumbers) =>
   ).map(formatStudyright)
 
 const inactiveStudyrights = async (studytrack, studentnumbers) => {
-  const now = moment(new Date())
-  const inactiveOrExpired = (
-    await Studyright.findAll({
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          include: {
-            model: ElementDetail,
+  const currentSemester = await getCurrentSemester()
+  const students = await Student.findAll({
+    attributes: ['studentnumber'],
+    include: [
+      {
+        model: Studyright,
+        required: true,
+        include: [
+          {
+            model: StudyrightElement,
             required: true,
             where: {
               code: studytrack,
             },
           },
+        ],
+        attributes: ['studyrightid', 'enddate'],
+        where: {
+          graduated: 0,
+          active: 0,
         },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-      where: {
-        student_studentnumber: whereStudents(studentnumbers),
-        graduated: 0,
       },
-    })
-  )
-    .filter(s => s.active === 0 || (s.enddate && moment(s.enddate).isBefore(now)))
-    .map(formatStudyright)
+      {
+        model: SemesterEnrollment,
+        attributes: ['semestercode', 'enrollmenttype'],
+      },
+    ],
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumbers,
+      },
+    },
+  })
 
-  return inactiveOrExpired
+  return students.filter(
+    student =>
+      student.studyrights[0].enddate <= new Date() ||
+      !student.semester_enrollments.find(enrollment => enrollment.semestercode === currentSemester) ||
+      student.semester_enrollments.find(enrollment => enrollment.semestercode === currentSemester).enrollmenttype === 3
+  )
 }
 
 const followingStudyrights = async (since, programmes, studentnumbers) =>
@@ -353,33 +403,20 @@ const allTransfers = async (studytrack, since) => {
   return [...transferredTo, ...transferredAway]
 }
 
-const getProgrammesStudyrights = async studyprogramme =>
-  (
-    await Studyright.findAll({
-      attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode'],
-      include: [
-        {
-          model: StudyrightElement,
-          required: true,
-          where: {
-            code: {
-              [Op.in]: [studyprogramme],
-            },
-          },
-        },
-        {
-          model: Student,
-          attributes: ['studentnumber'],
-          required: true,
-        },
-      ],
-    })
-  ).map(formatStudyright)
-
 const getStudyRights = async students =>
   (
     await Studyright.findAll({
-      attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode'],
+      attributes: [
+        'studyrightid',
+        'startdate',
+        'studystartdate',
+        'enddate',
+        'graduated',
+        'prioritycode',
+        'extentcode',
+        'cancelled',
+        'facultyCode',
+      ],
       where: {
         studentStudentnumber: students,
       },
@@ -402,62 +439,83 @@ const getStudyRights = async students =>
     })
   ).map(formatStudyright)
 
-const getCreditsForStudyProgramme = async (codes, since) => {
-  const res = await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
-    include: {
-      model: Course,
-      attributes: ['code'],
-      required: true,
+const getCreditsForStudyProgramme = async (codes, since) =>
+  (
+    await Credit.findAll({
+      attributes: ['course_code', 'credits', 'attainment_date', 'student_studentnumber'],
+      include: {
+        model: Course,
+        attributes: ['code'],
+        required: true,
+        where: {
+          code: codes,
+        },
+      },
       where: {
-        is_study_module: false,
-        code: codes,
+        credittypecode: 4,
+        isStudyModule: {
+          [Op.not]: true,
+        },
+        attainment_date: {
+          [Op.gte]: since,
+        },
       },
-    },
-    where: {
-      credittypecode: {
-        [Op.notIn]: [10, 9, 7],
-      },
-      isStudyModule: {
-        [Op.not]: true,
-      },
-      attainment_date: {
-        [Op.gte]: since,
-      },
-    },
-  })
-  return res
-}
+    })
+  ).map(formatCredit)
 
 const getCourseCodesForStudyProgramme = async provider => {
   const coursesByProvider = await Course.findAll({
-    attributes: ['code', 'substitutions'],
-    where: {
-      is_study_module: false,
-    },
+    attributes: ['id', 'code', 'substitutions'],
     include: {
       model: Organization,
       required: true,
       where: {
         code: provider,
       },
+      through: {
+        attributes: [],
+      },
     },
   })
+  // here is work in progress
+  // const courseIds = coursesByProvider.map(course => course.id)
+  // const coursesWithManyProviders = await CourseProvider.findAll({
+  //   attributes: ['coursecode', [sequelize.fn('COUNT', sequelize.col('organizationcode')), 'n_courses']],
+  //   where: {
+  //     coursecode: {
+  //       [Op.in]: courseIds,
+  //     },
+  //   },
+  //   group: 'coursecode',
+  // })
+
+  // const coursesProvidedByManyProgrammes = coursesWithManyProviders
+  //   .filter(course => course.n_courses >= 2)
+  //   .map(course => course.coursecode)
+  // // We are basically not interested other substitutions than those with open uni ones.
+  // const coursesToHave = coursesByProvider.filter(course => !coursesProvidedByManyProgrammes.includes(course.id))
   const coursesWithOpenUniSubstitutions = coursesByProvider.map(({ code, substitutions }) => {
     if (!substitutions || !substitutions.length) return [code]
     const alternatives = [`AY-${code}`, `AY${code}`, `A-${code}`]
     return [code].concat(substitutions.filter(sub => alternatives.includes(sub)))
   })
+  // const providedByManyProgrammes = coursesByProvider
+  //   .filter(course => coursesProvidedByManyProgrammes.includes(course.id))
+  //   .map(({ code, substitutions }) => {
+  //     if (!substitutions || !substitutions.length) return [code]
+  //     const alternatives = [`AY-${code}`, `AY${code}`, `A-${code}`]
+  //     return [code].concat(substitutions.filter(sub => alternatives.includes(sub)))
+  //   })
   return coursesWithOpenUniSubstitutions.flat()
 }
 
 const getAllProgrammeCourses = async providerCode => {
   const res = await Course.findAll({
-    attributes: ['code', 'substitutions'],
+    attributes: ['id', 'code', 'substitutions'],
     include: [
       {
         model: Organization,
-        attributes: [],
+        attributes: ['id'],
         required: true,
         where: {
           code: providerCode,
@@ -473,6 +531,7 @@ const getAllProgrammeCourses = async providerCode => {
 }
 
 const getStudentsForProgrammeCourses = async (from, to, programmeCourses) => {
+  if (!programmeCourses.length) return []
   try {
     const res = await sequelize.query(
       `
@@ -487,7 +546,7 @@ const getStudentsForProgrammeCourses = async (from, to, programmeCourses) => {
          )
          SELECT COUNT(student) AS total_students, SUM(credits) AS total_credits, code, course_name
          FROM Dist
-         GROUP BY dist.code, course_name;
+         GROUP BY dist.code, dist.course_name;
       `,
       {
         type: sequelize.QueryTypes.SELECT,
@@ -497,18 +556,120 @@ const getStudentsForProgrammeCourses = async (from, to, programmeCourses) => {
     return res.map(course => ({
       code: course.code,
       name: course.course_name,
-      year: course.year,
-      totalAllStudents: parseInt(course.total_students),
+      totalPassed: parseInt(course.total_students),
       totalAllcredits: parseInt(course.total_credits),
-      type: 'total',
+      type: 'passed',
     }))
   } catch (e) {
-    // eslint-disable-next-line no-console
-    console.log(e)
+    logger.error(`getStudentsForProgrammeCourses() function failed ${e}`)
   }
 }
 
+const getNotCompletedForProgrammeCourses = async (from, to, programmeCourses) => {
+  try {
+    const enrollmentsCourses = await Enrollment.findAll({
+      attributes: ['studentnumber', 'course_code'],
+      where: {
+        course_code: {
+          [Op.in]: programmeCourses,
+        },
+        studentnumber: {
+          [Op.ne]: null,
+        },
+        enrollment_date_time: {
+          [Op.between]: [from, to],
+        },
+        state: 'ENROLLED',
+      },
+    })
+    const getCourseCode = code => {
+      if (code.startsWith('AY')) return code.replace('AY', '')
+      if (code.startsWith('A')) return code.replace('AY', '')
+      return code
+    }
+    const allEnrollments = {}
+    for (const { studentnumber, course_code } of enrollmentsCourses) {
+      const code = getCourseCode(course_code)
+      if (!(code in allEnrollments)) {
+        allEnrollments[code] = []
+      }
+      if (!allEnrollments[code].includes(studentnumber)) allEnrollments[code].push(studentnumber)
+    }
+
+    const credits = await Credit.findAll({
+      attributes: ['course_code', 'student_studentnumber', 'credittypecode'],
+      include: {
+        model: Course,
+        attributes: ['code', 'name'],
+        required: true,
+        where: {
+          code: programmeCourses,
+        },
+      },
+      where: {
+        isStudyModule: {
+          [Op.not]: true,
+        },
+        attainment_date: {
+          [Op.between]: [from, to],
+        },
+      },
+    })
+
+    const creditsObj = credits.map(credit => {
+      return {
+        code: getCourseCode(credit.course_code),
+        studentnumber: credit.student_studentnumber,
+        credittype: credit.credittypecode,
+        courseName: credit.course.name,
+      }
+    })
+
+    const passedByCourseCodes = {}
+    const notCompletedByCourseCodes = {}
+    const courses = {}
+    for (const course of creditsObj) {
+      if (!(course.code in courses)) {
+        courses[course.code] = {
+          code: course.code,
+          name: course.courseName,
+        }
+
+        passedByCourseCodes[course.code] = []
+        notCompletedByCourseCodes[course.code] = []
+      }
+      if ([4, 7, 9].includes(course.credittype)) {
+        passedByCourseCodes[course.code].push(course.studentnumber)
+      }
+      if (course.credittype === 10 && !passedByCourseCodes[course.code].includes(course.studentnumber)) {
+        notCompletedByCourseCodes[course.code].push(course.studentnumber)
+      }
+    }
+    // If student has enrollments, but no attainment for a particular course, they have no credit info.
+    programmeCourses.forEach(courseCode => {
+      if (allEnrollments[courseCode]) {
+        allEnrollments[courseCode].forEach(studentnumber => {
+          if (passedByCourseCodes[courseCode] && !passedByCourseCodes[courseCode].includes(studentnumber)) {
+            notCompletedByCourseCodes[courseCode].push(studentnumber)
+          }
+        })
+      }
+    })
+
+    return Object.keys(courses)
+      .reduce((acc, val) => [...acc, { ...courses[val] }], [])
+      .map(course => ({
+        code: course.code,
+        name: course.name,
+        totalNotCompleted: [...new Set(notCompletedByCourseCodes[course.code])].length,
+        type: 'notCompleted',
+      }))
+  } catch (e) {
+    logger.error(`getNotCompletedForProgrammeCourses failed ${e}`)
+  }
+}
 const getOwnStudentsForProgrammeCourses = async (from, to, programmeCourses, studyprogramme) => {
+  if (!programmeCourses.length) return []
   const res = await sequelize.query(
     `
     WITH Dist AS (
@@ -534,7 +695,6 @@ const getOwnStudentsForProgrammeCourses = async (from, to, programmeCourses, stu
     }
   )
   return res.map(course => {
-    // console.log('course: ', course)
     const res = {
       code: course.code,
       name: course.course_name,
@@ -547,6 +707,7 @@ const getOwnStudentsForProgrammeCourses = async (from, to, programmeCourses, stu
 }
 
 const getOtherStudentsForProgrammeCourses = async (from, to, programmeCourses, studyprogramme) => {
+  if (!programmeCourses.length) return []
   const res = await sequelize.query(
     `
     WITH Dist AS (
@@ -589,6 +750,7 @@ const getOtherStudentsForProgrammeCourses = async (from, to, programmeCourses, s
 }
 
 const getTransferStudentsForProgrammeCourses = async (from, to, programmeCourses) => {
+  if (!programmeCourses.length) return []
   const res = await sequelize.query(
     `
     WITH Dist AS (
@@ -619,6 +781,7 @@ const getTransferStudentsForProgrammeCourses = async (from, to, programmeCourses
 }
 
 const getStudentsWithoutStudyrightForProgrammeCourses = async (from, to, programmeCourses) => {
+  if (!programmeCourses.length) return []
   const res = await sequelize.query(
     `
     WITH Dist AS (
@@ -732,6 +895,15 @@ const getCurrentStudyYearStartDate = _.memoize(
     )
 )
 
+const getProgrammeName = async code => {
+  return ProgrammeModule.findOne({
+    attributes: ['name'],
+    where: {
+      code,
+    },
+  })
+}
+
 module.exports = {
   studytrackStudents,
   enrolledStudents,
@@ -739,13 +911,13 @@ module.exports = {
   allStudyrights,
   startedStudyrights,
   graduatedStudyRights,
+  graduatedStudyRightsByStartDate,
   inactiveStudyrights,
   previousStudyrights,
   followingStudyrights,
   transfersAway,
   transfersTo,
   allTransfers,
-  getProgrammesStudyrights,
   getCreditsForStudyProgramme,
   getTransferredCredits,
   getThesisCredits,
@@ -758,4 +930,6 @@ module.exports = {
   getStudyRights,
   getCourseCodesForStudyProgramme,
   getTransferStudentsForProgrammeCourses,
+  getProgrammeName,
+  getNotCompletedForProgrammeCourses,
 }

services/backend/src/services/studyprogrammeBasics.js

@@ -12,38 +12,39 @@ const {
 const { graduatedStudyRights, startedStudyrights, transfersAway, transfersTo } = require('./studyprogramme')
 const { getYearStartAndEndDates } = require('../util/semester')
 
-const getStartedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+const getStartedStats = async ({ studyprogramme, years, isAcademicYear }) => {
   const { graphStats, tableStats } = getStatsBasis(years)
 
-  await Promise.all(
-    years.map(async year => {
-      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
-      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
-        codes: [studyprogramme],
-        startDate,
-        endDate,
-        includeAllSpecials,
-      })
-      const studyrights = await startedStudyrights(studyprogramme, since, studentnumbersOfTheYear)
-
-      studyrights.forEach(({ studystartdate }) => {
-        const startYear = defineYear(studystartdate, isAcademicYear)
-        graphStats[indexOf(years, startYear)] += 1
-        tableStats[startYear] += 1
-      })
+  for (const year of years) {
+    const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+    // Include not transferred to, but the transferred out students are fine as said in info box
+    const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+      codes: [studyprogramme],
+      startDate,
+      endDate,
+      includeAllSpecials: true,
+      includeTransferredTo: false,
+      includeGraduated: true,
     })
-  )
+    const studyrights = await startedStudyrights(studyprogramme, startDate, studentnumbersOfTheYear)
+    studyrights.forEach(({ studystartdate }) => {
+      const startYear = defineYear(studystartdate, isAcademicYear)
+      graphStats[indexOf(years, startYear)] += 1
+      tableStats[startYear] += 1
+    })
+  }
   return { graphStats, tableStats }
 }
 
 const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
   const { graphStats, tableStats } = getStatsBasis(years)
-
+  if (!studyprogramme) return { graphStats, tableStats }
   const studentnumbersOfTheYear = await getCorrectStudentnumbers({
     codes: [studyprogramme],
     startDate: alltimeStartDate,
     endDate: alltimeEndDate,
     includeAllSpecials,
+    includeTransferredTo: includeAllSpecials,
   })
 
   const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
@@ -62,8 +63,9 @@ const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear,
   return { graphStats, tableStats }
 }
 
-const getTransferredAwayStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+const getTransferredAwayStats = async ({ studyprogramme, since, years, isAcademicYear, combinedProgramme }) => {
   const studyrights = await transfersAway(studyprogramme, since)
+  const secondStudyrights = combinedProgramme ? await transfersAway(combinedProgramme, since) : []
   const { graphStats, tableStats } = getStatsBasis(years)
 
   studyrights.forEach(({ transferdate }) => {
@@ -72,11 +74,18 @@ const getTransferredAwayStats = async ({ studyprogramme, since, years, isAcademi
     tableStats[transferYear] += 1
   })
 
+  secondStudyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
   return { graphStats, tableStats }
 }
 
-const getTransferredToStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+const getTransferredToStats = async ({ studyprogramme, since, years, isAcademicYear, combinedProgramme }) => {
   const studyrights = await transfersTo(studyprogramme, since)
+  const secondStudyrights = combinedProgramme ? await transfersTo(combinedProgramme, since) : []
   const { graphStats, tableStats } = getStatsBasis(years)
 
   studyrights.forEach(({ transferdate }) => {
@@ -85,66 +94,163 @@ const getTransferredToStats = async ({ studyprogramme, since, years, isAcademicY
     tableStats[transferYear] += 1
   })
 
+  secondStudyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
   return { graphStats, tableStats }
 }
 
-const getBasicStatsForStudytrack = async ({ studyprogramme, settings }) => {
+const initializeGraphStats = (
+  includeAllSpecials,
+  combinedProgramme,
+  started,
+  startedSecondProg,
+  graduated,
+  transferredAway,
+  transferredTo,
+  graduatedSecondProg
+) => {
+  let basicTable = []
+  if (combinedProgramme) {
+    basicTable = [
+      {
+        name: 'Started bachelor',
+        data: started.graphStats,
+      },
+      {
+        name: 'Started licentiate',
+        data: startedSecondProg.graphStats,
+      },
+      {
+        name: 'Graduated bachelor',
+        data: graduated.graphStats,
+      },
+      {
+        name: 'Graduated licentiate',
+        data: graduatedSecondProg.graphStats,
+      },
+    ]
+  } else {
+    basicTable = [
+      {
+        name: 'Started',
+        data: started.graphStats,
+      },
+      {
+        name: 'Graduated',
+        data: graduated.graphStats,
+      },
+    ]
+  }
+
+  if (includeAllSpecials) {
+    basicTable.push({
+      name: 'Transferred away',
+      data: transferredAway.graphStats,
+    })
+    basicTable.push({
+      name: 'Transferred To',
+      data: transferredTo.graphStats,
+    })
+  }
+  return basicTable
+}
+
+const initializeTableStats = (
+  year,
+  combinedProgramme,
+  includeAllSpecials,
+  started,
+  startedSecondProg,
+  graduated,
+  graduatedSecondProg,
+  transferredAway,
+  transferredTo
+) => {
+  if (includeAllSpecials && combinedProgramme) {
+    return [
+      year,
+      started.tableStats[year],
+      startedSecondProg.tableStats[year],
+      graduated.tableStats[year],
+      graduatedSecondProg.tableStats[year],
+      transferredAway.tableStats[year],
+      transferredTo.tableStats[year],
+    ]
+  }
+  if (combinedProgramme) {
+    return [
+      year,
+      started.tableStats[year],
+      startedSecondProg.tableStats[year],
+      graduated.tableStats[year],
+      graduatedSecondProg.tableStats[year],
+    ]
+  }
+  if (includeAllSpecials) {
+    return [
+      year,
+      started.tableStats[year],
+      graduated.tableStats[year],
+      transferredAway.tableStats[year],
+      transferredTo.tableStats[year],
+    ]
+  }
+  return [year, started.tableStats[year], graduated.tableStats[year]]
+}
+
+const getBasicStatsForStudytrack = async ({ studyprogramme, combinedProgramme, settings }) => {
   const { includeAllSpecials, isAcademicYear } = settings
   const since = getStartDate(studyprogramme, isAcademicYear)
   const years = getYearsArray(since.getFullYear(), isAcademicYear)
-
-  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials, combinedProgramme }
+  const queryParametersCombinedProg = {
+    studyprogramme: combinedProgramme,
+    since,
+    years,
+    isAcademicYear,
+    includeAllSpecials,
+  }
   const started = await getStartedStats(queryParameters)
+  const startedSecondProg = await getStartedStats(queryParametersCombinedProg)
   const graduated = await getGraduatedStats(queryParameters)
+  const graduatedSecondProg = await getGraduatedStats(queryParametersCombinedProg)
   const transferredAway = await getTransferredAwayStats(queryParameters)
   const transferredTo = await getTransferredToStats(queryParameters)
 
   const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const titles = tableTitles['basics'][includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED']
+  const key = includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED'
+  const titles = tableTitles['basics'][combinedProgramme ? `${key}_COMBINED_PROGRAMME` : key]
   const tableStats = reversedYears.map(year =>
-    includeAllSpecials
-      ? [
-          year,
-          started.tableStats[year],
-          graduated.tableStats[year],
-          transferredAway.tableStats[year],
-          transferredTo.tableStats[year],
-        ]
-      : [year, started.tableStats[year], graduated.tableStats[year]]
+    initializeTableStats(
+      year,
+      combinedProgramme,
+      includeAllSpecials,
+      started,
+      startedSecondProg,
+      graduated,
+      graduatedSecondProg,
+      transferredAway,
+      transferredTo
+    )
   )
 
   return {
-    id: studyprogramme,
+    id: combinedProgramme ? `${studyprogramme}-${combinedProgramme}` : studyprogramme,
     years,
-    graphStats: includeAllSpecials
-      ? [
-          {
-            name: 'Started',
-            data: started.graphStats,
-          },
-          {
-            name: 'Graduated',
-            data: graduated.graphStats,
-          },
-          {
-            name: 'Transferred away',
-            data: transferredAway.graphStats,
-          },
-          {
-            name: 'Transferred To',
-            data: transferredTo.graphStats,
-          },
-        ]
-      : [
-          {
-            name: 'Started',
-            data: started.graphStats,
-          },
-          {
-            name: 'Graduated',
-            data: graduated.graphStats,
-          },
-        ],
+    graphStats: initializeGraphStats(
+      includeAllSpecials,
+      combinedProgramme,
+      started,
+      startedSecondProg,
+      graduated,
+      transferredAway,
+      transferredTo,
+      graduatedSecondProg
+    ),
     tableStats,
     titles,
   }

services/backend/src/services/studyprogrammeCourses.js

@@ -4,6 +4,7 @@ const {
   getStudentsForProgrammeCourses,
   getCurrentStudyYearStartDate,
   getOwnStudentsForProgrammeCourses,
+  getNotCompletedForProgrammeCourses,
   getStudentsWithoutStudyrightForProgrammeCourses,
   getTransferStudentsForProgrammeCourses,
   getOtherStudentsForProgrammeCourses,
@@ -33,15 +34,18 @@ const makeYearlyPromises = (years, academicYear, type, programmeCourses, studypr
   return years.map(
     year =>
       new Promise(async res => {
-        const from = academicYear == 'true' ? new Date(year, 7, 1, 0, 0, 0) : new Date(year, 0, 1, 0, 0, 0)
-        const to = academicYear == 'true' ? new Date(year + 1, 6, 31, 23, 59, 59) : new Date(year, 11, 31, 23, 59, 59)
-
+        const from = academicYear === 'ACADEMIC_YEAR' ? new Date(year, 7, 1, 0, 0, 0) : new Date(year, 0, 1, 0, 0, 0)
+        const to =
+          academicYear === 'ACADEMIC_YEAR' ? new Date(year + 1, 6, 31, 23, 59, 59) : new Date(year, 11, 31, 23, 59, 59)
         let result = null
 
         switch (type) {
-          case 'allStudents':
+          case 'passed':
             result = await getStudentsForProgrammeCourses(from, to, programmeCourses)
             break
+          case 'notCompleted':
+            result = await getNotCompletedForProgrammeCourses(from, to, programmeCourses)
+            break
           case 'ownStudents':
             result = await getOwnStudentsForProgrammeCourses(from, to, programmeCourses, studyprogramme)
             break
@@ -68,13 +72,22 @@ const makeYearlyPromises = (years, academicYear, type, programmeCourses, studypr
   )
 }
 
-const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme, academicYear) => {
-  const startDate = academicYear == 'true' ? await getCurrentStudyYearStartDate(unixMillis) : getCurrentYearStartDate()
+const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme, academicYear, combinedProgramme) => {
+  const startDate =
+    academicYear === 'ACADEMIC_YEAR' ? await getCurrentStudyYearStartDate(unixMillis) : getCurrentYearStartDate()
   const startYear = startDate.getFullYear()
   const yearRange = _.range(2017, startYear + 1)
-  const programmeCourses = await getAllStudyprogrammeCourses(studyprogramme)
+  const mainProgrammeCourses = await getAllStudyprogrammeCourses(studyprogramme)
+  const secondProgrammeCourses = combinedProgramme ? await getAllStudyprogrammeCourses(combinedProgramme) : []
+  const programmeCourses = [...mainProgrammeCourses, ...secondProgrammeCourses]
 
-  const yearlyStudentByCoursePromises = makeYearlyPromises(yearRange, academicYear, 'allStudents', programmeCourses)
+  const yearlyPassedStudentByCoursePromises = makeYearlyPromises(yearRange, academicYear, 'passed', programmeCourses)
+  const yearlyNotCompletedStudentByCoursePromises = makeYearlyPromises(
+    yearRange,
+    academicYear,
+    'notCompleted',
+    programmeCourses
+  )
   const yearlyProgrammeStudentsPromises = makeYearlyPromises(
     yearRange,
     academicYear,
@@ -95,7 +108,6 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
     programmeCourses,
     studyprogramme
   )
-
   const yearlyTransferStudentsPromises = makeYearlyPromises(
     yearRange,
     academicYear,
@@ -105,22 +117,24 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
   )
 
   const [
-    yearlyStudentByCourse,
+    yearlyPassedStudentByCourse,
+    yearlyNotCompletedStudentByCourse,
     yearlyProgrammeStudents,
     yearlyStudentsWithoutStudyright,
     yearlyOtherProgrammeStudents,
     yearlyTransferStudents,
   ] = await Promise.all([
-    Promise.all(yearlyStudentByCoursePromises),
+    Promise.all(yearlyPassedStudentByCoursePromises),
+    Promise.all(yearlyNotCompletedStudentByCoursePromises),
     Promise.all(yearlyProgrammeStudentsPromises),
     Promise.all(yearlyStudentsWithoutStudyrightPromises),
     Promise.all(yearlyOtherProgrammeStudentsPromises),
     Promise.all(yearlyTransferStudentsPromises),
   ])
-
   let maxYear = 0
   const allCourses = [
-    ...yearlyStudentByCourse.flat(),
+    ...yearlyPassedStudentByCourse.flat(),
+    ...yearlyNotCompletedStudentByCourse.flat(),
     ...yearlyProgrammeStudents.flat(),
     ...yearlyStudentsWithoutStudyright.flat(),
     ...yearlyOtherProgrammeStudents.flat(),
@@ -134,9 +148,12 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
         years: {},
       }
     }
+
     if (!acc[curr.code]['years'][curr.year]) {
       acc[curr.code]['years'][curr.year] = {
         totalAllStudents: 0,
+        totalPassed: 0,
+        totalNotCompleted: 0,
         totalAllCredits: 0,
         totalProgrammeStudents: 0,
         totalProgrammeCredits: 0,
@@ -149,10 +166,16 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
       }
     }
     switch (curr.type) {
-      case 'total':
-        acc[curr.code]['years'][curr.year]['totalAllStudents'] += curr.totalAllStudents
+      case 'passed':
+        acc[curr.code]['years'][curr.year]['totalPassed'] += curr.totalPassed
+        acc[curr.code]['years'][curr.year]['totalAllStudents'] += acc[curr.code]['years'][curr.year]['totalPassed']
         acc[curr.code]['years'][curr.year]['totalAllCredits'] += curr.totalAllcredits
         break
+      case 'notCompleted':
+        acc[curr.code]['years'][curr.year]['totalNotCompleted'] += curr.totalNotCompleted
+        acc[curr.code]['years'][curr.year]['totalAllStudents'] +=
+          acc[curr.code]['years'][curr.year]['totalNotCompleted']
+        break
       case 'ownProgramme':
         acc[curr.code]['years'][curr.year]['totalProgrammeStudents'] += curr.totalProgrammeStudents
         acc[curr.code]['years'][curr.year]['totalProgrammeCredits'] += curr.totalProgrammeCredits
@@ -175,6 +198,8 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
   const ayCourses = Object.keys(allCourses).filter(c => c.startsWith('AY'))
   const properties = [
     'totalAllStudents',
+    'totalPassed',
+    'totalNotCompleted',
     'totalAllCredits',
     'totalProgrammeStudents',
     'totalProgrammeCredits',
@@ -200,6 +225,8 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
           if (!allCourses[normCode]['years'][year]) {
             mergedCourse['years'][year] = {
               totalAllStudents: 0,
+              totalPassed: 0,
+              totalNotCompleted: 0,
               totalAllCredits: 0,
               totalProgrammeStudents: 0,
               totalProgrammeCredits: 0,

services/backend/src/services/studyprogrammeCredits.js

@@ -19,97 +19,28 @@ const {
   getCourseCodesForStudyProgramme,
 } = require('./studyprogramme')
 
-// Fetches all the credits for the studyprogramme and divides them into major-students and non-major students credits
-// Division is done on the basis that whether the student had a primary studyright to the programme on the attainment_date
-// If special groups are excluded, the transfer students are filtered away from the major-students as well
-const getRegularCreditStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
-  const providercode = mapToProviders([studyprogramme])[0]
-  const courses = await getCourseCodesForStudyProgramme(providercode)
-  const credits = await getCreditsForStudyProgramme(courses, since)
-  const students = [...new Set(credits.map(({ student_studentnumber }) => student_studentnumber))]
-
-  //let studyrights = await getProgrammesStudyrights(studyprogramme, since)
-  let studyrights = await getStudyRights(students, since)
-  const transfers = (await allTransfers(studyprogramme, since)).map(t => t.studyrightid)
-  if (!includeAllSpecials) {
-    studyrights = studyrights.filter(s => !transfers.includes(s.studyrightid))
-  }
-
-  let majors = getStatsBasis(years)
-  let nonMajors = getStatsBasis(years)
-  let nonDegree = getStatsBasis(years)
-
-  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
-    const studentStudyrights = studyrights.filter(studyright => studyright.studentnumber === student_studentnumber)
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-
-    if (!includeAllSpecials && isSpecialGroupCredit(studentStudyrights, attainment_date, transfers)) {
-      return
-    }
-
-    if (isMajorStudentCredit(studentStudyrights, attainment_date, studyprogramme)) {
-      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
-      majors.tableStats[attainmentYear] += credits || 0
-    } else if (isNonMajorCredit(studentStudyrights, attainment_date, studyprogramme)) {
-      nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
-      nonMajors.tableStats[attainmentYear] += credits || 0
-    } else {
-      nonDegree.graphStats[indexOf(years, attainmentYear)] += credits || 0
-      nonDegree.tableStats[attainmentYear] += credits || 0
-    }
-  })
-
-  if (
-    majors.graphStats.every(year => year === 0) &&
-    nonMajors.graphStats.every(year => year === 0) &&
-    nonDegree.graphStats.every(year => year === 0)
-  ) {
-    return {
-      majors: { graphStats: [], tableStats: {} },
-      nonMajors: { graphStats: [], tableStats: {} },
-      nonDegree: { graphStats: [], tablestats: {} },
-    }
-  }
-
-  return { majors, nonMajors, nonDegree }
+const createGraphStats = (majors, nonMajors, nonDegree, transferred) => {
+  return [
+    {
+      name: 'Major students credits',
+      data: majors.graphStats,
+    },
+    {
+      name: 'Non-major students credits',
+      data: nonMajors.graphStats,
+    },
+    {
+      name: 'Non-degree credits',
+      data: nonDegree.graphStats,
+    },
+    {
+      name: 'Transferred credits',
+      data: transferred.graphStats,
+    },
+  ]
 }
 
-// Fetches all the credits with the type "transferred / hyväksiluettu" and divides them by year
-const getTransferredCreditStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
-  const providercode = mapToProviders([studyprogramme])[0]
-  const credits = await getTransferredCredits(providercode, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  credits.forEach(({ attainment_date, credits }) => {
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-    graphStats[indexOf(years, attainmentYear)] += credits || 0
-    tableStats[attainmentYear] += credits || 0
-  })
-
-  if (graphStats.every(year => year === 0)) {
-    return { graphStats: [], tableStats: {} }
-  }
-
-  return { graphStats, tableStats }
-}
-
-// Fetches all credits for the studytrack and combines them into the statistics for the table
-// and graph in the studyprogramme overview
-const getCreditStatsForStudytrack = async ({ studyprogramme, settings }) => {
-  const { isAcademicYear, includeAllSpecials } = settings
-  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-
-  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
-  const { majors, nonMajors, nonDegree } = await getRegularCreditStats(queryParameters)
-  const transferred = await getTransferredCreditStats(queryParameters)
-
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const titles = tableTitles['credits'][includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED']
-  const dataFound = [majors, nonMajors, transferred].some(d => d.graphStats.length)
-
-  if (!dataFound) return null
-
+const createTableStats = (reversedYears, includeAllSpecials, majors, nonMajors, transferred, nonDegree) => {
   const tableStats = reversedYears.map(year =>
     includeAllSpecials
       ? [
@@ -133,31 +64,116 @@ const getCreditStatsForStudytrack = async ({ studyprogramme, settings }) => {
           transferred.tableStats[year],
         ]
   )
+  return tableStats
+}
 
-  const graphStats = [
-    {
-      name: 'Major students credits',
-      data: majors.graphStats,
-    },
-    {
-      name: 'Non-major students credits',
-      data: nonMajors.graphStats,
-    },
-    {
-      name: 'Non-degree credits',
-      data: nonDegree.graphStats,
-    },
-    {
-      name: 'Transferred credits',
-      data: transferred.graphStats,
-    },
-  ]
+// Fetches all the credits for the studyprogramme and divides them into major-students and non-major students credits
+// Division is done on the basis that whether the student had a primary studyright to the programme on the attainmentDate
+// If special groups are excluded, the transfer students are filtered away from the major-students as well
+const getRegularCreditStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const courses = await getCourseCodesForStudyProgramme(providercode)
+  const credits = await getCreditsForStudyProgramme(courses, since)
+  const students = [...new Set(credits.map(({ studentNumber }) => studentNumber))]
 
-  return {
-    id: studyprogramme,
+  let studyrights = await getStudyRights(students, since)
+  const transfers = (await allTransfers(studyprogramme, since)).map(t => t.studyrightid)
+  if (!includeAllSpecials) {
+    studyrights = studyrights.filter(s => !transfers.includes(s.studyrightid))
+  }
+
+  let majors = getStatsBasis(years)
+  let nonMajors = getStatsBasis(years)
+  let nonDegree = getStatsBasis(years)
+  credits.forEach(({ studentNumber, attainmentDate, credits }) => {
+    const studentStudyrights = studyrights.filter(studyright => studyright.studentNumber === studentNumber)
+    const attainmentYear = defineYear(attainmentDate, isAcademicYear)
+
+    if (!includeAllSpecials && isSpecialGroupCredit(studentStudyrights, attainmentDate, transfers)) {
+      return
+    }
+
+    if (isMajorStudentCredit(studentStudyrights, attainmentDate, studyprogramme)) {
+      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      majors.tableStats[attainmentYear] += credits || 0
+    } else if (isNonMajorCredit(studentStudyrights, attainmentDate, studyprogramme)) {
+      nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      nonMajors.tableStats[attainmentYear] += credits || 0
+    } else {
+      nonDegree.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      nonDegree.tableStats[attainmentYear] += credits || 0
+    }
+  })
+
+  return { majors, nonMajors, nonDegree }
+}
+
+// Fetches all the credits with the type "transferred / hyväksiluettu" and divides them by year
+const getTransferredCreditStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const credits = await getTransferredCredits(providercode, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  credits.forEach(({ attainment_date, credits }) => {
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+    graphStats[indexOf(years, attainmentYear)] += credits || 0
+    tableStats[attainmentYear] += credits || 0
+  })
+
+  return { graphStats, tableStats }
+}
+
+// Fetches all credits for the studytrack and combines them into the statistics for the table
+// and graph in the studyprogramme overview
+const getCreditStatsForStudytrack = async ({ studyprogramme, combinedProgramme, settings }) => {
+  const { isAcademicYear, includeAllSpecials } = settings
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const queryParametersSecondProg = {
+    studyprogramme: combinedProgramme,
+    since,
     years,
-    tableStats,
-    graphStats,
+    isAcademicYear,
+    includeAllSpecials,
+  }
+  const { majors, nonMajors, nonDegree } = await getRegularCreditStats(queryParameters)
+  const transferred = await getTransferredCreditStats(queryParameters)
+
+  const {
+    majors: majorsSecondProg,
+    nonMajors: nonMajorsSecondProg,
+    nonDegree: nonDegreeSecondProg,
+  } = await getRegularCreditStats(queryParametersSecondProg)
+  const transferredSecondProg = await getTransferredCreditStats(queryParametersSecondProg)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const titles = tableTitles['credits'][includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED']
+  const dataFound = [majors, nonMajors, transferred].some(d => d.graphStats.length)
+
+  if (!dataFound) return null
+  const dataFoundSecondProg = [majorsSecondProg, nonMajorsSecondProg, transferredSecondProg].some(
+    d => d.graphStats.length
+  )
+  return {
+    id: combinedProgramme ? `${studyprogramme}-${combinedProgramme}` : studyprogramme,
+    years,
+    tableStats: createTableStats(reversedYears, includeAllSpecials, majors, nonMajors, transferred, nonDegree),
+    graphStats: createGraphStats(majors, nonMajors, nonDegree, transferred),
+    tableStatsSecondProg: dataFoundSecondProg
+      ? createTableStats(
+          reversedYears,
+          includeAllSpecials,
+          majorsSecondProg,
+          nonMajorsSecondProg,
+          transferredSecondProg,
+          nonDegreeSecondProg
+        )
+      : [],
+    graphStatsSecondProg: dataFoundSecondProg
+      ? createGraphStats(majorsSecondProg, nonMajorsSecondProg, nonDegreeSecondProg, transferredSecondProg)
+      : [],
     titles,
   }
 }

services/backend/src/services/studyprogrammeGraduations.js

@@ -41,7 +41,7 @@ const addGraduation = async (studyright, isAcademicYear, amounts, times) => {
   const graduationYear = defineYear(studyright.enddate, isAcademicYear)
 
   const totalTimeToGraduation = moment(studyright.enddate).diff(moment(startdate), 'months')
-  const statutoryAbsences = await getStatutoryAbsences(studyright.studentnumber, startdate, studyright.enddate)
+  const statutoryAbsences = await getStatutoryAbsences(studyright.studentNumber, startdate, studyright.enddate)
   const timeToGraduation = totalTimeToGraduation - statutoryAbsences
 
   amounts[graduationYear] += 1
@@ -50,12 +50,13 @@ const addGraduation = async (studyright, isAcademicYear, amounts, times) => {
 
 const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
   const { graphStats, tableStats } = getStatsBasis(years)
-
+  if (!studyprogramme) return { graphStats, tableStats }
   const studentnumbers = await getCorrectStudentnumbers({
     codes: [studyprogramme],
     startDate: alltimeStartDate,
     endDate: alltimeEndDate,
     includeAllSpecials,
+    includeTransferredTo: includeAllSpecials,
   })
 
   const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbers)
@@ -75,15 +76,17 @@ const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear,
 }
 
 const getThesisStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+  if (!studyprogramme) return { graphStats, tableStats }
   const providercode = mapToProviders([studyprogramme])[0]
   const thesisType = getThesisType(studyprogramme)
-  const { graphStats, tableStats } = getStatsBasis(years)
 
   const studentnumbers = await getCorrectStudentnumbers({
     codes: [studyprogramme],
     startDate: alltimeStartDate,
     endDate: alltimeEndDate,
     includeAllSpecials,
+    includeTransferredTo: includeAllSpecials,
   })
 
   const credits = await getThesisCredits(providercode, since, thesisType, studentnumbers)
@@ -100,6 +103,7 @@ const getThesisStats = async ({ studyprogramme, since, years, isAcademicYear, in
 const getGraduationTimeStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
   let graduationAmounts = getYearsObject({ years })
   let graduationTimes = getYearsObject({ years, emptyArrays: true })
+  if (!studyprogramme) return { times: { medians: [], goal: 0 }, doCombo: false, comboTimes: { medians: [], goal: 0 } }
   // for bc+ms combo
   const doCombo = studyprogramme.startsWith('MH') && !['MH30_001', 'MH30_003'].includes(studyprogramme)
   let graduationAmountsCombo = getYearsObject({ years })
@@ -110,6 +114,7 @@ const getGraduationTimeStats = async ({ studyprogramme, since, years, isAcademic
     startDate: alltimeStartDate,
     endDate: alltimeEndDate,
     includeAllSpecials,
+    includeTransferredTo: includeAllSpecials,
   })
 
   const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbers)
@@ -167,7 +172,7 @@ const formatStats = (stats, years) => {
 
 const getProgrammesBeforeStarting = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
   const all = await allStudyrights(studyprogramme)
-  const allStudents = all.map(s => s.studentnumber)
+  const allStudents = all.map(s => s.studentNumber)
 
   // Get studyrights for bachelor studyprogrammes. This excludes studytracks.
   const programmes = await getAllProgrammes()
@@ -179,8 +184,8 @@ const getProgrammesBeforeStarting = async ({ studyprogramme, since, years, isAca
   // Map transfers to the Master's programme
   const transfers = await getTransferStudyrightMap(studyprogramme, since)
 
-  studyrightsBeforeMasters.forEach(({ studentnumber, code, givendate }) => {
-    const masterStudyright = all.find(s => s.studentnumber === studentnumber)
+  studyrightsBeforeMasters.forEach(({ studentNumber, code, givendate }) => {
+    const masterStudyright = all.find(s => s.studentNumber === studentNumber)
     const transfer = transfers.get(masterStudyright.studyrightid)
 
     // If special studyrights are excluded, transfers to the master's programme should be excluded
@@ -205,7 +210,7 @@ const getProgrammesBeforeStarting = async ({ studyprogramme, since, years, isAca
 
 const getProgrammesAfterGraduation = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
   const graduated = await graduatedStudyRights(studyprogramme, since)
-  const graduatedStudents = graduated.map(s => s.studentnumber)
+  const graduatedStudents = graduated.map(s => s.studentNumber)
 
   // Get studyrights for masters studyprogramme. This excludes studytracks.
   const programmes = await getAllProgrammes()
@@ -216,8 +221,8 @@ const getProgrammesAfterGraduation = async ({ studyprogramme, since, years, isAc
 
   const transfers = await getTransferStudyrightMap(studyprogramme, since)
 
-  studyrightsAfterThisProgramme.forEach(({ studentnumber, studystartdate, code, givendate }) => {
-    const bachelorStudyright = graduated.find(s => s.studentnumber === studentnumber)
+  studyrightsAfterThisProgramme.forEach(({ studentNumber, studystartdate, code, givendate }) => {
+    const bachelorStudyright = graduated.find(s => s.studentNumber === studentNumber)
 
     // If special studyrights are excluded, transfers to the bachelor programme should be excluded
     if (!includeAllSpecials && transfers.get(bachelorStudyright.studyrightid)) {
@@ -244,17 +249,26 @@ const getProgrammesBeforeOrAfter = async (studyprogramme, queryParameters) => {
   return null
 }
 
-const getGraduationStatsForStudytrack = async ({ studyprogramme, settings }) => {
+const getGraduationStatsForStudytrack = async ({ studyprogramme, combinedProgramme, settings }) => {
   const { isAcademicYear, includeAllSpecials } = settings
   const since = getStartDate(studyprogramme, isAcademicYear)
   const years = getYearsArray(since.getFullYear(), isAcademicYear)
-
   const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const combinedQueryParameters = {
+    studyprogramme: combinedProgramme,
+    since,
+    years,
+    isAcademicYear,
+    includeAllSpecials,
+  }
   const thesis = await getThesisStats(queryParameters)
+  const thesisSecondProgramme = await getThesisStats(combinedQueryParameters)
 
   const graduated = await getGraduatedStats(queryParameters)
+  const graduatedSecondProgramme = await getGraduatedStats(combinedQueryParameters)
 
   const graduationTimeStats = await getGraduationTimeStats(queryParameters)
+  const graduationTimeStatsSecondProg = await getGraduationTimeStats(combinedQueryParameters)
 
   const programmesBeforeOrAfter = await getProgrammesBeforeOrAfter(studyprogramme, queryParameters)
 
@@ -264,17 +278,64 @@ const getGraduationStatsForStudytrack = async ({ studyprogramme, settings }) =>
     studyprogramme.includes('LIS') || studyprogramme.includes('T')
       ? ['', 'Graduated']
       : ['', 'Graduated', 'Wrote thesis']
+  const combinedTitles = [
+    '',
+    'Graduated bachelor',
+    'Wrote thesis bachelor',
+    'Graduated licentiate',
+    'Wrote thesis licentiate',
+  ]
   const programmesBeforeOrAfterTitles = ['Code', 'Id', 'Programme', ...years]
+  const tableStatsDefault = combinedProgramme
+    ? reversedYears.map(year => [
+        year,
+        graduated.tableStats[year],
+        thesis.tableStats[year],
+        graduatedSecondProgramme.tableStats[year],
+        thesisSecondProgramme.tableStats[year],
+      ])
+    : reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
+
   const tableStats =
     studyprogramme.includes('LIS') || studyprogramme.includes('T')
       ? reversedYears.map(year => [year, graduated.tableStats[year]])
-      : reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
+      : tableStatsDefault
+
+  const graphStats = combinedProgramme
+    ? [
+        {
+          name: 'Graduated bachelor',
+          data: graduated.graphStats,
+        },
+        {
+          name: 'Wrote thesis bachelor',
+          data: thesis.graphStats,
+        },
+        {
+          name: 'Graduated licentiate',
+          data: graduatedSecondProgramme.graphStats,
+        },
+        {
+          name: 'Wrote thesis licentiate',
+          data: thesisSecondProgramme.graphStats,
+        },
+      ]
+    : [
+        {
+          name: 'Graduated students',
+          data: graduated.graphStats,
+        },
+        {
+          name: 'Wrote thesis',
+          data: thesis.graphStats,
+        },
+      ]
 
   return {
-    id: studyprogramme,
+    id: combinedProgramme ? `${studyprogramme}-${combinedProgramme}` : studyprogramme,
     years,
     tableStats,
-    titles,
+    titles: combinedProgramme ? combinedTitles : titles,
     graphStats:
       studyprogramme.includes('LIS') || studyprogramme.includes('T')
         ? [
@@ -283,19 +344,11 @@ const getGraduationStatsForStudytrack = async ({ studyprogramme, settings }) =>
               data: graduated.graphStats,
             },
           ]
-        : [
-            {
-              name: 'Graduated students',
-              data: graduated.graphStats,
-            },
-            {
-              name: 'Wrote thesis',
-              data: thesis.graphStats,
-            },
-          ],
+        : graphStats,
     graduationTimes: graduationTimeStats.times,
     doCombo: graduationTimeStats.doCombo,
     comboTimes: graduationTimeStats.comboTimes,
+    graduationTimesSecondProgramme: graduationTimeStatsSecondProg.comboTimes,
     programmesBeforeOrAfterTableStats: programmesBeforeOrAfter?.tableStats,
     programmesBeforeOrAfterGraphStats: programmesBeforeOrAfter?.graphStats,
     programmesBeforeOrAfterTitles,

services/backend/src/services/studyprogrammeHelpers.js

@@ -2,36 +2,36 @@ const { studentnumbersWithAllStudyrightElements } = require('./populations')
 const { codes } = require('../../config/programmeCodes')
 
 // Helper functions
-const getCorrectStudentnumbers = async ({ codes, startDate, endDate, includeAllSpecials, includeGraduated = true }) => {
+const getCorrectStudentnumbers = async ({
+  codes,
+  startDate,
+  endDate,
+  includeAllSpecials,
+  includeTransferredTo,
+  includeGraduated = true,
+}) => {
   let studentnumbers = []
   const exchangeStudents = includeAllSpecials
   const nondegreeStudents = includeAllSpecials
   const transferredOutStudents = includeAllSpecials
-  const transferredToStudents = !includeAllSpecials
-  const graduatedStudents = !includeGraduated
+  const transferredToStudents = includeTransferredTo
+  const graduatedStudents = includeGraduated
 
-  studentnumbers = await studentnumbersWithAllStudyrightElements(
-    codes,
+  studentnumbers = await studentnumbersWithAllStudyrightElements({
+    studyRights: codes,
     startDate,
     endDate,
     exchangeStudents,
     nondegreeStudents,
     transferredOutStudents,
-    null,
+    tag: null,
     transferredToStudents,
-    graduatedStudents
-  )
+    graduatedStudents,
+  })
 
   return studentnumbers
 }
 
-const defineStartDate = (studyright_elements, studystartdate) => {
-  if (studyright_elements?.length && studyright_elements[0].startdate > studystartdate) {
-    return studyright_elements[0].startdate
-  }
-  return studystartdate
-}
-
 const resolveStudyRightCode = studyright_elements => {
   if (!studyright_elements) return null
   const studyRightElement = studyright_elements
@@ -54,21 +54,25 @@ const formatStudyright = studyright => {
     extentcode,
     student,
     studyright_elements,
+    cancelled,
+    facultyCode,
   } = studyright
 
   return {
     studyrightid,
     startdate,
-    studystartdate: defineStartDate(studyright_elements, studystartdate),
+    studystartdate,
     enddate,
     givendate,
     graduated,
     active,
     prioritycode,
     extentcode,
-    studentnumber: student.studentnumber,
+    studentNumber: student.studentnumber,
     code: resolveStudyRightCode(studyright_elements),
     studyrightElements: studyright_elements,
+    cancelled,
+    facultyCode,
     name:
       studyright_elements?.length && studyright_elements[0].element_detail && studyright_elements[0].element_detail.name
         ? studyright_elements[0].element_detail.name
@@ -79,14 +83,26 @@ const formatStudyright = studyright => {
 const formatStudent = student => {
   const { studentnumber, gender_code, home_country_en, creditcount, credits } = student
   return {
-    studentnumber,
-    gender_code,
-    home_country_en,
+    studentNumber: studentnumber,
+    genderCode: gender_code,
+    homeCountryEn: home_country_en,
     creditcount,
     credits,
   }
 }
 
+const formatCredit = credit => {
+  const { student_studentnumber, course_code, credits, attainment_date } = credit
+  const code = course_code.replace('AY', '')
+  return {
+    id: `${student_studentnumber}-${code}`, // For getting unique credits for each course code and student number
+    studentNumber: student_studentnumber,
+    courseCode: code,
+    credits,
+    attainmentDate: attainment_date,
+  }
+}
+
 const formatTransfer = transfer => {
   const { sourcecode, targetcode, transferdate, studyrightid } = transfer
   return {
@@ -101,7 +117,9 @@ const getYearsArray = (since, isAcademicYear, yearsCombined) => {
   const years = []
   const allYears = 'Total'
   if (yearsCombined) years.push(allYears)
-  for (let i = since; i <= new Date().getFullYear(); i++) {
+  const today = new Date()
+  const until = isAcademicYear && today.getMonth() < 7 ? today.getFullYear() - 1 : today.getFullYear()
+  for (let i = since; i <= until; i++) {
     const year = isAcademicYear ? `${i} - ${i + 1}` : i
     years.push(year)
   }
@@ -130,38 +148,43 @@ const getStatsBasis = years => {
     tableStats: getYearsObject({ years }),
   }
 }
+// Take studystartdate for master students with Bachelor-Master studyright.
+const getCorrectStartDate = studyright => {
+  return studyright.studyrightid.slice(-2) === '-2' && studyright.extentcode === 2
+    ? studyright.studystartdate
+    : studyright.startdate
+}
 
-const isMajorStudentCredit = (studyrights, attainment_date, code) =>
+const isMajorStudentCredit = (studyrights, attainmentDate, code) =>
   studyrights.some(studyright => {
     if (!studyright) return false
     if (studyright.code !== code) return false
-    if (!studyright.graduated) return new Date(attainment_date) >= new Date(studyright.studystartdate)
-    return (
-      new Date(attainment_date) >= new Date(studyright.studystartdate) &&
-      new Date(attainment_date) <= new Date(studyright.enddate)
-    )
+    if ([6, 7, 9, 13, 14, 16, 18, 22, 23, 34, 99].includes(studyright.extentcode)) return false
+    const startDate = getCorrectStartDate(studyright)
+    if (!studyright.graduated) return new Date(attainmentDate) >= new Date(startDate)
+    return new Date(attainmentDate) >= new Date(startDate) && new Date(attainmentDate) <= new Date(studyright.enddate)
   })
 
-const isNonMajorCredit = (studyrights, attainment_date) =>
-  studyrights.some(studyright => {
+const isNonMajorCredit = (studyrights, attainmentDate) => {
+  return studyrights.some(studyright => {
     if (!studyright) return false
-    if (!studyright.graduated) return new Date(attainment_date) >= new Date(studyright.studystartdate)
-    return (
-      new Date(attainment_date) >= new Date(studyright.studystartdate) &&
-      new Date(attainment_date) <= new Date(studyright.enddate)
-    )
+    if ([6, 7, 9, 13, 14, 16, 18, 22, 23, 34, 99].includes(studyright.extentcode)) return false
+    const startDate = getCorrectStartDate(studyright)
+    if (!studyright.graduated) return new Date(attainmentDate) >= new Date(startDate)
+    return new Date(attainmentDate) >= new Date(startDate) && new Date(attainmentDate) <= new Date(studyright.enddate)
   })
+}
 
-const isSpecialGroupCredit = (studyrights, attainment_date, transfers) => {
+const isSpecialGroupCredit = (studyrights, attainment_date, transfers) =>
   studyrights.some(studyright => {
     if (!studyright) return true // If there is no studyright matching the credit, is not a major student credit
-    if (studyright.studystartdate > attainment_date) return true // Credits before the studyright started are not major student credits
-    if (studyright.enddate && attainment_date > studyright.enddate) return true // Credits after studyright are not major student credits
-    if ([7, 9, 16, 34, 22, 99, 14, 13].includes(studyright.extentcode)) return true // Excludes non-degree studyrights and exchange students
+    const startDate = getCorrectStartDate(studyright)
+    if (new Date(startDate) > new Date(attainment_date)) return true // Credits before the studyright started are not major student credits
+    if (studyright.enddate && new Date(attainment_date) > new Date(studyright.enddate)) return true // Credits after studyright are not major student credits
+    if ([6, 7, 9, 13, 14, 16, 18, 22, 23, 34, 99].includes(studyright.extentcode)) return true // Excludes non-degree studyrights and exchange students
     if (transfers.includes(studyright.studyrightid)) return true // Excludes both transfers in and out of the programme
     return false
   })
-}
 
 const getMedian = values => {
   if (values.length === 0) return 0
@@ -175,7 +198,9 @@ const defineYear = (date, isAcademicYear) => {
   if (!date) return ''
   const year = date.getFullYear()
   if (!isAcademicYear) return year
-  if (date < new Date(`${year}-07-31`)) return `${year - 1} - ${year}`
+  // Some dates are given in utc time in database, some not.
+  if (new Date(date).toISOString() < new Date(`${year}-07-31T21:00:00.000Z`).toISOString())
+    return `${year - 1} - ${year}`
   return `${year} - ${year + 1}`
 }
 
@@ -189,6 +214,9 @@ const getStartDate = (studyprogramme, isAcademicYear) => {
 const alltimeStartDate = new Date('1900-01-01')
 const alltimeEndDate = new Date()
 
+// In the object programmes should be {bachelorCode: masterCode}
+const combinedStudyprogrammes = { KH90_001: 'MH90_001' }
+
 // There are 9 course_unit_types
 // 1. urn:code:course-unit-type:regular
 // 2. urn:code:course-unit-type:bachelors-thesis
@@ -255,7 +283,7 @@ const getBachelorCreditGraphStats = years => ({
   },
 })
 
-const getMasterCreditGraphStats = years => ({
+const getBachelorMasterCreditGraphStats = years => ({
   lte200: {
     name: 'Less than 200 credits',
     data: getEmptyArray(years.length),
@@ -285,6 +313,7 @@ const getMasterCreditGraphStats = years => ({
     data: getEmptyArray(years.length),
   },
 })
+
 const getVetenaryCreditGraphStats = years => ({
   lte210: {
     name: 'Less than 210 credits',
@@ -316,7 +345,42 @@ const getVetenaryCreditGraphStats = years => ({
   },
 })
 
-const getOnlyMasterCreditGraphStats = years => ({
+const getCombinedVetenaryCreditGraphStats = years => ({
+  lte30: {
+    name: 'Less than 30 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte60: {
+    name: '30-59 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte120: {
+    name: '60-179 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte180: {
+    name: '180-239 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte240: {
+    name: '240-299 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte300: {
+    name: '300-359 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte360: {
+    name: '300-359 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte360: {
+    name: '360 or more credits',
+    data: getEmptyArray(years.length),
+  },
+})
+
+const getMasterCreditGraphStats = years => ({
   lte15: {
     name: 'Less than 15 credits',
     data: getEmptyArray(years.length),
@@ -366,30 +430,34 @@ const getDoctoralCreditGraphStats = years => ({
   },
 })
 
-const getCreditGraphStats = (studyprogramme, years, studyprog = false) => {
+const getCreditGraphStats = (studyprogramme, years, combinedVetenary, masterStudyRightOnly = false) => {
+  if (combinedVetenary) return getCombinedVetenaryCreditGraphStats(years)
   if (studyprogramme.includes('KH')) return getBachelorCreditGraphStats(years)
   if (studyprogramme === 'MH90_001')
-    return studyprog ? getBachelorCreditGraphStats(years) : getVetenaryCreditGraphStats(years)
-  if (studyprogramme.includes('MH')) return getOnlyMasterCreditGraphStats(years)
+    return masterStudyRightOnly ? getBachelorCreditGraphStats(years) : getVetenaryCreditGraphStats(years)
+  if (studyprogramme.includes('MH'))
+    return masterStudyRightOnly ? getMasterCreditGraphStats(years) : getBachelorMasterCreditGraphStats(years)
   return getDoctoralCreditGraphStats(years)
 }
 
 const bachelorCreditThresholds = ['lte15', 'lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'lte180', 'mte180']
-const masterCreditThresholds = ['lte200', 'lte220', 'lte240', 'lte260', 'lte280', 'lte300', 'mte300']
-const onlyMasterCreditThresholds = ['lte15', 'lte30', 'lte60', 'lte90', 'lte120', 'mte120']
+const bachelorMasterCreditThresholds = ['lte200', 'lte220', 'lte240', 'lte260', 'lte280', 'lte300', 'mte300']
+const masterCreditThresholds = ['lte15', 'lte30', 'lte60', 'lte90', 'lte120', 'mte120']
 const doctoralCreditThresholds = ['lte10', 'lte20', 'lte30', 'lte40', 'mte40']
 const bachelorCreditAmounts = [15, 30, 60, 90, 120, 150, 180, 180]
-const masterCreditAmounts = [200, 220, 240, 260, 280, 300, 300]
-const onlyMasterCreditAmounts = [15, 30, 60, 90, 120, 120]
+const bachelorMasterCreditAmounts = [200, 220, 240, 260, 280, 300, 300]
+const masterCreditAmounts = [15, 30, 60, 90, 120, 120]
 const doctoralCreditAmounts = [10, 20, 30, 40, 40]
 const vetenaryCreditThresholds = ['lte210', 'lte240', 'lte270', 'lte300', 'lte330', 'lte360', 'mte360']
 const vetenaryCreditAmounts = [210, 240, 270, 300, 330, 360, 360]
-const getBcMsThresholds = () => {
-  return { creditThresholdKeysBcMs: masterCreditThresholds, creditThresholdAmountsBcMs: masterCreditAmounts }
-}
-const getCreditThresholds = (studyprogramme, studyProg = false) => {
+const combinedVetenaryThresholds = ['lte30', 'lte60', 'lte120', 'lte180', 'lte240', 'lte300', 'lte360', 'mte360']
+const combinedVetenaryAmounts = [30, 60, 120, 180, 240, 300, 360, 360]
+
+const getCreditThresholds = (studyprogramme, combinedVetenary, masterStudyRightOnly = false) => {
+  if (combinedVetenary)
+    return { creditThresholdKeys: combinedVetenaryThresholds, creditThresholdAmounts: combinedVetenaryAmounts }
   if (studyprogramme === 'MH90_001') {
-    return studyProg
+    return masterStudyRightOnly
       ? { creditThresholdKeys: bachelorCreditThresholds, creditThresholdAmounts: bachelorCreditAmounts }
       : { creditThresholdKeys: vetenaryCreditThresholds, creditThresholdAmounts: vetenaryCreditAmounts }
   }
@@ -397,7 +465,9 @@ const getCreditThresholds = (studyprogramme, studyProg = false) => {
     return { creditThresholdKeys: bachelorCreditThresholds, creditThresholdAmounts: bachelorCreditAmounts }
   }
   if (studyprogramme.includes('MH')) {
-    return { creditThresholdKeys: onlyMasterCreditThresholds, creditThresholdAmounts: onlyMasterCreditAmounts }
+    return masterStudyRightOnly
+      ? { creditThresholdKeys: masterCreditThresholds, creditThresholdAmounts: masterCreditAmounts }
+      : { creditThresholdKeys: bachelorMasterCreditThresholds, creditThresholdAmounts: bachelorMasterCreditAmounts }
   }
   return { creditThresholdKeys: doctoralCreditThresholds, creditThresholdAmounts: doctoralCreditAmounts }
 }
@@ -406,6 +476,22 @@ const tableTitles = {
   basics: {
     SPECIAL_EXCLUDED: ['', 'Started studying', 'Graduated'],
     SPECIAL_INCLUDED: ['', 'Started studying', 'Graduated', 'Transferred away', 'Transferred to'],
+    SPECIAL_EXCLUDED_COMBINED_PROGRAMME: [
+      '',
+      'Started studying bachelor',
+      'Started studying licentiate',
+      'Graduated bachelor',
+      'Graduated licentiate',
+    ],
+    SPECIAL_INCLUDED_COMBINED_PROGRAMME: [
+      '',
+      'Started studying bachelor',
+      'Started studying licentiate',
+      'Graduated bachelor',
+      'Graduated licentiate',
+      'Transferred away',
+      'Transferred to',
+    ],
   },
   credits: {
     SPECIAL_EXCLUDED: ['', 'Total', 'Major students credits', 'Transferred credits'],
@@ -431,7 +517,7 @@ const tableTitles = {
       '150-179 credits',
       '180 ≤ credits',
     ],
-    master: [
+    bachelorMaster: [
       '',
       'All',
       '< 200 credits',
@@ -442,7 +528,7 @@ const tableTitles = {
       '280-299 credits',
       '300 ≤ credits',
     ],
-    masterOnly: [
+    master: [
       '',
       'All',
       '< 15 credits',
@@ -452,39 +538,34 @@ const tableTitles = {
       '90-119 credits',
       '120 ≤ credits',
     ],
+    combinedVetenary: [
+      '',
+      'All',
+      '< 30 credits',
+      '30-59 credits',
+      '60-119 credits',
+      '120-179 credits',
+      '180-239 credits',
+      '240-299 credits',
+      '300-359 credits',
+      '360 ≤ credits',
+    ],
     doctoral: ['', 'All', '< 10 credits', '10-19 credits', '20-29 credits', '30-39 credits', '40 ≤ credits'],
   },
-  studytracks: [
-    '',
-    'All',
-    'Started studying',
-    'Currently enrolled',
-    'Absent',
-    'Inactive',
-    'Graduated',
-    'Men',
-    'Women',
-    'Finland',
-  ],
-  programmes: [
-    '',
-    'All',
-    'Started studying',
-    'Currently enrolled',
-    'Absent',
-    'Inactive',
-    'Graduated',
-    'Men',
-    'Women',
-    'Other/Unknown',
-    'Finland',
-    'Other',
-  ],
+  studytracksStart: ['', 'All', 'Started studying', 'Currently enrolled', 'Absent', 'Inactive'],
+  studytracksBasic: ['Graduated'],
+  studytracksCombined: {
+    licentiate: ['Graduated bachelor', 'Graduated licentiate'],
+    master: ['Graduated bachelor', 'Graduated master'],
+  },
+  studytracksEnd: ['Men', 'Women', 'Other/\nUnknown', 'Finland', 'Other'],
 }
 
-const getCreditProgressTableTitles = studyprogramme => {
+const getCreditProgressTableTitles = (studyprogramme, combinedVetenaryProgramme, masterProgrammeOnly = false) => {
+  if (combinedVetenaryProgramme) return tableTitles.creditProgress.combinedVetenary
   if (studyprogramme.includes('KH')) return tableTitles.creditProgress.bachelor
-  if (studyprogramme.includes('MH')) return tableTitles.creditProgress.masterOnly
+  if (studyprogramme.includes('MH'))
+    return masterProgrammeOnly ? tableTitles.creditProgress.master : tableTitles.creditProgress.bachelorMaster
   return tableTitles.creditProgress.doctoral
 }
 
@@ -508,10 +589,12 @@ const getId = code => {
 }
 
 const getGoal = programme => {
+  if (!programme) return 0
   if (programme.startsWith('KH') || programme.endsWith('-ba')) {
     return 36
   }
   if (programme.startsWith('MH') || programme.endsWith('-ma')) {
+    if (programme === 'MH90_001') return 36 // vetenary programme's licentiate is 36 months.
     if (['MH30_004', '420420-ma'].includes(programme)) {
       return 24 + 6
     }
@@ -534,6 +617,7 @@ module.exports = {
   formatStudyright,
   formatStudent,
   formatTransfer,
+  formatCredit,
   getYearsArray,
   getYearsObject,
   getAcademicYearsObject,
@@ -545,18 +629,18 @@ module.exports = {
   getStartDate,
   alltimeStartDate,
   alltimeEndDate,
+  combinedStudyprogrammes,
   getThesisType,
   getPercentage,
   getBachelorCreditGraphStats,
+  getBachelorMasterCreditGraphStats,
   getMasterCreditGraphStats,
   getVetenaryCreditGraphStats,
   getDoctoralCreditGraphStats,
-  getOnlyMasterCreditGraphStats,
   getCreditGraphStats,
   getCreditThresholds,
   tableTitles,
   getCreditProgressTableTitles,
-  getBcMsThresholds,
   isNonMajorCredit,
   mapCodesToIds,
   getId,

services/backend/src/services/studyprogrammeUpdates.js

@@ -10,7 +10,7 @@ const {
 } = require('../services/analyticsService')
 const logger = require('../util/logger')
 
-const updateBasicView = async code => {
+const updateBasicView = async (code, combinedProgramme) => {
   const specialCalendar = {
     yearType: 'CALENDAR_YEAR',
     specialGroups: 'SPECIAL_INCLUDED',
@@ -34,6 +34,7 @@ const updateBasicView = async code => {
     try {
       const basicStats = await getBasicStatsForStudytrack({
         studyprogramme: code,
+        combinedProgramme,
         settings: {
           isAcademicYear: option.yearType === 'ACADEMIC_YEAR',
           includeAllSpecials: option.specialGroups === 'SPECIAL_INCLUDED',
@@ -43,6 +44,7 @@ const updateBasicView = async code => {
 
       const creditStats = await getCreditStatsForStudytrack({
         studyprogramme: code,
+        combinedProgramme,
         settings: {
           isAcademicYear: option.yearType === 'ACADEMIC_YEAR',
           includeAllSpecials: option.specialGroups === 'SPECIAL_INCLUDED',
@@ -52,6 +54,7 @@ const updateBasicView = async code => {
 
       const graduationStats = await getGraduationStatsForStudytrack({
         studyprogramme: code,
+        combinedProgramme,
         settings: {
           isAcademicYear: option.yearType === 'ACADEMIC_YEAR',
           includeAllSpecials: option.specialGroups === 'SPECIAL_INCLUDED',
@@ -59,13 +62,13 @@ const updateBasicView = async code => {
       })
       await setGraduationStats(graduationStats, option.yearType, option.specialGroups)
     } catch (e) {
-      logger.error(e)
+      logger.error(`Studytrack graduation stats failed: ${e}`)
     }
   }
   return 'OK'
 }
 
-const updateStudytrackView = async code => {
+const updateStudytrackView = async (code, combinedProgramme, associations) => {
   const specialGraduated = {
     graduated: 'GRADUATED_INCLUDED',
     specialGroups: 'SPECIAL_INCLUDED',
@@ -94,14 +97,16 @@ const updateStudytrackView = async code => {
     try {
       const stats = await getStudytrackStatsForStudyprogramme({
         studyprogramme: code,
+        combinedProgramme,
         settings: {
           specialGroups: option.specialGroups === 'SPECIAL_INCLUDED',
           graduated: option.graduated === 'GRADUATED_INCLUDED',
         },
+        associations,
       })
       await setStudytrackStats(stats, option.graduated, option.specialGroups)
     } catch (e) {
-      logger.error(e)
+      logger.error(`Studytrack stats update failed ${e}`)
     }
   }
 

services/backend/src/services/studyrights.js

@@ -6,7 +6,6 @@ const moment = require('moment')
 const { redisClient } = require('../services/redis')
 const _ = require('lodash')
 const { Op, col, where, fn } = require('sequelize')
-
 const REDIS_KEY = 'STUDYRIGHT_ASSOCIATIONS_V2'
 
 const byStudent = studentNumber => {
@@ -166,6 +165,9 @@ const associatedStudyrightElements = async (offset, limit) => {
         attributes: ['type', 'name', 'code'],
       },
     },
+    order: [['studyrightid', 'DESC']],
+    // ^Use order when chunking with offset & limit,
+    // otherwise results are random and there will be misses
     limit,
     offset,
   })
@@ -186,7 +188,6 @@ const StudyRightType = {
 }
 
 const calculateAssociationsFromDb = async (chunksize = 100000) => {
-  // bottlenecked by Studyright.findAll in associatedStudyrightElements()
   const transfers = await Transfer.findAll({
     attributes: ['studyrightid'],
   })
@@ -227,9 +228,10 @@ const calculateAssociationsFromDb = async (chunksize = 100000) => {
           const momentstartdate = moment(startdate)
           const enrollment = momentstartdate.isValid() ? getEnrollmentStartYear(momentstartdate) : null
           const enrollmentStartYears = associations.programmes[code].enrollmentStartYears
-          enrollmentStartYears[enrollment] = enrollmentStartYears[enrollment] || {
-            studyTracks: {},
-          }
+          if (!enrollmentStartYears[enrollment])
+            enrollmentStartYears[enrollment] = {
+              studyTracks: {},
+            }
           const enrollmentStartYear = enrollmentStartYears[enrollment]
 
           group
@@ -276,6 +278,7 @@ const calculateAssociationsFromDb = async (chunksize = 100000) => {
     })
     offset += chunksize
   }
+
   return associations
 }
 

services/backend/src/services/studytrackStats.js

@@ -1,7 +1,6 @@
 const { indexOf } = require('lodash')
 const moment = require('moment')
 
-const { getAssociations } = require('./studyrights')
 const {
   getMedian,
   getStartDate,
@@ -22,22 +21,32 @@ const {
   inactiveStudyrights,
   enrolledStudents,
   absentStudents,
+  graduatedStudyRightsByStartDate,
   graduatedStudyRights,
 } = require('./studyprogramme')
 const { getAcademicYearDates } = require('../util/semester')
-const { countTimeCategories } = require('./graduationHelpers')
+const { countTimeCategories, getStatutoryAbsences } = require('./graduationHelpers')
 
+const getUnique = studentnumbers => [...new Set(studentnumbers)]
 const getStudentData = (startDate, students, thresholdKeys, thresholdAmounts) => {
-  let data = { female: 0, male: 0, finnish: 0 }
+  let data = { female: 0, male: 0, otherUnkown: 0, finnish: 0, otherCountries: 0, otherCountriesCounts: {} }
   thresholdKeys.forEach(t => (data[t] = 0))
-  students.forEach(({ gender_code, home_country_en, credits }) => {
+  students.forEach(({ genderCode, homeCountryEn, credits }) => {
     const creditcount = credits
       .filter(credit => moment(credit.attainment_date).isAfter(startDate))
       .reduce((prev, curr) => (prev += curr.credits), 0)
 
-    data.male += gender_code === '1' ? 1 : 0
-    data.female += gender_code === '2' ? 1 : 0
-    data.finnish += home_country_en === 'Finland' ? 1 : 0
+    data.male += genderCode === '1' ? 1 : 0
+    data.female += genderCode === '2' ? 1 : 0
+    data.otherUnkown += ['0', '3'].includes(genderCode) ? 1 : 0
+    data.finnish += homeCountryEn === 'Finland' ? 1 : 0
+    data.otherCountries += homeCountryEn !== 'Finland' ? 1 : 0
+    if (homeCountryEn !== 'Finland') {
+      if (!(homeCountryEn in data.otherCountriesCounts)) {
+        data.otherCountriesCounts[homeCountryEn] = 0
+      }
+      data.otherCountriesCounts[homeCountryEn] += 1
+    }
     data[thresholdKeys[0]] += creditcount < thresholdAmounts[0] ? 1 : 0
     data[thresholdKeys[1]] += creditcount >= thresholdAmounts[0] && creditcount < thresholdAmounts[1] ? 1 : 0
     data[thresholdKeys[2]] += creditcount >= thresholdAmounts[1] && creditcount < thresholdAmounts[2] ? 1 : 0
@@ -60,33 +69,90 @@ const getStudentData = (startDate, students, thresholdKeys, thresholdAmounts) =>
   })
   return data
 }
+// type: either Bc + Ms combo or Bc/Ms/T/anything else
+const addGraduation = async ({ studentNumber, startdate, enddate, graduationAmounts, times, track, type, year }) => {
+  const totalTimeToGraduation = moment(enddate).diff(moment(startdate), 'months')
+  const statutoryAbsences = await getStatutoryAbsences(studentNumber, startdate, enddate)
+  const timeToGraduation = totalTimeToGraduation - statutoryAbsences
+  graduationAmounts[track][type][year] += 1
+  times[type] = [...times[type], timeToGraduation]
+}
 
-const getGraduationTimeStats = async ({ year, graduated, track, graduationAmounts, graduationTimes, classSize }) => {
+const getGraduationTimeStats = async ({
+  year,
+  graduated,
+  track,
+  graduationAmounts,
+  graduationTimes,
+  classSize,
+  doCombo,
+}) => {
   if (year === 'Total') {
     return
   }
-
   // Count how long each student took to graduate
-  let times = []
-  graduated.forEach(({ enddate, startdate }) => {
-    const timeToGraduation = moment(enddate).diff(moment(startdate), 'months')
-    graduationAmounts[track][year] += 1
-    times = [...times, timeToGraduation]
-  })
+  // Separate Bc + Ms and other
+  const times = { combo: [], basic: [] }
+  for (const { enddate, startdate, studyrightid, studentNumber } of graduated) {
+    if (doCombo && studyrightid.slice(-2) === '-2') {
+      await addGraduation({
+        studentNumber,
+        startdate,
+        enddate,
+        graduationAmounts,
+        times,
+        track,
+        type: 'combo',
+        year,
+      })
+    } else {
+      await addGraduation({
+        studentNumber,
+        startdate,
+        enddate,
+        graduationAmounts,
+        times,
+        track,
+        type: 'basic',
+        year,
+      })
+    }
+  }
 
-  const median = getMedian(times)
-  const statistics = countTimeCategories(times, graduationTimes.goal)
+  const median = getMedian(times.basic)
+  const medianCombo = getMedian(times.combo)
+  const statistics = countTimeCategories(times.basic, graduationTimes.goals.basic)
+  const statisticsCombo = countTimeCategories(times.combo, graduationTimes.goals.combo)
+  graduationTimes[track].medians.combo = [
+    ...graduationTimes[track].medians.combo,
+    {
+      y: medianCombo,
+      amount: graduationAmounts[track]['combo'][year],
+      name: year,
+      statistics: statisticsCombo,
+      classSize,
+    },
+  ]
 
-  graduationTimes[track].medians = [
-    ...graduationTimes[track].medians,
-    { y: median, amount: graduationAmounts[track][year], name: year, statistics, classSize },
+  graduationTimes[track].medians.basic = [
+    ...graduationTimes[track].medians.basic,
+    { y: median, amount: graduationAmounts[track]['basic'][year], name: year, statistics, classSize },
   ]
 }
 
-const getStats = (all, started, enrolled, absent, inactive, graduated, studentData) => {
-  return [
+const getStats = (
+  all,
+  started,
+  enrolled,
+  absent,
+  inactive,
+  graduated,
+  graduatedSecondProg,
+  studentData,
+  combinedProgramme
+) => {
+  const beginTablestats = [
     all.length,
-    getPercentage(all.length, all.length),
     started.length,
     getPercentage(started.length, all.length),
     enrolled.length,
@@ -97,18 +163,29 @@ const getStats = (all, started, enrolled, absent, inactive, graduated, studentDa
     getPercentage(inactive.length, all.length),
     graduated.length,
     getPercentage(graduated.length, all.length),
+  ]
+  const combinedTableStats = combinedProgramme
+    ? [graduatedSecondProg.length, getPercentage(graduatedSecondProg.length, all.length)]
+    : []
+  const endTableStats = [
     studentData.male,
     getPercentage(studentData.male, all.length),
     studentData.female,
     getPercentage(studentData.female, all.length),
+    studentData.otherUnkown,
+    getPercentage(studentData.otherUnkown, all.length),
     studentData.finnish,
     getPercentage(studentData.finnish, all.length),
+    studentData.otherCountries,
+    getPercentage(studentData.otherCountries, all.length),
   ]
+  return [...beginTablestats, ...combinedTableStats, ...endTableStats]
 }
 
 // Goes through the programme and all its studytracks for the said year and adds the wanted stats to the data objects
 const getStudytrackDataForTheYear = async ({
   studyprogramme,
+  combinedProgramme,
   since,
   settings,
   studytracks,
@@ -118,6 +195,7 @@ const getStudytrackDataForTheYear = async ({
   data,
   creditThresholdKeys,
   creditThresholdAmounts,
+  doCombo,
 }) => {
   const { specialGroups: includeAllSpecials, graduated: includeGraduated } = settings
   const { startDate, endDate } = getAcademicYearDates(year, since)
@@ -129,9 +207,12 @@ const getStudytrackDataForTheYear = async ({
     creditTableStats,
     graduationAmounts,
     graduationTimes,
+    graduationTimesSecondProg,
+    graduationAmountsSecondProg,
     totalAmounts,
     emptyTracks,
     totals,
+    otherCountriesCount,
   } = data
 
   await Promise.all(
@@ -142,6 +223,16 @@ const getStudytrackDataForTheYear = async ({
         startDate,
         endDate,
         includeAllSpecials,
+        includeTransferredTo: includeAllSpecials,
+        includeGraduated,
+      })
+
+      const startedStudentnumbers = await getCorrectStudentnumbers({
+        codes,
+        startDate,
+        endDate,
+        includeAllSpecials,
+        includeTransferredTo: false,
         includeGraduated,
       })
       const students = await studytrackStudents(studentnumbers)
@@ -153,24 +244,54 @@ const getStudytrackDataForTheYear = async ({
       let absent = []
       let inactive = []
       let graduated = []
+      let graduatedSecondProg = []
+      let graduatedByStartdate = []
+      let graduatedByStartSecondProg = []
       // Get all the studyrights and students for the calculations
       if (year !== 'Total') {
         all = await allStudyrights(track, studentnumbers)
         studentData = getStudentData(startDate, students, creditThresholdKeys, creditThresholdAmounts)
-        started = await startedStudyrights(track, startDate, studentnumbers)
-        enrolled = await enrolledStudents(track, startDate, studentnumbers)
+        otherCountriesCount[track][year] = studentData.otherCountriesCounts
+        started = await startedStudyrights(track, startDate, startedStudentnumbers)
+        enrolled = await enrolledStudents(track, studentnumbers)
         absent = await absentStudents(track, studentnumbers)
         inactive = await inactiveStudyrights(track, studentnumbers)
         graduated = await graduatedStudyRights(track, startDate, studentnumbers)
+
+        const enrolledSecondProgramme = combinedProgramme
+          ? await enrolledStudents(combinedProgramme, studentnumbers)
+          : []
+        const absentSecondProgramme = combinedProgramme ? await absentStudents(combinedProgramme, studentnumbers) : []
+        const inactiveSecondProgramme = combinedProgramme
+          ? await inactiveStudyrights(combinedProgramme, studentnumbers)
+          : []
+        // Studentnumbers are fetched based on studystartdate, if it is greater than startdate
+        // Thus, computing the bc+ms graduated by startdate based on these studentnumbers does not work.
+        const academicEnddate = `${year.slice(-4)}-07-31T23:59:59`
+        graduatedByStartdate = await graduatedStudyRightsByStartDate(track, startDate, academicEnddate)
+        if (combinedProgramme) {
+          enrolled = getUnique([...enrolled, ...enrolledSecondProgramme].map(student => student.studentnumber))
+          absent = getUnique([...absent, ...absentSecondProgramme].map(student => student.studentnumber))
+          inactive = getUnique([...inactive, ...inactiveSecondProgramme].map(student => student.studentnumber))
+          graduatedSecondProg = await graduatedStudyRights(combinedProgramme, startDate, studentnumbers)
+          graduatedByStartSecondProg = await graduatedStudyRightsByStartDate(
+            combinedProgramme,
+            startDate,
+            academicEnddate
+          )
+        }
         totals[track].all = [...totals[track].all, ...all]
         totals[track].studentData.male += studentData.male
         totals[track].studentData.female += studentData.female
+        totals[track].studentData.otherUnkown += studentData.otherUnkown
         totals[track].studentData.finnish += studentData.finnish
+        totals[track].studentData.otherCountries += studentData.otherCountries
         totals[track].started = [...totals[track].started, ...started]
         totals[track].enrolled = [...totals[track].enrolled, ...enrolled]
         totals[track].absent = [...totals[track].absent, ...absent]
         totals[track].inactive = [...totals[track].inactive, ...inactive]
         totals[track].graduated = [...totals[track].graduated, ...graduated]
+        totals[track].graduatedSecondProg = [...totals[track].graduatedSecondProg, ...graduatedSecondProg]
       } else {
         all = totals[track].all
         studentData = totals[track].studentData
@@ -179,6 +300,20 @@ const getStudytrackDataForTheYear = async ({
         absent = totals[track].absent
         inactive = totals[track].inactive
         graduated = totals[track].graduated
+        graduatedSecondProg = totals[track].graduatedSecondProg
+        const countryStatsFromYears = years.reduce((acc, year) => {
+          if (year === 'Total') return acc
+          const countries = otherCountriesCount[track][year]
+          Object.keys(countries).forEach(country => {
+            if (!(country in acc)) {
+              acc[country] = 0
+            }
+            acc[country] += countries[country]
+          })
+          return acc
+        }, {})
+
+        otherCountriesCount[track]['Total'] = countryStatsFromYears
       }
 
       // If the track has no stats for that year, it should be removed from the table and dropdown options
@@ -189,12 +324,11 @@ const getStudytrackDataForTheYear = async ({
 
       if (year !== 'Total') {
         // Count stats for creditgraph for the year per track
-        creditThresholdKeys.forEach(
-          threshold => (creditGraphStats[track][threshold].data[indexOf(years, year)] += studentData[threshold])
-        )
-        creditThresholdKeys.forEach(
-          threshold => (creditGraphStats[track][threshold].data[indexOf(years, 'Total')] += studentData[threshold])
-        )
+        creditThresholdKeys.forEach(threshold => {
+          creditGraphStats[track][threshold].data[indexOf(years, year)] += studentData[threshold]
+          creditGraphStats[track][threshold].data[indexOf(years, 'Total')] += studentData[threshold]
+        })
+
         // Count stats for the credit progress table for the year per track
         creditTableStats[track] = [
           [year, all.length, ...creditThresholdKeys.map(threshold => studentData[threshold])],
@@ -207,10 +341,22 @@ const getStudytrackDataForTheYear = async ({
         )
         creditTableStats[track] = [[year, ...totalCredits.slice(1)], ...creditTableStats[track]]
       }
-
       // Count stats for the main studytrack table grouped by tracks
       mainStatsByTrack[track] = [
-        [year, ...getStats(all, started, enrolled, absent, inactive, graduated, studentData)],
+        [
+          year,
+          ...getStats(
+            all,
+            started,
+            enrolled,
+            absent,
+            inactive,
+            graduated,
+            graduatedSecondProg,
+            studentData,
+            combinedProgramme
+          ),
+        ],
         ...mainStatsByTrack[track],
       ]
 
@@ -218,30 +364,62 @@ const getStudytrackDataForTheYear = async ({
       mainStatsByYear[year] = [
         [
           studytrackNames[track]?.name['fi'] ? `${studytrackNames[track]?.name['fi']}, ${track}` : year,
-          ...getStats(all, started, enrolled, absent, inactive, graduated, studentData),
+          ...getStats(
+            all,
+            started,
+            enrolled,
+            absent,
+            inactive,
+            graduated,
+            graduatedSecondProg,
+            studentData,
+            combinedProgramme
+          ),
         ],
         ...mainStatsByYear[year],
       ]
 
       // Count stats for the graduation time charts grouped by year
       if (!(track in graduationTimes)) {
-        graduationTimes[track] = { medians: [] }
+        graduationTimes[track] = { medians: { basic: [], combo: [] } }
       }
       totalAmounts[track][year] = all.length
       await getGraduationTimeStats({
         year,
-        graduated,
+        graduated: graduatedByStartdate,
         track,
         graduationAmounts,
         graduationTimes,
         classSize: totalAmounts[track][year],
+        doCombo,
       })
+      if (combinedProgramme) {
+        await getGraduationTimeStats({
+          year,
+          graduated: graduatedByStartSecondProg,
+          track: combinedProgramme,
+          graduationAmounts: graduationAmountsSecondProg,
+          graduationTimes: graduationTimesSecondProg,
+          classSize: totalAmounts[studyprogramme][year],
+          doCombo: true,
+        })
+      }
     })
   )
+
   Object.keys(graduationTimes).forEach(track => {
-    if (track !== 'goal') {
-      const sortedMedians = graduationTimes[track].medians.sort((a, b) => b.name.localeCompare(a.name))
-      graduationTimes[track].medians = sortedMedians
+    if (track !== 'goals') {
+      const sortedMedians = graduationTimes[track].medians.basic.sort((a, b) => b.name.localeCompare(a.name))
+      const sortedMediansCombo = graduationTimes[track].medians.combo.sort((a, b) => b.name.localeCompare(a.name))
+      graduationTimes[track].medians.basic = sortedMedians
+      graduationTimes[track].medians.combo = sortedMediansCombo
+    }
+  })
+
+  Object.keys(graduationTimesSecondProg).forEach(track => {
+    if (track !== 'goals') {
+      const sortedMedians = graduationTimesSecondProg[track].medians.combo.sort((a, b) => b.name.localeCompare(a.name))
+      graduationTimesSecondProg[track].medians.combo = sortedMedians
     }
   })
 }
@@ -251,7 +429,7 @@ const getStudytrackDataForTheYear = async ({
 const getStudytrackOptions = (studyprogramme, studytrackNames, studytracks, emptyTracks, years) => {
   const names = { [studyprogramme]: 'All students of the programme' }
   studytracks.forEach(track => {
-    const trackName = studytrackNames[track]?.name['fi']
+    const trackName = studytrackNames[track]?.name
     if (trackName && emptyTracks.get(track) !== years.length) {
       names[track] = trackName
     }
@@ -260,21 +438,34 @@ const getStudytrackOptions = (studyprogramme, studytrackNames, studytracks, empt
 }
 
 // Creates empty objects for each statistic type, which are then updated with the studytrack data
-const getEmptyStatsObjects = (years, studytracks, studyprogramme) => {
+const getEmptyStatsObjects = (years, studytracks, studyprogramme, combinedProgramme) => {
   const mainStatsByYear = getYearsObject({ years, emptyArrays: true })
   const mainStatsByTrack = {}
   const creditGraphStats = {}
   const creditTableStats = {}
   const graduationAmounts = {}
-  const graduationTimes = { goal: getGoal(studyprogramme) }
+  const goal = getGoal(studyprogramme)
+  const goalSecondProg = getGoal(combinedProgramme)
+  const graduationTimes = { goals: { basic: goal, combo: goal + 36 } }
+  const graduationTimesSecondProg = {
+    [combinedProgramme]: { medians: { basic: [], combo: [] } },
+    goals: { basic: goal, combo: goal + goalSecondProg },
+  }
+  const otherCountriesCount = {}
+  const graduationAmountsSecondProg = {
+    [combinedProgramme]: { basic: getYearsObject({ years }), combo: getYearsObject({ years }) },
+  }
+
   const totalAmounts = {}
   const emptyTracks = new Map()
   const totals = {}
+
   studytracks.forEach(async track => {
+    otherCountriesCount[track] = {}
     mainStatsByTrack[track] = []
-    creditGraphStats[track] = getCreditGraphStats(studyprogramme, years, true)
+    creditGraphStats[track] = getCreditGraphStats(studyprogramme, years, combinedProgramme, true)
     creditTableStats[track] = []
-    graduationAmounts[track] = getYearsObject({ years })
+    graduationAmounts[track] = { basic: getYearsObject({ years }), combo: getYearsObject({ years }) }
     totalAmounts[track] = getYearsObject({ years })
     totals[track] = {
       all: [],
@@ -283,7 +474,8 @@ const getEmptyStatsObjects = (years, studytracks, studyprogramme) => {
       absent: [],
       inactive: [],
       graduated: [],
-      studentData: { male: 0, female: 0, finnish: 0 },
+      graduatedSecondProg: [],
+      studentData: { male: 0, female: 0, otherUnkown: 0, finnish: 0, otherCountries: 0 },
     }
   })
 
@@ -294,34 +486,42 @@ const getEmptyStatsObjects = (years, studytracks, studyprogramme) => {
     creditTableStats,
     graduationAmounts,
     graduationTimes,
+    graduationAmountsSecondProg,
+    graduationTimesSecondProg,
     totalAmounts,
     emptyTracks,
     totals,
+    otherCountriesCount,
   }
 }
 
 // Combines all the data for the Populations and Studytracks -view
-const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, settings }) => {
+// At the moment combined programme is thought to have only one track, the programme itself
+const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, combinedProgramme, settings, associations }) => {
   const isAcademicYear = true
   const includeYearsCombined = true
   const since = getStartDate(studyprogramme, isAcademicYear)
   const years = getYearsArray(since.getFullYear(), isAcademicYear, includeYearsCombined)
 
-  const associations = await getAssociations()
   const studytracks = associations.programmes[studyprogramme]
     ? [studyprogramme, ...associations.programmes[studyprogramme].studytracks]
     : [studyprogramme]
 
   const studytrackNames = associations.studyTracks
+  const data = getEmptyStatsObjects(years, studytracks, studyprogramme, combinedProgramme)
+  const onlyMasterStudyrights = true
 
-  const data = getEmptyStatsObjects(years, studytracks, studyprogramme)
-  const { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(studyprogramme, true)
-
+  const { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(
+    studyprogramme,
+    combinedProgramme,
+    onlyMasterStudyrights
+  )
+  const doCombo = studyprogramme.startsWith('MH') && !['MH30_001', 'MH30_003'].includes(studyprogramme)
   const yearsReversed = [...years].reverse()
-
   for (const year of yearsReversed) {
     await getStudytrackDataForTheYear({
       studyprogramme,
+      combinedProgramme,
       since,
       settings,
       studytracks,
@@ -331,21 +531,28 @@ const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, settings })
       data,
       creditThresholdKeys,
       creditThresholdAmounts,
+      doCombo,
     })
   }
 
+  const getCorrectCombinedTitles = () => {
+    if (['MH90_001'].includes(combinedProgramme)) return tableTitles.studytracksCombined.licentiate
+    return tableTitles.studytracksCombined.master
+  }
   const studytrackOptions = getStudytrackOptions(studyprogramme, studytrackNames, studytracks, data.emptyTracks, years)
+  const graduatedTitles = combinedProgramme ? getCorrectCombinedTitles() : tableTitles.studytracksBasic
   return {
-    id: studyprogramme,
+    id: combinedProgramme ? `${studyprogramme}-${combinedProgramme}` : studyprogramme,
     years: years,
     ...data,
+    doCombo,
     studytrackOptions,
     includeGraduated: settings.graduated,
-    populationTitles: tableTitles['studytracks'],
+    populationTitles: [...tableTitles.studytracksStart, ...graduatedTitles, ...tableTitles.studytracksEnd],
     creditTableTitles:
       studyprogramme === 'MH90_001'
         ? tableTitles['creditProgress']['bachelor']
-        : getCreditProgressTableTitles(studyprogramme),
+        : getCreditProgressTableTitles(studyprogramme, combinedProgramme, true),
   }
 }
 

services/backend/src/services/trends/statusAttainments.js

@@ -346,7 +346,7 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
     if (!regexresult) return code
     return regexresult[1]
   }
-  // console.log('yearly open uni acc credits: ', yearlyOpenUniAccCredits)
+
   const fromAyToNormalCourses = [
     ...new Set(_.flatten(yearlyOpenUniAccCredits).map(c => unifyOpenUniversity(c.code))),
     ...new Set(_.flatten(yearlyOpenUniTotalCredits).map(c => unifyOpenUniversity(c.code))),

services/backend/src/services/trends/statusGraduated.js

@@ -100,30 +100,33 @@ const calculateStatusGraduated = async (unixMillis, showByYear) => {
           Sentry.captureException(new Error(`Failed to find org for faculty code: ${curr.faculty_code}`))
         }
       }
-      if (!acc[curr.faculty_code]['yearly'][data.year]) {
-        acc[curr.faculty_code]['yearly'][data.year] = { acc: 0, total: 0 }
-      }
-      // init programme
-      if (!acc[curr.faculty_code]['drill'][curr.code]) {
-        const element = elements.find(e => e.code === curr.code)
-
-        acc[curr.faculty_code]['drill'][curr.code] = {
-          code: curr.code,
-          name: element.name,
-          current: 0,
-          previous: 0,
-          yearly: {},
+      // run following only if the faculty code is there
+      if (acc[curr.faculty_code]) {
+        if (!acc[curr.faculty_code]['yearly'][data.year]) {
+          acc[curr.faculty_code]['yearly'][data.year] = { acc: 0, total: 0 }
         }
-      }
-      acc[curr.faculty_code]['drill'][curr.code]['yearly'][data.year] = { acc: Number(curr.sum) }
-      acc[curr.faculty_code]['drill'][curr.code]['current'] =
-        _.get(acc, [curr.faculty_code, 'drill', curr.code, 'yearly', startYear, 'acc']) || 0
-      acc[curr.faculty_code]['drill'][curr.code]['previous'] =
-        _.get(acc, [curr.faculty_code, 'drill', curr.code, 'yearly', startYear - 1, 'acc']) || 0
+        // init programme
+        if (!acc[curr.faculty_code]['drill'][curr.code]) {
+          const element = elements.find(e => e.code === curr.code)
 
-      acc[curr.faculty_code]['yearly'][data.year].acc += Number(curr.sum)
-      acc[curr.faculty_code]['current'] = _.get(acc, [curr.faculty_code, 'yearly', startYear, 'acc']) || 0
-      acc[curr.faculty_code]['previous'] = _.get(acc, [curr.faculty_code, 'yearly', startYear - 1, 'acc']) || 0
+          acc[curr.faculty_code]['drill'][curr.code] = {
+            code: curr.code,
+            name: element.name,
+            current: 0,
+            previous: 0,
+            yearly: {},
+          }
+        }
+        acc[curr.faculty_code]['drill'][curr.code]['yearly'][data.year] = { acc: Number(curr.sum) }
+        acc[curr.faculty_code]['drill'][curr.code]['current'] =
+          _.get(acc, [curr.faculty_code, 'drill', curr.code, 'yearly', startYear, 'acc']) || 0
+        acc[curr.faculty_code]['drill'][curr.code]['previous'] =
+          _.get(acc, [curr.faculty_code, 'drill', curr.code, 'yearly', startYear - 1, 'acc']) || 0
+
+        acc[curr.faculty_code]['yearly'][data.year].acc += Number(curr.sum)
+        acc[curr.faculty_code]['current'] = _.get(acc, [curr.faculty_code, 'yearly', startYear, 'acc']) || 0
+        acc[curr.faculty_code]['previous'] = _.get(acc, [curr.faculty_code, 'yearly', startYear - 1, 'acc']) || 0
+      }
     })
     return acc
   }, {})
@@ -141,20 +144,22 @@ const calculateStatusGraduated = async (unixMillis, showByYear) => {
         }
       }
       // if no accumulated init programme
-      if (!mankeled[curr.faculty_code]['drill'][curr.code]) {
-        const element = elements.find(e => e.code === curr.code)
-        mankeled[curr.faculty_code]['drill'][curr.code] = { name: element.name, code: curr.code, yearly: {} }
-      }
-      // if no year in programme yearly add year
-      if (!mankeled[curr.faculty_code]['drill'][curr.code]['yearly'][data.year])
-        mankeled[curr.faculty_code]['drill'][curr.code]['yearly'][data.year] = { total: 0, acc: 0 }
-      // if no year in yearly total for faculty
-      if (!mankeled[curr.faculty_code]['yearly'][data.year])
-        mankeled[curr.faculty_code]['yearly'][data.year] = { total: 0, acc: 0 }
-      // do not add anything to total if current year. might fuck up in fall :D
-      if (data.year !== startYear) {
-        mankeled[curr.faculty_code]['yearly'][data.year].total += Number(curr.sum)
-        mankeled[curr.faculty_code]['drill'][curr.code]['yearly'][data.year].total = Number(curr.sum)
+      if (mankeled[curr.faculty_code]) {
+        if (!mankeled[curr.faculty_code]['drill'][curr.code]) {
+          const element = elements.find(e => e.code === curr.code)
+          mankeled[curr.faculty_code]['drill'][curr.code] = { name: element.name, code: curr.code, yearly: {} }
+        }
+        // if no year in programme yearly add year
+        if (!mankeled[curr.faculty_code]['drill'][curr.code]['yearly'][data.year])
+          mankeled[curr.faculty_code]['drill'][curr.code]['yearly'][data.year] = { total: 0, acc: 0 }
+        // if no year in yearly total for faculty
+        if (!mankeled[curr.faculty_code]['yearly'][data.year])
+          mankeled[curr.faculty_code]['yearly'][data.year] = { total: 0, acc: 0 }
+        // do not add anything to total if current year. might fuck up in fall :D
+        if (data.year !== startYear) {
+          mankeled[curr.faculty_code]['yearly'][data.year].total += Number(curr.sum)
+          mankeled[curr.faculty_code]['drill'][curr.code]['yearly'][data.year].total = Number(curr.sum)
+        }
       }
     })
   })