Refactor mocking under hook

Resolves #3242

.env.template

@@ -0,0 +1,4 @@
+# Populate and change this to .env
+IMPORTER_DB_API_TOKEN=
+IMPORTER_DB_API_URL=https://importer.cs.helsinki.fi
+PATE_TOKEN=

.eslintrc.json

@@ -8,17 +8,10 @@
   "parserOptions": {
     "requireConfigFile": false
   },
-  "extends": [
-    "eslint:recommended",
-    "plugin:prettier/recommended",
-    "plugin:cypress/recommended"
-  ],
-  "plugins": [
-    "prettier",
-    "cypress"
-  ],
+  "extends": ["eslint:recommended", "plugin:prettier/recommended", "plugin:cypress/recommended"],
+  "plugins": ["prettier", "cypress"],
   "rules": {
-    "no-console": "off",
+    "no-console": "error",
     "react/no-unescaped-entities": "off",
     "no-async-promise-executor": "off",
     "prettier/prettier": "error"

.github/workflows/release-oodikone-staging.yaml

@@ -1,143 +0,0 @@
-name: "Release oodikone to staging"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-oodikone-staging.yaml'
-      - 'services/**'
-
-jobs:
-
-  build_and_release_backend:
-    name: "Build and release backend"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
-            type=registry,ref=toska/oodikone2-backend:staging
-            type=registry,ref=toska/oodikone2-backend:latest
-          cache-to: type=inline
-          context: services/backend
-          push: true
-          tags: toska/oodikone2-backend:staging
-          target: production
-
-  build_and_release_frontend:
-    name: "Build and release frontend"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Get current time"
-        id: get_current_time
-        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-          cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
-            type=registry,ref=toska/oodikone2-frontend:staging
-            type=registry,ref=toska/oodikone2-frontend:latest
-          cache-to: type=inline
-          context: services/frontend
-          push: true
-          tags: toska/oodikone2-frontend:staging
-          target: production
-      - name: "Create a container from built image "
-        run: docker create --name frontend toska/oodikone2-frontend:staging
-      - name: "Copy assets from the container"
-        run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
-        uses: getsentry/action-release@v1
-        env:
-          SENTRY_URL: ${{ secrets.SENTRY_URL }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_ORG: toska
-          SENTRY_PROJECT: oodikone
-        with:
-          environment: staging
-          set_commits: 'skip'
-          sourcemaps: './build/static/js'
-          url_prefix: '~/oodikone/static/js'
-          version: ${{ github.sha }}
-
-  build_and_release_userservice:
-    name: "Build and release userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: true
-          tags: toska/oodikone2-userservice:staging
-          target: production
-
-  report_success:
-    runs-on: ubuntu-latest
-    name: "Report deployment success to oodikone notifications"
-    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
-    if: success()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment
-          softa-url: https://oodikone-staging.cs.helsinki.fi/
-          deployment-target: staging
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: staging

.github/workflows/release-oodikone.yaml

@@ -29,7 +29,6 @@ jobs:
             SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
             SENTRY_ENVIRONMENT=production
           cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
             type=registry,ref=toska/oodikone2-backend:staging
             type=registry,ref=toska/oodikone2-backend:latest
           cache-to: type=inline
@@ -66,7 +65,6 @@ jobs:
             SENTRY_ENVIRONMENT=production
             BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
             type=registry,ref=toska/oodikone2-frontend:latest
           cache-to: type=inline
@@ -103,11 +101,19 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Extract release version"
+        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
+        # This steps extracts release version from GITHUB_REF env var
+        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
+        id: extract_version
+        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
       - name: "Build and push"
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
+            SENTRY_ENVIRONMENT=production
           cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
             type=registry,ref=toska/oodikone2-userservice:staging
             type=registry,ref=toska/oodikone2-userservice:latest
           cache-to: type=inline

.github/workflows/release-sis-updater-scheduler.yaml

@@ -3,7 +3,7 @@ name: "Release sis-updater-scheduler"
 on:
   push:
     branches:
-      - trunk
+      - master
     paths:
       - '.github/workflows/release-sis-updater-scheduler.yaml'
       - 'updater/sis-updater-scheduler/**'

.github/workflows/release-sis-updater-worker.yaml

@@ -3,7 +3,7 @@ name: "Release sis-updater-worker"
 on:
   push:
     branches:
-      - trunk
+      - master
     paths:
       - '.github/workflows/release-sis-updater-worker.yaml'
       - 'updater/sis-updater-worker/**'

.github/workflows/test-oodikone.yaml

@@ -1,149 +1,161 @@
-name: "Test oodikone"
+name: 'Test oodikone and release to staging'
 
 on:
   push:
-    branches-ignore:
+    branches:
       - master
     paths:
-      - .github/workflows/test-oodikone.yaml
-      - cypress/**
-      - cypress.json
-      - services/**
-      - docker-compose.ci.yml
-      - nginx.conf
-      - package*.json
+      - '.github/workflows/test-oodikone.yaml'
+      - 'cypress/**'
+      - 'cypress.json'
+      - 'services/**'
+      - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
+      - 'nginx.conf'
+      - 'package*.json'
 
 jobs:
   # === Static tests / linting ===
 
   lint_backend:
-    name: "Lint backend"
+    name: 'Lint backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/backend && npm ci
-      - name: "Run backend linter"
+      - name: 'Run backend linter'
         run: npm run eslint services/backend
 
   lint_frontend:
-    name: "Lint frontend"
+    name: 'Lint frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/frontend && npm ci
-      - name: "Run frontend linter"
+      - name: 'Run frontend linter'
         run: npm run concurrently \
-              'eslint --ext .jsx,.js services/frontend' \
-              'stylelint services/frontend/src/**/*.css'
+          'eslint --ext .jsx,.js services/frontend' \
+          'stylelint services/frontend/src/**/*.css'
 
   lint_userservice:
-    name: "Lint userservice"
+    name: 'Lint userservice'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/userservice && npm ci
-      - name: "Run frontend linter"
+      - name: 'Run frontend linter'
         run: npm run eslint services/userservice
 
   # === Build images ===
 
   build_backend:
-    name: "Build backend"
+    name: 'Build backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
           cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
             type=registry,ref=toska/oodikone2-backend:staging
             type=registry,ref=toska/oodikone2-backend:latest
           cache-to: type=inline
           context: services/backend
           push: false
-          tags: toska/oodikone2-backend:trunk
+          tags: toska/oodikone2-backend:staging
           target: production
           outputs: type=docker,dest=/tmp/backend.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/backend.tar
 
   build_frontend:
-    name: "Build frontend"
+    name: 'Build frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Get current time'
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
             type=registry,ref=toska/oodikone2-frontend:latest
           cache-to: type=inline
           context: services/frontend
           push: false
-          tags: toska/oodikone2-frontend:trunk
+          tags: toska/oodikone2-frontend:staging
           target: production
           outputs: type=docker,dest=/tmp/frontend.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/frontend.tar
 
   build_userservice:
-    name: "Build userservice"
+    name: 'Build userservice'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
           cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
             type=registry,ref=toska/oodikone2-userservice:staging
             type=registry,ref=toska/oodikone2-userservice:latest
           cache-to: type=inline
           context: services/userservice
           push: false
-          tags: toska/oodikone2-userservice:trunk
+          tags: toska/oodikone2-userservice:staging
           target: production
           outputs: type=docker,dest=/tmp/userservice.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/userservice.tar
 
   # === Integration tests ===
+  #
   run_userservice_integration_tests:
-    name: "Run userservice integration tests"
+    name: 'Run userservice integration tests'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Run tests included in docker build"
+      - name: 'Run tests included in docker build'
         run: docker-compose --file docker-compose.test.yml up --build userservice-test
 
   # === End-to-end tests ===
 
   test_course_statistics:
-    name: "Test course statistics"
+    name: 'Test course statistics'
     needs:
       - build_backend
       - build_frontend
@@ -151,42 +163,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Course_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_custom_population:
-    name: "Test custom population"
+    name: 'Test custom population'
     needs:
       - build_backend
       - build_frontend
@@ -194,42 +206,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Custom_population.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_population_statistics:
-    name: "Test population statistics"
+    name: 'Test population statistics'
     needs:
       - build_backend
       - build_frontend
@@ -237,42 +249,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Population_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_filters:
-    name: "Test filters"
+    name: 'Test filters'
     needs:
       - build_backend
       - build_frontend
@@ -280,42 +292,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Filters.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_student_statistics:
-    name: "Test student statistics"
+    name: 'Test student statistics'
     needs:
       - build_backend
       - build_frontend
@@ -323,42 +335,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Student_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_studyprogramme_overview:
-    name: "Test studyprogramme overview"
+    name: 'Test studyprogramme overview'
     needs:
       - build_backend
       - build_frontend
@@ -366,42 +378,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Studyprogramme_overview.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_teachers_and_users:
-    name: "Test teachers and users"
+    name: 'Test teachers and users'
     needs:
       - build_backend
       - build_frontend
@@ -409,24 +421,24 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
@@ -435,20 +447,20 @@ jobs:
             cypress/integration/Users.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   # === Other tests ===
 
   test_service_exit_statuses:
-    name: "Check services run correctly"
+    name: 'Check services run correctly'
     needs:
       - build_backend
       - build_frontend
@@ -456,91 +468,127 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Check that exit statuses are not errors"
+      - name: 'Check that exit statuses are not errors'
         run: |
           docker ps -a
           if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
             echo 'Some process had nonzero exit code';
             exit 1;
           fi
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
-  push_to_dockerhub:
-    name: "Push images to docker hub for caching"
-    needs:
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - run_userservice_integration_tests
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_population_statistics
-    - test_student_statistics
-    - test_studyprogramme_overview
-    - test_teachers_and_users
-    - test_service_exit_statuses
+  # === Release ===
+
+  release_images:
+    name: 'Release built images'
     runs-on: ubuntu-latest
+    needs:
+      - lint_backend
+      - lint_frontend
+      - lint_userservice
+      - run_userservice_integration_tests
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Push to docker hub"
-        run: docker-compose --file docker-compose.ci.yml push
+      - name: 'Create a container from frontend'
+        run: docker create --name frontend toska/oodikone2-frontend:staging
+      - name: 'Copy assets from the container'
+        run: docker cp frontend:/usr/src/app/build ./build
+      - name: 'Create Sentry release'
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: oodikone
+        with:
+          environment: staging
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/oodikone/static/js'
+          version: ${{ github.sha }}
+      - name: 'Push to docker hub'
+        run: |
+          docker push toska/oodikone2-backend:staging
+          docker push toska/oodikone2-frontend:staging
+          docker push toska/oodikone2-userservice:staging
+      - name: 'Send deployment success notification to slack'
+        if: success()
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone-staging.cs.helsinki.fi/
+          deployment-target: staging
+      - name: 'Send deployment failure notification to slack'
+        if: failure()
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment-failure
+          deployment-target: staging
 
-  report_failure:
+  report_test_failure:
     runs-on: ubuntu-latest
-    name: "Report test failure to oodikone notifications"
+    name: 'Report test failure to oodikone notifications'
     needs:
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - run_userservice_integration_tests
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_population_statistics
-    - test_student_statistics
-    - test_studyprogramme_overview
-    - test_teachers_and_users
-    - test_service_exit_statuses
-    - push_to_dockerhub
+      - lint_backend
+      - lint_frontend
+      - lint_userservice
+      - run_userservice_integration_tests
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     if: failure()
     steps:
       - uses: actions/checkout@v2
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: test-failure

.lintstagedrc.js

@@ -0,0 +1,26 @@
+const path = require('path')
+const cwd = process.cwd()
+const dockerCmdBase = `docker run --rm --volume ${cwd}:/oodikone --workdir /oodikone`
+const relativeFilePaths = files => [...files.map(file => path.relative(cwd, file))].join(' ')
+
+module.exports = {
+  '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
+  '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
+  '*.css': files => `stylelint --fix ${files.join(' ')}`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0-alpine hadolint ${relativeFilePaths(files)}`,
+  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck-alpine:v0.7.2 ${relativeFilePaths(files)} -x`,
+  '.github/workflows/*': files => `${dockerCmdBase} rhysd/actionlint:1.6.2 ${relativeFilePaths(files)}`,
+  'docker-compose*': files => {
+    const composeFiles = file => {
+      if (['docker-compose.ci.yml', 'docker-compose.test.yml'].some(f => file.includes(f))) {
+        return `--file ${file}`
+      }
+      if (file.includes('docker-compose.real.yml')) {
+        return `--file ${cwd}/docker-compose.yml --file ${file}`
+      }
+      return ''
+    }
+
+    return files.map(file => `docker-compose ${composeFiles(file)} config --quiet`)
+  },
+}

README.md

@@ -10,12 +10,19 @@ To run Oodikone locally, you will need the following:
    - [Docker](https://docs.docker.com/install/) (version 20.10+),
    - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
    - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
-2. Access to Toska Docker Hub, see Toska Gitlabs password document for more information on how to login
-3. (Optional) For real data setup, access to Toskas production servers.
+2. Access to Toska Docker Hub,
+3. For real data setup, access to Toskas production servers.
+4. For some features to work, access to Toskas service tokens
 
 ## 🚀 Installation
 
-Launch the CLI and follow the instructions:
+Pre-installation steps:
+
+- Clone this repo
+- Login to toskas docker hub (see Toskas Gitlab for more information)
+- Populate `.env.template` file with correct tokens and rename it to `.env`
+
+Then launch the CLI and follow the instructions:
 
 ```bash
 npm run cli
@@ -108,10 +115,10 @@ There are three types of tests in this project: static tests with eslint/prettie
 Continuous integration (CI) works with Github actions and is defined in workflow files in `.github/workflows` folder:
 
 - oodikone setup for cypress and other tests in CI is defined in `docker-compose.ci.yml`. Take a look at this too if debugging github action workflows.
-- tests are run on every push to branches other than `master`. Pull requests can't be merged to master unless tests are ok.
-- after successful merge to `master`, oodikone is deployed to staging
+- tests are run on every push.
+- after successful test run, oodikone is deployed to staging
 - after creating a release, oodikone is deployed to production
-- updater is deployed to production on every push to `trunk`.
+- updater is deployed to production on every push, no tests included
 
 ## ❓FAQ
 
@@ -119,6 +126,10 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 
 You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
 
+### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data
+
+Make sure you have your VPN on
+
 ### Everything is broken, can't get oodikone running, data is not there etc.
 
 Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.
@@ -128,3 +139,7 @@ Just do clean install by launching cli with `npm cli` and running option 1: _Set
 Toska of course.
 
 University of Helsinki
+
+## Tietosuoja / data protection
+
+See [tietosuoja.md](tietosuoja.md).

adminer-plugins/login-servers.php

@@ -8,7 +8,6 @@ $driver = "pgsql";
 return new AdminerLoginServers(
     $servers = array(
         "kone-db" => array("server" => "kone-db", "driver" => $driver),
-        "oodi-db" => array("server" => "oodi-db", "driver" => $driver),
         "sis-db" => array("server" => "sis-db", "driver" => $driver),
         "sis-importer-db" => array("server" => "sis-importer-db", "driver" => $driver),
         "user-db" => array("server" => "user-db", "driver" => $driver)

cypress/integration/Filters.js

@@ -120,7 +120,8 @@ describe('Population Statistics', () => {
     })
   })
 
-  it('Admission type filter works', () => {
+  // Return when we have +2020 population with admission types in data
+  it.skip('Admission type filter works', () => {
     runTestStepWithPreAndPostParts('admissionTypeFilter-header', () => {
       cy.selectFromDropdown('admissionTypeFilter-dropdown', 0)
       checkFilteringResult(5)
@@ -283,9 +284,9 @@ describe('Custom Population Statistics', () => {
     runTestStepWithPreAndPostParts('courseFilter-header', () => {
       const courses = ['MAT11001', 'TKT20004']
       cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
-      checkFilteringResult(4)
+      checkFilteringResult(5)
       cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
-      checkFilteringResult(4)
+      checkFilteringResult(5)
       cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
       checkFilteringResult(3)
       cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)

cypress/integration/Studyprogramme_overview.js

@@ -13,33 +13,33 @@ describe('Studyprogramme overview', () => {
     // Column amount depends on user. Note that with dev user, you'll see even more
     // stuff than with admin or basic.
     const CSBachPopulationProgress2018ForAdmin = [
-      '160',
-      '42 (26%)',
-      '118 (73%)',
-      '159 (99%)',
-      '159',
-      '11',
-      '11',
+      '162',
+      '44 (27%)',
+      '118 (72%)',
+      '161 (99%)',
+      '161',
       '1',
-      '0',
-      '150',
-      '123',
-      '76',
-      '41',
+      '11',
+      '11',
+      '152',
+      '125',
+      '78',
+      '42',
       '16',
     ]
     const CSBachPopulationProgress2018ForBasic = [
-      '160',
-      '42 (26%)',
-      '118 (73%)',
-      '159 (99%)',
-      '159',
-      '11',
+      '162',
+      '44 (27%)',
+      '118 (72%)',
+      '161 (99%)',
+      '161',
       '1',
-      '150',
-      '123',
-      '76',
-      '41',
+      '11',
+      '11',
+      '152',
+      '125',
+      '78',
+      '42',
       '16',
     ] // cancelled and transferred from are missing
 
@@ -52,7 +52,7 @@ describe('Studyprogramme overview', () => {
       .each((elem, index) => {
         cy.wrap(elem).contains(population[index])
       })
-    const populationproductivity2019 = ['7388.0', '17', '7329.00', '59.00', '349.00']
+    const populationproductivity2019 = ['7388.0', '17', '7388.00', '0.00', '349.00']
     cy.get('table')
       .eq(1)
       .contains('2019')
@@ -79,7 +79,7 @@ describe('Studyprogramme overview', () => {
     let originalProductivityCalculatedText
     const doRecalculation = () => {
       // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
-      cy.wait(2000)
+      cy.wait(5000)
       cy.get('.attached').contains('Admin').click()
       cy.contains('recalculate productivity').click()
       cy.contains('recalculate throughput').click()

cypress/support/commands.js

@@ -10,6 +10,7 @@ const adminUserHeaders = {
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
   edupersonaffiliation: 'member;employee;faculty',
   mail: 'grp-toska+mockadminuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-6666666',
 }
 
 const basicUserHeaders = {
@@ -19,6 +20,7 @@ const basicUserHeaders = {
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
   edupersonaffiliation: 'member',
   mail: 'grp-toska+mockbasicuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-1234567',
 }
 
 const onlycoursestatisticsUserHeaders = {
@@ -28,6 +30,7 @@ const onlycoursestatisticsUserHeaders = {
   hyGroupCn: 'grp-oodikone-basic-users',
   eduPersonAffiliation: 'member',
   mail: 'grp-toska+mockonlycoursestatisticsuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-7654321',
 }
 
 const norightsUserHeaders = {
@@ -37,6 +40,7 @@ const norightsUserHeaders = {
   hygroupcn: 'grp-oodikone-users',
   edupersonaffiliation: 'member',
   mail: 'grp-toska+mocknorightuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-0000000',
 }
 
 const userHeaders = [adminUserHeaders, basicUserHeaders, onlycoursestatisticsUserHeaders, norightsUserHeaders]

docker-compose.ci.yml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 x-defaults: &defaults
   restart: unless-stopped
 
@@ -13,6 +11,7 @@ services:
       - redis
       - sis-db
     environment:
+      CI: "true"
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:1337
       REDIS: redis
@@ -22,12 +21,12 @@ services:
       TOKEN_SECRET: IWannaBeTheVeryBest
       USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       USERSERVICE_URL: http://userservice:4567
-    image: toska/oodikone2-backend:trunk
+    image: toska/oodikone2-backend:staging
 
   frontend:
     <<: *defaults
     container_name: frontend
-    image: toska/oodikone2-frontend:trunk
+    image: toska/oodikone2-frontend:staging
 
   kone-db:
     <<: *defaults
@@ -58,10 +57,11 @@ services:
     depends_on:
       - user-db
     environment:
+      CI: "true"
       DB_URL: postgres://postgres@user-db:5432/user-db
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       TOKEN_SECRET: IWannaBeTheVeryBest
-    image: toska/oodikone2-userservice:trunk
+    image: toska/oodikone2-userservice:staging
 
   nginx:
     image: nginx:latest

docker-compose.yml

@@ -5,7 +5,6 @@ x-build_defaults: &build_defaults
   target: development
 
 services:
-
   adminer:
     <<: *defaults
     container_name: adminer
@@ -27,6 +26,8 @@ services:
       - kone-db
       - redis
       - sis-db
+    env_file:
+      - .env
     environment:
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:3000
@@ -70,11 +71,11 @@ services:
 
   redis:
     <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
     container_name: redis
     image: redis
     ports:
-      - "6379:6379"
+      - '6379:6379'
     volumes:
       - redis-data:/data
 
@@ -84,6 +85,7 @@ services:
     image: registry.toska.cs.helsinki.fi/sis-db
     volumes:
       - sis-db-data:/data
+    shm_size: 2gb
 
   sis-importer-db:
     <<: *defaults
@@ -96,7 +98,7 @@ services:
 
   sis-updater-nats:
     <<: *defaults
-    command: "-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore"
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:

package-lock.json

@@ -5,24 +5,25 @@
   "requires": true,
   "packages": {
     "": {
+      "name": "oodikone",
       "license": "ISC",
       "devDependencies": {
         "@babel/eslint-parser": "^7.15.0",
         "@babel/plugin-syntax-jsx": "^7.14.5",
         "concurrently": "^6.2.1",
-        "cypress": "^8.2.0",
+        "cypress": "^8.3.1",
         "eslint": "^7.32.0",
         "eslint-config-airbnb": "^18.2.1",
         "eslint-config-prettier": "^8.3.0",
         "eslint-plugin-babel": "^5.3.1",
         "eslint-plugin-cypress": "^2.11.3",
-        "eslint-plugin-import": "^2.24.0",
+        "eslint-plugin-import": "^2.24.2",
         "eslint-plugin-jsx-a11y": "^6.4.1",
         "eslint-plugin-prefer-object-spread": "^1.2.1",
-        "eslint-plugin-prettier": "^3.4.0",
+        "eslint-plugin-prettier": "^3.4.1",
         "eslint-plugin-react": "^7.24.0",
         "eslint-plugin-react-hooks": "^4.2.0",
-        "husky": "^7.0.1",
+        "husky": "^7.0.2",
         "lint-staged": "^11.1.2",
         "prettier": "^2.3.2",
         "stylelint": "^13.13.1",
@@ -535,9 +536,9 @@
       }
     },
     "node_modules/@cypress/request": {
-      "version": "2.88.5",
-      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.5.tgz",
-      "integrity": "sha512-TzEC1XMi1hJkywWpRfD2clreTa/Z+lOrXDCxxBTBPEcY5azdPi56A6Xw+O4tWJnaJH3iIE7G5aDXZC6JgRZLcA==",
+      "version": "2.88.6",
+      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.6.tgz",
+      "integrity": "sha512-z0UxBE/+qaESAHY9p9sM2h8Y4XqtsbDCt0/DPOrqA/RZgKi4PkxdpXyK4wCCnSk1xHqWHZZAE+gV6aDAR6+caQ==",
       "dev": true,
       "dependencies": {
         "aws-sign2": "~0.7.0",
@@ -553,13 +554,12 @@
         "isstream": "~0.1.2",
         "json-stringify-safe": "~5.0.1",
         "mime-types": "~2.1.19",
-        "oauth-sign": "~0.9.0",
         "performance-now": "^2.1.0",
         "qs": "~6.5.2",
         "safe-buffer": "^5.1.2",
         "tough-cookie": "~2.5.0",
         "tunnel-agent": "^0.6.0",
-        "uuid": "^3.3.2"
+        "uuid": "^8.3.2"
       },
       "engines": {
         "node": ">= 6"
@@ -686,6 +686,12 @@
         "postcss-syntax": ">=0.36.2"
       }
     },
+    "node_modules/@types/json5": {
+      "version": "0.0.29",
+      "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz",
+      "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=",
+      "dev": true
+    },
     "node_modules/@types/mdast": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.4.tgz",
@@ -1468,19 +1474,6 @@
         "node": ">=10.0.0"
       }
     },
-    "node_modules/concurrently/node_modules/date-fns": {
-      "version": "2.22.1",
-      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.22.1.tgz",
-      "integrity": "sha512-yUFPQjrxEmIsMqlHhAhmxkuH769baF21Kk+nZwZGyrMoyLA+LugaQtC0+Tqf9CBUUULWwUJt6Q5ySI3LJDDCGg==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.11"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/date-fns"
-      }
-    },
     "node_modules/concurrently/node_modules/supports-color": {
       "version": "8.1.1",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
@@ -1580,13 +1573,13 @@
       }
     },
     "node_modules/cypress": {
-      "version": "8.2.0",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.2.0.tgz",
-      "integrity": "sha512-jg7S5VxxslwsgEyAkCE9ZCkFADxOUY1bSWScp1cWnga88K0TZgFQ0zdxyG9Mw/4spLGuvkriIZ62am+TR6C04w==",
+      "version": "8.3.1",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.3.1.tgz",
+      "integrity": "sha512-1v6pfx+/5cXhaT5T6QKOvnkawmEHWHLiVzm3MYMoQN1fkX2Ma1C32STd3jBStE9qT5qPSTILjGzypVRxCBi40g==",
       "dev": true,
       "hasInstallScript": true,
       "dependencies": {
-        "@cypress/request": "^2.88.5",
+        "@cypress/request": "^2.88.6",
         "@cypress/xvfb": "^1.2.4",
         "@types/node": "^14.14.31",
         "@types/sinonjs__fake-timers": "^6.0.2",
@@ -1665,6 +1658,19 @@
         "node": ">=0.10"
       }
     },
+    "node_modules/date-fns": {
+      "version": "2.23.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.23.0.tgz",
+      "integrity": "sha512-5ycpauovVyAk0kXNZz6ZoB9AYMZB4DObse7P3BPWmyEjXNORTI8EJ6X0uaSAq4sCHzM1uajzrkr6HnsLQpxGXA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.11"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/date-fns"
+      }
+    },
     "node_modules/dayjs": {
       "version": "1.10.5",
       "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.5.tgz",
@@ -2067,9 +2073,9 @@
       }
     },
     "node_modules/eslint-import-resolver-node": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.5.tgz",
-      "integrity": "sha512-XMoPKjSpXbkeJ7ZZ9icLnJMTY5Mc1kZbCakHquaFsXPpyWOwK0TK6CODO+0ca54UoM9LKOxyUNnoVZRl8TeaAg==",
+      "version": "0.3.6",
+      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz",
+      "integrity": "sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw==",
       "dev": true,
       "dependencies": {
         "debug": "^3.2.7",
@@ -2144,26 +2150,26 @@
       }
     },
     "node_modules/eslint-plugin-import": {
-      "version": "2.24.0",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.0.tgz",
-      "integrity": "sha512-Kc6xqT9hiYi2cgybOc0I2vC9OgAYga5o/rAFinam/yF/t5uBqxQbauNPMC6fgb640T/89P0gFoO27FOilJ/Cqg==",
+      "version": "2.24.2",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.2.tgz",
+      "integrity": "sha512-hNVtyhiEtZmpsabL4neEj+6M5DCLgpYyG9nzJY8lZQeQXEn5UPW1DpUdsMHMXsq98dbNm7nt1w9ZMSVpfJdi8Q==",
       "dev": true,
       "dependencies": {
         "array-includes": "^3.1.3",
         "array.prototype.flat": "^1.2.4",
         "debug": "^2.6.9",
         "doctrine": "^2.1.0",
-        "eslint-import-resolver-node": "^0.3.5",
+        "eslint-import-resolver-node": "^0.3.6",
         "eslint-module-utils": "^2.6.2",
         "find-up": "^2.0.0",
         "has": "^1.0.3",
-        "is-core-module": "^2.4.0",
+        "is-core-module": "^2.6.0",
         "minimatch": "^3.0.4",
-        "object.values": "^1.1.3",
+        "object.values": "^1.1.4",
         "pkg-up": "^2.0.0",
         "read-pkg-up": "^3.0.0",
         "resolve": "^1.20.0",
-        "tsconfig-paths": "^3.9.0"
+        "tsconfig-paths": "^3.11.0"
       },
       "engines": {
         "node": ">=4"
@@ -2237,9 +2243,9 @@
       }
     },
     "node_modules/eslint-plugin-prettier": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.4.0.tgz",
-      "integrity": "sha512-UDK6rJT6INSfcOo545jiaOwB701uAIt2/dR7WnFQoGCVl1/EMqdANBmwUaqqQ45aXprsTGzSa39LI1PyuRBxxw==",
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.4.1.tgz",
+      "integrity": "sha512-htg25EUYUeIhKHXjOinK4BgCcDwtLHjqaxCDsMy5nbnUMkKFvIhMVCp+5GFUXQ4Nr8lBsPqtGAqBenbpFqAA2g==",
       "dev": true,
       "dependencies": {
         "prettier-linter-helpers": "^1.0.0"
@@ -3034,6 +3040,7 @@
       "version": "5.1.5",
       "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz",
       "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==",
+      "deprecated": "this library is no longer supported",
       "dev": true,
       "dependencies": {
         "ajv": "^6.12.3",
@@ -3148,9 +3155,9 @@
       }
     },
     "node_modules/husky": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/husky/-/husky-7.0.1.tgz",
-      "integrity": "sha512-gceRaITVZ+cJH9sNHqx5tFwbzlLCVxtVZcusME8JYQ8Edy5mpGDOqD8QBCdMhpyo9a+JXddnujQ4rpY2Ff9SJA==",
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/husky/-/husky-7.0.2.tgz",
+      "integrity": "sha512-8yKEWNX4z2YsofXAMT7KvA1g8p+GxtB1ffV8XtpAEGuXNAbCV5wdNKH+qTpw8SM9fh4aMPDR+yQuKfgnreyZlg==",
       "dev": true,
       "bin": {
         "husky": "lib/bin.js"
@@ -3355,9 +3362,9 @@
       }
     },
     "node_modules/is-core-module": {
-      "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.5.0.tgz",
-      "integrity": "sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg==",
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.6.0.tgz",
+      "integrity": "sha512-wShG8vs60jKfPWpF2KZRaAtvt3a20OAn7+IJ6hLPECpSABLcKtFKTTI4ZtH5QcBruBHlq+WsdHWyz0BCZW7svQ==",
       "dev": true,
       "dependencies": {
         "has": "^1.0.3"
@@ -4345,21 +4352,21 @@
       }
     },
     "node_modules/mime-db": {
-      "version": "1.47.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz",
-      "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==",
+      "version": "1.49.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz",
+      "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==",
       "dev": true,
       "engines": {
         "node": ">= 0.6"
       }
     },
     "node_modules/mime-types": {
-      "version": "2.1.30",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz",
-      "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==",
+      "version": "2.1.32",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz",
+      "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==",
       "dev": true,
       "dependencies": {
-        "mime-db": "1.47.0"
+        "mime-db": "1.49.0"
       },
       "engines": {
         "node": ">= 0.6"
@@ -4487,15 +4494,6 @@
       "integrity": "sha1-b2gragJ6Tp3fpFZM0lidHU5mnt4=",
       "dev": true
     },
-    "node_modules/oauth-sign": {
-      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
-      "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
-      "dev": true,
-      "engines": {
-        "node": "*"
-      }
-    },
     "node_modules/object-assign": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
@@ -4761,9 +4759,9 @@
       }
     },
     "node_modules/path-parse": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
-      "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
       "dev": true
     },
     "node_modules/path-type": {
@@ -5625,6 +5623,11 @@
         "safer-buffer": "^2.0.2",
         "tweetnacl": "~0.14.0"
       },
+      "bin": {
+        "sshpk-conv": "bin/sshpk-conv",
+        "sshpk-sign": "bin/sshpk-sign",
+        "sshpk-verify": "bin/sshpk-verify"
+      },
       "engines": {
         "node": ">=0.10.0"
       }
@@ -6135,16 +6138,29 @@
       }
     },
     "node_modules/tsconfig-paths": {
-      "version": "3.10.1",
-      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz",
-      "integrity": "sha512-rETidPDgCpltxF7MjBZlAFPUHv5aHH2MymyPvh+vEyWAED4Eb/WeMbsnD/JDr4OKPOA1TssDHgIcpTN5Kh0p6Q==",
+      "version": "3.11.0",
+      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.11.0.tgz",
+      "integrity": "sha512-7ecdYDnIdmv639mmDwslG6KQg1Z9STTz1j7Gcz0xa+nshh/gKDAHcPxRbWOsA3SPp0tXP2leTcY9Kw+NAkfZzA==",
       "dev": true,
       "dependencies": {
-        "json5": "^2.2.0",
+        "@types/json5": "^0.0.29",
+        "json5": "^1.0.1",
         "minimist": "^1.2.0",
         "strip-bom": "^3.0.0"
       }
     },
+    "node_modules/tsconfig-paths/node_modules/json5": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz",
+      "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==",
+      "dev": true,
+      "dependencies": {
+        "minimist": "^1.2.0"
+      },
+      "bin": {
+        "json5": "lib/cli.js"
+      }
+    },
     "node_modules/tslib": {
       "version": "1.10.0",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz",
@@ -6327,12 +6343,12 @@
       "dev": true
     },
     "node_modules/uuid": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
-      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
       "dev": true,
       "bin": {
-        "uuid": "bin/uuid"
+        "uuid": "dist/bin/uuid"
       }
     },
     "node_modules/v8-compile-cache": {
@@ -6928,9 +6944,9 @@
       }
     },
     "@cypress/request": {
-      "version": "2.88.5",
-      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.5.tgz",
-      "integrity": "sha512-TzEC1XMi1hJkywWpRfD2clreTa/Z+lOrXDCxxBTBPEcY5azdPi56A6Xw+O4tWJnaJH3iIE7G5aDXZC6JgRZLcA==",
+      "version": "2.88.6",
+      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.6.tgz",
+      "integrity": "sha512-z0UxBE/+qaESAHY9p9sM2h8Y4XqtsbDCt0/DPOrqA/RZgKi4PkxdpXyK4wCCnSk1xHqWHZZAE+gV6aDAR6+caQ==",
       "dev": true,
       "requires": {
         "aws-sign2": "~0.7.0",
@@ -6946,13 +6962,12 @@
         "isstream": "~0.1.2",
         "json-stringify-safe": "~5.0.1",
         "mime-types": "~2.1.19",
-        "oauth-sign": "~0.9.0",
         "performance-now": "^2.1.0",
         "qs": "~6.5.2",
         "safe-buffer": "^5.1.2",
         "tough-cookie": "~2.5.0",
         "tunnel-agent": "^0.6.0",
-        "uuid": "^3.3.2"
+        "uuid": "^8.3.2"
       }
     },
     "@cypress/xvfb": {
@@ -7055,6 +7070,12 @@
         "unist-util-find-all-after": "^3.0.2"
       }
     },
+    "@types/json5": {
+      "version": "0.0.29",
+      "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz",
+      "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=",
+      "dev": true
+    },
     "@types/mdast": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.4.tgz",
@@ -7653,12 +7674,6 @@
         "yargs": "^16.2.0"
       },
       "dependencies": {
-        "date-fns": {
-          "version": "2.22.1",
-          "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.22.1.tgz",
-          "integrity": "sha512-yUFPQjrxEmIsMqlHhAhmxkuH769baF21Kk+nZwZGyrMoyLA+LugaQtC0+Tqf9CBUUULWwUJt6Q5ySI3LJDDCGg==",
-          "dev": true
-        },
         "supports-color": {
           "version": "8.1.1",
           "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
@@ -7736,12 +7751,12 @@
       "dev": true
     },
     "cypress": {
-      "version": "8.2.0",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.2.0.tgz",
-      "integrity": "sha512-jg7S5VxxslwsgEyAkCE9ZCkFADxOUY1bSWScp1cWnga88K0TZgFQ0zdxyG9Mw/4spLGuvkriIZ62am+TR6C04w==",
+      "version": "8.3.1",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.3.1.tgz",
+      "integrity": "sha512-1v6pfx+/5cXhaT5T6QKOvnkawmEHWHLiVzm3MYMoQN1fkX2Ma1C32STd3jBStE9qT5qPSTILjGzypVRxCBi40g==",
       "dev": true,
       "requires": {
-        "@cypress/request": "^2.88.5",
+        "@cypress/request": "^2.88.6",
         "@cypress/xvfb": "^1.2.4",
         "@types/node": "^14.14.31",
         "@types/sinonjs__fake-timers": "^6.0.2",
@@ -7810,6 +7825,12 @@
         "assert-plus": "^1.0.0"
       }
     },
+    "date-fns": {
+      "version": "2.23.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.23.0.tgz",
+      "integrity": "sha512-5ycpauovVyAk0kXNZz6ZoB9AYMZB4DObse7P3BPWmyEjXNORTI8EJ6X0uaSAq4sCHzM1uajzrkr6HnsLQpxGXA==",
+      "dev": true
+    },
     "dayjs": {
       "version": "1.10.5",
       "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.5.tgz",
@@ -8142,9 +8163,9 @@
       "requires": {}
     },
     "eslint-import-resolver-node": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.5.tgz",
-      "integrity": "sha512-XMoPKjSpXbkeJ7ZZ9icLnJMTY5Mc1kZbCakHquaFsXPpyWOwK0TK6CODO+0ca54UoM9LKOxyUNnoVZRl8TeaAg==",
+      "version": "0.3.6",
+      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz",
+      "integrity": "sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw==",
       "dev": true,
       "requires": {
         "debug": "^3.2.7",
@@ -8210,26 +8231,26 @@
       }
     },
     "eslint-plugin-import": {
-      "version": "2.24.0",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.0.tgz",
-      "integrity": "sha512-Kc6xqT9hiYi2cgybOc0I2vC9OgAYga5o/rAFinam/yF/t5uBqxQbauNPMC6fgb640T/89P0gFoO27FOilJ/Cqg==",
+      "version": "2.24.2",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.2.tgz",
+      "integrity": "sha512-hNVtyhiEtZmpsabL4neEj+6M5DCLgpYyG9nzJY8lZQeQXEn5UPW1DpUdsMHMXsq98dbNm7nt1w9ZMSVpfJdi8Q==",
       "dev": true,
       "requires": {
         "array-includes": "^3.1.3",
         "array.prototype.flat": "^1.2.4",
         "debug": "^2.6.9",
         "doctrine": "^2.1.0",
-        "eslint-import-resolver-node": "^0.3.5",
+        "eslint-import-resolver-node": "^0.3.6",
         "eslint-module-utils": "^2.6.2",
         "find-up": "^2.0.0",
         "has": "^1.0.3",
-        "is-core-module": "^2.4.0",
+        "is-core-module": "^2.6.0",
         "minimatch": "^3.0.4",
-        "object.values": "^1.1.3",
+        "object.values": "^1.1.4",
         "pkg-up": "^2.0.0",
         "read-pkg-up": "^3.0.0",
         "resolve": "^1.20.0",
-        "tsconfig-paths": "^3.9.0"
+        "tsconfig-paths": "^3.11.0"
       },
       "dependencies": {
         "debug": {
@@ -8285,9 +8306,9 @@
       "requires": {}
     },
     "eslint-plugin-prettier": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.4.0.tgz",
-      "integrity": "sha512-UDK6rJT6INSfcOo545jiaOwB701uAIt2/dR7WnFQoGCVl1/EMqdANBmwUaqqQ45aXprsTGzSa39LI1PyuRBxxw==",
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.4.1.tgz",
+      "integrity": "sha512-htg25EUYUeIhKHXjOinK4BgCcDwtLHjqaxCDsMy5nbnUMkKFvIhMVCp+5GFUXQ4Nr8lBsPqtGAqBenbpFqAA2g==",
       "dev": true,
       "requires": {
         "prettier-linter-helpers": "^1.0.0"
@@ -8960,9 +8981,9 @@
       "dev": true
     },
     "husky": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/husky/-/husky-7.0.1.tgz",
-      "integrity": "sha512-gceRaITVZ+cJH9sNHqx5tFwbzlLCVxtVZcusME8JYQ8Edy5mpGDOqD8QBCdMhpyo9a+JXddnujQ4rpY2Ff9SJA==",
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/husky/-/husky-7.0.2.tgz",
+      "integrity": "sha512-8yKEWNX4z2YsofXAMT7KvA1g8p+GxtB1ffV8XtpAEGuXNAbCV5wdNKH+qTpw8SM9fh4aMPDR+yQuKfgnreyZlg==",
       "dev": true
     },
     "ignore": {
@@ -9091,9 +9112,9 @@
       }
     },
     "is-core-module": {
-      "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.5.0.tgz",
-      "integrity": "sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg==",
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.6.0.tgz",
+      "integrity": "sha512-wShG8vs60jKfPWpF2KZRaAtvt3a20OAn7+IJ6hLPECpSABLcKtFKTTI4ZtH5QcBruBHlq+WsdHWyz0BCZW7svQ==",
       "dev": true,
       "requires": {
         "has": "^1.0.3"
@@ -9817,18 +9838,18 @@
       }
     },
     "mime-db": {
-      "version": "1.47.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz",
-      "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==",
+      "version": "1.49.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz",
+      "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==",
       "dev": true
     },
     "mime-types": {
-      "version": "2.1.30",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz",
-      "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==",
+      "version": "2.1.32",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz",
+      "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==",
       "dev": true,
       "requires": {
-        "mime-db": "1.47.0"
+        "mime-db": "1.49.0"
       }
     },
     "mimic-fn": {
@@ -9932,12 +9953,6 @@
       "integrity": "sha1-b2gragJ6Tp3fpFZM0lidHU5mnt4=",
       "dev": true
     },
-    "oauth-sign": {
-      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
-      "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
-      "dev": true
-    },
     "object-assign": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
@@ -10127,9 +10142,9 @@
       "dev": true
     },
     "path-parse": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
-      "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
       "dev": true
     },
     "path-type": {
@@ -11174,14 +11189,26 @@
       "dev": true
     },
     "tsconfig-paths": {
-      "version": "3.10.1",
-      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz",
-      "integrity": "sha512-rETidPDgCpltxF7MjBZlAFPUHv5aHH2MymyPvh+vEyWAED4Eb/WeMbsnD/JDr4OKPOA1TssDHgIcpTN5Kh0p6Q==",
+      "version": "3.11.0",
+      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.11.0.tgz",
+      "integrity": "sha512-7ecdYDnIdmv639mmDwslG6KQg1Z9STTz1j7Gcz0xa+nshh/gKDAHcPxRbWOsA3SPp0tXP2leTcY9Kw+NAkfZzA==",
       "dev": true,
       "requires": {
-        "json5": "^2.2.0",
+        "@types/json5": "^0.0.29",
+        "json5": "^1.0.1",
         "minimist": "^1.2.0",
         "strip-bom": "^3.0.0"
+      },
+      "dependencies": {
+        "json5": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz",
+          "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==",
+          "dev": true,
+          "requires": {
+            "minimist": "^1.2.0"
+          }
+        }
       }
     },
     "tslib": {
@@ -11333,9 +11360,9 @@
       "dev": true
     },
     "uuid": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
-      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
       "dev": true
     },
     "v8-compile-cache": {

package.json

@@ -2,10 +2,11 @@
   "name": "oodikone",
   "description": "An application for analyzing university data.",
   "scripts": {
-    "concurrently": "concurrently",
     "cypress": "cypress",
+    "concurrently": "concurrently",
     "eslint": "eslint",
     "stylelint": "stylelint",
+    "prettier": "prettier",
     "cli": "./cli.sh",
     "oodikone": "./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
@@ -13,29 +14,8 @@
     "updater:real": "./run.sh updater real up --detach",
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
-    "docker:down": "./run.sh both down --remove-orphans",
-    "diff:students": "docker exec backend node /usr/src/app/src/sisoodi_diff/students/index.js",
-    "diff:populations": "docker exec backend node /usr/src/app/src/sisoodi_diff/populations.js",
-    "diff:courses": "docker exec backend node /usr/src/app/src/sisoodi_diff/courses/index.js",
-    "diff:transfers": "docker exec backend node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js",
-    "prepare": "husky install",
-    "lint:docker-compose": "docker-compose config -q && :",
-    "lint:docker-compose:real": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q && :",
-    "lint:docker-compose:ci": "docker-compose -f docker-compose.ci.yml config -q && :",
-    "lint:docker-compose:test": "docker-compose -f docker-compose.test.yml config -q && :"
-  },
-  "lint-staged": {
-    "{services,updater}/**/*.{js,jsx}": "eslint --fix",
-    "*.{js,json,md}": "prettier --write",
-    "*.css": "stylelint --fix",
-    ".github/workflows/*": "actionlint",
-    "Dockerfile": "hadolint",
-    "*.sh": "shellcheck -x",
-    "docker-compose.yml": "npm run lint:docker-compose",
-    "docker-compose.real.yml": "npm run lint:docker-compose:real",
-    "docker-compose.ci.yml": "npm run lint:docker-compose:ci",
-    "docker-compose.test.yml": "npm run lint:docker-compose:test"
+    "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
+    "prepare": "husky install"
   },
   "repository": {
     "type": "git",
@@ -50,19 +30,19 @@
     "@babel/eslint-parser": "^7.15.0",
     "@babel/plugin-syntax-jsx": "^7.14.5",
     "concurrently": "^6.2.1",
-    "cypress": "^8.2.0",
+    "cypress": "^8.3.1",
     "eslint": "^7.32.0",
     "eslint-config-airbnb": "^18.2.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-babel": "^5.3.1",
     "eslint-plugin-cypress": "^2.11.3",
-    "eslint-plugin-import": "^2.24.0",
+    "eslint-plugin-import": "^2.24.2",
     "eslint-plugin-jsx-a11y": "^6.4.1",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
-    "eslint-plugin-prettier": "^3.4.0",
+    "eslint-plugin-prettier": "^3.4.1",
     "eslint-plugin-react": "^7.24.0",
     "eslint-plugin-react-hooks": "^4.2.0",
-    "husky": "^7.0.1",
+    "husky": "^7.0.2",
     "lint-staged": "^11.1.2",
     "prettier": "^2.3.2",
     "stylelint": "^13.13.1",

run.sh

@@ -64,8 +64,7 @@ parse_profiles() {
   return 0
 }
 
-# Set which docker-compose files to use based on version. In ci, profiles are also
-# emptied, since they're passed from github actions.
+# Set which docker-compose files to use based on version.
 parse_env() {
   env=""
   if [[ "$version" == "real" ]]; then
@@ -85,7 +84,7 @@ parse_env
 if [[ "$compose_command" == "" ]]; then
   final_command="docker-compose ${env}"
 elif [[ "$compose_command" == *"down"* ]]; then
-  final_command="docker-compose ${compose_command}"
+  final_command="docker-compose ${env} ${compose_command}"
 else
   final_command="docker-compose ${env} ${profiles} ${compose_command}"
 fi

scripts/functions_for_setup.sh

@@ -20,10 +20,10 @@ USER_DB_NAME="user-db"
 DATABASES=("$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")
 
 ## Urls should be in same order as databases as both are iterated through by indexes.
-KONE_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-kone-pg.sqz"
+KONE_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-kone-db.sqz"
 SIS_DB_REAL_DUMP_URL="svm-96.cs.helsinki.fi:/home/updater_user/backups/latest-sis.sqz"
-SIS_IMPORTER_DB_REAL_DUMP_URL="importer:/home/importer_user/importer-db/backup/importer-db.sqz"
-USER_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-user-pg.sqz"
+SIS_IMPORTER_DB_REAL_DUMP_URL="svm-92.cs.helsinki.fi:/home/importer_user/importer-db/backup/importer-db.sqz"
+USER_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-user-db.sqz"
 REAL_DUMP_URLS=("$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")
 
 # Source utility functions
@@ -71,7 +71,7 @@ reset_databases() {
 
   infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
 
-  docker-compose down
+  npm run docker:down
   docker-compose up -d "${databases[@]}"
 
   for database in "${databases[@]}"; do
@@ -152,10 +152,10 @@ linting and formatting"
   npm run prepare
 
   infomsg "Pulling images"
-  "$PROJECT_ROOT"/run.sh oodikone anon pull
+  "$PROJECT_ROOT"/run.sh both anon pull
 
   infomsg "Building images"
-  "$PROJECT_ROOT"/run.sh oodikone anon build
+  "$PROJECT_ROOT"/run.sh both anon build
 
   successmsg "Setup ready, oodikone can be started! See README for more info."
 }

services/backend/index.js

@@ -1,5 +1,6 @@
 require('./src/app')
+const logger = require('./src/util/logger')
 
 process.on('unhandledRejection', reason => {
-  console.log(reason)
+  logger.error({ message: 'unhandledRejection: ', meta: reason })
 })

services/backend/package.json

@@ -12,65 +12,30 @@
   },
   "dependencies": {
     "@sentry/node": "^5.30.0",
-    "async": "^3.2.0",
-    "async-lock": "^1.2.8",
-    "axios": "^0.21.1",
+    "@sentry/tracing": "^6.11.0",
+    "async": "^3.2.1",
+    "async-lock": "^1.3.0",
+    "axios": "^0.21.2",
     "bluebird": "^3.7.2",
-    "body-parser": "^1.19.0",
     "cors": "^2.8.5",
     "cron": "^1.8.2",
     "express": "^4.17.1",
+    "express-async-errors": "^3.1.1",
     "jsonwebtoken": "^8.5.1",
-    "kafka-node": "^4.1.3",
     "lodash": "^4.17.21",
     "lru-cache": "^5.1.1",
-    "mkdirp": "^0.5.5",
     "moment": "^2.29.1",
     "moment-timezone": "^0.5.33",
     "morgan": "^1.10.0",
-    "nodemailer": "^4.7.0",
-    "pg": "^8.6.0",
-    "redis": "^3.1.1",
-    "sequelize": "^6.6.2",
+    "pg": "^8.7.1",
+    "redis": "^3.1.2",
+    "sequelize": "^6.6.5",
     "umzug": "^2.3.0",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "uuid": "^3.4.0",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0",
-    "winston-transport": "^4.4.0"
+    "winston": "^3.3.3"
   },
   "devDependencies": {
-    "ava": "^2.4.0",
-    "faker": "^4.1.0",
-    "jest": "^24.9.0",
-    "nock": "^10.0.6",
-    "node-status": "^1.0.0",
-    "nodemon": "^1.19.4",
-    "nyc": "^13.3.0",
-    "supertest": "^3.4.2"
-  },
-  "jest": {
-    "setupFilesAfterEnv": [
-      "./test/jest/globals.js"
-    ],
-    "testEnvironment": "node"
-  },
-  "ava": {
-    "files": [
-      "test/**/*_test.js"
-    ],
-    "source": [
-      "**/*.{js,jsx}",
-      "!test/**/*"
-    ],
-    "concurrency": 5,
-    "failFast": true,
-    "failWithoutAssertions": true,
-    "tap": true,
-    "powerAssert": false,
-    "require": [
-      "babel-register"
-    ],
-    "babel": "inherit"
+    "nodemon": "^1.19.4"
   }
 }

services/backend/requests/courses_yearly.rest

@@ -1,7 +0,0 @@
-GET /api/courseyearlystats/?start=2013&end=2018&code=TKT10002 HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token generated from login.rest to use

services/backend/requests/enableUser.rest

@@ -1,6 +0,0 @@
-POST http://localhost:8080/api/users/enable
-Content-Type: application/json
-
-{
-  "id": 2
-}

services/backend/requests/get_all_duplicate_codes.rest

@@ -1,6 +0,0 @@
-GET /api/courses/duplicatecodes/all HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/get_duplicate_codes.rest

@@ -1,7 +0,0 @@
-GET /api/courses/duplicatecodes/?code=TKT10005 HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token:
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/get_mandatory_courses.rest

@@ -1,7 +0,0 @@
-GET /api/v2/studyprogrammes/{insertstudyprogrammecodehere}/mandatory_courses
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest and a student id to use

services/backend/requests/get_student.rest

@@ -1,7 +0,0 @@
-GET /api/students/{insert student id}
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest and a student id to use

services/backend/requests/login.rest

@@ -1,7 +0,0 @@
-POST /api/login HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-Cache-Control: no-cache
-Postman-Token: d8a05d0f-fd91-e4e0-54a0-172d18ff3fad
-Content-Type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW

services/backend/requests/newUser.rest

@@ -1,4 +0,0 @@
-GET http://localhost:8080/api/login
-Content-Type: application/json
-eduPersonPrincipalName: totutotu@
-shib-session-id: IDEEE

services/backend/requests/post_duplicate_codes.rest

@@ -1,7 +0,0 @@
-POST /api/courses/duplicatecodes/TKT10005/205044 HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token:
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/remove_duplicate_code.rest

@@ -1,7 +0,0 @@
-DELETE /api/courses/duplicatecodes/TKT10005/58123 HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/update.rest

@@ -1,11 +0,0 @@
-POST /api/updatedatabase HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-content-type: application/json
-
-[
-  "01234562",
-  "12302459"
-]

services/backend/scripts/create_fmsei_report.js

@@ -1,132 +0,0 @@
-/* eslint-disable no-console */
-
-const { Student, Credit, Studyright } = require('../src/models')
-const Sequelize = require('sequelize')
-const Op = Sequelize.Op
-
-const startDate = new Date(2019, 1, 1, 0, 0, 0, 0)
-
-const wasFMSEIStudent = (student, date) => {
-  const degreeright = student.studyrights.find(sr => sr.extentcode !== 9 && sr.startdate <= date && sr.enddate >= date)
-  return !degreeright
-}
-
-const courseCodes = [
-  'AYTKT200041',
-  'AYTKT21007',
-  'TKT21007',
-  'AYTKT21025en',
-  'AYTKT21009',
-  'TKT21009',
-  'AYTKT21010',
-  'TKT21010',
-  'AYTKT21018',
-  'AYTKT21018fi',
-  'AYTKT21018sv',
-  'AY5823951',
-  'AY5823952',
-  'AY5823953',
-  'AY5823954',
-  'AY5823955',
-  'AY5823956',
-]
-
-const createReport = async () => {
-  console.log('SPLIT')
-  const { fmseiStudentnumbers, otherStudentnumbers } = await courseCodes.reduce(
-    async (accPromise, courseCode) => {
-      const credits = await Credit.findAll({
-        where: {
-          course_code: courseCode,
-          attainment_date: {
-            [Op.gte]: startDate,
-          },
-        },
-      })
-
-      const { fmseiStudentnumbers, otherStudentnumbers } = await credits.reduce(
-        async (accPromise, credit) => {
-          const student = await Student.findOne({
-            where: { studentnumber: credit.student_studentnumber },
-            include: [{ model: Studyright }],
-          })
-          const acc = await accPromise
-
-          if (wasFMSEIStudent(student, credit.attainment_date)) {
-            // FMSEI student fmsei credits
-            console.log(
-              `${credit.student_studentnumber};${credit.course_code};${credit.grade};${credit.credits};${new Date(
-                credit.attainment_date
-              ).toISOString()}`
-            )
-            return {
-              fmseiStudentnumbers: acc.fmseiStudentnumbers.concat(student.studentnumber),
-              otherStudentnumbers: acc.otherStudentnumbers,
-            }
-          }
-          return {
-            fmseiStudentnumbers: acc.fmseiStudentnumbers,
-            otherStudentnumbers: acc.otherStudentnumbers.concat(student.studentnumber),
-          }
-        },
-        { fmseiStudentnumbers: [], otherStudentnumbers: [] }
-      )
-
-      const acc = await accPromise
-
-      return {
-        courseStats: acc.courseStats.concat({
-          courseCode: courseCode,
-          fmsei: fmseiStudentnumbers.length,
-          other: otherStudentnumbers.length,
-        }),
-        fmseiStudentnumbers: acc.fmseiStudentnumbers.concat(fmseiStudentnumbers),
-        otherStudentnumbers: acc.otherStudentnumbers.concat(otherStudentnumbers),
-      }
-    },
-    { courseStats: [], fmseiStudentnumbers: [], otherStudentnumbers: [] }
-  )
-  const fmseiStudents = await Student.findAll({
-    where: { studentnumber: fmseiStudentnumbers },
-    include: [
-      {
-        model: Credit,
-        separate: true,
-      },
-    ],
-  })
-
-  const otherStudents = await Student.findAll({
-    where: { studentnumber: otherStudentnumbers },
-  })
-  console.log('SPLIT')
-  otherStudents.forEach(s => console.log(`${new Date(s.birthdate).getFullYear()};${s.gender_fi}`))
-  console.log('SPLIT')
-
-  const getFirstName = namestring => {
-    const names = namestring.split(' ')
-    const i = names.findIndex(n => n.substring(0, 1) === '*')
-    return i != -1 ? names[i].substring(1) : names[0]
-  }
-
-  fmseiStudents.forEach(s =>
-    console.log(
-      `${s.studentnumber};${s.email || ''};${s.zipcode || ''};${s.city_fi || ''};${new Date(
-        s.birthdate
-      ).getFullYear()};${s.country_fi || ''};${s.home_country_fi || ''};${s.gender_fi};${getFirstName(s.firstnames)}`
-    )
-  )
-  console.log('SPLIT')
-  // FMSEI all credits
-  fmseiStudents.forEach(s => {
-    s.credits.forEach(c =>
-      console.log(
-        `${c.student_studentnumber};${c.course_code};${c.grade};${c.credits};${new Date(
-          c.attainment_date
-        ).toISOString()}`
-      )
-    )
-  })
-}
-
-createReport()

services/backend/scripts/force_sync_database.js

@@ -1,16 +0,0 @@
-const { seedAllMigrations } = require('../src/database/seed_migrations')
-const { forceSyncDatabase } = require('../src/database/connection')
-
-const sync = async () => {
-  try {
-    await forceSyncDatabase()
-    await seedAllMigrations()
-    console.log('Force sync succeeded. ')
-    process.exit(0)
-  } catch (e) {
-    console.log(`Force sync failed: ${e}`)
-    process.exit(1)
-  }
-}
-
-sync()

services/backend/scripts/refresh_studyright_associations.js

@@ -1,8 +0,0 @@
-const { refreshAssociationsInRedis } = require('../src/services/studyrights')
-
-const run = async () => {
-  await refreshAssociationsInRedis()
-  process.exit(0)
-}
-
-run()

services/backend/scripts/seed_migrations.js

@@ -1,13 +0,0 @@
-const { seedAllMigrations } = require('../src/database/seed_migrations')
-
-const run = async () => {
-  try {
-    await seedAllMigrations()
-    process.exit(0)
-  } catch (e) {
-    console.log(`Seeding migrations failed: ${e}`)
-    process.exit(1)
-  }
-}
-
-run()

services/backend/scripts/update_top_teachers.js

@@ -1,38 +0,0 @@
-const { findAndSaveTopTeachers } = require('../src/services/topteachers')
-const logger = require('../src/util/logger')
-
-const findAndSaveTeachers = async (startcode, endcode) => {
-  for (let code = startcode; code <= endcode; code++) {
-    logger.info(`Saving top teachers from year ${code}`)
-    await findAndSaveTopTeachers(code)
-  }
-}
-
-const parseargs = (args, SEPARATOR = '=') =>
-  args
-    .filter(arg => arg.includes(SEPARATOR))
-    .reduce((acc, arg) => {
-      const [key, val] = arg.split(SEPARATOR)
-      return {
-        ...acc,
-        [key.trim()]: val.trim(),
-      }
-    }, {})
-
-const run = async () => {
-  try {
-    const { from, to } = parseargs(process.argv)
-    const start = from
-    const end = to || from
-    if (end < start) {
-      throw Error('"to" -argument has to be less than "from" -argument. ')
-    }
-    await findAndSaveTeachers(start, end)
-    process.exit(0)
-  } catch (error) {
-    logger.error(`Failed to update top teachers: ${error.message}`)
-    process.exit(1)
-  }
-}
-
-run()

services/backend/src/app.js

@@ -1,66 +1,56 @@
+require('express-async-errors')
 const express = require('express')
 const cors = require('cors')
-const bodyParser = require('body-parser')
 const Sentry = require('@sentry/node')
-const conf = require('./conf-backend')
+const { baseUrl, backendPort, frontUrl } = require('./conf-backend')
 const routes = require('./routes')
 const { startCron } = require('./events')
-const { PORT } = conf
 const { initializeDatabaseConnection, dbConnections } = require('./database/connection')
-
-const SENTRY_ENVIRONMENT = process.env.SENTRY_ENVIRONMENT || ''
-const SENTRY_RELEASE = process.env.SENTRY_ENVIRONMENT || ''
+const initializeSentry = require('./util/sentry')
+const errorMiddleware = require('./middleware/errormiddleware')
+const logger = require('./util/logger')
 
 initializeDatabaseConnection()
   .then(() => {
     dbConnections.connect()
 
     dbConnections.on('connect', () => {
-      console.log('Connected to sis db successfully')
+      logger.info('Connected to sis db successfully')
     })
 
     dbConnections.on('error', () => {
-      console.log('Failed connecting to sis db')
+      logger.error('Failed to connect to sis db!')
     })
 
     const app = express()
-
-    if (SENTRY_ENVIRONMENT && SENTRY_RELEASE) {
-      Sentry.init({
-        dsn: 'https://020b79f0cbb14aad94cc9d69a1ea9d52@sentry.cs.helsinki.fi/2',
-        environment: SENTRY_ENVIRONMENT,
-        release: SENTRY_RELEASE,
-      })
-      app.use(Sentry.Handlers.requestHandler())
-    }
+    initializeSentry(app)
+    app.use(Sentry.Handlers.requestHandler())
+    app.use(Sentry.Handlers.tracingHandler())
 
     startCron()
 
-    app.use(cors({ credentials: true, origin: conf.frontend_addr }))
-    app.use(bodyParser.json())
+    app.use(cors({ credentials: true, origin: frontUrl }))
+    app.use(express.json())
 
-    let BASE_URL = ''
-    if (process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test') {
-      BASE_URL = '/api'
-    }
+    routes(app, baseUrl)
 
-    routes(app, BASE_URL)
-
-    app.get('*', async (req, res) => {
+    app.get('*', async (_, res) => {
       const results = { error: 'unknown endpoint' }
       res.status(404).json(results)
     })
 
     app.use(Sentry.Handlers.errorHandler())
 
-    const server = app.listen(PORT, () => console.log(`Backend listening on port ${PORT}!`))
+    app.use(errorMiddleware)
+
+    const server = app.listen(backendPort, () => logger.info(`Backend listening on port ${backendPort}!`))
     process.on('SIGTERM', () => {
       server.close(() => {
-        console.log('Process terminated')
+        logger.info('Process terminated')
       })
     })
   })
   .catch(e => {
     process.exitCode = 1
-    console.log(e)
+    logger.error(e)
   })

services/backend/src/conf-backend.js

@@ -1,4 +1,41 @@
-const isTest = process.env.NODE_ENV === 'test'
+// Node env to use
+const isStaging = process.env.NODE_ENV === 'staging'
+const isDev = process.env.NODE_ENV === 'development'
+const isProduction = process.env.NODE_ENV === 'production'
+
+// Sentry
+const sentryRelease = process.env.SENTRY_RELEASE || ''
+const sentryEnvironment = process.env.SENTRY_ENVIRONMENT || ''
+const runningInCI = process.env.CI === 'true'
+
+// IAM group
+const requiredGroup = isStaging
+  ? ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
+  : ['grp-oodikone-users', 'grp-oodikone-basic-users']
+
+// Pate
+const pateToken = process.env.PATE_API_TOKEN || ''
+
+// Importer client
+const importerToken = process.env.IMPORTER_DB_API_TOKEN || ''
+const importerUrl = process.env.IMPORTER_DB_API_URL || ''
+
+// Networking: Urls & ports
+const baseUrl = isDev ? '/api' : ''
+const frontUrl = process.env.FRONT_URL
+const backendPort = 8080
+
+// Analytics
+const ANALYTICS_INFLUXDB_URL = process.env.ANALYTICS_INFLUXDB_URL
+const ANALYTICS_INFLUXDB_USER = process.env.ANALYTICS_INFLUXDB_USER
+const ANALYTICS_INFLUXDB_PASSWORD = process.env.ANALYTICS_INFLUXDB_PASSWORD
+const ANALYTICS_INFLUXDB_DB = process.env.ANALYTICS_INFLUXDB_DB
+
+// Matomo
+const MATOMO_SITE_ID = process.env.MATOMO_SITE_ID
+const MATOMO_URL = process.env.MATOMO_URL
+
+// Other stuff
 const DB_URL_KONE = process.env.DB_URL_KONE
 
 let DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS, 10)
@@ -10,20 +47,12 @@ if (DB_MAX_CRON_CONNECTIONS < 1) {
   DB_MAX_CRON_CONNECTIONS = 1
 }
 
-const frontend_addr = process.env.FRONT_URL
 const redis = process.env.REDIS
 const TOKEN_SECRET = process.env.TOKEN_SECRET
 const SECRET_TOKEN = process.env.SECRET_TOKEN
 const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
 const USERSERVICE_URL = process.env.USERSERVICE_URL
 const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL
-const PORT = isTest ? 8079 : 8080
-const ANALYTICS_INFLUXDB_URL = process.env.ANALYTICS_INFLUXDB_URL
-const ANALYTICS_INFLUXDB_USER = process.env.ANALYTICS_INFLUXDB_USER
-const ANALYTICS_INFLUXDB_PASSWORD = process.env.ANALYTICS_INFLUXDB_PASSWORD
-const ANALYTICS_INFLUXDB_DB = process.env.ANALYTICS_INFLUXDB_DB
-const MATOMO_SITE_ID = process.env.MATOMO_SITE_ID
-const MATOMO_URL = process.env.MATOMO_URL
 
 const formatURL = url => {
   return !!url && !url.startsWith('http') ? `http://${url}` : url
@@ -33,14 +62,7 @@ const addSlashToEnd = url => (url.endsWith('/') ? url : url + '/')
 
 const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
 
-const isStaging = process.env.NODE_ENV === 'staging'
-
-const requiredGroup = isStaging
-  ? ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
-  : ['grp-oodikone-users', 'grp-oodikone-basic-users']
-
 module.exports = {
-  frontend_addr,
   DB_URL_KONE,
   DB_MAX_CONNECTIONS,
   DB_MAX_CRON_CONNECTIONS,
@@ -50,9 +72,7 @@ module.exports = {
   DB_SCHEMA_KONE,
   USERSERVICE_URL: formatURL(USERSERVICE_URL),
   ACCESS_TOKEN_HEADER_KEY,
-  PORT,
   requiredGroup,
-  isTest,
   SIS_UPDATER_URL,
   ANALYTICS_INFLUXDB_URL: formatURL(ANALYTICS_INFLUXDB_URL),
   ANALYTICS_INFLUXDB_USER,
@@ -60,4 +80,14 @@ module.exports = {
   ANALYTICS_INFLUXDB_DB,
   MATOMO_SITE_ID,
   MATOMO_URL: MATOMO_URL && addSlashToEnd(MATOMO_URL),
+  runningInCI,
+  sentryRelease,
+  sentryEnvironment,
+  isProduction,
+  pateToken,
+  baseUrl,
+  frontUrl,
+  backendPort,
+  importerToken,
+  importerUrl,
 }

services/backend/src/database/connection.js

@@ -3,6 +3,8 @@ const EventEmitter = require('events')
 const Umzug = require('umzug')
 const conf = require('../conf-backend')
 const { SIS_DB_URL } = process.env
+const logger = require('../util/logger')
+
 class DbConnection extends EventEmitter {
   constructor() {
     super()
@@ -31,8 +33,7 @@ class DbConnection extends EventEmitter {
         this.emit('error', e)
         return
       }
-      console.log(`Sis database connection failed! Attempt ${attempt}/${this.RETRY_ATTEMPTS}`)
-      console.log(e)
+      logger.error({ message: `Sis database connection failed! Attempt ${attempt}/${this.RETRY_ATTEMPTS}`, meta: e })
       setTimeout(() => this.connect(attempt + 1), 1000 * attempt)
     }
   }
@@ -54,47 +55,33 @@ const initializeDatabaseConnection = async () => {
       break
     } catch (e) {
       if (i === waitSeconds) {
-        console.log(`Could not connect to database in ${waitSeconds} seconds`)
+        logger.error(`Could not connect to database in ${waitSeconds} seconds`)
         throw e
       }
-      console.log('.')
       await sleep(1000)
     }
   }
-  if (!conf.isTest) {
-    try {
-      const migrator = new Umzug({
-        storage: 'sequelize',
-        storageOptions: {
-          sequelize: sequelizeKone,
-          tableName: 'migrations',
-          schema: conf.DB_SCHEMA_KONE,
-        },
-        logging: console.log,
-        migrations: {
-          params: [sequelizeKone.getQueryInterface(), Sequelize],
-          path: `${process.cwd()}/src/database/migrations_kone`,
-          pattern: /\.js$/,
-          schema: conf.DB_SCHEMA_KONE,
-        },
-      })
-      const migrations = await migrator.up()
-
-      console.log('Kone Migrations up to date', migrations)
-    } catch (e) {
-      console.log('Kone Migration error')
-      throw e
-    }
-  }
-}
-
-const forceSyncDatabase = async () => {
   try {
-    await sequelizeKone.getQueryInterface().createSchema(conf.DB_SCHEMA_KONE)
+    const migrator = new Umzug({
+      storage: 'sequelize',
+      storageOptions: {
+        sequelize: sequelizeKone,
+        tableName: 'migrations',
+        schema: conf.DB_SCHEMA_KONE,
+      },
+      migrations: {
+        params: [sequelizeKone.getQueryInterface(), Sequelize],
+        path: `${process.cwd()}/src/database/migrations_kone`,
+        pattern: /\.js$/,
+        schema: conf.DB_SCHEMA_KONE,
+      },
+    })
+    const migrations = await migrator.up()
+    logger.info({ message: 'Kone Migrations up to date', meta: migrations })
   } catch (e) {
-    //console.log(e)
+    logger.error('Kone Migration error')
+    throw e
   }
-  await sequelizeKone.sync({ force: true })
 }
 
 const dbConnections = new DbConnection()
@@ -102,5 +89,4 @@ module.exports = {
   dbConnections,
   sequelizeKone,
   initializeDatabaseConnection,
-  forceSyncDatabase,
 }

services/backend/src/database/migrations_kone/20210817_00_remove_filters.js

@@ -0,0 +1,6 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('filters')
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_kone/20210817_01_remove_usagestats.js

@@ -0,0 +1,6 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('usage_statistics')
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_kone/20210817_02_remove_unit_tags.js

@@ -0,0 +1,6 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('unit_tag')
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_kone/20210817_03_remove_course_duplicates.js

@@ -0,0 +1,6 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('course_duplicates')
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_kone/20210823_00_remove_programme_modules_and_programme_module_children.js

@@ -0,0 +1,7 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('programme_module_children')
+    await queryInterface.dropTable('programme_modules')
+  },
+  down: async () => {},
+}

services/backend/src/events.js

@@ -9,204 +9,168 @@ const {
   getStartYears,
 } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
-const { getAllProgrammes, nonGraduatedStudentsOfElementDetail } = require('./services/studyrights')
+const { getAllProgrammes } = require('./services/studyrights')
 const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studyprogramme')
 const { findAndSaveTeachers } = require('./services/topteachers')
-const {
-  setProductivity,
-  setThroughput,
-  patchProductivity,
-  patchThroughput,
-  patchNonGraduatedStudents,
-} = require('./services/analyticsService')
-const { isNewHYStudyProgramme } = require('./util')
+const { setProductivity, setThroughput, patchProductivity, patchThroughput } = require('./services/analyticsService')
+const { isProduction } = require('./conf-backend')
+const { getCurrentSemester } = require('./services/semesters')
+const logger = require('./util/logger')
 
 const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
 
 const refreshStudyrightAssociations = async () => {
-  try {
-    console.log('Refreshing studyright associations...')
-    await refreshAssociationsInRedis()
-  } catch (e) {
-    console.error(e)
-  }
+  await refreshAssociationsInRedis()
+  logger.info('Studyright associations refreshed!')
 }
 
 const refreshOverview = async () => {
-  try {
-    console.log('Refreshing throughput and productivity for programmes...')
-    const codes = (await getAllProgrammes()).map(p => p.code)
-    let ready = 0
-    for (const code of codes) {
-      let programmeStatsSince = new Date('2017-07-31')
-      if (code.includes('MH') || code.includes('KH')) {
-        programmeStatsSince = new Date('2017-07-31')
-      } else {
-        programmeStatsSince = new Date('2000-07-31')
-      }
-      try {
-        await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-        const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
-        await setThroughput(data)
-      } catch (e) {
-        try {
-          await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-        } catch (e) {
-          console.error(e)
-        }
-        console.error(e)
-        console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
-      }
-      try {
-        await patchProductivity({ [code]: { status: 'RECALCULATING' } })
-        const data = await productivityStatsForStudytrack(code, programmeStatsSince)
-        await setProductivity(data)
-      } catch (e) {
-        try {
-          await patchProductivity({
-            [code]: { status: 'RECALCULATION ERRORED' },
-          })
-        } catch (e) {
-          console.error(e)
-        }
-        console.error(e)
-        console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-      }
-      ready += 1
-      console.log(`${ready}/${codes.length} programmes done`)
+  logger.info('Refreshing throughput and productivity for programmes...')
+  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH') || code.includes('MH'))
+  let ready = 0
+  for (const code of codes) {
+    let programmeStatsSince = new Date('2017-07-31')
+    if (code.includes('MH') || code.includes('KH')) {
+      programmeStatsSince = new Date('2017-07-31')
+    } else {
+      programmeStatsSince = new Date('2000-07-31')
     }
-  } catch (e) {
-    console.error(e)
+    try {
+      await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+      const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
+      await setThroughput(data)
+    } catch (e) {
+      logger.error({ message: `failed to recalculate throughput for ${code}`, meta: e })
+      try {
+        await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
+      } catch (e) {
+        logger.error({ message: 'failed to update throughtput status to error', meta: e })
+      }
+    }
+    try {
+      await patchProductivity({ [code]: { status: 'RECALCULATING' } })
+      const data = await productivityStatsForStudytrack(code, programmeStatsSince)
+      await setProductivity(data)
+    } catch (e) {
+      logger.error({ message: `failed to recalculate productivity for ${code}`, meta: e })
+      try {
+        await patchProductivity({
+          [code]: { status: 'RECALCULATION ERRORED' },
+        })
+      } catch (e) {
+        logger.error({ message: 'failed to update productivity status to error', meta: e })
+      }
+    }
+    ready += 1
+    logger.info(`${ready}/${codes.length} programmes done`)
   }
+  logger.info('Throughput and productivity for programmes refreshed')
 }
 
 const refreshTeacherLeaderboard = async () => {
-  try {
-    const startyearcode = new Date().getFullYear() - 1950
-    const endyearcode = startyearcode + 1
-    console.log('Refreshing teacher leaderboard...')
-    await findAndSaveTeachers(startyearcode, endyearcode)
-  } catch (e) {
-    console.log(e)
-  }
-}
-
-const refreshNonGraduatedStudentsOfOldProgrammes = async () => {
-  try {
-    const oldProgrammeCodes = (await getAllProgrammes()).map(p => p.code).filter(c => !isNewHYStudyProgramme(c))
-    let i = 0
-    console.log('Refreshing non-graduated students of old programmes...')
-    await Promise.all(
-      oldProgrammeCodes.map(
-        c =>
-          new Promise(async res => {
-            try {
-              const [nonGraduatedStudents, studentnumbers] = await nonGraduatedStudentsOfElementDetail(c)
-              await patchNonGraduatedStudents({ [c]: { formattedData: nonGraduatedStudents, studentnumbers } })
-              console.log(`${++i}/${oldProgrammeCodes.length}`)
-            } catch (e) {
-              console.log(`Failed refreshing non-graduated students of programme ${c}!`)
-            }
-            res()
-          })
-      )
-    )
-  } catch (e) {
-    console.log(e)
-  }
+  // refresh this and previous year
+  const currentSemestersYearCode = (await getCurrentSemester()).getDataValue('yearcode')
+  await findAndSaveTeachers(currentSemestersYearCode - 1, currentSemestersYearCode)
 }
 
 const refreshProtoCtoRedis = async () => {
-  try {
-    const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false' }
-    const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false' }
-    const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true' }
-    const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true' }
-    console.log('Refreshing CDS ProtoC')
-    await refreshProtoC(defaultQuery)
-    await refreshProtoC(onlyOld)
-    await refreshProtoC(onlyEnr)
-    await refreshProtoC(bothToggles)
-  } catch (e) {
-    console.log(e)
-  }
+  logger.info('Refreshing CDS ProtoC')
+  const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false' }
+  const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false' }
+  const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true' }
+  const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true' }
+  await refreshProtoC(defaultQuery)
+  await refreshProtoC(onlyOld)
+  await refreshProtoC(onlyEnr)
+  await refreshProtoC(bothToggles)
+  logger.info('Refreshing CDS ProtoC doned')
 }
 
 const refreshStatusToRedis = async () => {
-  try {
-    const unixMillis = moment().valueOf()
-    const date = new Date(Number(unixMillis))
+  const unixMillis = moment().valueOf()
+  const date = new Date(Number(unixMillis))
 
-    date.setHours(23, 59, 59, 999)
-    const showByYearOff = 'false'
-    const showByYear = 'true'
-    console.log('Refreshing CDS Status')
-    await refreshStatus(date.getTime(), showByYearOff)
-    await refreshStatus(date.getTime(), showByYear)
+  date.setHours(23, 59, 59, 999)
+  const showByYearOff = 'false'
+  const showByYear = 'true'
+  logger.info('Refreshing CDS Status')
+  await refreshStatus(date.getTime(), showByYearOff)
+  await refreshStatus(date.getTime(), showByYear)
+  logger.info('Refreshing CDS Status doned')
 
-    console.log('Refreshing CDS Graduated')
-    await refreshStatusGraduated(date.getTime(), showByYearOff)
-    await refreshStatusGraduated(date.getTime(), showByYear)
-  } catch (e) {
-    console.log(e)
-  }
+  logger.info('Refreshing CDS Graduated')
+  await refreshStatusGraduated(date.getTime(), showByYearOff)
+  await refreshStatusGraduated(date.getTime(), showByYear)
+  logger.info('Refreshing CDS graduated doned')
 }
 
 const refreshUberToRedis = async () => {
-  try {
-    const years = await getStartYears()
-    const mappedYears = years.map(({ studystartdate }) => studystartdate)
-    mappedYears.forEach(async year => {
-      console.log('Refreshing CDS Uber data for date', year)
-      const defaultQuery = { include_old_attainments: 'false', start_date: year }
-      const oldAttainmentsQuery = { include_old_attainments: 'true', start_date: year }
+  const years = (await getStartYears()).map(({ studystartdate }) => studystartdate)
+  for (const year of years) {
+    const formattedYear = new Date(year).getFullYear()
+    logger.info(`Refreshing CDS uber year ${formattedYear}`)
+    const defaultQuery = { include_old_attainments: 'false', start_date: year }
+    const oldAttainmentsQuery = { include_old_attainments: 'true', start_date: year }
+    try {
       await refreshUber(defaultQuery)
       await refreshUber(oldAttainmentsQuery)
-    })
-  } catch (e) {
-    console.log(e)
+      logger.info(`Refreshing CDS uber year ${formattedYear} doned`)
+    } catch (e) {
+      logger.error({ message: `Error when refreshing CDS uber year ${formattedYear}`, meta: e })
+    }
   }
+  logger.info(`Refreshing CDS Uber data doned`)
 }
 
 const refreshProtoCProgrammeToRedis = async () => {
-  try {
-    const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH'))
-    codes.forEach(async code => {
-      console.log('refreshing code', code)
-      const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
-      const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
-      const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
-      const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true', code }
-      console.log('Refreshing CDS ProtoCProgramme for code ', code)
-      await refreshProtoCProgramme(defaultQuery)
-      await refreshProtoCProgramme(onlyOld)
-      await refreshProtoCProgramme(onlyEnr)
-      await refreshProtoCProgramme(bothToggles)
-    })
-  } catch (e) {
-    console.log(e)
+  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH'))
+  for (const code of codes) {
+    logger.info(`Refreshing ProtoCProgramme code ${code}`)
+    const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
+    const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
+    const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
+    const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true', code }
+    const queries = [defaultQuery, onlyOld, onlyEnr, bothToggles]
+    for (const query of queries) {
+      try {
+        await refreshProtoCProgramme(query)
+      } catch (e) {
+        logger.error({
+          message: `Error when refreshing ProtocProgramme code ${code}, query ${Object.keys({ query })[0]}`,
+          meta: e,
+        }) // object.keys is hax to print query variable name
+      }
+    }
+    logger.info(`Refreshing ProtoCProgramme code ${code} doned`)
   }
+  logger.info(`Refreshing protocprogramme doned`)
 }
 
 const refreshStatistics = async () => {
-  await refreshStudyrightAssociations()
-  await refreshOverview()
-  await refreshNonGraduatedStudentsOfOldProgrammes()
-  await refreshTeacherLeaderboard()
+  const statfuncs = [refreshStudyrightAssociations, refreshOverview, refreshTeacherLeaderboard]
+  logger.info('Refreshing statistics')
+  for (const func of statfuncs) {
+    await func()
+  }
+  logger.info('Statistics refreshed!')
 }
 
 const refreshTrends = async () => {
-  await refreshProtoCtoRedis()
-  await refreshStatusToRedis()
-  await refreshUberToRedis()
-  await refreshProtoCProgrammeToRedis()
+  const trendfuncs = [refreshProtoCtoRedis, refreshStatusToRedis, refreshUberToRedis, refreshProtoCProgrammeToRedis]
+  logger.info('Refreshing trends')
+  for (const func of trendfuncs) {
+    await func()
+  }
+  logger.info('Trends refreshed!')
 }
 
 const startCron = () => {
-  if (process.env.NODE_ENV === 'production') {
-    schedule('0 6 * * *', async () => {
-      await refreshStatistics()
-      await refreshTrends()
+  if (isProduction) {
+    // refresh 3am every day
+    schedule('0 3 * * *', async () => {
+      for (const func of [refreshStatistics, refreshTrends]) {
+        await func()
+      }
     })
   }
 }
@@ -214,4 +178,5 @@ const startCron = () => {
 module.exports = {
   startCron,
   refreshStatistics,
+  refreshTrends,
 }

services/backend/src/middleware/auth.js

@@ -5,7 +5,7 @@ const { getUserDataFor } = require('../services/userService')
 const { ACCESS_TOKEN_HEADER_KEY } = require('../conf-backend')
 const { hasRequiredGroup, parseHyGroups } = require('../util/utils')
 
-const TOKEN_VERSION = 1 // When token structure changes, increment in userservice, backend and frontend
+const TOKEN_VERSION = 1.1 // When token structure changes, increment in userservice, backend and frontend
 
 const checkAuth = async (req, res, next) => {
   const token = req.headers[ACCESS_TOKEN_HEADER_KEY]
@@ -75,17 +75,9 @@ const checkUserBlacklisting = async (req, res, next) => {
   next()
 }
 
-const checkIfNonAdminAskingOldOodiData = async (req, res, next) => {
-  if (req.headers['x-sis'] === 'false' && !req.roles.includes('admin')) {
-    return res.status(403).json({ error: 'Requesting data from old oodi backend without admin rights' })
-  }
-  next()
-}
-
 module.exports = {
   checkAuth,
   roles,
   checkRequiredGroup,
   checkUserBlacklisting,
-  checkIfNonAdminAskingOldOodiData,
 }

services/backend/src/middleware/errormiddleware.js

@@ -0,0 +1,19 @@
+const Sentry = require('@sentry/node')
+const { ApplicationError } = require('../util/customErrors')
+const logger = require('../util/logger')
+
+const errorHandler = (error, req, res, next) => {
+  logger.error(`${error.message} ${error.name} ${error.stack}`)
+
+  Sentry.captureException(error)
+
+  if (res.headersSent) {
+    return next(error)
+  }
+
+  const normalizedError = error instanceof ApplicationError ? error : new ApplicationError(error.message)
+
+  return res.status(normalizedError.status).json(normalizedError)
+}
+
+module.exports = errorHandler

services/backend/src/models/index.js

@@ -30,6 +30,9 @@ Student.hasMany(SemesterEnrollment, { foreignKey: 'studentnumber', sourceKey: 's
 SemesterEnrollment.belongsTo(Semester, { foreignKey: 'semestercomposite', targetKey: 'composite' })
 Semester.hasMany(SemesterEnrollment, { foreignKey: 'semestercomposite', sourceKey: 'composite' })
 
+ProgrammeModule.belongsTo(Organization, { foreignKey: 'organization_id' })
+Organization.hasMany(ProgrammeModule, { foreignKey: 'organization_id' })
+
 Credit.notUnnecessary = credit => {
   return credit.credits > 0 && credit.credits <= 12
 }
@@ -96,6 +99,8 @@ ProgrammeModule.belongsToMany(ProgrammeModule, {
   foreignKey: 'parent_id',
 })
 
+Organization.hasMany(Organization, { foreignKey: 'parent_id', as: 'children' })
+
 module.exports = {
   Organization,
   Course,

services/backend/src/models/models_kone.js

@@ -44,62 +44,6 @@ const MandatoryCourse = sequelizeKone.define('mandatory_courses', {
   },
 })
 
-const Filters = sequelizeKone.define('filters', {
-  id: {
-    type: Sequelize.STRING,
-    primaryKey: true,
-  },
-  name: {
-    type: Sequelize.STRING,
-  },
-  description: {
-    type: Sequelize.STRING,
-  },
-  filters: {
-    type: Sequelize.JSONB,
-  },
-  population: {
-    type: Sequelize.JSONB,
-  },
-})
-
-const UsageStatistic = sequelizeKone.define(
-  'usage_statistics',
-  {
-    id: {
-      primaryKey: true,
-      type: Sequelize.STRING,
-    },
-    username: {
-      type: Sequelize.STRING,
-    },
-    name: {
-      type: Sequelize.STRING,
-    },
-    time: {
-      type: Sequelize.INTEGER,
-    },
-    admin: {
-      type: Sequelize.BOOLEAN,
-    },
-    method: {
-      type: Sequelize.STRING,
-    },
-    URL: {
-      type: Sequelize.STRING,
-    },
-    status: {
-      type: Sequelize.INTEGER,
-    },
-    data: {
-      type: Sequelize.JSONB,
-    },
-  },
-  {
-    timestamps: false,
-  }
-)
-
 const TagStudent = sequelizeKone.define(
   'tag_student',
   {
@@ -160,30 +104,6 @@ const MandatoryCourseLabels = sequelizeKone.define('mandatory_course_labels', {
   },
 })
 
-const MigrationKone = sequelizeKone.define(
-  'migrations',
-  {
-    name: {
-      type: Sequelize.STRING,
-      primaryKey: true,
-    },
-  },
-  {
-    tableName: 'migrations',
-    timestamps: false,
-  }
-)
-
-const CourseDuplicates = sequelizeKone.define('course_duplicates', {
-  groupid: {
-    primaryKey: true,
-    type: Sequelize.INTEGER,
-  },
-  coursecode: {
-    type: Sequelize.STRING,
-  },
-})
-
 const ThesisCourse = sequelizeKone.define('thesis_courses', {
   programmeCode: {
     primaryKey: true,
@@ -231,13 +151,9 @@ module.exports = {
   MandatoryCourse,
   TagStudent,
   Tag,
-  Filters,
-  UsageStatistic,
-  CourseDuplicates,
   ThesisCourse,
   ThesisTypeEnums,
   MandatoryCourseLabels,
-  MigrationKone,
   CustomPopulationSearch,
   sequelizeKone,
   ExcludedCourse,

services/backend/src/models/organization.js

@@ -19,6 +19,9 @@ Organization.init(
     name: {
       type: JSONB,
     },
+    parent_id: {
+      type: STRING,
+    },
     createdAt: {
       type: DATE,
     },

services/backend/src/models/programmeModule.js

@@ -18,6 +18,16 @@ ProgrammeModule.init(
     type: {
       type: STRING,
     },
+    studyLevel: {
+      type: STRING,
+    },
+    organization_id: {
+      type: STRING,
+      references: {
+        model: 'organization',
+        key: 'id',
+      },
+    },
     createdAt: {
       type: DATE,
     },

services/backend/src/routes.js

@@ -25,21 +25,26 @@ const matomoInit = require('./routes/matomo-init')
 const customPopulationSearch = require('./routes/customPopulationSearch')
 const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
+const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
 
 module.exports = (app, url) => {
   app.use(url, ping)
-  app.use(shibbolethHeadersFix(['hyGroupCn', 'SHIB_LOGOUT_URL', 'eduPersonAffiliation', 'uid', 'displayName', 'mail']))
+  app.use(
+    shibbolethHeadersFix([
+      'hyGroupCn',
+      'SHIB_LOGOUT_URL',
+      'eduPersonAffiliation',
+      'uid',
+      'displayName',
+      'mail',
+      'hyPersonSisuId',
+      'preferredLanguage',
+    ])
+  )
   app.use(url, matomoInit)
   app.use(url, login)
   app.use(`${url}/superlogin`, superlogin)
-  app.use(
-    auth.checkAuth,
-    auth.checkRequiredGroup,
-    auth.checkUserBlacklisting,
-    auth.checkIfNonAdminAskingOldOodiData,
-    accessLogger,
-    sentryUserId
-  )
+  app.use(auth.checkAuth, auth.checkRequiredGroup, auth.checkUserBlacklisting, accessLogger, sentryUserId)
   app.use(url, elementdetails)
   app.use(url, courses)
   app.use(url, students)
@@ -52,11 +57,12 @@ module.exports = (app, url) => {
   app.use(`${url}/faculties`, faculties)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
-  app.use(`${url}/users`, auth.roles(['users']), users)
+  app.use(`${url}/users`, auth.roles(['admin']), users)
   app.use(`${url}/feedback`, feedback)
   app.use(`${url}/mandatory_courses`, mandatoryCourses)
   app.use(`${url}/mandatory-course-labels`, mandatoryCourseLabels)
   app.use(`${url}/tsa`, tsaAnalytics)
   app.use(`${url}/custom-population-search`, customPopulationSearch)
   app.use(`${url}/cool-data-science`, trends)
+  app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
 }

services/backend/src/routes/courses.js

@@ -2,7 +2,6 @@ const router = require('express').Router()
 const crypto = require('crypto')
 const Course = require('../services/courses')
 const { validateParamLength } = require('../util')
-const logger = require('../util/logger')
 
 router.get('/v2/coursesmulti', async (req, res) => {
   let results = { courses: [] }
@@ -17,35 +16,29 @@ router.get('/v2/coursesmulti', async (req, res) => {
 })
 
 router.get('/v3/courseyearlystats', async (req, res) => {
-  try {
-    const { rights, roles } = req
+  const { rights, roles } = req
 
-    const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
+  const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
 
-    // If user has rights to see at least one programme, then they are allowed
-    // to see all of them
-    if (!allowedRoles && rights.length < 1) {
-      return res.status(403).json({ error: 'No programmes so no access to course stats' })
-    }
+  // If user has rights to see at least one programme, then they are allowed
+  // to see all of them
+  if (!allowedRoles && rights.length < 1) {
+    return res.status(403).json({ error: 'No programmes so no access to course stats' })
+  }
 
-    const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
+  const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
 
-    const separate = !sep ? false : JSON.parse(sep)
-    const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
-    if (!codes) {
-      res.status(422).send('Missing required query parameters')
-    } else {
-      // Studentnumbers should be obfuscated to all other users except admins
-      // and users with rights to any specific study programmes
-      const anonymize = allowedRoles !== 'admin' && rights.length < 1
-      const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
-      const results = await Course.courseYearlyStats(codes, separate, unifyOpenUniCourses, anonymizationSalt)
-      res.json(results)
-    }
-  } catch (e) {
-    logger.error(e.message)
-    console.log(e)
-    res.status(500).send('Something went wrong with handling the request.')
+  const separate = !sep ? false : JSON.parse(sep)
+  const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
+  if (!codes) {
+    res.status(422).send('Missing required query parameters')
+  } else {
+    // Studentnumbers should be obfuscated to all other users except admins
+    // and users with rights to any specific study programmes
+    const anonymize = allowedRoles !== 'admin' && rights.length < 1
+    const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
+    const results = await Course.courseYearlyStats(codes, separate, unifyOpenUniCourses, anonymizationSalt)
+    res.json(results)
   }
 })
 

services/backend/src/routes/customPopulationSearch.js

@@ -7,76 +7,56 @@ const {
 } = require('../services/customPopulationSearch')
 
 router.get('/', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id },
-    } = req
-    const customPopulationSearches = await getCustomPopulationSearchesByUser(id)
-    res.json(customPopulationSearches)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  const {
+    decodedToken: { id },
+  } = req
+  const customPopulationSearches = await getCustomPopulationSearchesByUser(id)
+  res.json(customPopulationSearches)
 })
 
 router.post('/', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id },
-      body: { name, students },
-    } = req
+  const {
+    decodedToken: { id },
+    body: { name, students },
+  } = req
 
-    if (!name) return res.status(400).json({ error: 'name missing' })
-    if (students && !Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
+  if (!name) return res.status(400).json({ error: 'name missing' })
+  if (students && !Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
 
-    const customPopulationSearch = await createCustomPopulationSearch(name, id, students || [])
-    res.json(customPopulationSearch)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  const customPopulationSearch = await createCustomPopulationSearch(name, id, students || [])
+  res.json(customPopulationSearch)
 })
 
 router.put('/:id', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id: userId },
-      body: { students },
-    } = req
-    const { id } = req.params
+  const {
+    decodedToken: { id: userId },
+    body: { students },
+  } = req
+  const { id } = req.params
 
-    if (!id) return res.status(400).json({ error: 'id missing' })
-    if (!students) return res.status(400).json({ error: 'students missing' })
-    if (!Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
+  if (!id) return res.status(400).json({ error: 'id missing' })
+  if (!students) return res.status(400).json({ error: 'students missing' })
+  if (!Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
 
-    const updatedPopulationSearch = await updateCustomPopulationSearch(userId, id, students)
-    if (!updatedPopulationSearch) return res.status(404).json({ error: 'custom population search not found' })
+  const updatedPopulationSearch = await updateCustomPopulationSearch(userId, id, students)
+  if (!updatedPopulationSearch) return res.status(404).json({ error: 'custom population search not found' })
 
-    res.json(updatedPopulationSearch)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  res.json(updatedPopulationSearch)
 })
 
 router.delete('/:id', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id: userId },
-    } = req
-    const { id } = req.params
+  const {
+    decodedToken: { id: userId },
+  } = req
+  const { id } = req.params
 
-    if (!id) return res.status(400).json({ error: 'id missing' })
+  if (!id) return res.status(400).json({ error: 'id missing' })
 
-    const deletedSuccessfully = (await deleteCustomPopulationSearch(userId, id)) > 0
+  const deletedSuccessfully = (await deleteCustomPopulationSearch(userId, id)) > 0
 
-    if (!deletedSuccessfully) return res.status(404).json({ error: 'custom population search not found' })
+  if (!deletedSuccessfully) return res.status(404).json({ error: 'custom population search not found' })
 
-    res.json(id)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  res.json(id)
 })
 
 module.exports = router

services/backend/src/routes/elementdetails.js

@@ -1,13 +1,11 @@
 const router = require('express').Router()
-const { getStudentsUserCanAccess } = require('../services/userService')
 const MandatoryCourses = require('../services/mandatoryCourses')
 const {
   productivityStatsForStudytrack,
   throughputStatsForStudytrack,
   optionData,
 } = require('../services/studyprogramme')
-const { findProgrammeTheses } = require('../services/thesis')
-
+const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
 const {
   getProductivity,
   setProductivity,
@@ -15,11 +13,22 @@ const {
   setThroughput,
   patchProductivity,
   patchThroughput,
-  getNonGraduatedStudents,
 } = require('../services/analyticsService')
+const { getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
+const logger = require('../util/logger')
 
 const programmeStatsSince = new Date('2017-07-31')
 
+router.get('/elementdetails/all', async (req, res) => {
+  const elementdetails = await getAllElementDetails()
+  res.json(elementdetails)
+})
+
+router.get('/studyprogrammes', async (req, res) => {
+  const studyrights = await getAllProgrammes()
+  res.json(studyrights)
+})
+
 router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
   const code = req.params.id
 
@@ -36,34 +45,6 @@ router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
   res.json(data)
 })
 
-router.get('/v2/studyprogrammes/:id/present_students', async (req, res) => {
-  try {
-    const {
-      roles,
-      decodedToken,
-      params: { id },
-    } = req
-    if (!id) return res.status(400).json({ error: 'programme id missing' })
-
-    const nonGraduatedStudents = await getNonGraduatedStudents(id)
-    if (!nonGraduatedStudents) return res.status(200).json({})
-    const {
-      data: { formattedData, studentnumbers },
-    } = nonGraduatedStudents
-
-    const filteredData = { ...formattedData }
-    const studentsUserCanAccess = await getStudentsUserCanAccess(studentnumbers, roles, decodedToken.userId)
-    Object.keys(filteredData).forEach(year => {
-      filteredData[year] = filteredData[year].filter(({ studentNumber }) => studentsUserCanAccess.has(studentNumber))
-    })
-
-    res.json(filteredData)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
 router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
   if (req.params.id) {
     const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
@@ -80,14 +61,14 @@ router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
     try {
       data = await getProductivity(code)
     } catch (e) {
-      console.error(e)
+      logger.error(`Failed to get code ${code} productivity`)
     }
     if (!data) {
       try {
         const stats = await productivityStatsForStudytrack(code, programmeStatsSince)
         data = await setProductivity(stats)
       } catch (e) {
-        console.error(e)
+        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
       }
     }
     return res.json(data)
@@ -99,14 +80,8 @@ router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
 router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
   const code = req.query.code
 
-  console.log('Productivity stats recalculation starting')
-  try {
-    await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-    res.status(200).end()
-  } catch (e) {
-    console.error(e)
-    return res.status(500).end()
-  }
+  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+  res.status(200).end()
 
   try {
     if (code.includes('MH') || code.includes('KH')) {
@@ -122,13 +97,13 @@ router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
         [code]: { status: 'RECALCULATION ERRORED' },
       })
     } catch (e) {
-      console.error(e)
+      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
       return
     }
-    console.error(e)
-    console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
+
+    logger.error(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
   }
-  console.log(`Productivity stats recalculation for studyprogramme ${code} done`)
+  logger.info(`Produdtiviy stats recalculation for studyprogramme ${code} done`)
 })
 
 router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
@@ -140,14 +115,14 @@ router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
     try {
       data = await getThroughput(code)
     } catch (e) {
-      console.error(e)
+      logger.error(`Failed to get code ${code} throughput`)
     }
     if (!data) {
       try {
         const result = await throughputStatsForStudytrack(req.params.id, programmeStatsSince.getFullYear())
         data = await setThroughput(result)
       } catch (e) {
-        console.error(e)
+        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
       }
     }
     return res.json({ ...data, thesis: await thesisPromise })
@@ -159,14 +134,8 @@ router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
 router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
   const code = req.query.code
 
-  console.log('Throughput stats recalculation starting')
-  try {
-    await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-    res.status(200).end()
-  } catch (e) {
-    console.error(e)
-    return res.status(500).end()
-  }
+  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+  res.status(200).end()
 
   try {
     if (code.includes('MH') || code.includes('KH')) {
@@ -180,13 +149,43 @@ router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
     try {
       await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
     } catch (e) {
-      console.error(e)
+      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
       return
     }
-    console.error(e)
-    console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
+    logger.error(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
+  }
+  logger.info(`Throughput stats recalculation for studyprogramme ${code} done`)
+})
+
+router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {
+  const { id } = req.params
+  if (id) {
+    const thesis = await findProgrammeTheses(id)
+    res.json(thesis)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.post('/v2/studyprogrammes/:id/thesis', async (req, res) => {
+  const { id } = req.params
+  const { course, thesisType } = req.body
+  if (id && course && thesisType) {
+    const thesis = await createThesisCourse(id, course, thesisType)
+    res.status(201).json(thesis)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
+  const { id, course } = req.params
+  if (id && course) {
+    const deleted = await deleteThesisCourse(id, course)
+    res.status(204).json(deleted)
+  } else {
+    res.status(422).end()
   }
-  console.log(`Throughput stats recalculation for studyprogramme ${code} done`)
 })
 
 router.use('*', (req, res, next) => next())

services/backend/src/routes/faculties.js

@@ -3,7 +3,6 @@ const { faculties } = require('../services/organisations')
 
 router.get('/', async (req, res) => {
   const facultyList = await faculties()
-  console.log('faculties')
   res.json(facultyList)
 })
 

services/backend/src/routes/feedback.js

@@ -1,29 +1,27 @@
 const router = require('express').Router()
-const mailservice = require('../services/mailservice')
 const userService = require('../services/userService')
+const { sendFeedbackToToska } = require('../services/mailservice')
+const logger = require('../util/logger')
 
 router.post('/email', async (req, res) => {
   const { content } = req.body
-  const { uid } = req.headers
-  const { email, full_name } = await userService.byUsername(uid)
-  const formattedEmail = content
-    .split('\n\n')
-    .map(line => `<p> ${line} </p>`)
-    .join('')
-  const feedback = mailservice.feedback(formattedEmail, uid, email, full_name)
+  const {
+    decodedToken: { userId },
+  } = req
+  const { email, full_name } = await userService.byUsername(userId)
 
-  if (process.env.SMTP !== undefined && content) {
-    await mailservice.transporter.sendMail(feedback, error => {
-      if (error) {
-        console.log('Error occured')
-        res.status(400).json(error).end()
-      } else {
-        console.log('Message sent succesfully!')
-        res.status(200).json('success').end()
-      }
-      mailservice.transporter.close()
-    })
+  const result = await sendFeedbackToToska({
+    feedbackContent: content,
+    userId,
+    userEmail: email,
+    userFullName: full_name,
+  })
+  if (result.error) {
+    return res.status(400).json(result).end()
   }
+  logger.info(`${userId} succesfully sent some feedback mail to toska`)
+
+  res.status(200).json('success').end()
 })
 
 module.exports = router

services/backend/src/routes/login.js

@@ -1,66 +1,47 @@
 const router = require('express').Router()
-const mailservice = require('../services/mailservice')
 const userService = require('../services/userService')
 const { parseHyGroups } = require('../util/utils')
-
-const sendEmail = async uid => {
-  if (process.env.SMTP !== undefined) {
-    const message = mailservice.message1(uid)
-    await mailservice.transporter.sendMail(message, error => {
-      if (error) {
-        console.log('Error occurred')
-        console.log(error.message)
-      } else {
-        console.log('Message sent successfully!')
-      }
-      // only needed when using pooled connections
-      mailservice.transporter.close()
-    })
-  }
-}
+const { sendNotificationAboutNewUser } = require('../services/mailservice')
+const logger = require('../util/logger')
 
 router.post('/login', async (req, res) => {
-  try {
-    const uid = req.headers['uid']
-    if (req.headers['shib-session-id'] && uid) {
-      const full_name = req.headers.displayname || ''
-      const mail = req.headers.mail || ''
-      const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-      const affiliations = parseHyGroups(req.headers['edupersonaffiliation'])
-
-      console.log(uid, 'trying to login, referring to userservice.')
-      let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail)
-      isNew && sendEmail(uid)
-      res.status(200).json({ token })
-    } else {
-      res
-        .status(401)
-        .json({
-          message: `Not enough headers login, uid: ${req.headers.uid}
-        session-id ${req.headers['shib-session-id']}`,
-        })
-        .end()
+  const uid = req.headers['uid']
+  if (req.headers['shib-session-id'] && uid) {
+    const full_name = req.headers.displayname || ''
+    const mail = req.headers.mail || ''
+    const hyGroups = parseHyGroups(req.headers['hygroupcn'])
+    const affiliations = parseHyGroups(req.headers['edupersonaffiliation'])
+    const hyPersonSisuId = req.headers.hypersonsisuid || ''
+    logger.info(`${uid} trying to login, referring to userservice`)
+    let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail, hyPersonSisuId)
+    if (isNew) {
+      const result = await sendNotificationAboutNewUser({ userId: uid, userFullName: full_name })
+      if (result.error) {
+        return res.status(400).json(result).end()
+      }
     }
-  } catch (err) {
-    console.log(err)
-    res.status(401).json({ message: 'problem with login', err })
+    res.status(200).json({ token })
+  } else {
+    res
+      .status(401)
+      .json({
+        message: `Not enough headers login, uid: ${req.headers.uid}
+        session-id ${req.headers['shib-session-id']}`,
+      })
+      .end()
   }
 })
 
 router.delete('/logout', async (req, res) => {
-  try {
-    const logoutUrl = req.headers.shib_logout_url
-    const { returnUrl } = req.body
-    if (logoutUrl) {
-      return res
-        .status(200)
-        .send({ logoutUrl: `${logoutUrl}?return=${returnUrl}` })
-        .end()
-    }
-    res.status(200).send({ logoutUrl: returnUrl }).end()
-  } catch (err) {
-    res.status(500).json({ message: 'Error with logout', err })
+  const logoutUrl = req.headers.shib_logout_url
+  const { returnUrl } = req.body
+  if (logoutUrl) {
+    return res
+      .status(200)
+      .send({ logoutUrl: `${logoutUrl}?return=${returnUrl}` })
+      .end()
   }
+  res.status(200).send({ logoutUrl: returnUrl }).end()
 })
 
 module.exports = router

services/backend/src/routes/mandatorycourselabels.js

@@ -1,5 +1,6 @@
 const router = require('express').Router()
 const MandatoryCourseLabels = require('../services/mandatoryCourseLabels')
+const logger = require('../util/logger')
 
 router.get('/:programme/labels', async (req, res) => {
   const { programme } = req.params
@@ -8,7 +9,7 @@ router.get('/:programme/labels', async (req, res) => {
       const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
       res.status(200).json(labels)
     } catch (err) {
-      console.error(err)
+      logger.error(err)
       res.status(400).json(err.message)
     }
   } else {
@@ -25,7 +26,7 @@ router.delete('/:programme/labels', async (req, res) => {
       const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
       res.status(200).json(labels)
     } catch (err) {
-      console.error(err)
+      logger.error(err)
       res.status(400).json(err.message)
     }
   } else {
@@ -42,7 +43,7 @@ router.post('/:programme/move', async (req, res) => {
       const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
       res.status(200).json(labels)
     } catch (err) {
-      console.error(err)
+      logger.error(err)
       res.status(400).json(err.message)
     }
   } else {
@@ -59,7 +60,7 @@ router.post('/:programme/labels', async (req, res) => {
       const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
       res.status(200).json(labels)
     } catch (err) {
-      console.error(err)
+      logger.error(err)
       res.status(400).json(err.message)
     }
   } else {

services/backend/src/routes/mandatorycourses.js

@@ -5,47 +5,32 @@ router.delete('/:programme', async (req, res) => {
   const { programme } = req.params
   const { course } = req.body
   if (!programme || !course) return res.status(400).end()
-  try {
-    await MandatoryCourses.remove(programme, course)
-    res.status(200).json(course)
-  } catch (err) {
-    console.error(err)
-    res.status(500).json(err.message)
-  }
+  await MandatoryCourses.remove(programme, course)
+  res.status(200).json(course)
 })
 
 router.post('/:programme', async (req, res) => {
   const { programme } = req.params
   const { course } = req.body
   if (!programme || !course) return res.status(400).end()
-  try {
-    await MandatoryCourses.create(programme, course)
-    const newCourse = await MandatoryCourses.find(programme, course)
-    if (!newCourse) return res.status(400).end()
-    res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
-  } catch (err) {
-    console.error(err)
-    res.status(500).json(err.message)
-  }
+  await MandatoryCourses.create(programme, course)
+  const newCourse = await MandatoryCourses.find(programme, course)
+  if (!newCourse) return res.status(400).end()
+  res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
 })
 
 router.post('/:programme/label/:course', async (req, res) => {
   const { programme, course } = req.params
   const { label } = req.body
   if (!programme || !course) return res.status(400).end()
-  try {
-    await MandatoryCourses.updateLabel(programme, course, label)
-    const updatedCourse = await MandatoryCourses.find(programme, course)
-    if (!updatedCourse) return res.status(400).end()
-    res.status(200).json({
-      name: updatedCourse.course.name,
-      code: updatedCourse.course_code,
-      label: updatedCourse.mandatory_course_label,
-    })
-  } catch (err) {
-    console.error(err)
-    res.status(500).json(err.message)
-  }
+  await MandatoryCourses.updateLabel(programme, course, label)
+  const updatedCourse = await MandatoryCourses.find(programme, course)
+  if (!updatedCourse) return res.status(400).end()
+  res.status(200).json({
+    name: updatedCourse.course.name,
+    code: updatedCourse.course_code,
+    label: updatedCourse.mandatory_course_label,
+  })
 })
 
 module.exports = router

services/backend/src/routes/population.js

@@ -2,13 +2,14 @@ const crypto = require('crypto')
 const Sentry = require('@sentry/node')
 const router = require('express').Router()
 const Population = require('../services/populations')
-const Filters = require('../services/filters')
 const Student = require('../services/students')
 const StudyrightService = require('../services/studyrights')
 const UserService = require('../services/userService')
 const TagService = require('../services/tags')
 const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
+const { getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')
+const { logger } = require('../util/logger')
 
 const filterPersonalTags = (population, userId) => {
   return {
@@ -22,266 +23,238 @@ const filterPersonalTags = (population, userId) => {
 
 // POST instead of GET because of too long params and "sensitive" data
 router.post('/v2/populationstatistics/courses', async (req, res) => {
-  try {
-    if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
-      Sentry.captureException(new Error('The body should have a year, semester and study rights defined'))
-      res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
+  if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
+    Sentry.captureException(new Error('The body should have a year, semester and study rights defined'))
+    res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
+    return
+  }
+  if (!Array.isArray(req.body.studyRights)) {
+    // studyRights should always be an array
+    req.body.studyRights = [req.body.studyRights]
+  }
+
+  if (req.body.months == null) {
+    req.body.months = 12
+  }
+
+  if (req.body.years) {
+    const upperYearBound = new Date().getFullYear() + 1
+    const multicoursestatPromises = Promise.all(
+      req.body.years.map(year => {
+        if (req.body.selectedStudentsByYear) {
+          req.body.selectedStudents = req.body.selectedStudentsByYear[year]
+        }
+        const newMonths = (upperYearBound - Number(year)) * 12
+        const query = { ...req.body, year, months: newMonths }
+        const coursestatistics = Population.bottlenecksOf(query)
+        return coursestatistics
+      })
+    )
+    const multicoursestats = await multicoursestatPromises
+    const result = StatMergeService.populationCourseStatsMerger(multicoursestats)
+    if (result.error) {
+      res.status(400).json(result)
       return
     }
-    if (!Array.isArray(req.body.studyRights)) {
-      // studyRights should always be an array
-      req.body.studyRights = [req.body.studyRights]
+
+    res.json(result)
+  } else {
+    const result = await Population.bottlenecksOf(req.body)
+
+    if (result.error) {
+      Sentry.captureException(new Error(result.error))
+      res.status(400).json(result)
+      return
     }
 
-    if (req.body.months == null) {
-      req.body.months = 12
-    }
-
-    if (req.body.years) {
-      console.log('years väylä')
-      const upperYearBound = new Date().getFullYear() + 1
-      const multicoursestatPromises = Promise.all(
-        req.body.years.map(year => {
-          if (req.body.selectedStudentsByYear) {
-            req.body.selectedStudents = req.body.selectedStudentsByYear[year]
-          }
-          const newMonths = (upperYearBound - Number(year)) * 12
-          const query = { ...req.body, year, months: newMonths }
-          const coursestatistics = Population.bottlenecksOf(query)
-          return coursestatistics
-        })
-      )
-      const multicoursestats = await multicoursestatPromises
-      const result = StatMergeService.populationCourseStatsMerger(multicoursestats)
-      console.log('result: ', result)
-      if (result.error) {
-        res.status(400).json(result)
-        return
-      }
-
-      res.json(result)
-    } else {
-      console.log('bottleneck väylä')
-      const result = await Population.bottlenecksOf(req.body)
-
-      if (result.error) {
-        Sentry.captureException(new Error(result.error))
-        res.status(400).json(result)
-        return
-      }
-
-      res.json(result)
-    }
-  } catch (e) {
-    console.log(e)
-    Sentry.captureException(e)
-    res.status(500).json({ error: e })
+    res.json(result)
   }
 })
 
 router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => {
-  try {
-    const { from, to, coursecodes } = req.body
-    if (!from || !to || !coursecodes) {
-      return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
-    }
-    const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(coursecodes)
-    if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
-      return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
-    }
-    let studentnumberlist
-    const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
-    const {
-      decodedToken: { userId },
-      roles,
-    } = req
-
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = studentnumbers
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-    }
-
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-      },
-      studentnumberlist
-    )
-
-    if (result.error) {
-      res.status(400).json(result)
-      return
-    }
-
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  const { from, to, coursecodes } = req.body
+  if (!from || !to || !coursecodes) {
+    return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
   }
+  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(coursecodes)
+  if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
+    return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
+  }
+  let studentnumberlist
+  const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
+  const {
+    decodedToken: { userId },
+    roles,
+  } = req
+
+  if (roles && roles.includes('admin')) {
+    studentnumberlist = studentnumbers
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
+  }
+
+  const result = await Population.bottlenecksOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+    },
+    studentnumberlist
+  )
+
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).json(result)
+    return
+  }
+
+  res.json(result)
 })
 
 router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
-  try {
-    const { tag } = req.body
-    if (!tag) {
-      res.status(400).json({ error: 'The body should have a tag defined' })
-      return
-    }
-    let studentnumberlist
-    const studentnumbers = await Student.findByTag(tag)
-    const {
-      decodedToken: { userId },
-      roles,
-    } = req
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = studentnumbers
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-    }
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-        tag,
-      },
-      studentnumberlist
-    )
-
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  const { tag } = req.body
+  if (!tag) {
+    res.status(400).json({ error: 'The body should have a tag defined' })
+    return
   }
+  let studentnumberlist
+  const studentnumbers = await Student.findByTag(tag)
+  const {
+    decodedToken: { userId },
+    roles,
+  } = req
+  if (roles && roles.includes('admin')) {
+    studentnumberlist = studentnumbers
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
+  }
+  const result = await Population.bottlenecksOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+      tag,
+    },
+    studentnumberlist
+  )
+
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).end()
+    return
+  }
+
+  res.json(result)
 })
 
 router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
-  try {
-    if (!req.body.studentnumberlist) {
-      res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
-      return
-    }
-    let studentnumberlist
-    const {
-      decodedToken: { userId },
-      roles,
-    } = req
-
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = req.body.studentnumberlist
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
-    }
-
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-      },
-      studentnumberlist
-    )
-    if (result.error) {
-      res.status(400).json(result)
-      return
-    }
-
-    res.json(result)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: err })
+  if (!req.body.studentnumberlist) {
+    res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
+    return
   }
+  let studentnumberlist
+  const {
+    decodedToken: { userId, sisPersonId },
+    roles,
+  } = req
+
+  if (roles && roles.includes('admin')) {
+    studentnumberlist = req.body.studentnumberlist
+  } else if (req.body.usingStudyGuidanceGroups) {
+    const studentsUserCanAccess = await getAllStudentsUserHasInGroups(sisPersonId)
+    studentnumberlist = req.body.studentnumberlist.filter(s => studentsUserCanAccess.has(s))
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
+  }
+
+  const result = await Population.bottlenecksOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+    },
+    studentnumberlist
+  )
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).json(result)
+    return
+  }
+
+  res.json(result)
 })
 
 router.get('/v3/populationstatistics', async (req, res) => {
   const { year, semesters, studyRights: studyRightsJSON } = req.query
   const { decodedToken } = req
 
+  if (!year || !semesters || !studyRightsJSON) {
+    res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
+    return
+  }
+  let studyRights = null
   try {
-    if (!year || !semesters || !studyRightsJSON) {
-      res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
-      return
-    }
-    let studyRights = null
-    try {
-      studyRights = JSON.parse(studyRightsJSON)
-      const { rights, roles } = req
+    studyRights = JSON.parse(studyRightsJSON)
+    const { rights, roles } = req
 
-      if (!roles || !roles.includes('admin')) {
-        if (!rights.includes(studyRights.programme)) {
-          res.status(403).json([])
-          return
-        }
-      }
-    } catch (e) {
-      console.error(e)
-      res.status(400).json({ error: 'The query had invalid studyRights' })
-      return
-    }
-
-    if (req.query.months == null) {
-      req.query.months = 12
-    }
-
-    if (req.query.years) {
-      const upperYearBound = new Date().getFullYear() + 1
-      const multipopulationstudentPromises = Promise.all(
-        req.query.years.map(year => {
-          const newMonths = (upperYearBound - Number(year)) * 12
-          const populationStudents = Population.optimizedStatisticsOf({
-            ...req.query,
-            studyRights,
-            year,
-            months: newMonths,
-          })
-          return populationStudents
-        })
-      )
-      const multipopulationstudents = await multipopulationstudentPromises
-
-      const result = StatMergeService.populationStudentsMerger(multipopulationstudents)
-
-      if (result.error) {
-        console.log(result.error)
-        res.status(400).end()
+    if (!roles || !roles.includes('admin')) {
+      if (!rights.includes(studyRights.programme)) {
+        res.status(403).json([])
         return
       }
-
-      console.log(`request completed ${new Date()}`)
-      res.json(filterPersonalTags(result, decodedToken.id))
-    } else {
-      const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
-      if (result.error) {
-        console.log(result.error)
-        res.status(400).end()
-        return
-      }
-
-      console.log(`request completed ${new Date()}`)
-      res.json(filterPersonalTags(result, decodedToken.id))
     }
   } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+    res.status(400).json({ error: 'The query had invalid studyRights' })
+    return
+  }
+
+  if (req.query.months == null) {
+    req.query.months = 12
+  }
+
+  if (req.query.years) {
+    const upperYearBound = new Date().getFullYear() + 1
+    const multipopulationstudentPromises = Promise.all(
+      req.query.years.map(year => {
+        const newMonths = (upperYearBound - Number(year)) * 12
+        const populationStudents = Population.optimizedStatisticsOf({
+          ...req.query,
+          studyRights,
+          year,
+          months: newMonths,
+        })
+        return populationStudents
+      })
+    )
+    const multipopulationstudents = await multipopulationstudentPromises
+
+    const result = StatMergeService.populationStudentsMerger(multipopulationstudents)
+
+    if (result.error) {
+      Sentry.captureException(new Error(result.error))
+      res.status(400).end()
+      return
+    }
+
+    res.json(filterPersonalTags(result, decodedToken.id))
+  } else {
+    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
+    if (result.error) {
+      Sentry.captureException(new Error(result.error))
+      res.status(400).end()
+      return
+    }
+
+    res.json(filterPersonalTags(result, decodedToken.id))
   }
 })
 
@@ -307,34 +280,28 @@ router.get('/v3/populationstatisticsbytag', async (req, res) => {
     const codes = unitsUserCanAccess.map(unit => unit.id)
     studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
   }
-  try {
-    const studyRights = JSON.parse(studyRightsJSON)
-    const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
-    const yearDifference = Number(year) - Number(newStartYear)
-    const newMonths = Number(months) + 12 * yearDifference
-    const result = await Population.optimizedStatisticsOf(
-      {
-        year: newStartYear,
-        studyRights,
-        semesters,
-        months: newMonths,
-        tag: foundTag,
-      },
-      studentnumberlist
-    )
+  const studyRights = JSON.parse(studyRightsJSON)
+  const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
+  const yearDifference = Number(year) - Number(newStartYear)
+  const newMonths = Number(months) + 12 * yearDifference
+  const result = await Population.optimizedStatisticsOf(
+    {
+      year: newStartYear,
+      studyRights,
+      semesters,
+      months: newMonths,
+      tag: foundTag,
+    },
+    studentnumberlist
+  )
 
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(filterPersonalTags(result, decodedToken.id))
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).end()
+    return
   }
+
+  res.json(filterPersonalTags(result, decodedToken.id))
 })
 
 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
@@ -356,166 +323,108 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
 
   const semesters = ['FALL', 'SPRING']
   const studentnumbers = await Student.findByCourseAndSemesters(JSON.parse(coursecodes), from, to, separate)
-  try {
-    const result = await Population.optimizedStatisticsOf(
-      {
-        // Useless, because studentnumbers are already filtered above by from & to.
-        // We should probably refactor this to avoid more confusement.
-        year: 1900,
-        studyRights: [],
-        semesters,
-        months: 10000,
-      },
-      studentnumbers
-    )
+  const result = await Population.optimizedStatisticsOf(
+    {
+      // Useless, because studentnumbers are already filtered above by from & to.
+      // We should probably refactor this to avoid more confusement.
+      year: 1900,
+      studyRights: [],
+      semesters,
+      months: 10000,
+    },
+    studentnumbers
+  )
 
-    let studentsUserCanAccess
-    if (roles && roles.includes('admin')) {
-      studentsUserCanAccess = new Set(studentnumbers)
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(decodedToken.userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentsUserCanAccess = new Set(await Student.filterStudentnumbersByAccessrights(studentnumbers, codes))
-    }
-
-    const randomHash = crypto.randomBytes(12).toString('hex')
-    const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
-      courses,
-      studyrights,
-      gender_code,
-      studentNumber: crypto.createHash('md5').update(`${studentNumber}${randomHash}`).digest('hex'),
-      firstnames: '',
-      lastname: '',
-      name: '',
-      email: '',
-      started: null,
-      credits: '',
-      tags: [],
-      obfuscated: true,
-    })
-
-    result.students = result.students.map(s => (studentsUserCanAccess.has(s.studentNumber) ? s : obfuscateStudent(s)))
-
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(filterPersonalTags(result, decodedToken.id))
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  let studentsUserCanAccess
+  if (roles && roles.includes('admin')) {
+    studentsUserCanAccess = new Set(studentnumbers)
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(decodedToken.userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentsUserCanAccess = new Set(await Student.filterStudentnumbersByAccessrights(studentnumbers, codes))
   }
+
+  const randomHash = crypto.randomBytes(12).toString('hex')
+  const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
+    courses,
+    studyrights,
+    gender_code,
+    studentNumber: crypto.createHash('md5').update(`${studentNumber}${randomHash}`).digest('hex'),
+    firstnames: '',
+    lastname: '',
+    name: '',
+    email: '',
+    started: null,
+    credits: '',
+    tags: [],
+    obfuscated: true,
+  })
+
+  result.students = result.students.map(s => (studentsUserCanAccess.has(s.studentNumber) ? s : obfuscateStudent(s)))
+
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).end()
+    return
+  }
+
+  res.json(filterPersonalTags(result, decodedToken.id))
 })
 
 router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
-  const { studentnumberlist } = req.body
+  const { studentnumberlist, usingStudyGuidanceGroups } = req.body
   const { roles, decodedToken } = req
 
-  const studentsUserCanAccess = await UserService.getStudentsUserCanAccess(
-    studentnumberlist,
-    roles,
-    decodedToken.userId
-  )
+  if (usingStudyGuidanceGroups) {
+    if (!decodedToken.sisPersonId) {
+      logger.error(
+        `User ${decodedToken.userId} tried to get person groups but personId was ${decodedToken.sisPersonId} in header`
+      )
+      return res.status(400).json({ error: 'Not possible to get groups without personId' })
+    }
+  }
+
+  const studentsUserCanAccess = usingStudyGuidanceGroups
+    ? await getAllStudentsUserHasInGroups(decodedToken.sisPersonId)
+    : await UserService.getStudentsUserCanAccess(studentnumberlist, roles, decodedToken.userId)
   const filteredStudentNumbers = studentnumberlist.filter(s => studentsUserCanAccess.has(s))
 
-  try {
-    const result = await Population.optimizedStatisticsOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-      },
-      filteredStudentNumbers
-    )
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-    res.status(200).json(filterPersonalTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(400).end()
-  }
-})
-
-router.get('/v2/populationstatistics/filters', async (req, res) => {
-  let results = []
-  let rights = req.query.studyRights
-  if (!Array.isArray(rights)) {
-    // studyRights should always be an array
-    rights = [rights]
-  }
-  try {
-    results = await Filters.findForPopulation(rights)
-    res.status(200).json(results)
-  } catch (err) {
-    console.log(err)
-    res.status(400).end()
-  }
-})
-
-router.post('/v2/populationstatistics/filters', async (req, res) => {
-  let results = []
-  const filter = req.body
-
-  try {
-    results = await Filters.createNewFilter(filter)
-    res.status(200).json(results)
-  } catch (err) {
-    console.log(err)
-    res.status(400).end()
-  }
-})
-
-router.delete('/v2/populationstatistics/filters', async (req, res) => {
-  let results = []
-  const filter = req.body
-  try {
-    results = await Filters.deleteFilter(filter)
-    res.status(200).json(results)
-  } catch (err) {
-    res.status(400).end()
+  const result = await Population.optimizedStatisticsOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+    },
+    filteredStudentNumbers
+  )
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    return
   }
+  res.status(200).json(filterPersonalTags(result, decodedToken.id))
 })
 
 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
-  try {
-    const { rights, roles } = req
-    if (roles && roles.includes('admin')) {
-      const studyrights = await StudyrightService.getAssociations()
-      res.json(studyrights)
-    } else {
-      const studyrights = await StudyrightService.getFilteredAssociations(rights)
-      res.json(studyrights)
-    }
-  } catch (err) {
-    res.status(500).json(err)
+  const { rights, roles } = req
+  if (roles && roles.includes('admin')) {
+    const studyrights = await StudyrightService.getAssociations()
+    res.json(studyrights)
+  } else {
+    const studyrights = await StudyrightService.getFilteredAssociations(rights)
+    res.json(studyrights)
   }
 })
 
 router.get('/v3/populationstatistics/studyprogrammes/unfiltered', async (req, res) => {
-  try {
-    const studyrights = await StudyrightService.getAssociations()
-    res.json(studyrights)
-  } catch (err) {
-    res.status(500).json(err)
-  }
+  const studyrights = await StudyrightService.getAssociations()
+  res.json(studyrights)
 })
 
 router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
-  try {
-    const { courseCodes } = req.query
-    const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(courseCodes))
-    return res.json(maxYearsToCreatePopulationFrom)
-  } catch (err) {
-    console.log('err', err)
-    res.status(500).json(err)
-  }
+  const { courseCodes } = req.query
+  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(courseCodes))
+  return res.json(maxYearsToCreatePopulationFrom)
 })
 
 router.use('*', (req, res, next) => next())

services/backend/src/routes/programmeModules.js

@@ -8,17 +8,20 @@ const {
 
 router.get('/v3/programme_modules/:code', async (req, res) => {
   const { code } = req.params
-  const module = await byProgrammeCode(code)
-  if (!module) {
-    res.status(400).end()
-    return
-  }
-  res.json(module)
+  const result = await byProgrammeCode(code)
+  res.json(result)
 })
 
 router.get('/v3/programme_modules/:code/modules', async (req, res) => {
   const { code } = req.params
   const result = await modulesByProgrammeCode(code)
+  res.json(result)
+})
+
+router.delete('/v3/programme_modules', async (req, res) => {
+  const { programmecode, ids } = req.body
+  await removeExcludedCourses(ids)
+  const result = await byProgrammeCode(programmecode)
   if (!result) {
     res.status(400).end()
     return
@@ -26,36 +29,15 @@ router.get('/v3/programme_modules/:code/modules', async (req, res) => {
   res.json(result)
 })
 
-router.delete('/v3/programme_modules', async (req, res) => {
-  const { programmecode, ids } = req.body
-  try {
-    await removeExcludedCourses(ids)
-    const result = await byProgrammeCode(programmecode)
-    if (!result) {
-      res.status(400).end()
-      return
-    }
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.json(500).json(e)
-  }
-})
-
 router.post('/v3/programme_modules/:programmecode/', async (req, res) => {
   const { programmecode, coursecodes } = req.body
-  try {
-    await addExcludedCourses(programmecode, coursecodes)
-    const result = await byProgrammeCode(programmecode)
-    if (!result) {
-      res.status(400).end()
-      return
-    }
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.json(500).json(e)
+  await addExcludedCourses(programmecode, coursecodes)
+  const result = await byProgrammeCode(programmecode)
+  if (!result) {
+    res.status(400).end()
+    return
   }
+  res.json(result)
 })
 
 module.exports = router

services/backend/src/routes/studyGuidanceGroups.js

@@ -0,0 +1,16 @@
+const router = require('express').Router()
+const { logger } = require('../util/logger')
+const { getAllGroupsAndStudents } = require('../services/studyGuidanceGroups')
+
+router.get('/', async (req, res) => {
+  const {
+    decodedToken: { uid, sisPersonId },
+  } = req
+  if (!sisPersonId) {
+    logger.error(`User ${uid} tried to get person groups but personId was ${sisPersonId} in header`)
+    return res.status(400).json({ error: 'Not possible to get groups without personId' })
+  }
+  return res.json(await getAllGroupsAndStudents(sisPersonId))
+})
+
+module.exports = router

services/backend/src/routes/superlogin.js

@@ -2,24 +2,19 @@ const router = require('express').Router()
 const userService = require('../services/userService')
 
 router.post('/:uid', async (req, res) => {
-  try {
-    const uid = req.headers['uid']
-    if (req.headers['shib-session-id'] && uid) {
-      const asUser = req.params.uid
-      const token = await userService.superlogin(uid, asUser)
-      res.status(200).json(token)
-    } else {
-      res
-        .status(401)
-        .json({
-          message: `Not enough headers login, uid:
+  const uid = req.headers['uid']
+  if (req.headers['shib-session-id'] && uid) {
+    const asUser = req.params.uid
+    const token = await userService.superlogin(uid, asUser)
+    res.status(200).json(token)
+  } else {
+    res
+      .status(401)
+      .json({
+        message: `Not enough headers login, uid:
         ${req.headers.uid} session-id ${req.headers['shib-session-id']}`,
-        })
-        .end()
-    }
-  } catch (err) {
-    console.log(err.message)
-    res.status(500).json({ message: 'problem with login' })
+      })
+      .end()
   }
 })
 

services/backend/src/routes/tags.js

@@ -13,187 +13,132 @@ const filterRelevantStudentTags = (studentTags, userId) => {
 }
 
 router.get('/tags', async (req, res) => {
-  try {
-    const tags = await Tags.findTags()
-    res.status(200).json(tags)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const tags = await Tags.findTags()
+  res.status(200).json(tags)
 })
 
 router.get('/tags/:studytrack', async (req, res) => {
-  try {
-    const { studytrack } = req.params
-    const { rights, roles, decodedToken } = req
+  const { studytrack } = req.params
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    const tags = await Tags.findTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantTags(tags, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const tags = await Tags.findTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantTags(tags, decodedToken.id))
 })
 
 router.post('/tags', async (req, res) => {
-  try {
-    const {
-      tag: { studytrack, tagname, year, personal_user_id },
-    } = req.body
-    const { rights, roles, decodedToken } = req
+  const {
+    tag: { studytrack, tagname, year, personal_user_id },
+  } = req.body
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
-    const tags = await Tags.findTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantTags(tags, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err.message)
-  }
+  await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
+  const tags = await Tags.findTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantTags(tags, decodedToken.id))
 })
 
 router.delete('/tags', async (req, res) => {
-  try {
-    const { tag } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tag } = req.body
+  const { rights, roles, decodedToken } = req
 
-    if (
-      !rights.includes(tag.studytrack) &&
-      !(roles && roles.includes('admin')) &&
-      !(tag.personal_user_id === decodedToken.id)
-    )
-      return res.status(403).end()
+  if (
+    !rights.includes(tag.studytrack) &&
+    !(roles && roles.includes('admin')) &&
+    !(tag.personal_user_id === decodedToken.id)
+  )
+    return res.status(403).end()
 
-    await Tags.deleteTag(tag)
-    const t = await Tags.findTagsByStudytrack(tag.studytrack)
-    res.status(200).json(filterRelevantTags(t, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  await Tags.deleteTag(tag)
+  const t = await Tags.findTagsByStudytrack(tag.studytrack)
+  res.status(200).json(filterRelevantTags(t, decodedToken.id))
 })
 
 router.get('/studenttags', async (req, res) => {
-  try {
-    const result = await TagStudent.getStudentTags()
-    res.status(200).json(result)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const result = await TagStudent.getStudentTags()
+  res.status(200).json(result)
 })
 
 router.get('/studenttags/:studytrack', async (req, res) => {
-  try {
-    const { studytrack } = req.params
-    const { rights, roles, decodedToken } = req
+  const { studytrack } = req.params
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
 })
 
 router.get('/studenttags/:studentnumber', async (req, res) => {
-  try {
-    const { studentnumber } = req.params
-    const { decodedToken } = req
-    const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const { studentnumber } = req.params
+  const { decodedToken } = req
+  const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
+  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
 })
 
 router.post('/studenttags/:studentnumber', async (req, res) => {
-  try {
-    const { tag, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tag, studytrack } = req.body
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    await TagStudent.createStudentTag(tag)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  await TagStudent.createStudentTag(tag)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
 })
 
 router.post('/studenttags', async (req, res) => {
-  try {
-    const { tags, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tags, studytrack } = req.body
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    const existingtags = await Tags.findTagsByStudytrack(studytrack)
-    const existingTagids = existingtags.map(t => t.tag_id)
-    const tagids = [...new Set(tags.map(t => t.tag_id))]
-    if (!tagids.find(t => existingTagids.includes(t))) return res.status(400).json({ error: 'The tag does not exist' })
+  const existingtags = await Tags.findTagsByStudytrack(studytrack)
+  const existingTagids = existingtags.map(t => t.tag_id)
+  const tagids = [...new Set(tags.map(t => t.tag_id))]
+  if (!tagids.find(t => existingTagids.includes(t))) return res.status(400).json({ error: 'The tag does not exist' })
 
-    const studentnumbers = tags.map(t => t.studentnumber)
-    const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
-    const missingStudents = difference(studentnumbers, students)
-    if (missingStudents.length !== 0)
-      return res
-        .status(400)
-        .json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
+  const studentnumbers = tags.map(t => t.studentnumber)
+  const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
+  const missingStudents = difference(studentnumbers, students)
+  if (missingStudents.length !== 0)
+    return res
+      .status(400)
+      .json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
 
-    await TagStudent.createMultipleStudentTags(tags)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  await TagStudent.createMultipleStudentTags(tags)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
 })
 
 router.delete('/studenttags/delete_one', async (req, res) => {
-  try {
-    const { tag_id, studentnumber, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tag_id, studentnumber, studytrack } = req.body
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
-    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
+  const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
+  if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
 
-    await TagStudent.deleteStudentTag(studentnumber, tag_id)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  await TagStudent.deleteStudentTag(studentnumber, tag_id)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
 })
 
 router.delete('/studenttags/delete_many', async (req, res) => {
-  try {
-    const { tagId, studentnumbers, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tagId, studentnumbers, studytrack } = req.body
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
-    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
+  const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
+  if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
 
-    await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(400).json(err)
-  }
+  await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
 })
 
 router.use('*', (req, res, next) => next())

services/backend/src/routes/teachers.js

@@ -1,5 +1,6 @@
 const teachers = require('../services/teachers')
 const topteachers = require('../services/topteachers')
+const { isFaculty, providersOfFaculty } = require('../services/organisations')
 const router = require('express').Router()
 const { mapToProviders } = require('../util/utils')
 
@@ -24,6 +25,7 @@ router.get('/top', async (req, res) => {
   if (!yearcode) {
     return res.status(422).send('Missing required yearcode query param')
   }
+
   const result = await topteachers.getTeacherStats(category, yearcode)
   res.json(result)
 })
@@ -51,7 +53,19 @@ router.get('/stats', async (req, res) => {
   if (!(providers.every(p => providerRights.includes(p)) || roles.includes('admin'))) {
     return res.status(403).send('You do not have rights to see this data')
   }
-  const result = await teachers.yearlyStatistics(providers, semesterStart, semesterEnd || semesterStart + 1)
+
+  // combine provider list that may include programmes and faculties
+  const parsedProviders = [
+    ...new Set(
+      await providers.reduce(
+        async (acc, curr) =>
+          isFaculty(curr) ? [...(await acc), ...(await providersOfFaculty(curr))] : [...(await acc), curr],
+        []
+      )
+    ),
+  ]
+
+  const result = await teachers.yearlyStatistics(parsedProviders, semesterStart, semesterEnd || semesterStart + 1)
   res.json(result)
 })
 

services/backend/src/routes/updater.js

@@ -8,116 +8,63 @@ const {
   abort,
   updateCoursesByCourseCode,
 } = require('../services/sisUpdaterService')
-const { refreshStatistics } = require('../events')
+const { refreshStatistics, refreshTrends } = require('../events')
 
 router.get('/update/v2/meta', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISMetadata(req)
-    if (response) {
-      res.status(200).json('Update SIS meta scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateSISMetadata(req)
+  if (response) {
+    res.status(200).json('Update SIS meta scheduled')
   }
 })
 
 router.get('/update/v2/students', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISStudents()
-    if (response) {
-      res.status(200).json('Update SIS students scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateSISStudents()
+  if (response) {
+    res.status(200).json('Update SIS students scheduled')
   }
 })
 
 router.post('/update/v2/students', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateStudentsByStudentNumber(req.body)
-    if (response) {
-      res.status(200).json('Update SIS students scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateStudentsByStudentNumber(req.body)
+  if (response) {
+    res.status(200).json('Update SIS students scheduled')
   }
 })
 
 router.post('/update/v2/courses', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateCoursesByCourseCode(req.body)
-    if (response) {
-      res.status(200).json('Update SIS courses scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateCoursesByCourseCode(req.body)
+  if (response) {
+    res.status(200).json('Update SIS courses scheduled')
   }
 })
 
 router.get('/update/v2/programmes', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISProgrammes()
-    if (response) {
-      res.status(200).json('Update SIS programmes scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateSISProgrammes()
+  if (response) {
+    res.status(200).json('Update SIS programmes scheduled')
   }
 })
 
 router.get('/refresh_redis_cache', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISRedisCache()
-    if (response) {
-      res.status(200).json('Refreshing SIS redis cache scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateSISRedisCache()
+  if (response) {
+    res.status(200).json('Refreshing SIS redis cache scheduled')
   }
 })
 
 router.post('/refresh_statistic_v2', async (req, res) => {
-  try {
-    refreshStatistics()
-    res.status(200).json('Refreshing sis statistics')
-  } catch (err) {
-    console.error(err)
-    res.status(500).json({ error: 'error' })
-  }
+  await refreshStatistics()
+  res.status(200).json('Refreshing sis statistics')
+})
+
+router.post('/refresh_trends', async (req, res) => {
+  await refreshTrends()
+  res.status(200).json('Refreshing trends')
 })
 
 router.get('/abort', async (req, res) => {
-  try {
-    await abort()
-    res.status(200).json()
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
+  await abort()
+  res.status(200).json()
 })
 
 module.exports = router

services/backend/src/routes/users.js

@@ -1,8 +1,9 @@
 const router = require('express').Router()
 const userService = require('../services/userService')
-const mailservice = require('../services/mailservice')
 const blacklist = require('../services/blacklist')
 const { userDataCache } = require('../services/cache')
+const { sendNotificationAboutAccessToUser, previewNotificationAboutAccessToUser } = require('../services/mailservice')
+const logger = require('../util/logger')
 
 const addUserToBlacklist = async (user, decodedToken) => {
   if (user) {
@@ -35,29 +36,22 @@ router.post('/modifyaccess', async (req, res) => {
 })
 
 router.get('/email/preview', (req, res) => {
-  const { subject, html } = mailservice.message2(null)
-  res.json({ subject, html })
+  const { accessMessageSubject, accessMessageText } = previewNotificationAboutAccessToUser()
+  res.json({ subject: accessMessageSubject, html: accessMessageText })
 })
 
 router.post('/email', async (req, res) => {
-  if (!process.env.SMTP) {
-    return res.status(500).json({ error: 'Email system has not been configured' })
-  }
-
-  const email = req.body.email
-  if (!email) {
+  const userEmail = req.body.email
+  if (!userEmail) {
     return res.status(400).json({ error: 'email is missing' })
   }
 
-  const message = mailservice.message2(email)
-  try {
-    const info = await mailservice.transporter.sendMail(message)
-    console.log('Message sent successfully', info)
-    res.status(200).end()
-  } catch (e) {
-    console.error('Error occurred while sending user email', e)
-    res.status(500).json({ error: e.message })
+  const result = await sendNotificationAboutAccessToUser(userEmail)
+  if (result.error) {
+    return res.status(400).json(result).end()
   }
+  logger.info('Succesfully sent message about oodikone access to user')
+  res.status(200).end()
 })
 
 router.post('/:uid/elements', async (req, res) => {

services/backend/src/services/analyticsService.js

@@ -3,7 +3,6 @@ const moment = require('moment')
 
 const createRedisKeyForProductivity = id => `PRODUCTIVITY_${id}`
 const createRedisKeyForThroughput = id => `THROUGHPUT_${id}`
-const createRedisKeyForNonGraduatedStudents = id => `NONGRADUATEDSTUDENTS_${id}`
 
 const getProductivity = async id => {
   const redisKey = createRedisKeyForProductivity(id)
@@ -88,31 +87,6 @@ const patchThroughput = async data => {
   }
 }
 
-const patchNonGraduatedStudents = async data => {
-  const [id, dataToPatch] = Object.entries(data)[0]
-  const redisKey = createRedisKeyForNonGraduatedStudents(id)
-  const dataFromRedis = JSON.parse(await redisClient.getAsync(redisKey))
-  const patchedData = {
-    data: {
-      ...dataFromRedis.data,
-      ...dataToPatch,
-    },
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(patchedData))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: patchedData,
-  }
-}
-
-const getNonGraduatedStudents = async id => {
-  const redisKey = createRedisKeyForNonGraduatedStudents(id)
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return JSON.parse(dataFromRedis)
-}
-
 module.exports = {
   getProductivity,
   setProductivity,
@@ -120,6 +94,4 @@ module.exports = {
   getThroughput,
   setThroughput,
   patchThroughput,
-  patchNonGraduatedStudents,
-  getNonGraduatedStudents,
 }

services/backend/src/services/blacklist.js

@@ -1,10 +1,11 @@
 const { redisClient } = require('./redis')
 const moment = require('moment')
+const logger = require('../util/logger')
 
 const MAKE_REDIS_HASH_KEY = id => `BLACKLIST ${id}`
 
 const addUserToBlacklist = async userId => {
-  console.log('ADDED BLACKLIST', userId)
+  logger.info(`user ${userId} added to token blacklist`)
   await redisClient.setAsync(MAKE_REDIS_HASH_KEY(userId), moment().toISOString(false))
 }
 

services/backend/src/services/course_stats_counter.js

@@ -16,6 +16,7 @@ class CourseStatsCounter {
       name,
       disciplines: {},
       coursetypes: {},
+      substitutions: [],
     }
     this.students = {
       all: {},
@@ -142,6 +143,10 @@ class CourseStatsCounter {
     this.course.coursetypes[typecode] = name
   }
 
+  addCourseSubstitutions(subs) {
+    this.course.substitutions = subs
+  }
+
   addCourseDiscipline(id, name) {
     this.course.disciplines[id] = name
   }

services/backend/src/services/courses.js

@@ -172,7 +172,7 @@ const allCodeAltenatives = async code => {
   const temp = courses
     .map(c => c.code)
     .map(c => {
-      if (c.match(/^A/)) return [c, 4] // open university codes come last
+      if (c.match(/^AY?/)) return [c, 4] // open university codes come last
       if (c.match(/^\d/)) return [c, 2] // old numeric codes come second
       if (c.match(/^[A-Za-z]/)) return [c, 1] // new letter based codes come first
       return [c, 3] // unknown, comes before open uni?
@@ -424,7 +424,6 @@ const byNameAndOrCodeLike = async (name, code) => {
       organizeSubgroups(course)
     }
   })
-  console.log(courses)
   return { courses }
 }
 

services/backend/src/services/filters.js

@@ -1,32 +0,0 @@
-const Sequelize = require('sequelize')
-const { Filters } = require('../models/models_kone')
-
-const Op = Sequelize.Op
-
-const findForPopulation = pops =>
-  Filters.findAll({
-    where: {
-      population: {
-        [Op.eq]: pops,
-      },
-    },
-  })
-
-const createNewFilter = async filter => {
-  return Filters.create(filter)
-}
-const deleteFilter = async filter => {
-  Filters.destroy({
-    where: {
-      id: {
-        [Op.eq]: filter.id,
-      },
-    },
-  })
-  return filter
-}
-module.exports = {
-  createNewFilter,
-  deleteFilter,
-  findForPopulation,
-}

services/backend/src/services/mailservice.js

@@ -1,81 +1,110 @@
-const nodemailer = require('nodemailer')
+const axios = require('axios')
+const { isProduction, pateToken } = require('../conf-backend')
+const logger = require('../util/logger')
 
-// NB! Store the account object values somewhere if you want
-// to re-use the same account for future mail deliveries
-
-// Create a SMTP transporter object
-const transporter = nodemailer.createTransport(
-  {
-    from: 'Oodikone Robot <noreply@helsinki.fi>',
-    host: `${process.env.SMTP}`,
-    port: 587,
-    secure: false, // false -> TLS, true -> SSL
-    logger: false,
-    debug: false, // include SMTP traffic in the logs
+const pateClient = axios.create({
+  baseURL: 'https://pate.toska.cs.helsinki.fi',
+  params: {
+    token: pateToken,
   },
-  {
-    from: 'Oodikone Robot <noreply@helsinki.fi>',
-  }
-)
+})
 
-const message1 = user => {
-  return {
-    to: 'Toska <grp-toska@helsinki.fi>',
-    subject: 'New user in Oodikone ✔',
-    text: `${user} just logged into oodikone for the first time!`,
-    html: `<p>${user} just logged into oodikone for the first time! </p><br />
-          <img src="cid:toskalogoustcid"/>`,
-    attachments: [
-      {
-        filename: 'toska.png',
-        path: `${process.cwd()}/assets/toska.png`,
-        cid: 'toskalogoustcid',
-      },
-    ],
-  }
-}
-const message2 = email => {
-  return {
-    to: [email, 'grp-toska@helsinki.fi'],
-    replyTo: 'Toska <grp-toska@helsinki.fi>',
-    subject: 'Authorized in oodikone',
-    text: "You've been authorized to use oodikone.",
-    html: `<p>You've been authorized to use <a href="https://oodikone.cs.helsinki.fi">oodikone</a> or 
-    your privileges in <a href="https://oodikone.cs.helsinki.fi">oodikone</a> have changed.</p>
-    <p>You may need to log out and back in before your changes take place.</p>
-    <p>NOTE! If the automatic logging out from all services was not successful when
-    logging out of oodikone, you can log out one by one or close all of your browser windows.</p>
-    <p>If you have defined your browser to restore your previous session in connection
-    with starting the browser, logging out may also require deleting the cookies
-    of the listed services in addition to closing the browser windows.</p>
-    <p>See the instructions for clearing the browser cache: <a href="https://helpdesk.it.helsinki.fi/en/help/1002">
-    https://helpdesk.it.helsinki.fi/en/help/1002</a></p>
-    <br />
-    <hr />
-    <br />
-    <p>Helsingin Yliopisto, TOSKA.</p>
-    <p>grp-toska@helsinki.fi</p>
-    <p>Pietari Kalmin katu 5, Exactum BK113</p>
-    <img style="max-width: 13.5%;height: auto;" src="https://i.imgur.com/tnNDAJk.png" /> `,
-  }
-}
-const feedback = (content, user, email, full_name) => {
-  return {
-    to: 'Toska <grp-toska@helsinki.fi>',
-    replyTo: email,
-    subject: 'New message from Oodikone feedback',
-    text: `New message from user ${user}`,
-    html: `${content}
-    <p>sent by ${full_name}, userid: ${user}, email: ${email}</p>
-    <img src="cid:toskalogoustcid"/>`,
-    attachments: [
-      {
-        filename: 'toska.png',
-        path: `${process.cwd()}/assets/toska.png`,
-        cid: 'toskalogoustcid',
-      },
-    ],
-  }
+const template = {
+  from: 'Oodikone Robot',
 }
 
-module.exports = { transporter, message1, message2, feedback }
+const baseSettings = {
+  color: 'orange',
+  header: 'Sent by Oodikone',
+  dryrun: false,
+}
+
+const sendEmail = async (options = {}) => {
+  if (!isProduction) {
+    const errorMessage = 'Skipped sending email in non-production environment'
+    logger.error({ message: errorMessage, meta: options })
+    return { error: errorMessage }
+  }
+
+  if (!pateToken) {
+    const errorMessage = 'Skipped sending email since pate token was not available'
+    logger.error({ message: errorMessage, meta: options })
+    return { error: errorMessage }
+  }
+
+  const { data } = await pateClient.post('/', options)
+
+  return data
+}
+
+const sendFeedbackToToska = ({ feedbackContent, userId, userEmail, userFullName }) => {
+  const userDetails = `Sent by ${userFullName}, userid: ${userId}, email: ${userEmail}`
+  const text = [feedbackContent, userDetails].join('<br />')
+  return sendEmail({
+    template,
+    emails: [
+      {
+        to: 'Toska <grp-toska@helsinki.fi>',
+        replyTo: userEmail,
+        subject: `Oodikone feedback from ${userFullName}`,
+        text,
+      },
+    ],
+    settings: {
+      ...baseSettings,
+      hideToska: true,
+      disableToska: true,
+    },
+  })
+}
+
+const sendNotificationAboutNewUser = ({ userId, userFullName }) =>
+  sendEmail({
+    template,
+    emails: [
+      {
+        to: 'Toska <grp-toska@helsinki.fi>',
+        subject: 'New user in Oodikone ✔',
+        text: `${userFullName} (${userId}) just logged into Oodikone for the first time!`,
+      },
+    ],
+    settings: {
+      ...baseSettings,
+      hideToska: true,
+      disableToska: true,
+    },
+  })
+
+const accessMessageSubject = 'Authorized in Oodikone'
+const accessMessageText = `<p>You've been authorized to use <a href="https://oodikone.cs.helsinki.fi">Oodikone</a> or your privileges in <a href="https://oodikone.cs.helsinki.fi">Oodikone</a> have changed.</p>
+<p>You may need to log out and back in before your changes take place.</p>
+<p>NOTE! If the automatic logging out from all services was not successful when logging out of oodikone, you can log out one by one or close all of your browser windows.</p>
+<p>If you have defined your browser to restore your previous session in connection with starting the browser, logging out may also require deleting the cookies of the listed services in addition to closing the browser windows.</p>
+<p>See the instructions for clearing the browser cache: <a href="https://helpdesk.it.helsinki.fi/en/help/1002">https://helpdesk.it.helsinki.fi/en/help/1002</a></p>`
+
+const previewNotificationAboutAccessToUser = () => ({ accessMessageSubject, accessMessageText })
+
+const sendNotificationAboutAccessToUser = userEmail =>
+  sendEmail({
+    template,
+    emails: [
+      {
+        to: userEmail,
+        replyTo: 'Toska <grp-toska@helsinki.fi>',
+        subject: accessMessageSubject,
+        text: accessMessageText,
+      },
+    ],
+    settings: {
+      ...baseSettings,
+      hideToska: false,
+      disableToska: false,
+    },
+  })
+
+module.exports = {
+  sendFeedbackToToska,
+  sendNotificationAboutNewUser,
+  sendNotificationAboutAccessToUser,
+  previewNotificationAboutAccessToUser,
+}

services/backend/src/services/organisations.js

@@ -1,6 +1,7 @@
 const Sequelize = require('sequelize')
-const { Organization } = require('../models')
+const { Organization, ProgrammeModule } = require('../models')
 const Op = Sequelize.Op
+const { dbConnections } = require('../database/connection')
 
 // Have facultyfetching to work like it worked during oodi-db time
 const facultiesInOodi = [
@@ -33,6 +34,36 @@ const faculties = () => {
   })
 }
 
+const degreeProgrammeCodesOfFaculty = async facultyCode =>
+  (
+    await ProgrammeModule.findAll({
+      attributes: ['code'],
+      include: {
+        model: Organization,
+        where: {
+          code: facultyCode,
+        },
+      },
+    })
+  ).map(({ code }) => code)
+
+const providersOfFaculty = async facultyCode => {
+  const [result] = await dbConnections.sequelize.query(
+    `SELECT childOrg.code
+     FROM organization parentOrg
+     INNER JOIN organization childOrg 
+     ON childOrg.parent_id = parentOrg.id
+     WHERE parentOrg.code = ?`,
+    { replacements: [facultyCode] }
+  )
+  return result.map(({ code }) => code)
+}
+
+const isFaculty = facultyCode => facultiesInOodi.includes(facultyCode)
+
 module.exports = {
   faculties,
+  degreeProgrammeCodesOfFaculty,
+  isFaculty,
+  providersOfFaculty,
 }

services/backend/src/services/populations.js

@@ -19,7 +19,7 @@ const {
   dbConnections: { sequelize },
 } = require('../database/connection')
 const { Tag, TagStudent } = require('../models/models_kone')
-const { allCodeAltenatives, findOneByCode } = require('./courses')
+const { /*allCodeAltenatives, */ findOneByCode } = require('./courses')
 const { CourseStatsCounter } = require('./course_stats_counter')
 const { getPassingSemester, semesterEnd, semesterStart } = require('../util/semester')
 const { getAllProgrammes } = require('./studyrights')
@@ -654,11 +654,10 @@ const formatStudentsForApi = async (
   elementdetails = elementdetails.reduce(
     (acc, e) => {
       acc.data[e.code] = e
-      if (e.type === 10) acc.degrees.push(e.code)
       if (e.type === 20) acc.programmes.push(e.code)
       return acc
     },
-    { programmes: [], degrees: [], data: {} }
+    { programmes: [], data: {} }
   )
   credits = credits.reduce((acc, e) => {
     acc[e.student_studentnumber] = acc[e.student_studentnumber] || []
@@ -689,7 +688,6 @@ const formatStudentsForApi = async (
         sources: {},
       },
       studyrights: {
-        degrees: [],
         programmes: [],
       },
     }
@@ -803,7 +801,7 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
 
 const findCourses = async (studentnumbers, beforeDate) => {
   const res = await Course.findAll({
-    attributes: ['code', 'name', 'coursetypecode'],
+    attributes: ['code', 'name', 'coursetypecode', 'substitutions'],
     include: [
       {
         required: true,
@@ -844,6 +842,14 @@ const parseCreditInfo = credit => ({
   date: credit.attainment_date,
 })
 
+const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+
+const unifyOpenUniversity = code => {
+  const regexresult = isOpenUniCourseCode(code)
+  if (!regexresult) return code
+  return regexresult[1]
+}
+
 const bottlenecksOf = async (query, studentnumberlist) => {
   const isValidRequest = async (query, params) => {
     const {
@@ -855,6 +861,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
       nondegreeStudents,
       transferredStudents,
     } = params
+
     if (!query.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
       return { error: 'Semester should be either SPRING OR FALL' }
     }
@@ -885,6 +892,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
     }
     return null
   }
+
   const getStudentsAndCourses = async (selectedStudents, studentnumberlist) => {
     if (!studentnumberlist) {
       const {
@@ -945,13 +953,15 @@ const bottlenecksOf = async (query, studentnumberlist) => {
 
   for (const course of courses) {
     let { course_type } = course
-    const substitutions = allCodeAltenatives(course.code)
     let maincourse = course
+    if (isOpenUniCourseCode) {
+      const nonOpen = unifyOpenUniversity(course.code)
+      const response = await findOneByCode(nonOpen)
 
-    if (course.code !== substitutions[0]) {
-      maincourse = await findOneByCode(course.code)
+      if (response) {
+        maincourse = response
+      }
     }
-
     if (!stats[maincourse.code]) {
       stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudentslength)
     }
@@ -959,6 +969,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
     const coursestats = stats[maincourse.code]
 
     coursestats.addCourseType(course_type.coursetypecode, course_type.name)
+    coursestats.addCourseSubstitutions(course.substitutions)
     bottlenecks.coursetypes[course_type.coursetypecode] = course_type.name
 
     course.credits.forEach(credit => {

services/backend/src/services/programmeModules.js

@@ -1,13 +1,13 @@
 const Sequelize = require('sequelize')
 const { ExcludedCourse } = require('../models/models_kone')
 const { Op } = Sequelize
+const logger = require('../util/logger')
 
-const { dbConnections: sisConnections } = require('../database/connection')
-const { sequelizeKone } = require('../database/connection')
+const { dbConnections } = require('../database/connection')
 
 const recursivelyGetModuleAndChildren = async (code, type) => {
   // TODO use only sis connection once sis is stable
-  const connection = sisConnections.established ? sisConnections.sequelize : sequelizeKone
+  const connection = dbConnections.sequelize
   try {
     const [result] = await connection.query(
       `WITH RECURSIVE children as (
@@ -23,15 +23,14 @@ const recursivelyGetModuleAndChildren = async (code, type) => {
     )
     return result
   } catch (e) {
-    console.error('Error when searching with code: ', code)
-    console.error('Details: ', e)
+    logger.error(`Error when searching modules and children with code: ${code}`)
     return []
   }
 }
 
 const byProgrammeCode = async code => {
   const result = await recursivelyGetModuleAndChildren(code, 'course')
-
+  if (!result.length) return []
   const excluded = await ExcludedCourse.findAll({
     where: {
       programme_code: {

services/backend/src/services/statMerger.js

@@ -82,7 +82,6 @@ const populationStudentsMerger = multiyearstudents => {
   samples.courses = []
   samples.elementdetails = {}
   samples.elementdetails.data = {}
-  samples.elementdetails.degrees = []
   samples.elementdetails.programmes = []
   samples.transfers = {}
   samples.transfers.targets = {}
@@ -94,7 +93,6 @@ const populationStudentsMerger = multiyearstudents => {
     samples.semesters = samples.semesters.concat(year.semesters)
     samples.courses = samples.courses.concat(year.courses)
     samples.elementdetails.data = Object.assign(samples.elementdetails.data, year.elementdetails.data)
-    samples.elementdetails.degrees = samples.elementdetails.degrees.concat(year.elementdetails.degrees)
     samples.elementdetails.programmes = samples.elementdetails.programmes.concat(year.elementdetails.programmes)
     samples.transfers.targets = { ...samples.transfers.targets, ...year.transfers.targets }
     samples.transfers.sources = { ...samples.transfers.sources, ...year.transfers.sources }

services/backend/src/services/students.js

@@ -11,9 +11,11 @@ const {
   StudyrightElement,
   ElementDetail,
   SemesterEnrollment,
+  Semester,
 } = require('../models')
 const { TagStudent, Tag } = require('../models/models_kone')
 const Op = Sequelize.Op
+const logger = require('../util/logger')
 
 const createStudent = student => Student.create(student)
 
@@ -75,6 +77,21 @@ const byId = async id => {
       },
     },
   })
+
+  const semesters = await Semester.findAll()
+
+  const mappedEnrollments = student.semester_enrollments.map(enrollment => {
+    const semester = semesters.find(sem => sem.semestercode === enrollment.semestercode)
+    return {
+      ...enrollment.dataValues,
+      name: semester.name,
+      yearname: semester.yearname,
+      startYear: semester.startYear,
+    }
+  })
+
+  student.semester_enrollments = mappedEnrollments
+
   student.tags = tags.map(t => ({
     ...t.get(),
     programme: tagprogrammes.find(p => p.code === t.tag.studytrack),
@@ -161,19 +178,25 @@ const formatStudent = ({
       },
       date: attainment_date,
       course_code: course.code,
-      passed: Credit.passed({ credittypecode }),
+      passed: Credit.passed({ credittypecode }) || Credit.improved({ credittypecode }),
       grade,
       credits,
+      credittypecode,
       isStudyModuleCredit: isStudyModule,
     }
   }
 
   studyrights = studyrights || []
   semester_enrollments = semester_enrollments || []
-  const semesterenrollments = semester_enrollments.map(({ semestercode, enrollmenttype }) => ({
-    semestercode,
-    enrollmenttype,
-  }))
+  const semesterenrollments = semester_enrollments.map(
+    ({ semestercode, enrollmenttype, name, yearname, startYear }) => ({
+      yearname,
+      name,
+      startYear,
+      semestercode,
+      enrollmenttype,
+    })
+  )
 
   const courseByDate = (a, b) => {
     return moment(a.attainment_date).isSameOrBefore(b.attainment_date) ? -1 : 1
@@ -211,7 +234,7 @@ const withId = async id => {
     const result = await byId(id)
     return formatStudent(result)
   } catch (e) {
-    console.log(e)
+    logger.error({ message: 'error when fetching single student', meta: e })
     return {
       error: e,
     }
@@ -307,16 +330,6 @@ const filterStudentnumbersByAccessrights = async (studentnumbers, codes) => {
   return students.map(student => student.studentnumber)
 }
 
-const hasEnrolledForSemester = async (studentnumber, semestercode) => {
-  const enrollment = await SemesterEnrollment.findOne({
-    where: {
-      studentnumber,
-      semestercode,
-    },
-  })
-  return !!(enrollment && enrollment.enrollmenttype === 1)
-}
-
 module.exports = {
   withId,
   bySearchTerm,
@@ -327,5 +340,4 @@ module.exports = {
   findByCourseAndSemesters,
   findByTag,
   splitByEmptySpace,
-  hasEnrolledForSemester,
 }

services/backend/src/services/studyGuidanceGroups.js

@@ -0,0 +1,28 @@
+const { getImporterClient } = require('../util/importerClient')
+const importerClient = getImporterClient()
+
+const getAllStudentsUserHasInGroups = async sisPersonId => {
+  if (!importerClient) {
+    return new Set()
+  }
+  const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
+  return new Set(
+    Object.values(data)
+      .map(group => group.members)
+      .flat()
+      .map(member => member.personStudentNumber)
+  )
+}
+
+const getAllGroupsAndStudents = async sisPersonId => {
+  if (!importerClient) {
+    return []
+  }
+  const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
+  return Object.values(data)
+}
+
+module.exports = {
+  getAllStudentsUserHasInGroups,
+  getAllGroupsAndStudents,
+}

services/backend/src/services/studyprogramme.js

@@ -268,7 +268,17 @@ const productivityStatsForStudytrack = async (studytrack, since) => {
   const year = 1950
   const startDate = `${year}-${semesterStart['FALL']}`
   const endDate = `${moment(new Date(), 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`
-  const studentnumbers = await studentnumbersWithAllStudyrightElements([studytrack], startDate, endDate, false, true)
+  // This includes ALL students: exchange students, the ones that have transferred to program, the ones
+  // with non-degree studyright and the ones that have cancelled their studyright
+  const studentnumbers = await studentnumbersWithAllStudyrightElements(
+    [studytrack],
+    startDate,
+    endDate,
+    true, // exchange students
+    true, // cancelled students
+    true, // non-degree students
+    true // transferred to students
+  )
   const promises = [
     graduatedStatsForStudytrack(studytrack, since),
     productivityStatsForProvider(providercode, since),
@@ -375,7 +385,9 @@ const graduationsFromClass = async (studentnumbers, studytrack) => {
 const genderCodeToValue = code => {
   if (code === '1') return 'Male'
   if (code === '2') return 'Female'
-  return 'Other'
+  if (code === '3') return 'Other'
+  if (code === '0') return 'Unknown'
+  return 'Unknown'
 }
 
 const gendersFromClass = async studentnumbers => {
@@ -430,22 +442,6 @@ const tranferredToStudyprogram = async (studentnumbers, startDate, studytrack, e
   })
 }
 
-const transferredFromStudyprogram = async (studentnumbers, startDate, studytrack, endDate) => {
-  return await Transfer.count({
-    where: {
-      studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-      transferdate: {
-        [Op.between]: [startDate, endDate],
-      },
-      sourcecode: studytrack,
-    },
-    distinct: true,
-    col: 'studentnumber',
-  })
-}
-
 const formatCreditsForProductivity = credits => {
   return credits.map(formatCredit).reduce(function (acc, curr) {
     var key = curr['year']
@@ -522,7 +518,17 @@ const startedStudyright = async (studentnumbers, startDate, studytrack, endDate)
 
 const optionData = async (startDate, endDate, code, level) => {
   const programmes = await getAllProgrammes()
-  const students = await studentnumbersWithAllStudyrightElements([code], startDate, endDate, false, true)
+  // This includes ALL students: exchange students, the ones that have transferred to program, the ones
+  // with non-degree studyright and the ones that have cancelled their studyright
+  const students = await studentnumbersWithAllStudyrightElements(
+    [code],
+    startDate,
+    endDate,
+    true, // exchange students
+    true, // cancelled students
+    true, // non-degree students
+    true // transferred to students
+  )
 
   let graduated
   let currentExtent
@@ -643,7 +649,6 @@ const statsForClass = async (studentnumbers, startDate, studyprogramme, endDate)
     gendersFromClass(studentnumbers),
     tranferredToStudyprogram(studentnumbers, startDate, studyprogramme, endDate),
     nationalitiesFromClass(studentnumbers),
-    transferredFromStudyprogram(studentnumbers, startDate, studyprogramme, new Date()),
     cancelledStudyright(studentnumbers, startDate, studyprogramme, endDate),
     startedStudyright(studentnumbers, startDate, studyprogramme, endDate),
   ])
@@ -679,7 +684,6 @@ const throughputStatsForStudytrack = async (studyprogramme, since) => {
     inTargetTime: 0,
     transferred: 0,
     nationalities: {},
-    transferredFrom: 0,
     cancelled: 0,
     started: 0,
   }
@@ -712,15 +716,17 @@ const throughputStatsForStudytrack = async (studyprogramme, since) => {
           [studyprogramme, curr],
           startDate,
           endDate,
-          false,
-          true
+          true, // exchange students
+          true, // cancelled students
+          true, // non-degree students
+          true // transferred to students
         )
         const creditsForStudyprogramme = await productivityCreditsFromStudyprogrammeStudents(
           studyprogramme,
           startDate,
           studentnumbers
         )
-        const [credits, graduated, theses, genders, transferredTo, nationalities, transferredFrom, cancelled, started] =
+        const [credits, graduated, theses, genders, transferredTo, nationalities, cancelled, started] =
           await statsForClass(studentnumbers, startDate, studyprogramme, endDate)
 
         delete genders[null]
@@ -753,7 +759,6 @@ const throughputStatsForStudytrack = async (studyprogramme, since) => {
             inTargetTime: 0,
             transferred: 0,
             nationalities: {},
-            transferredFrom: 0,
             cancelled: 0,
             started: 0,
           }
@@ -782,7 +787,6 @@ const throughputStatsForStudytrack = async (studyprogramme, since) => {
         stTotals[curr].medianGraduationTime = median(allGraduationTimes)
         stTotals[curr].inTargetTime = stTotals[curr].inTargetTime + inTargetTime
         stTotals[curr].transferred = stTotals[curr].transferred + transferredTo
-        stTotals[curr].transferredFrom += transferredFrom
         stTotals[curr].cancelled += cancelled
         stTotals[curr].started += started
         return {
@@ -800,7 +804,6 @@ const throughputStatsForStudytrack = async (studyprogramme, since) => {
             creditValues,
             transferred: transferredTo,
             nationalities,
-            transferredFrom,
             cancelled,
             started,
           },
@@ -811,15 +814,17 @@ const throughputStatsForStudytrack = async (studyprogramme, since) => {
         [studyprogramme],
         startDate,
         endDate,
-        false,
-        true
+        true, // exchange students
+        true, // cancelled students
+        true, // non-degree students
+        true // transferred to students
       )
       const creditsForStudyprogramme = await productivityCreditsFromStudyprogrammeStudents(
         studyprogramme,
         startDate,
         studentnumbers
       )
-      const [credits, graduated, theses, genders, transferredTo, nationalities, transferredFrom, cancelled, started] =
+      const [credits, graduated, theses, genders, transferredTo, nationalities, cancelled, started] =
         await statsForClass(studentnumbers, startDate, studyprogramme, endDate)
       // theres so much shit in the data that transefferFrom doesnt rly mean anything
       delete genders[null]
@@ -857,7 +862,6 @@ const throughputStatsForStudytrack = async (studyprogramme, since) => {
       totals.medianGraduationTime = median(allGraduationTimes)
       totals.inTargetTime = totals.inTargetTime + inTargetTime
       totals.transferred = totals.transferred + transferredTo
-      totals.transferredFrom += transferredFrom
       totals.cancelled += cancelled
       totals.started += started
       return {
@@ -873,7 +877,6 @@ const throughputStatsForStudytrack = async (studyprogramme, since) => {
         creditValues,
         transferred: transferredTo,
         nationalities,
-        transferredFrom,
         cancelled,
         started,
         studytrackdata,

services/backend/src/services/studyrights.js

@@ -3,15 +3,11 @@ const {
 } = require('../database/connection')
 const { Studyright, StudyrightElement, ElementDetail } = require('../models')
 const { getUserElementDetails } = require('./userService')
-const { hasEnrolledForSemester } = require('./students')
-const { getCurrentSemester } = require('./semesters')
 const moment = require('moment')
 const { redisClient } = require('../services/redis')
 const _ = require('lodash')
 const { Op, col, where, fn } = require('sequelize')
 
-const createStudyright = apiData => Studyright.create(apiData)
-
 const REDIS_KEY = 'STUDYRIGHT_ASSOCIATIONS_V2'
 
 const byStudent = studentNumber => {
@@ -24,97 +20,6 @@ const byStudent = studentNumber => {
   })
 }
 
-const getActiveStudyrightElementsOfCodeBeforeDate = async (code, date) =>
-  await StudyrightElement.findAll({
-    where: {
-      code,
-      enddate: {
-        [Op.gte]: date,
-      },
-    },
-  })
-
-const getActiveStudyrightsFromIdsBeforeDate = async (studyrightIds, date) =>
-  await Studyright.findAll({
-    where: {
-      graduated: 0,
-      studyrightid: {
-        [Op.in]: studyrightIds,
-      },
-      canceldate: null,
-      enddate: {
-        [Op.gte]: date,
-      },
-    },
-  })
-
-const nonGraduatedStudentsOfElementDetail = async code => {
-  const today = new Date()
-  const currentSemesterCode = (await getCurrentSemester()).semestercode
-
-  // Get studyrights of target code that haven't ended yet
-  const studyrightElements = await getActiveStudyrightElementsOfCodeBeforeDate(code, today)
-  const studentToDatesMap = {}
-
-  studyrightElements.forEach(({ startdate, enddate, studentnumber }) => {
-    studentToDatesMap[studentnumber] = {
-      startdate,
-      enddate,
-    }
-  })
-
-  const studyrights = await getActiveStudyrightsFromIdsBeforeDate(
-    studyrightElements.map(sE => sE.studyrightid),
-    today
-  )
-
-  // Filter out students that are in new master's programmes,
-  // see if student is currently enrolled and format the result
-  // to shape of { year1: [studentnumbers], ... }
-  const result = {}
-  const studentnumbers = new Set()
-  const studentsToBeFiltered = new Set()
-  await Promise.all(
-    studyrights.map(
-      ({ studentStudentnumber: student_studentnumber, studyrightid }) =>
-        new Promise(async res => {
-          const [enrolled, studentElementDetails] = await Promise.all([
-            await hasEnrolledForSemester(student_studentnumber, currentSemesterCode),
-            await StudyrightElement.findAll({
-              where: {
-                studyrightid,
-              },
-            }),
-          ])
-
-          // If student is in new master's programme,
-          // then don't include them in the result
-          if (studentElementDetails.find(e => e.code.match(/^M[A-Z]*[0-9]*_[0-9]*$/))) {
-            studentsToBeFiltered.add(student_studentnumber)
-            return res()
-          }
-
-          if (studentnumbers.has(student_studentnumber)) return res()
-          const year = moment(studentToDatesMap[student_studentnumber].startdate).tz('Europe/Helsinki').year()
-          if (!result[year]) result[year] = []
-          studentnumbers.add(student_studentnumber)
-          result[year].push({
-            studentNumber: student_studentnumber,
-            enrolled,
-          })
-          res()
-        })
-    )
-  )
-
-  // Filter out some special cases
-  Object.keys(result).forEach(year => {
-    result[year] = result[year].filter(({ studentNumber }) => !studentsToBeFiltered.has(studentNumber))
-  })
-
-  return [result, [...studentnumbers]]
-}
-
 const studentNumbersWithAllStudyRightElements = async (codes, startedAfter, startedBefore) => {
   const studyrights = await StudyrightElement.findAll({
     attributes: ['studentnumber'],
@@ -244,17 +149,6 @@ const getStudyrightElementsAndAssociationsForUser = async username => {
   return formatStudyrightElements(studyrightelementcodes, associations)
 }
 
-const getAllDegreesAndProgrammes = async () => {
-  const elementDetails = ElementDetail.findAll({
-    where: {
-      type: {
-        [Op.in]: [10, 20],
-      },
-    },
-  })
-  return formatStudyrightElements(elementDetails)
-}
-
 const getAllProgrammes = async () => {
   const elementDetails = ElementDetail.findAll({
     where: {
@@ -297,7 +191,6 @@ const associatedStudyrightElements = async (offset, limit) => {
 }
 
 const StudyRightType = {
-  DEGREE: 10,
   PROGRAMME: 20,
   STUDYTRACK: 30,
 }
@@ -314,22 +207,15 @@ const calculateAssociationsFromDb = async (chunksize = 100000) => {
   }
   const total = await Studyright.count()
   let offset = 0
-  const types = new Set([StudyRightType.DEGREE, StudyRightType.PROGRAMME, StudyRightType.STUDYTRACK])
+  const types = new Set([StudyRightType.PROGRAMME, StudyRightType.STUDYTRACK])
   const isValid = ({ type }) => types.has(type)
-  const associations = { programmes: {}, degrees: {}, studyTracks: {} }
+  const associations = { programmes: {}, studyTracks: {} }
   while (offset <= total) {
     const elementgroups = await associatedStudyrightElements(offset, chunksize)
     elementgroups.forEach(fullgroup => {
       const group = fullgroup.filter(isValid)
       group.forEach(({ type, code, name, studyrightid, startdate, enddate }) => {
-        if (type === StudyRightType.DEGREE) {
-          associations.degrees[code] = associations.degrees[code] || {
-            type,
-            name,
-            code,
-            programmes: {},
-          }
-        } else if (type === StudyRightType.STUDYTRACK) {
+        if (type === StudyRightType.STUDYTRACK) {
           associations.studyTracks[code] = associations.studyTracks[code] || {
             type,
             name,
@@ -348,7 +234,6 @@ const calculateAssociationsFromDb = async (chunksize = 100000) => {
           const enrollment = momentstartdate.isValid() ? getEnrollmentStartYear(momentstartdate) : null
           const enrollmentStartYears = associations.programmes[code].enrollmentStartYears
           enrollmentStartYears[enrollment] = enrollmentStartYears[enrollment] || {
-            degrees: {},
             studyTracks: {},
           }
           const enrollmentStartYear = enrollmentStartYears[enrollment]
@@ -356,24 +241,7 @@ const calculateAssociationsFromDb = async (chunksize = 100000) => {
           group
             .filter(e => e.studyrightid === studyrightid && e.code !== code)
             .forEach(e => {
-              if (e.type === StudyRightType.DEGREE) {
-                enrollmentStartYear.degrees[e.code] = {
-                  type: e.type,
-                  name: e.name,
-                  code: e.code,
-                }
-                associations.degrees[e.code] = associations.degrees[e.code] || {
-                  type: e.type,
-                  name: e.name,
-                  code: e.code,
-                  programmes: {},
-                }
-                associations.degrees[e.code].programmes[code] = {
-                  type: type,
-                  name: name,
-                  code: code,
-                }
-              } else if (e.type === StudyRightType.STUDYTRACK) {
+              if (e.type === StudyRightType.STUDYTRACK) {
                 const momentenddate = moment(enddate)
                 const estartdate = moment(e.startdate)
                 const eenddate = moment(e.enddate)
@@ -439,23 +307,16 @@ const getAssociations = async (doRefresh = false) => {
 }
 
 const getFilteredAssociations = async codes => {
-  console.log(codes)
   const associations = await getAssociations()
   associations.programmes = _.pick(associations.programmes, codes)
 
-  const degrees = []
   const studyTracks = []
   Object.keys(associations.programmes).forEach(k => {
     Object.keys(associations.programmes[k].enrollmentStartYears).forEach(year => {
       const yearData = associations.programmes[k].enrollmentStartYears[year]
-      degrees.push(...Object.keys(yearData.degrees))
       studyTracks.push(...Object.keys(yearData.studyTracks))
     })
   })
-  associations.degrees = _.pick(associations.degrees, degrees)
-  Object.keys(associations.degrees).forEach(k => {
-    associations.degrees[k].programmes = _.pick(associations.degrees[k].programmes, codes)
-  })
   associations.studyTracks = _.pick(associations.studyTracks, studyTracks)
   Object.keys(associations.studyTracks).forEach(k => {
     associations.studyTracks[k].programmes = _.pick(associations.studyTracks[k].programmes, codes)
@@ -471,17 +332,14 @@ const getUserAssociations = async userid => {
 
 module.exports = {
   byStudent,
-  createStudyright,
   studentNumbersWithAllStudyRightElements,
   getAssociatedStudyrights,
   getAllStudyrightElementsAndAssociations,
   getStudyrightElementsAndAssociationsForUser,
-  getAllDegreesAndProgrammes,
   getAssociations,
   getFilteredAssociations,
   getUserAssociations,
   refreshAssociationsInRedis,
   getAllProgrammes,
   getAllElementDetails,
-  nonGraduatedStudentsOfElementDetail,
 }

services/backend/src/services/tester.js

@@ -1,59 +0,0 @@
-const { bottlenecksOf, optimizedStatisticsOf } = require('./populations')
-
-const moment = require('moment')
-
-// eslint-disable-next-line no-unused-vars
-const run2 = async () => {
-  const o = {
-    semesters: ['FALL', 'SPRING'],
-    year: 2017, // 2015
-    studyRights: ['KH10_001'], // ['KH50_005'], // ['320001']
-    months: 16, // 41
-  }
-
-  const result = await bottlenecksOf(o) // eslint-disable-line
-
-  //console.log(JSON.stringify(result.coursestatistics[0].stats, null, 2))
-
-  //console.log(result.coursestatistics.length)
-
-  process.exit()
-}
-
-// eslint-disable-next-line no-unused-vars
-const run1 = async () => {
-  // eslint-disable-line
-  const o = {
-    semesters: ['FALL', 'SPRING'],
-    year: 2018, // 2015
-    studyRights: ['KH50_001'], // ['KH50_005'], // ['320001']
-    months: 41, // 41
-  }
-
-  const a = await optimizedStatisticsOf(o)
-
-  const start = moment('2018-07-31')
-
-  a.students.forEach(s1 => {
-    // eslint-disable-next-line no-constant-condition
-    if (true || s1.studentNumber === '014612688') {
-      const studyright = s1.studyrights.find(s =>
-        s.studyright_elements.map(d => d.element_detail.code).includes('KH50_001')
-      )
-
-      console.log(studyright.startdate, start.isAfter(moment(studyright.startdate)), s1.changedStudyright)
-
-      //console.log(JSON.stringify(studyright, null, 2))
-    }
-  })
-
-  process.exit()
-}
-
-const run3 = async () => {
-  const userStatistics = require('./userService')
-  const resp = await userStatistics.ping()
-  console.log(resp)
-}
-
-run3()

services/backend/src/services/topteachers.js

@@ -1,7 +1,8 @@
 const { redisClient } = require('../services/redis')
-const { getSemestersAndYears, getMaxYearcode } = require('./semesters')
+const { getSemestersAndYears, getCurrentSemester } = require('./semesters')
 const { Teacher, Semester, Credit, Course } = require('../models')
 const { Op } = require('sequelize')
+const logger = require('../util/logger')
 
 const category = (name, rediskey) => ({ name, rediskey })
 
@@ -128,11 +129,13 @@ const findTopTeachers = async yearcode => {
   }
 }
 
-const findAndSaveTeachers = async (startcode = 50, to) => {
-  const endcode = to ? to : await getMaxYearcode()
+const findAndSaveTeachers = async (startcode = 1, to) => {
+  const endcode = to || (await getCurrentSemester()).getDataValue('yearcode')
   for (let code = startcode; code <= endcode; code++) {
     await findAndSaveTopTeachers(code)
+    logger.info(`Teacher leaderboard for yearcode ${code} calculated`)
   }
+  logger.info(`Teacher leaderboard calculations done`)
 }
 
 const findAndSaveTopTeachers = async yearcode => {

services/backend/src/services/trends/index.js

@@ -14,7 +14,6 @@ const CURRENT_DATE = new Date()
 
 const withErr = handler => (req, res, next) =>
   handler(req, res, next).catch(e => {
-    console.error(e)
     res.status(500).json({ error: { message: e.message, stack: e.stack } })
   })
 

services/backend/src/services/trends/statusAttainments.js

@@ -95,7 +95,10 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
     yearRange,
     diff => ({
       from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
-      to: new Date(unixMillis - diff * YEAR_TO_MILLISECONDS),
+      to:
+        showByYear === 'true'
+          ? new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS)
+          : new Date(unixMillis - diff * YEAR_TO_MILLISECONDS),
     }),
     'acc',
     'students'

services/backend/src/services/trends/statusGraduated.js

@@ -56,7 +56,7 @@ const calculateStatusGraduated = async (unixMillis, showByYear) => {
 
   const yearlyRange = diff => ({
     from: new Date(startTime - diff * Y_TO_MS),
-    to: new Date(unixMillis - diff * Y_TO_MS),
+    to: showByYear === 'true' ? new Date(startTime - (diff - 1) * Y_TO_MS) : new Date(unixMillis - diff * Y_TO_MS),
   })
 
   const totalRange = diff => ({

services/backend/src/services/userService.js

@@ -5,6 +5,8 @@ const elementDetailService = require('./elementdetails')
 const { filterStudentnumbersByAccessrights } = require('./students')
 const { userDataCache } = require('./cache')
 const client = axios.create({ baseURL: USERSERVICE_URL, headers: { secret: process.env.USERSERVICE_SECRET } })
+const { getImporterClient } = require('../util/importerClient')
+const logger = require('../util/logger')
 
 const ping = async () => {
   const url = '/ping'
@@ -17,13 +19,27 @@ const findAll = async () => {
   return response.data
 }
 
-const login = async (uid, full_name, hyGroups, affiliations, email) => {
+const checkStudyGuidanceGroupsAccess = async hyPersonSisuId => {
+  if (!hyPersonSisuId) {
+    logger.error('Not possible to get groups without personId header')
+    return false
+  }
+  const importerClient = getImporterClient()
+  if (!importerClient) return false
+  const { data } = await importerClient.get(`/person-groups/person/${hyPersonSisuId}`)
+  return data && Object.values(data).length > 0
+}
+
+const login = async (uid, full_name, hyGroups, affiliations, email, hyPersonSisuId) => {
+  const hasStudyGuidanceGroupAccess = await checkStudyGuidanceGroupsAccess(hyPersonSisuId)
   const response = await client.post('/login', {
     uid,
     full_name,
     hyGroups,
     affiliations,
     email,
+    hyPersonSisuId,
+    hasStudyGuidanceGroupAccess,
   })
   return response.data
 }
@@ -69,10 +85,8 @@ const updateUser = async (uid, fields) => {
 }
 
 const getRolesFor = async user => {
-  console.log('roles for', user)
   const url = `/get_roles/${user}`
   const response = await client.get(url)
-  console.log(response.data)
   return response.data
 }
 

services/backend/src/util/add_json_duplicates_to_redis.js

@@ -1,18 +0,0 @@
-const { setDuplicateCode } = require('../services/courses')
-const newToOld = require('./newToOld.json')
-const oldToNew = require('./oldToNew.json')
-
-const mapJson = async () => {
-  for (const i in newToOld) {
-    await setDuplicateCode(i, newToOld[i])
-    await setDuplicateCode(newToOld[i], i)
-  }
-  for (const i in oldToNew) {
-    await setDuplicateCode(i, oldToNew[i])
-    await setDuplicateCode(oldToNew[i], i)
-  }
-  console.log('Mapped codes from newToOld.json and oldToNew.json')
-  process.exit()
-}
-
-mapJson()

services/backend/src/util/customErrors.js

@@ -0,0 +1,25 @@
+class ApplicationError extends Error {
+  constructor(message, status, extra) {
+    super()
+
+    Error.captureStackTrace(this, this.constructor)
+
+    this.name = this.constructor.name
+
+    this.message = message || 'Something went wrong. Please try again.'
+
+    this.status = status || 500
+
+    this.extra = extra || {}
+  }
+
+  toJSON() {
+    return {
+      error: this.message,
+    }
+  }
+}
+
+module.exports = {
+  ApplicationError,
+}

services/backend/src/util/importerClient.js

@@ -0,0 +1,22 @@
+const axios = require('axios')
+const { importerToken, importerUrl } = require('../conf-backend')
+const logger = require('./logger')
+
+const importerClient = axios.create({
+  headers: {
+    token: importerToken,
+  },
+  baseURL: importerUrl,
+})
+
+const getImporterClient = () => {
+  if (!importerToken || !importerUrl) {
+    logger.error("Importer token or importer url not set, can't return client!")
+    return null
+  }
+  return importerClient
+}
+
+module.exports = {
+  getImporterClient,
+}

services/backend/src/util/index.js

@@ -1,82 +1,5 @@
-const LRU = require('lru-cache')
-const oldToNewMap = require('./oldToNew.json')
-
-const arrayUnique = (value, index, self) => {
-  return self.indexOf(value) === index
-}
-
-const getStudentNumberChecksum = studentNumber => {
-  let checksumNumbers = [7, 3, 1]
-  let checksum = 0
-
-  for (let i = 0; i < studentNumber.length; i++) {
-    // go from end t start
-    let currentNumber = studentNumber[studentNumber.length - (i + 1)]
-    checksum += currentNumber * checksumNumbers[i % checksumNumbers.length]
-  }
-
-  return (10 - (checksum % 10)) % 10
-}
-
-const isValidStudentId = id => {
-  if (/^0\d{8}$/.test(id)) {
-    // is a 9 digit number
-    const multipliers = [7, 1, 3, 7, 1, 3, 7]
-    const checksum = id
-      .substring(1, 8)
-      .split('')
-      .reduce((sum, curr, index) => {
-        return (sum + curr * multipliers[index]) % 10
-      }, 0)
-    return (10 - checksum) % 10 == id[8]
-  }
-  return false
-}
-
-// helpers to print complex sequelizedata
-const plainPrint = sequelizeData => console.log(JSON.parse(JSON.stringify(sequelizeData)))
-
-const plainify = sequelizeData => JSON.parse(JSON.stringify(sequelizeData))
-
-const oldToNew = code => {
-  const mappedCode = oldToNewMap[code]
-  return mappedCode ? mappedCode : code
-}
-
 const validateParamLength = (param, minLength) => param && param.trim().length >= minLength
 
-const isNewHYStudyProgramme = code => !!(code && code.match(/^[A-Z]*[0-9]*_[0-9]*$/))
-
-/**
- * @param {(...args: any[]) => Promise<any>} fn
- * @param {(...args: any[]) => string | number} keyPicker
- * @param {LRU.Options} cacheOpts
- * @returns {(...args: any[]) => Promise<any>}
- */
-const lruMemoize = (fn, keyPicker, cacheOpts) => {
-  const cache = new LRU(cacheOpts)
-
-  return async (...args) => {
-    const key = keyPicker(args)
-    const cached = cache.get(key)
-    if (cached) {
-      return cached
-    }
-
-    const ret = await fn(...args)
-    cache.set(key, ret)
-    return ret
-  }
-}
-
 module.exports = {
-  arrayUnique,
-  getStudentNumberChecksum,
-  plainPrint,
-  plainify,
-  isValidStudentId,
-  oldToNew,
   validateParamLength,
-  isNewHYStudyProgramme,
-  lruMemoize,
 }

services/backend/src/util/init_redis.js

@@ -1,9 +0,0 @@
-const { redisClient } = require('../services/redis')
-
-const initializeDuplicates = async () => {
-  await redisClient.setAsync('duplicates', '{}')
-  console.log('Initiated redis')
-  process.exit()
-}
-
-initializeDuplicates()

services/backend/src/util/logger.js

@@ -1,28 +1,45 @@
 const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
+const { isProduction } = require('../conf-backend')
+
+const { combine, timestamp, printf, splat } = winston.format
 
 const transports = []
 
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
+transports.push(new winston.transports.File({ filename: 'debug.log' }))
+
+if (!isProduction) {
+  const devFormat = printf(
+    ({ level, message, timestamp, ...rest }) => `${timestamp} ${level}: ${message} ${JSON.stringify(rest)}`
+  )
+
   transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'oodikone-backend',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf',
-      },
+    new winston.transports.Console({
+      level: 'debug',
+      format: combine(splat(), timestamp(), devFormat),
     })
   )
 }
 
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
+if (isProduction) {
+  const levels = {
+    error: 0,
+    warn: 1,
+    info: 2,
+    http: 3,
+    verbose: 4,
+    debug: 5,
+    silly: 6,
+  }
 
-transports.push(new winston.transports.Console({ level: 'debug' }))
+  const prodFormat = winston.format.printf(({ level, ...rest }) =>
+    JSON.stringify({
+      level: levels[level],
+      ...rest,
+    })
+  )
+
+  transports.push(new winston.transports.Console({ format: prodFormat }))
+}
 
 const logger = winston.createLogger({ transports })
 

services/backend/src/util/map_course_codes.js

@@ -1,36 +0,0 @@
-const Course = require('../services/courses')
-const { setDuplicateCode } = require('../services/courses')
-const { redisClient } = require('../services/redis')
-
-// two tables of prefixes, for example new prefixes of  Matlu courses:
-// ['CSM', 'DATA', 'FYS', 'MAT', 'TKT', 'KEK']
-// and old prefixes
-// ['5', '7']
-
-const mapCourseCodes = async (newPrefixes, oldPrefixes) => {
-  const all = await redisClient.getAsync('duplicates')
-  if (!all) {
-    await redisClient.setAsync('duplicates', '{}')
-  }
-  const res = await Course.findDuplicates(newPrefixes, oldPrefixes)
-  const mappedCodes = res ? res[0] : []
-  const formattedCodes = mappedCodes.map(r => {
-    return { [r.code1]: r.code2 }
-  })
-  await updateRedis(formattedCodes)
-  console.log('Mapped course codes with similar names with codes starting with ', newPrefixes, ' to  ', oldPrefixes)
-  process.exit()
-}
-
-const updateRedis = async codes => {
-  for (const i in codes) {
-    let code = codes[i]
-    const newCode = Object.keys(code)[0]
-    const oldCode = code[newCode]
-
-    await setDuplicateCode(newCode, oldCode)
-    await setDuplicateCode(oldCode, newCode)
-  }
-}
-
-mapCourseCodes(['CSM', 'TKT', 'MAT', 'FYS', 'KEK'], ['5', '7'])

services/backend/src/util/newToOld.json

@@ -1,114 +0,0 @@
-{
-  "TKT20002": "582101",
-  "TKT20002": "582104",
-  "TKT20001": "58131",
-  "TKT20010": "58161",
-  "TKT10005": "581305",
-  "TKT10004": "581328",
-  "TKT20006": "581259",
-  "TKT20007": "581260",
-  "TKT20005": "582206",
-  "MAT11001": "57033",
-  "TKT50003": "581324",
-  "TKT10002": "581325",
-  "80088": "582514",
-  "TKT10001": "582102",
-  "TKT10003": "582103",
-  "TKT20004": "582202",
-  "TKT20011": "582203",
-  "MAT11002": "57043",
-  "MAT21014": "57160",
-  "TKT21015": "582335",
-  "TKT20003": "582219",
-  "TKT21002": "582733",
-  "MAT21001": "57047",
-  "MAT11003": "57116",
-  "DATA15001": "582216",
-  "MAT21015": "57161",
-  "TKT50005": "582509",
-  "MAT11004": "57117",
-  "FYS2041": "53861",
-  "TKT50002": "582506",
-  "TKT20013": "582204",
-  "TKT50001": "582505",
-  "TKT1": "584111",
-  "TKT21001": "582482",
-  "TKT21013": "582326",
-  "CSM14101": "581358",
-  "CSM14203": "581362",
-  "CSM14103": "582663",
-  "FYS4012": "53905",
-  "CSM14105": "581360",
-  "KEK101": "55052",
-  "65122": "64162",
-  "KEK223": "55059",
-  "MAT21003": "57121",
-  "FYS1001": "530281",
-  "MAT21020": "57122",
-  "CSM13202": "582749",
-  "ON-160": "200017",
-  "770270": "770044",
-  "MAT21012": "57044",
-  "MAT21007": "57101",
-  "MAT21013": "57048",
-  "99501Mat-lu": "99501Kaikki",
-  "57595": "530146",
-  "FYS1002": "530282",
-  "78134": "57714",
-  "582221": "58160",
-  "KEK103": "55053",
-  "KEK110": "55395",
-  "68153Mat": "68153Fys",
-  "PED111": "68152",
-  "MFK-403": "57169",
-  "55521": "55056",
-  "FYS1013": "53398",
-  "FYS2010": "53723",
-  "MFK-M303": "57124",
-  "FYS2011": "53724",
-  "MFK-M305": "57136",
-  "FYS2083": "530177",
-  "530148": "300123",
-  "55328": "55055",
-  "FYS2013": "53726",
-  "FYS2012": "53725",
-  "200230_WT": "200230T",
-  "90093": "3750501",
-  "90096": "3750502",
-  "20012_WasaT": "20012",
-  "FYS2077": "530289",
-  "200250_WasaT": "200250",
-  "55306": "55077",
-  "50031": "30300",
-  "53112": "530217",
-  "ME-004": "80144",
-  "992993": "99299",
-  "200251_WasaT": "200251",
-  "200203C_WT": "200203C",
-  "MAT11005": "57118",
-  "MAT21002": "57119",
-  "MAT21005": "57152",
-  "590190": "50099",
-  "FYS1003": "530283",
-  "FYS1010": "53704",
-  "KK-ENVALT": "KK-ENMALU",
-  "KK-RUMAME": "KK-RUMALU",
-  "FYS1011": "53705",
-  "57084": "50000",
-  "FYS2001": "530286",
-  "KK-ENMALU1": "KK-ENLAAK1",
-  "20016T": "20016",
-  "20014T": "20014",
-  "FYS2004": "530000",
-  "BIO-104": "52073",
-  "FYS2031": "535026",
-  "ON-330": "MAT20001",
-  "99291Oik": "99291Mat-lu",
-  "KK-ENMALU2": "KK-ENLAAK2",
-  "FYS2085": "53399",
-  "PROV-907": "590367",
-  "790Y330": "732290",
-  "KEK225": "55075",
-  "68207": "68166",
-  "TKT20008": "DATA15001"
-}